def get_gomofs(date_time, lat, lon, depth='bottom', mindistance=20): # JiM's simple version for bottom temp """ JiM's simplified version of Lei Zhao's function the format time(GMT) is: datetime.datetime(2019, 2, 27, 11, 56, 51, 666857) lat and lon use decimal degrees return the temperature of specify location HARDCODED TO RETURN BOTTOM TEMP """ rho_index = 0 # for bottom layer if depth == 99999: depth = 'bottom' rho_index = 0 if not gomofs_coordinaterange(lat, lon): print('lat and lon out of range in gomofs') return np.nan if date_time < datetime.datetime.strptime('2018-07-01 00:00:00', '%Y-%m-%d %H:%M:%S'): print('Time out of range, time start :2018-07-01 00:00:00z') return np.nan if date_time > datetime.datetime.utcnow() + datetime.timedelta( days=3): #forecast time under 3 days print('forecast time under 3 days') return np.nan #start download data if (datetime.datetime.utcnow() - date_time) < datetime.timedelta(days=25): url = get_gomofs_url( date_time) #this url get data within 25 days recently else: url = get_gomofs_url_new(date_time) #this url get data 25 days ago nc = netCDF4.Dataset(str(url)) gomofs_lons = nc.variables['lon_rho'][:] gomofs_lats = nc.variables['lat_rho'][:] gomofs_rho = nc.variables['s_rho'] gomofs_h = nc.variables['h'] gomofs_temp = nc.variables['temp'] #caculate the index of the nearest four points using a "find_nd" function in Lei Zhao's conversion module target_distance = 2 * zl.dist(lat1=gomofs_lats[0][0], lon1=gomofs_lons[0][0], lat2=gomofs_lats[0][1], lon2=gomofs_lons[0][1]) eta_rho, xi_rho = zl.find_nd(target=target_distance, lat=lat, lon=lon, lats=gomofs_lats, lons=gomofs_lons) if zl.dist(lat1=lat, lon1=lon, lat2=gomofs_lats[eta_rho][xi_rho], lon2=gomofs_lons[eta_rho][xi_rho]) > mindistance: print('THE location is out of range') return np.nan temperature = gomofs_temp[0][rho_index][eta_rho][xi_rho] depth = gomofs_h[eta_rho][xi_rho] #temperature=float(gomofs_temp[0,rho_index,eta_rho,xi_rho].data) return temperature, depth
def find_nd(target,lat,lon,lats,lons): """ Bisection method:find the index of nearest distance""" row=0 maxrow=len(lats)-1 col=len(lats[0])-1 while col>=0 and row<=maxrow: distance=zl.dist(lat1=lats[row,col],lat2=lat,lon1=lons[row,col],lon2=lon) if distance<=target: break elif abs(lats[row,col]-lat)<abs(lons[row,col]-lon): col-=1 else: row+=1 distance=zl.dist(lat1=lats[row,col],lat2=lat,lon1=lons[row,col],lon2=lon) row_md,col_md=row,col #row_md the row of minimum distance #avoid row,col out of range in next step if row<3: row=3 if col<3: col=3 if row>maxrow-3: row=maxrow-3 if col>len(lats[0])-4: col=len(lats[0])-4 for i in range(row-3,row+3,1): for j in range(col-3,col+3,1): distance_c=zl.dist(lat1=lats[i,j],lat2=lat,lon1=lons[i,j],lon2=lon) if distance_c<=distance: distance=distance_c row_md,col_md=i,j return row_md,col_md
def get_doppio(lat, lon, time, depth): """ notice: the format of time is like "%Y-%m-%d %H:%M:%S" the default depth is under the bottom depth the module only output the temperature of point location """ time = dt.strptime(time, '%Y-%m-%d %H:%M:%S') # transform time format if (time - datetime.datetime(2017, 11, 1, 0, 0, 0)).total_seconds() < 0: print('the date can\'t be earlier than 2017-11-1') return np.nan url_time = time.strftime('%Y-%m-%d') # url = get_doppio_url(url_time) nc = netCDF4.Dataset(url).variables #first find the index of the grid lons = nc['lon_rho'][:] lats = nc['lat_rho'][:] temp = nc['temp'] #second find the index of time doppio_time = nc['time'] itime = netCDF4.date2index(time, doppio_time, select='nearest') # where startime in datetime # figure out layer from depth min_distance = zl.dist(lat1=lat, lon1=lon, lat2=lats[0][0], lon2=lons[0][0]) index_1, index_2 = 0, 0 for i in range(len(lons)): for j in range(len(lons[i])): if min_distance > zl.dist( lat1=lat, lon1=lon, lat2=lats[i][j], lon2=lons[i][j]): min_distance = zl.dist(lat1=lat, lon1=lon, lat2=lats[i][j], lon2=lons[i][j]) index_1 = i index_2 = j doppio_depth = nc['h'][index_1][index_2] if depth > doppio_depth: # case of bottom S_coordinate = 1 else: S_coordinate = float(depth) / float(doppio_depth) if 0 <= S_coordinate < 1: doppio_temp = temp[ itime, 39 - int(S_coordinate / 0.025), index_1, index_2] # because there are 0.025 between each later elif S_coordinate == 1: doppio_temp = temp[itime][0][index_1][index_2] else: doppio_temp = temp[itime][0][index_1][index_2] return doppio_temp
def get_gomofs_zl(date_time,lat,lon,depth='bottom',mindistance=20,autocheck=True,fortype='temperature'): """ the format time(GMT) is: datetime.datetime(2019, 2, 27, 11, 56, 51, 666857) lat and lon use decimal degrees if the depth is under the water, please must add the marker of '-' input time,lat,lon,depth return the temperature of specify location (or return temperature,nc,rho_index,ocean_time_index) the unit is mile of distance return the temperature of specify location """ #print(depth) if depth==99999: depth='bottom' if not gomofs_coordinaterange(lat,lon): print('lat and lon out of range in gomofs') return np.nan if date_time<datetime.datetime.strptime('2018-07-01 00:00:00','%Y-%m-%d %H:%M:%S'): print('Time out of range, time start :2018-07-01 00:00:00z') return np.nan if date_time>datetime.datetime.now()+datetime.timedelta(days=3): #forecast time under 3 days print('forecast time under 3 days') return np.nan #start download data forecastdate=date_time #forecast time equal input date_time changefile,filecheck=1,1 #changefile means whether we need to change the file to get data, filecheck means check the file exist or not. while(changefile==1): count=1 while(filecheck==1): #download the data try: if forecastdate==date_time: #the forcastdate is input date_time, if the date_time changed yet,we will use the forecast data url=get_gomofs_url(date_time) nc=netCDF4.Dataset(str(url)) #print('download nowcast data.') else: url=get_gomofs_url_forecast(date_time,forecastdate) nc=netCDF4.Dataset(str(url)) #print('download forecast data.') filecheck,readcheck=0,1 # if the file is there, filecheck=0,readcheck use to check the file whether read successfully except OSError: try: url=get_gomofs_url_forecast(date_time,forecastdate) nc=netCDF4.Dataset(str(url)) #print('download forecast data.') filecheck,readcheck=0,1 except OSError: date_time=date_time-datetime.timedelta(hours=6) if (forecastdate-date_time)>datetime.timedelta(days=3): #every file only have 3 days data. print('please check the website or file is exist!') print(url) return np.nan except: return np.nan except: return np.nan # print('start read data.') while(readcheck==1): #read data, if readcheck==1 start loop try: while True: #print('connecting the web.') if zl.isConnected(address=url): break print('check the website is well or internet is connected?') time.sleep(5) gomofs_lons=nc.variables['lon_rho'][:] gomofs_lats=nc.variables['lat_rho'][:] gomofs_rho=nc.variables['s_rho'] #gomofs_h=nc.variables['h'] gomofs_h=nc.variables['h'][:] gomofs_temp=nc.variables['temp'] readcheck,changefile=0,0 #if read data successfully, we do not need to loop # print('end read data.') except RuntimeError: count=count+1 if count>8: if autocheck==True: return np.nan while True: print('it will return nan, if you do not need read again.') cmd = input("whether need read again(y/n)?:") if cmd.lower() == "y": count=1 break elif cmd.lower() == "n": cmd2 = input("whether need change file(y/n)?:") if cmd2.lower()=="y": date_time=date_time-datetime.timedelta(hours=6) readcheck,filecheck=0,1 break else: print('interrupt read data.') return np.nan else: break time.sleep(20) #every time to reread data need stop 20s print('the '+str(int(count))+' times to read data.') except: return np.nan #caculate the index of the nearest four points target_distance=2*zl.dist(lat1=gomofs_lats[0][0],lon1=gomofs_lons[0][0],lat2=gomofs_lats[0][1],lon2=gomofs_lons[0][1]) eta_rho,xi_rho=zl.find_nd(target=target_distance,lat=lat,lon=lon,lats=gomofs_lats,lons=gomofs_lons) if zl.dist(lat1=lat,lon1=lon,lat2=gomofs_lats[eta_rho][xi_rho],lon2=gomofs_lons[eta_rho][xi_rho])>mindistance: print('THE location is out of range') return np.nan # estimate the bottom depth of point location if eta_rho==0: eta_rho=1 if eta_rho==len(gomofs_lats)-1: eta_rho=len(gomofs_lats)-2 if xi_rho==len(gomofs_lats[0])-1: eta_rho=len(gomofs_lats[0])-2 # print('start caculate the bottom depth of point location!') while True: points_h=[[gomofs_lats[eta_rho][xi_rho],gomofs_lons[eta_rho][xi_rho],gomofs_h[eta_rho,xi_rho]], [gomofs_lats[eta_rho,(xi_rho-1)],gomofs_lons[eta_rho,(xi_rho-1)],gomofs_h[eta_rho,(xi_rho-1)]], [gomofs_lats[eta_rho,(xi_rho+1)],gomofs_lons[eta_rho,(xi_rho+1)],gomofs_h[eta_rho,(xi_rho+1)]], [gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_h[(eta_rho-1),xi_rho]], [gomofs_lats[(eta_rho+1),xi_rho],gomofs_lons[(eta_rho+1),xi_rho],gomofs_h[(eta_rho+1),xi_rho]]] break point_h=zl.fitting(points_h,lat,lon) # caculate the rho index if depth=='bottom': rho_index=0 else: distance_h=gomofs_rho[0]*point_h-depth for k in range(len(gomofs_rho)): if abs(distance_h)>=abs(gomofs_rho[k]*point_h-depth): distance_h=gomofs_rho[k]*point_h-depth rho_index=k #estimate the temperature of point location while True: points_temp=[[gomofs_lats[eta_rho,xi_rho],gomofs_lons[eta_rho,xi_rho],gomofs_temp[0][rho_index][eta_rho][xi_rho]], [gomofs_lats[eta_rho,(xi_rho-1)],gomofs_lons[eta_rho,(xi_rho-1)],gomofs_temp[0][rho_index][eta_rho,(xi_rho-1)]], [gomofs_lats[eta_rho,(xi_rho+1)],gomofs_lons[eta_rho,(xi_rho+1)],gomofs_temp[0][rho_index][eta_rho][(xi_rho+1)]], [gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_temp[0][rho_index][(eta_rho-1)][xi_rho]], [gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_temp[0][rho_index][(eta_rho-1)][xi_rho]]] break temperature=zl.fitting(points_temp,lat,lon) # if input depth out of the bottom, print the prompt message if depth!='bottom': if abs(point_h)<abs(depth): print ("the depth is out of the bottom:"+str(point_h)) return np.nan if fortype=='tempdepth': return temperature,point_h else: return temperature
def get_doppio(lat=0,lon=0,depth='bottom',time='2018-11-12 12:00:00',fortype='temperature'): """ notice: the format of time is like "%Y-%m-%d %H:%M:%S" this time is utctime or it can also be datetime the depth is under the bottom depth the module only output the temperature of point location if fortype ='temperature',only return temperature, else return temperature and depth """ if depth==99999: depth='bottom' if not doppio_coordinate(lat,lon): print('the lat and lon out of range in doppio') return np.nan,np.nan if type(time)==str: date_time=datetime.datetime.strptime(time,'%Y-%m-%d %H:%M:%S') # transform time format elif type(time)==datetime.datetime: date_time=time else: print('check the type of input time in get_doppio') for m in range(0,7): try: url_time=(date_time-datetime.timedelta(days=m)).strftime('%Y-%m-%d')# url=zl.get_doppio_url(url_time) #get the data nc=netCDF4.Dataset(url) lons=nc.variables['lon_rho'][:] lats=nc.variables['lat_rho'][:] doppio_time=nc.variables['time'] doppio_rho=nc.variables['s_rho'] doppio_temp=nc.variables['temp'] doppio_h=nc.variables['h'] except: continue min_diff_time=abs(datetime.datetime(2017,11,1,0,0,0)+datetime.timedelta(hours=int(doppio_time[0]))-date_time) min_diff_index=0 for i in range(1,len(doppio_time)): diff_time=abs(datetime.datetime(2017,11,1,0,0,0)+datetime.timedelta(hours=int(doppio_time[i]))-date_time) if diff_time<min_diff_time: min_diff_time=diff_time min_diff_index=i #calculate the min,second small and third small distance and index target_distance=zl.dist(lat1=lats[0][0],lon1=lons[0][0],lat2=lats[0][1],lon2=lons[0][1]) index_1,index_2=zl.find_nd(target=target_distance,lat=lat,lon=lon,lats=lats,lons=lons) #calculate the optimal layer index added this section Feb 2020 doppio_depth=nc['h'][index_1][index_2] if depth > doppio_depth:# case of bottom S_coordinate=1 else: S_coordinate=float(depth)/float(doppio_depth) if 0<=S_coordinate<1: layer_index=39-int(S_coordinate/0.025)#doppio_temp=temp[itime,39-int(S_coordinate/0.025),index_1,index_2]# because there are 0.025 between each later elif S_coordinate==1: layer_index=0#doppio_temp=temp[itime][0][index_1][index_2] else: layer_index=0#doppio_temp=temp[itime][0][index_1][index_2] #return doppio_temp #layer_index=0 #specify the initial layer index '''if depth!='bottom': h_distance=depth+doppio_rho[0]*doppio_h[index_1,index_2] #specify the initial distanc of high for i in range(len(doppio_rho)): if abs(depth+doppio_rho[0]*doppio_h[index_1,index_2])<=h_distance: h_distance=depth+doppio_rho[i]*doppio_h[index_1,index_2] layer_index=i if depth>doppio_h[index_1,index_2]: print ("the depth is out of the depth of bottom:"+str(doppio_h[index_1,index_2])) ''' if index_1==0: index_1=1 if index_1==len(lats)-1: index_1=len(lats)-2 if index_2==0: index_2=1 if index_2==len(lats[0])-1: index_2=len(lats[0])-2 while True: point=[[lats[index_1][index_2],lons[index_1][index_2],doppio_temp[min_diff_index,layer_index,index_1,index_2]],\ [lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_temp[min_diff_index,layer_index,(index_1-1),index_2]],\ [lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_temp[min_diff_index,layer_index,(index_1+1),index_2]],\ [lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_temp[min_diff_index,layer_index,index_1,(index_2-1)]],\ [lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_temp[min_diff_index,layer_index,index_1,(index_2+1)]]] break point_temp=fitting(point,lat,lon) if np.isnan(point_temp): continue if min_diff_time<datetime.timedelta(hours=1): break if fortype=='temperature': return point_temp else: return point_temp,doppio_h[index_1,index_2]
lat, lon = cv.dm2dd(value_data_df['Lat'][len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) - 1]) #write the data of raw file to dict for i in range(len(telemetrystatus_df)): if telemetrystatus_df['Vessel#'][i] == vessel_number: raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\ fname,float(mean_temp),float(mean_depth)]],columns=['time','filename','mean_temp','mean_depth']).iloc[0],ignore_index=True) #caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file for i in range(len(valuable_tele_df)): if valuable_tele_df['vessel_n'][i].split('_')[1] == str( vessel_number): if abs(valuable_tele_df['time'][i] - time_gmt) <= acceptable_time_diff: #time match if zl.dist(lat1=lat, lon1=lon, lat2=float(valuable_tele_df['lat'][i]), lon2=float(valuable_tele_df['lon'][i]) ) <= acceptable_distance_diff: #distance match for j in range(len(record_file_df)): if record_file_df['Vessel#'][j] == vessel_number: diff_temp = abs( float(mean_temp) - float(valuable_tele_df['temp'][i])) diff_depth = abs( float(mean_depth) - float(valuable_tele_df['depth'][i])) if record_file_df['matched_number'].isnull( )[j]: record_file_df['matched_number'][j] = 1 record_file_df['sum_diff_temp'][ j] = diff_temp
def get_doppio_fitting(latp=0, lonp=0, depth='bottom', dtime=datetime.datetime.now(), fortype='temperature', hour_allowed=1): """ notice: the format of time is like "%Y-%m-%d %H:%M:%S" this time is utctime or the type of time is datetime.datetime the depth is under the bottom depth the module only output the temperature of point location """ if not doppio_coordinate(latp, lonp): print('the lat and lon out of range in doppio') return np.nan, np.nan if type(dtime) == str: date_time = datetime.datetime.strptime( dtime, '%Y-%m-%d %H:%M:%S') # transform time format else: date_time = dtime for m in range(0, 7): try: url_time = (date_time - datetime.timedelta(days=m)).strftime('%Y-%m-%d') url = zl.get_doppio_url(url_time) #get the data nc = netCDF4.Dataset(url) lons = nc.variables['lon_rho'][:] lats = nc.variables['lat_rho'][:] doppio_time = nc.variables['time'] doppio_rho = nc.variables['s_rho'] doppio_temp = nc.variables['temp'] doppio_h = nc.variables['h'] except: continue #calculate the index of the minimum timedelta parameter = (datetime.datetime(2017, 11, 1, 0, 0, 0) - date_time).days * 24 + (datetime.datetime( 2017, 11, 1, 0, 0, 0) - date_time).seconds / 3600. time_delta = abs(doppio_time[:] + parameter) min_diff_index = np.argmin(time_delta) #calculate the min distance and index target_distance = 2 * zl.dist( lat1=lats[0, 0], lon1=lons[0, 0], lat2=lats[0, 1], lon2=lons[0, 1]) index_1, index_2 = find_nd(target=target_distance, lat=latp, lon=lonp, lats=lats, lons=lons) #calculate the optimal layer index if depth == 'bottom': layer_index = 0 #specify the initial layer index else: h_distance = abs(doppio_rho[:] * doppio_h[index_1, index_2] + abs(depth)) layer_index = np.argmin(h_distance) # fitting the data through the 5 points if index_1 == 0: index_1 = 1 if index_1 == len(lats) - 1: index_1 = len(lats) - 2 if index_2 == 0: index_2 = 1 if index_2 == len(lats[0]) - 1: index_2 = len(lats[0]) - 2 while True: point=[[lats[index_1][index_2],lons[index_1][index_2],doppio_temp[min_diff_index,layer_index,index_1,index_2]],\ [lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_temp[min_diff_index,layer_index,(index_1-1),index_2]],\ [lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_temp[min_diff_index,layer_index,(index_1+1),index_2]],\ [lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_temp[min_diff_index,layer_index,index_1,(index_2-1)]],\ [lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_temp[min_diff_index,layer_index,index_1,(index_2+1)]]] break point_temp = fitting(point, latp, lonp) while True: points_h=[[lats[index_1][index_2],lons[index_1][index_2],doppio_h[index_1,index_2]],\ [lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_h[(index_1-1),index_2]],\ [lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_h[(index_1+1),index_2]],\ [lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_h[index_1,(index_2-1)]],\ [lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_h[index_1,(index_2+1)]]] break point_temp = fitting(point, latp, lonp) point_h = fitting(points_h, latp, lonp) if np.isnan(point_temp): continue if time_delta[min_diff_index] < hour_allowed: break if fortype == 'tempdepth': return point_temp, point_h else: return point_temp
stemp = s['temp'] t = pd.read_csv() # orginal data file t_id = t['PTT'] tlat = t['lat_gps'] tlon = t['lon_gps'] ttime = pd.Series( datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t['gps_date']) tdepth = t['depth'] ttemp = t['temp'] index = [] #index of turtle indx = [] #index of shipboard for i in tqdm(range(len(t))): for j in range(len(s)): l = zl.dist(slat[j], slon[j], tlat[i], tlon[i]) if l < r2 and l >= r1: try: #print l #distance maxtime = ttime[i] + timedelta(days=day) mintime = ttime[i] - timedelta(days=day) mx = stime[j] < maxtime mn = stime[j] > mintime TF = mx * mn if TF == 1: #time index.append(i) #turtle index indx.append(j) #ship index except: continue data = pd.DataFrame(range(len(indx)))
def get_doppio(lat=0, lon=0, depth=99999, time='2018-11-12 12:00:00'): """ notice: the format of time is like "%Y-%m-%d %H:%M:%S" the depth is under the bottom depth the module only output the temperature of point location """ date_time = datetime.datetime.strptime( time, '%Y-%m-%d %H:%M:%S') # transform time format for m in range(0, 7): try: url_time = (date_time - datetime.timedelta(days=m)).strftime( '%Y-%m-%d') # url = zl.get_doppio_url(url_time) #get the data nc = netCDF4.Dataset(url) except: continue lons = nc.variables['lon_rho'][:] lats = nc.variables['lat_rho'][:] temp = nc.variables['temp'] doppio_time = nc.variables['time'] doppio_depth = nc.variables['h'][:] min_diff_time = abs( datetime.datetime(2017, 11, 1, 0, 0, 0) + datetime.timedelta(hours=int(doppio_time[0])) - date_time) min_diff_index = 0 for i in range(1, len(doppio_time)): diff_time = abs( datetime.datetime(2017, 11, 1, 0, 0, 0) + datetime.timedelta(hours=int(doppio_time[i])) - date_time) if diff_time < min_diff_time: min_diff_time = diff_time min_diff_index = i #calculate the min,second small and third small distance and index min_distance = zl.dist(lat1=lat, lon1=lon, lat2=lats[0][0], lon2=lons[0][0]) # secondmin_distance=min_distance # thirdmin_distance=min_distance # index_1, index_2 = 0, 0 secondindex_1, secondindex_2 = 0, 0 thirdindex_1, thirdindex_2 = 0, 0 fourthindex_1, fourthindex_2 = 0, 0 fifthindex_1, fifthindex_2 = 0, 0 # sixthindex_1,sixthindex_2=0,0 for i in range(len(lons)): for j in range(len(lons[i])): distance = zl.dist(lat1=lat, lon1=lon, lat2=lats[i][j], lon2=lons[i][j]) if min_distance >= distance: # thirdmin_distance=secondmin_distance # secondmin_distance=min_distance min_distance = distance # sixthindex_1,sixthindex_2=fifthindex_1,fifthindex_2 fifthindex_1, fifthindex_2 = fourthindex_1, fourthindex_2 fourthindex_1, fourthindex_2 = thirdindex_1, thirdindex_2 thirdindex_1, thirdindex_2 = secondindex_1, secondindex_2 secondindex_1, secondindex_2 = index_1, index_2 index_1, index_2 = i, j if depth == 99999: S_coordinate = 1 else: S_coordinate = float(depth) / float(doppio_depth[index_1][index_2]) if 0 <= S_coordinate < 1: layer_index = 39 - int(S_coordinate / 0.025) elif S_coordinate == 1: layer_index = 39 else: return 9999 point=[[lats[index_1][index_2],lons[index_1][index_2],temp[min_diff_index][layer_index][index_1][index_2]],\ [lats[secondindex_1][secondindex_2],lons[secondindex_1][secondindex_2],temp[min_diff_index][layer_index][secondindex_1][secondindex_2]],\ [lats[thirdindex_1][thirdindex_2],lons[thirdindex_1][thirdindex_2],temp[min_diff_index][layer_index][thirdindex_1][thirdindex_2]],\ [lats[fourthindex_1][fourthindex_2],lons[fourthindex_1][fourthindex_2],temp[min_diff_index][layer_index][fourthindex_1][fourthindex_2]],\ [lats[fifthindex_1][fifthindex_2],lons[fifthindex_1][fifthindex_2],temp[min_diff_index][layer_index][fifthindex_1][fifthindex_2]]] point_temp = fitting(point, lat, lon) if np.isnan(point_temp): continue if min_diff_time < datetime.timedelta(hours=1): break return point_temp, index_1, index_2
def match_tele_raw( input_dir, path_save, telemetry_status, start_time, end_time, telemetry_path='https://www.nefsc.noaa.gov/drifter/emolt.dat', accept_minutes_diff=20, acceptable_distance_diff=2, dpi=300): """ match the file and telementy. we can known how many file send to the satallite and output the figure """ #read the file of the telementry_status telemetrystatus_df = read_telemetrystatus(telemetry_status) #st the record file use to write minmum maxmum and average of depth and temperature,the numbers of file, telemetry and successfully matched record_file_df=telemetrystatus_df.loc[:,['Boat','Vessel#']].reindex(columns=['Boat','Vessel#','matched_number','file_number','tele_num','max_diff_depth',\ 'min_diff_depth','average_diff_depth','max_diff_temp','min_diff_temp','average_diff_temp','sum_diff_depth','sum_diff_temp',\ 'min_lat','max_lat','min_lon','max_lon'],fill_value=None) #transfer the time format of string to datetime start_time_local = datetime.strptime(start_time, '%Y-%m-%d') end_time_local = datetime.strptime(end_time, '%Y-%m-%d') allfile_lists = zl.list_all_files(input_dir) ###################### file_lists = [] for file in allfile_lists: if file[len(file) - 4:] == '.csv': file_lists.append(file) #download the data of telementry tele_df = read_telemetry(telemetry_path) #screen out the data of telemetry in interval valuable_tele_df = pd.DataFrame( data=None, columns=['vessel_n', 'esn', 'time', 'lon', 'lat', 'depth', 'temp']) #use to save the data during start time and end time for i in range(len(tele_df)): tele_time=datetime.strptime(str(tele_df['year'].iloc[i])+'-'+str(tele_df['month'].iloc[i])+'-'+str(tele_df['day'].iloc[i])+' '+\ str(tele_df['Hours'].iloc[i])+':'+str(tele_df['minates'].iloc[i])+':'+'00','%Y-%m-%d %H:%M:%S') if zl.local2utc(start_time_local) <= tele_time < zl.local2utc( end_time_local): valuable_tele_df=valuable_tele_df.append(pd.DataFrame(data=[[tele_df['vessel_n'][i],tele_df['esn'][i],tele_time,tele_df['lon'][i],tele_df['lat'][i],tele_df['depth'][i],\ tele_df['temp'][i]]],columns=['vessel_n','esn','time','lon','lat','depth','temp'])) valuable_tele_df.index = range(len(valuable_tele_df)) #whether the data of file and telemetry is exist if len(valuable_tele_df) == 0 and len(file_lists) == 0: print( 'please check the data website of telementry and the directory of raw_data is exist!' ) sys.exit() elif len(valuable_tele_df) == 0: print('please check the data website of telementry!') sys.exit() elif len(file_lists) == 0: print('please check the directory raw_data is exist!') sys.exit() #match the file index = telemetrystatus_df['Boat'] #set the index for dictionary raw_dict = { } #the dictinary about raw data, use to write the data about 'time','filename','mean_temp','mean_depth' tele_dict = { } #the dictionary about telementry data,use to write the data about'time','mean_temp','mean_depth' for i in range(len(index)): #loop every boat raw_dict[index[i]] = pd.DataFrame(data=None, columns=[ 'time', 'filename', 'mean_temp', 'mean_depth', 'mean_lat', 'mean_lon' ]) tele_dict[index[i]] = pd.DataFrame(data=None, columns=[ 'time', 'mean_temp', 'mean_depth', 'mean_lat', 'mean_lon' ]) for file in file_lists: # loop raw files fpath, fname = os.path.split(file) #get the file's path and name # now, read header and data of every file header_df = zl.nrows_len_to(file, 2, name=['key', 'value']) #only header data_df = zl.skip_len_to(file, 2) #only data #caculate the mean temperature and depth of every file value_data_df = data_df.ix[( data_df['Depth(m)'] > 0.85 * mean(data_df['Depth(m)']))] #filter the data value_data_df = value_data_df.ix[ 2:] #delay several minutes to let temperature sensor record the real bottom temp value_data_df=value_data_df.ix[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \ (value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error value_data_df.index = range(len(value_data_df)) #reindex for i in range(len(value_data_df['Lat'])): value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd( value_data_df['Lat'][i], value_data_df['Lon'][i]) min_lat = min(value_data_df['Lat'].values) max_lat = max(value_data_df['Lat'].values) min_lon = min(value_data_df['Lon'].values) max_lon = max(value_data_df['Lon'].values) mean_lat = str(round(mean(value_data_df['Lat'].values), 4)) mean_lon = str(round(mean(value_data_df['Lon'].values), 4)) #caculate the mean depth mean_temp = str( round(mean(value_data_df['Temperature(C)'][1:len(value_data_df)]), 2)) mean_depth = str( abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill( 3) #caculate the mean depth #get the vessel number of every file for i in range(len(header_df)): if header_df['key'][i].lower() == 'vessel number'.lower(): vessel_number = int(header_df['value'][i]) break #caculate the number of raw files in every vessel,and min,max of lat and lon for i in range(len(record_file_df)): if record_file_df['Vessel#'][i] == vessel_number: if record_file_df['file_number'].isnull()[i]: record_file_df['min_lat'][i] = min_lat record_file_df['max_lat'][i] = max_lat record_file_df['min_lon'][i] = min_lon record_file_df['max_lon'][i] = max_lon record_file_df['file_number'][i] = 1 else: record_file_df['file_number'][i] = int( record_file_df['file_number'][i] + 1) if record_file_df['min_lat'][i] > min_lat: record_file_df['min_lat'][i] = min_lat if record_file_df['max_lat'][i] < max_lat: record_file_df['max_lat'][i] = max_lat if record_file_df['min_lon'][i] > min_lon: record_file_df['min_lon'][i] = min_lon if record_file_df['max_lon'][i] < max_lon: record_file_df['max_lon'][i] = max_lon #match rawdata and telementry data time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split( '.')[0].split('_')[3] #GMT time to local time of file time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] + '-' + time_str[6:8] + ' ' + time_str[9:11] + ':' + time_str[11:13] + ':' + time_str[13:15]) time_gmt = datetime.strptime(time_str, "%Y%m%d %H%M%S") #transfer the format latitude and longitude lat, lon = value_data_df['Lat'][ len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) - 1] #write the data of raw file to dict for i in range(len(telemetrystatus_df)): if telemetrystatus_df['Vessel#'][i] == vessel_number: raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\ fname,float(mean_temp),float(mean_depth),float(mean_lat),float(mean_lon)]],columns=['time','filename','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True) #caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file for i in range(len(valuable_tele_df)): if valuable_tele_df['vessel_n'][i].split('_')[1] == str( vessel_number): if abs(valuable_tele_df['time'][i] - time_gmt) <= timedelta( minutes=accept_minutes_diff): #time match if zl.dist(lat1=lat, lon1=lon, lat2=float(valuable_tele_df['lat'][i]), lon2=float(valuable_tele_df['lon'][i]) ) <= acceptable_distance_diff: #distance match for j in range(len(record_file_df)): if record_file_df['Vessel#'][j] == vessel_number: diff_temp = round( (float(mean_temp) - float(valuable_tele_df['temp'][i])), 4) diff_depth = round( (float(mean_depth) - float(valuable_tele_df['depth'][i])), 4) if record_file_df['matched_number'].isnull( )[j]: record_file_df['matched_number'][j] = 1 record_file_df['sum_diff_temp'][ j] = diff_temp record_file_df['max_diff_temp'][ j] = diff_temp record_file_df['min_diff_temp'][ j] = diff_temp record_file_df['sum_diff_depth'][ j] = diff_depth record_file_df['max_diff_depth'][ j] = diff_depth record_file_df['min_diff_depth'][ j] = diff_depth break else: record_file_df['matched_number'][j] = int( record_file_df['matched_number'][j] + 1) record_file_df['sum_diff_temp'][ j] = record_file_df['sum_diff_temp'][ j] + diff_temp record_file_df['sum_diff_depth'][ j] = record_file_df['sum_diff_depth'][ j] + diff_depth if record_file_df['max_diff_temp'][ j] < diff_temp: record_file_df['max_diff_temp'][ j] = diff_temp if record_file_df['min_diff_temp'][ j] > diff_temp: record_file_df['min_diff_temp'][ j] = diff_temp if record_file_df['max_diff_depth'][ j] < diff_depth: record_file_df['max_diff_depth'][ j] = diff_depth if record_file_df['min_diff_depth'][ j] > diff_depth: record_file_df['min_diff_depth'][ j] = diff_depth break #write 'time','mean_temp','mean_depth' of the telementry to tele_dict for i in range( len(valuable_tele_df) ): #valuable_tele_df is the valuable telemetry data during start time and end time for j in range(len(telemetrystatus_df)): if int(valuable_tele_df['vessel_n'][i].split('_') [1]) == telemetrystatus_df['Vessel#'][j]: #count the numbers by boats if record_file_df['tele_num'].isnull()[j]: record_file_df['tele_num'][j] = 1 else: record_file_df['tele_num'][ j] = record_file_df['tele_num'][j] + 1 if record_file_df['max_lat'].isnull()[j]: record_file_df['min_lat'][j] = valuable_tele_df['lat'][i] record_file_df['max_lat'][j] = valuable_tele_df['lat'][i] record_file_df['min_lon'][j] = valuable_tele_df['lon'][i] record_file_df['max_lon'][j] = valuable_tele_df['lon'][i] else: if record_file_df['min_lat'][j] > valuable_tele_df['lat'][ i]: record_file_df['min_lat'][j] = valuable_tele_df['lat'][ i] if record_file_df['max_lat'][j] < valuable_tele_df['lat'][ i]: record_file_df['max_lat'][j] = valuable_tele_df['lat'][ i] if record_file_df['min_lon'][j] > valuable_tele_df['lon'][ i]: record_file_df['min_lon'][j] = valuable_tele_df['lon'][ i] if record_file_df['max_lon'][j] < valuable_tele_df['lon'][ i]: record_file_df['max_lon'][j] = valuable_tele_df['lon'][ i] #write 'time','mean_temp','mean_depth' of the telementry to tele_dict tele_dict[telemetrystatus_df['Boat'][j]]=tele_dict[telemetrystatus_df['Boat'][j]].append(pd.DataFrame(data=[[valuable_tele_df['time'][i],\ float(valuable_tele_df['temp'][i]),float(valuable_tele_df['depth'][i]),float(valuable_tele_df['lat'][i]),float(valuable_tele_df['lon'][i])]],columns=['time','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True) print("finish the calculate of min_lat and min_lon!") for i in range(len(record_file_df)): if not record_file_df['matched_number'].isnull()[i]: record_file_df['average_diff_depth'][i] = round( record_file_df['sum_diff_depth'][i] / record_file_df['matched_number'][i], 4) record_file_df['average_diff_temp'][i] = round( record_file_df['sum_diff_temp'][i] / record_file_df['matched_number'][i], 4) else: record_file_df['matched_number'][i] = 0 if record_file_df['tele_num'].isnull()[i]: record_file_df['tele_num'][i] = 0 if record_file_df['file_number'].isnull()[i]: record_file_df['file_number'][i] = 0 for i in index: #loop every boat, i represent the name of boat raw_dict[i] = raw_dict[i].sort_values(by=['time']) raw_dict[i].index = range(len(raw_dict[i])) record_file_df = record_file_df.drop(['sum_diff_depth', 'sum_diff_temp'], axis=1) #save the record file record_file_df.to_csv(path_save + '/' + start_time + '_' + end_time + ' statistics.csv', index=0) return raw_dict, tele_dict, record_file_df, index, start_time_local, end_time_local, path_save