def get_gomofs(date_time,
lat,
lon,
depth='bottom',
mindistance=20): # JiM's simple version for bottom temp
"""
JiM's simplified version of Lei Zhao's function
the format time(GMT) is: datetime.datetime(2019, 2, 27, 11, 56, 51, 666857)
lat and lon use decimal degrees
return the temperature of specify location
HARDCODED TO RETURN BOTTOM TEMP
"""
rho_index = 0 # for bottom layer
if depth == 99999:
depth = 'bottom'
rho_index = 0
if not gomofs_coordinaterange(lat, lon):
print('lat and lon out of range in gomofs')
return np.nan
if date_time < datetime.datetime.strptime('2018-07-01 00:00:00',
'%Y-%m-%d %H:%M:%S'):
print('Time out of range, time start :2018-07-01 00:00:00z')
return np.nan
if date_time > datetime.datetime.utcnow() + datetime.timedelta(
days=3): #forecast time under 3 days
print('forecast time under 3 days')
return np.nan
#start download data
if (datetime.datetime.utcnow() - date_time) < datetime.timedelta(days=25):
url = get_gomofs_url(
date_time) #this url get data within 25 days recently
else:
url = get_gomofs_url_new(date_time) #this url get data 25 days ago
nc = netCDF4.Dataset(str(url))
gomofs_lons = nc.variables['lon_rho'][:]
gomofs_lats = nc.variables['lat_rho'][:]
gomofs_rho = nc.variables['s_rho']
gomofs_h = nc.variables['h']
gomofs_temp = nc.variables['temp']
#caculate the index of the nearest four points using a "find_nd" function in Lei Zhao's conversion module
target_distance = 2 * zl.dist(lat1=gomofs_lats[0][0],
lon1=gomofs_lons[0][0],
lat2=gomofs_lats[0][1],
lon2=gomofs_lons[0][1])
eta_rho, xi_rho = zl.find_nd(target=target_distance,
lat=lat,
lon=lon,
lats=gomofs_lats,
lons=gomofs_lons)
if zl.dist(lat1=lat,
lon1=lon,
lat2=gomofs_lats[eta_rho][xi_rho],
lon2=gomofs_lons[eta_rho][xi_rho]) > mindistance:
print('THE location is out of range')
return np.nan
temperature = gomofs_temp[0][rho_index][eta_rho][xi_rho]
depth = gomofs_h[eta_rho][xi_rho]
#temperature=float(gomofs_temp[0,rho_index,eta_rho,xi_rho].data)
return temperature, depth
def find_nd(target,lat,lon,lats,lons):
""" Bisection method:find the index of nearest distance"""
row=0
maxrow=len(lats)-1
col=len(lats[0])-1
while col>=0 and row<=maxrow:
distance=zl.dist(lat1=lats[row,col],lat2=lat,lon1=lons[row,col],lon2=lon)
if distance<=target:
break
elif abs(lats[row,col]-lat)<abs(lons[row,col]-lon):
col-=1
else:
row+=1
distance=zl.dist(lat1=lats[row,col],lat2=lat,lon1=lons[row,col],lon2=lon)
row_md,col_md=row,col #row_md the row of minimum distance
#avoid row,col out of range in next step
if row<3:
row=3
if col<3:
col=3
if row>maxrow-3:
row=maxrow-3
if col>len(lats[0])-4:
col=len(lats[0])-4
for i in range(row-3,row+3,1):
for j in range(col-3,col+3,1):
distance_c=zl.dist(lat1=lats[i,j],lat2=lat,lon1=lons[i,j],lon2=lon)
if distance_c<=distance:
distance=distance_c
row_md,col_md=i,j
return row_md,col_md
def get_doppio(lat, lon, time, depth):
"""
notice:
the format of time is like "%Y-%m-%d %H:%M:%S"
the default depth is under the bottom depth
the module only output the temperature of point location
"""
time = dt.strptime(time, '%Y-%m-%d %H:%M:%S') # transform time format
if (time - datetime.datetime(2017, 11, 1, 0, 0, 0)).total_seconds() < 0:
print('the date can\'t be earlier than 2017-11-1')
return np.nan
url_time = time.strftime('%Y-%m-%d') #
url = get_doppio_url(url_time)
nc = netCDF4.Dataset(url).variables
#first find the index of the grid
lons = nc['lon_rho'][:]
lats = nc['lat_rho'][:]
temp = nc['temp']
#second find the index of time
doppio_time = nc['time']
itime = netCDF4.date2index(time, doppio_time,
select='nearest') # where startime in datetime
# figure out layer from depth
min_distance = zl.dist(lat1=lat,
lon1=lon,
lat2=lats[0][0],
lon2=lons[0][0])
index_1, index_2 = 0, 0
for i in range(len(lons)):
for j in range(len(lons[i])):
if min_distance > zl.dist(
lat1=lat, lon1=lon, lat2=lats[i][j], lon2=lons[i][j]):
min_distance = zl.dist(lat1=lat,
lon1=lon,
lat2=lats[i][j],
lon2=lons[i][j])
index_1 = i
index_2 = j
doppio_depth = nc['h'][index_1][index_2]
if depth > doppio_depth: # case of bottom
S_coordinate = 1
else:
S_coordinate = float(depth) / float(doppio_depth)
if 0 <= S_coordinate < 1:
doppio_temp = temp[
itime, 39 - int(S_coordinate / 0.025), index_1,
index_2] # because there are 0.025 between each later
elif S_coordinate == 1:
doppio_temp = temp[itime][0][index_1][index_2]
else:
doppio_temp = temp[itime][0][index_1][index_2]
return doppio_temp
def get_gomofs_zl(date_time,lat,lon,depth='bottom',mindistance=20,autocheck=True,fortype='temperature'):
"""
the format time(GMT) is: datetime.datetime(2019, 2, 27, 11, 56, 51, 666857)
lat and lon use decimal degrees
if the depth is under the water, please must add the marker of '-'
input time,lat,lon,depth return the temperature of specify location (or return temperature,nc,rho_index,ocean_time_index)
the unit is mile of distance
return the temperature of specify location
"""
#print(depth)
if depth==99999:
depth='bottom'
if not gomofs_coordinaterange(lat,lon):
print('lat and lon out of range in gomofs')
return np.nan
if date_time<datetime.datetime.strptime('2018-07-01 00:00:00','%Y-%m-%d %H:%M:%S'):
print('Time out of range, time start :2018-07-01 00:00:00z')
return np.nan
if date_time>datetime.datetime.now()+datetime.timedelta(days=3): #forecast time under 3 days
print('forecast time under 3 days')
return np.nan
#start download data
forecastdate=date_time #forecast time equal input date_time
changefile,filecheck=1,1 #changefile means whether we need to change the file to get data, filecheck means check the file exist or not.
while(changefile==1):
count=1
while(filecheck==1): #download the data
try:
if forecastdate==date_time: #the forcastdate is input date_time, if the date_time changed yet,we will use the forecast data
url=get_gomofs_url(date_time)
nc=netCDF4.Dataset(str(url))
#print('download nowcast data.')
else:
url=get_gomofs_url_forecast(date_time,forecastdate)
nc=netCDF4.Dataset(str(url))
#print('download forecast data.')
filecheck,readcheck=0,1 # if the file is there, filecheck=0,readcheck use to check the file whether read successfully
except OSError:
try:
url=get_gomofs_url_forecast(date_time,forecastdate)
nc=netCDF4.Dataset(str(url))
#print('download forecast data.')
filecheck,readcheck=0,1
except OSError:
date_time=date_time-datetime.timedelta(hours=6)
if (forecastdate-date_time)>datetime.timedelta(days=3): #every file only have 3 days data.
print('please check the website or file is exist!')
print(url)
return np.nan
except:
return np.nan
except:
return np.nan
# print('start read data.')
while(readcheck==1): #read data, if readcheck==1 start loop
try:
while True:
#print('connecting the web.')
if zl.isConnected(address=url):
break
print('check the website is well or internet is connected?')
time.sleep(5)
gomofs_lons=nc.variables['lon_rho'][:]
gomofs_lats=nc.variables['lat_rho'][:]
gomofs_rho=nc.variables['s_rho']
#gomofs_h=nc.variables['h']
gomofs_h=nc.variables['h'][:]
gomofs_temp=nc.variables['temp']
readcheck,changefile=0,0 #if read data successfully, we do not need to loop
# print('end read data.')
except RuntimeError:
count=count+1
if count>8:
if autocheck==True:
return np.nan
while True:
print('it will return nan, if you do not need read again.')
cmd = input("whether need read again(y/n)?：")
if cmd.lower() == "y":
count=1
break
elif cmd.lower() == "n":
cmd2 = input("whether need change file(y/n)?：")
if cmd2.lower()=="y":
date_time=date_time-datetime.timedelta(hours=6)
readcheck,filecheck=0,1
break
else:
print('interrupt read data.')
return np.nan
else:
break
time.sleep(20) #every time to reread data need stop 20s
print('the '+str(int(count))+' times to read data.')
except:
return np.nan
#caculate the index of the nearest four points
target_distance=2*zl.dist(lat1=gomofs_lats[0][0],lon1=gomofs_lons[0][0],lat2=gomofs_lats[0][1],lon2=gomofs_lons[0][1])
eta_rho,xi_rho=zl.find_nd(target=target_distance,lat=lat,lon=lon,lats=gomofs_lats,lons=gomofs_lons)
if zl.dist(lat1=lat,lon1=lon,lat2=gomofs_lats[eta_rho][xi_rho],lon2=gomofs_lons[eta_rho][xi_rho])>mindistance:
print('THE location is out of range')
return np.nan
# estimate the bottom depth of point location
if eta_rho==0:
eta_rho=1
if eta_rho==len(gomofs_lats)-1:
eta_rho=len(gomofs_lats)-2
if xi_rho==len(gomofs_lats[0])-1:
eta_rho=len(gomofs_lats[0])-2
# print('start caculate the bottom depth of point location!')
while True:
points_h=[[gomofs_lats[eta_rho][xi_rho],gomofs_lons[eta_rho][xi_rho],gomofs_h[eta_rho,xi_rho]],
[gomofs_lats[eta_rho,(xi_rho-1)],gomofs_lons[eta_rho,(xi_rho-1)],gomofs_h[eta_rho,(xi_rho-1)]],
[gomofs_lats[eta_rho,(xi_rho+1)],gomofs_lons[eta_rho,(xi_rho+1)],gomofs_h[eta_rho,(xi_rho+1)]],
[gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_h[(eta_rho-1),xi_rho]],
[gomofs_lats[(eta_rho+1),xi_rho],gomofs_lons[(eta_rho+1),xi_rho],gomofs_h[(eta_rho+1),xi_rho]]]
break
point_h=zl.fitting(points_h,lat,lon)
# caculate the rho index
if depth=='bottom':
rho_index=0
else:
distance_h=gomofs_rho[0]*point_h-depth
for k in range(len(gomofs_rho)):
if abs(distance_h)>=abs(gomofs_rho[k]*point_h-depth):
distance_h=gomofs_rho[k]*point_h-depth
rho_index=k
#estimate the temperature of point location
while True:
points_temp=[[gomofs_lats[eta_rho,xi_rho],gomofs_lons[eta_rho,xi_rho],gomofs_temp[0][rho_index][eta_rho][xi_rho]],
[gomofs_lats[eta_rho,(xi_rho-1)],gomofs_lons[eta_rho,(xi_rho-1)],gomofs_temp[0][rho_index][eta_rho,(xi_rho-1)]],
[gomofs_lats[eta_rho,(xi_rho+1)],gomofs_lons[eta_rho,(xi_rho+1)],gomofs_temp[0][rho_index][eta_rho][(xi_rho+1)]],
[gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_temp[0][rho_index][(eta_rho-1)][xi_rho]],
[gomofs_lats[(eta_rho-1),xi_rho],gomofs_lons[(eta_rho-1),xi_rho],gomofs_temp[0][rho_index][(eta_rho-1)][xi_rho]]]
break
temperature=zl.fitting(points_temp,lat,lon)
# if input depth out of the bottom, print the prompt message
if depth!='bottom':
if abs(point_h)<abs(depth):
print ("the depth is out of the bottom:"+str(point_h))
return np.nan
if fortype=='tempdepth':
return temperature,point_h
else:
return temperature
def get_doppio(lat=0,lon=0,depth='bottom',time='2018-11-12 12:00:00',fortype='temperature'):
"""
notice:
the format of time is like "%Y-%m-%d %H:%M:%S" this time is utctime or it can also be datetime
the depth is under the bottom depth
the module only output the temperature of point location
if fortype ='temperature',only return temperature, else return temperature and depth
"""
if depth==99999:
depth='bottom'
if not doppio_coordinate(lat,lon):
print('the lat and lon out of range in doppio')
return np.nan,np.nan
if type(time)==str:
date_time=datetime.datetime.strptime(time,'%Y-%m-%d %H:%M:%S') # transform time format
elif type(time)==datetime.datetime:
date_time=time
else:
print('check the type of input time in get_doppio')
for m in range(0,7):
try:
url_time=(date_time-datetime.timedelta(days=m)).strftime('%Y-%m-%d')#
url=zl.get_doppio_url(url_time)
#get the data
nc=netCDF4.Dataset(url)
lons=nc.variables['lon_rho'][:]
lats=nc.variables['lat_rho'][:]
doppio_time=nc.variables['time']
doppio_rho=nc.variables['s_rho']
doppio_temp=nc.variables['temp']
doppio_h=nc.variables['h']
except:
continue
min_diff_time=abs(datetime.datetime(2017,11,1,0,0,0)+datetime.timedelta(hours=int(doppio_time[0]))-date_time)
min_diff_index=0
for i in range(1,len(doppio_time)):
diff_time=abs(datetime.datetime(2017,11,1,0,0,0)+datetime.timedelta(hours=int(doppio_time[i]))-date_time)
if diff_time<min_diff_time:
min_diff_time=diff_time
min_diff_index=i
#calculate the min,second small and third small distance and index
target_distance=zl.dist(lat1=lats[0][0],lon1=lons[0][0],lat2=lats[0][1],lon2=lons[0][1])
index_1,index_2=zl.find_nd(target=target_distance,lat=lat,lon=lon,lats=lats,lons=lons)
#calculate the optimal layer index added this section Feb 2020
doppio_depth=nc['h'][index_1][index_2]
if depth > doppio_depth:# case of bottom
S_coordinate=1
else:
S_coordinate=float(depth)/float(doppio_depth)
if 0<=S_coordinate<1:
layer_index=39-int(S_coordinate/0.025)#doppio_temp=temp[itime,39-int(S_coordinate/0.025),index_1,index_2]# because there are 0.025 between each later
elif S_coordinate==1:
layer_index=0#doppio_temp=temp[itime][0][index_1][index_2]
else:
layer_index=0#doppio_temp=temp[itime][0][index_1][index_2]
#return doppio_temp
#layer_index=0 #specify the initial layer index
'''if depth!='bottom':
h_distance=depth+doppio_rho[0]*doppio_h[index_1,index_2] #specify the initial distanc of high
for i in range(len(doppio_rho)):
if abs(depth+doppio_rho[0]*doppio_h[index_1,index_2])<=h_distance:
h_distance=depth+doppio_rho[i]*doppio_h[index_1,index_2]
layer_index=i
if depth>doppio_h[index_1,index_2]:
print ("the depth is out of the depth of bottom:"+str(doppio_h[index_1,index_2]))
'''
if index_1==0:
index_1=1
if index_1==len(lats)-1:
index_1=len(lats)-2
if index_2==0:
index_2=1
if index_2==len(lats[0])-1:
index_2=len(lats[0])-2
while True:
point=[[lats[index_1][index_2],lons[index_1][index_2],doppio_temp[min_diff_index,layer_index,index_1,index_2]],\
[lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_temp[min_diff_index,layer_index,(index_1-1),index_2]],\
[lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_temp[min_diff_index,layer_index,(index_1+1),index_2]],\
[lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_temp[min_diff_index,layer_index,index_1,(index_2-1)]],\
[lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_temp[min_diff_index,layer_index,index_1,(index_2+1)]]]
break
point_temp=fitting(point,lat,lon)
if np.isnan(point_temp):
continue
if min_diff_time<datetime.timedelta(hours=1):
break
if fortype=='temperature':
return point_temp
else:
return point_temp,doppio_h[index_1,index_2]
lat, lon = cv.dm2dd(value_data_df['Lat'][len(value_data_df) - 1],
value_data_df['Lon'][len(value_data_df) - 1])
#write the data of raw file to dict
for i in range(len(telemetrystatus_df)):
if telemetrystatus_df['Vessel#'][i] == vessel_number:
raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\
fname,float(mean_temp),float(mean_depth)]],columns=['time','filename','mean_temp','mean_depth']).iloc[0],ignore_index=True)
#caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file
for i in range(len(valuable_tele_df)):
if valuable_tele_df['vessel_n'][i].split('_')[1] == str(
vessel_number):
if abs(valuable_tele_df['time'][i] -
time_gmt) <= acceptable_time_diff: #time match
if zl.dist(lat1=lat,
lon1=lon,
lat2=float(valuable_tele_df['lat'][i]),
lon2=float(valuable_tele_df['lon'][i])
) <= acceptable_distance_diff: #distance match
for j in range(len(record_file_df)):
if record_file_df['Vessel#'][j] == vessel_number:
diff_temp = abs(
float(mean_temp) -
float(valuable_tele_df['temp'][i]))
diff_depth = abs(
float(mean_depth) -
float(valuable_tele_df['depth'][i]))
if record_file_df['matched_number'].isnull(
)[j]:
record_file_df['matched_number'][j] = 1
record_file_df['sum_diff_temp'][
j] = diff_temp
def get_doppio_fitting(latp=0,
lonp=0,
depth='bottom',
dtime=datetime.datetime.now(),
fortype='temperature',
hour_allowed=1):
"""
notice:
the format of time is like "%Y-%m-%d %H:%M:%S" this time is utctime or the type of time is datetime.datetime
the depth is under the bottom depth
the module only output the temperature of point location
"""
if not doppio_coordinate(latp, lonp):
print('the lat and lon out of range in doppio')
return np.nan, np.nan
if type(dtime) == str:
date_time = datetime.datetime.strptime(
dtime, '%Y-%m-%d %H:%M:%S') # transform time format
else:
date_time = dtime
for m in range(0, 7):
try:
url_time = (date_time -
datetime.timedelta(days=m)).strftime('%Y-%m-%d')
url = zl.get_doppio_url(url_time)
#get the data
nc = netCDF4.Dataset(url)
lons = nc.variables['lon_rho'][:]
lats = nc.variables['lat_rho'][:]
doppio_time = nc.variables['time']
doppio_rho = nc.variables['s_rho']
doppio_temp = nc.variables['temp']
doppio_h = nc.variables['h']
except:
continue
#calculate the index of the minimum timedelta
parameter = (datetime.datetime(2017, 11, 1, 0, 0, 0) -
date_time).days * 24 + (datetime.datetime(
2017, 11, 1, 0, 0, 0) - date_time).seconds / 3600.
time_delta = abs(doppio_time[:] + parameter)
min_diff_index = np.argmin(time_delta)
#calculate the min distance and index
target_distance = 2 * zl.dist(
lat1=lats[0, 0], lon1=lons[0, 0], lat2=lats[0, 1], lon2=lons[0, 1])
index_1, index_2 = find_nd(target=target_distance,
lat=latp,
lon=lonp,
lats=lats,
lons=lons)
#calculate the optimal layer index
if depth == 'bottom':
layer_index = 0 #specify the initial layer index
else:
h_distance = abs(doppio_rho[:] * doppio_h[index_1, index_2] +
abs(depth))
layer_index = np.argmin(h_distance)
# fitting the data through the 5 points
if index_1 == 0:
index_1 = 1
if index_1 == len(lats) - 1:
index_1 = len(lats) - 2
if index_2 == 0:
index_2 = 1
if index_2 == len(lats[0]) - 1:
index_2 = len(lats[0]) - 2
while True:
point=[[lats[index_1][index_2],lons[index_1][index_2],doppio_temp[min_diff_index,layer_index,index_1,index_2]],\
[lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_temp[min_diff_index,layer_index,(index_1-1),index_2]],\
[lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_temp[min_diff_index,layer_index,(index_1+1),index_2]],\
[lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_temp[min_diff_index,layer_index,index_1,(index_2-1)]],\
[lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_temp[min_diff_index,layer_index,index_1,(index_2+1)]]]
break
point_temp = fitting(point, latp, lonp)
while True:
points_h=[[lats[index_1][index_2],lons[index_1][index_2],doppio_h[index_1,index_2]],\
[lats[index_1-1][index_2],lons[index_1-1][index_2],doppio_h[(index_1-1),index_2]],\
[lats[index_1+1][index_2],lons[index_1+1][index_2],doppio_h[(index_1+1),index_2]],\
[lats[index_1][index_2-1],lons[index_1][index_2-1],doppio_h[index_1,(index_2-1)]],\
[lats[index_1][index_2+1],lons[index_1][index_2+1],doppio_h[index_1,(index_2+1)]]]
break
point_temp = fitting(point, latp, lonp)
point_h = fitting(points_h, latp, lonp)
if np.isnan(point_temp):
continue
if time_delta[min_diff_index] < hour_allowed:
break
if fortype == 'tempdepth':
return point_temp, point_h
else:
return point_temp
stemp = s['temp']
t = pd.read_csv() # orginal data file
t_id = t['PTT']
tlat = t['lat_gps']
tlon = t['lon_gps']
ttime = pd.Series(
datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t['gps_date'])
tdepth = t['depth']
ttemp = t['temp']
index = [] #index of turtle
indx = [] #index of shipboard
for i in tqdm(range(len(t))):
for j in range(len(s)):
l = zl.dist(slat[j], slon[j], tlat[i], tlon[i])
if l < r2 and l >= r1:
try:
#print l #distance
maxtime = ttime[i] + timedelta(days=day)
mintime = ttime[i] - timedelta(days=day)
mx = stime[j] < maxtime
mn = stime[j] > mintime
TF = mx * mn
if TF == 1: #time
index.append(i) #turtle index
indx.append(j) #ship index
except:
continue
data = pd.DataFrame(range(len(indx)))
def get_doppio(lat=0, lon=0, depth=99999, time='2018-11-12 12:00:00'):
"""
notice:
the format of time is like "%Y-%m-%d %H:%M:%S"
the depth is under the bottom depth
the module only output the temperature of point location
"""
date_time = datetime.datetime.strptime(
time, '%Y-%m-%d %H:%M:%S') # transform time format
for m in range(0, 7):
try:
url_time = (date_time - datetime.timedelta(days=m)).strftime(
'%Y-%m-%d') #
url = zl.get_doppio_url(url_time)
#get the data
nc = netCDF4.Dataset(url)
except:
continue
lons = nc.variables['lon_rho'][:]
lats = nc.variables['lat_rho'][:]
temp = nc.variables['temp']
doppio_time = nc.variables['time']
doppio_depth = nc.variables['h'][:]
min_diff_time = abs(
datetime.datetime(2017, 11, 1, 0, 0, 0) +
datetime.timedelta(hours=int(doppio_time[0])) - date_time)
min_diff_index = 0
for i in range(1, len(doppio_time)):
diff_time = abs(
datetime.datetime(2017, 11, 1, 0, 0, 0) +
datetime.timedelta(hours=int(doppio_time[i])) - date_time)
if diff_time < min_diff_time:
min_diff_time = diff_time
min_diff_index = i
#calculate the min,second small and third small distance and index
min_distance = zl.dist(lat1=lat,
lon1=lon,
lat2=lats[0][0],
lon2=lons[0][0])
# secondmin_distance=min_distance
# thirdmin_distance=min_distance
#
index_1, index_2 = 0, 0
secondindex_1, secondindex_2 = 0, 0
thirdindex_1, thirdindex_2 = 0, 0
fourthindex_1, fourthindex_2 = 0, 0
fifthindex_1, fifthindex_2 = 0, 0
# sixthindex_1,sixthindex_2=0,0
for i in range(len(lons)):
for j in range(len(lons[i])):
distance = zl.dist(lat1=lat,
lon1=lon,
lat2=lats[i][j],
lon2=lons[i][j])
if min_distance >= distance:
# thirdmin_distance=secondmin_distance
# secondmin_distance=min_distance
min_distance = distance
# sixthindex_1,sixthindex_2=fifthindex_1,fifthindex_2
fifthindex_1, fifthindex_2 = fourthindex_1, fourthindex_2
fourthindex_1, fourthindex_2 = thirdindex_1, thirdindex_2
thirdindex_1, thirdindex_2 = secondindex_1, secondindex_2
secondindex_1, secondindex_2 = index_1, index_2
index_1, index_2 = i, j
if depth == 99999:
S_coordinate = 1
else:
S_coordinate = float(depth) / float(doppio_depth[index_1][index_2])
if 0 <= S_coordinate < 1:
layer_index = 39 - int(S_coordinate / 0.025)
elif S_coordinate == 1:
layer_index = 39
else:
return 9999
point=[[lats[index_1][index_2],lons[index_1][index_2],temp[min_diff_index][layer_index][index_1][index_2]],\
[lats[secondindex_1][secondindex_2],lons[secondindex_1][secondindex_2],temp[min_diff_index][layer_index][secondindex_1][secondindex_2]],\
[lats[thirdindex_1][thirdindex_2],lons[thirdindex_1][thirdindex_2],temp[min_diff_index][layer_index][thirdindex_1][thirdindex_2]],\
[lats[fourthindex_1][fourthindex_2],lons[fourthindex_1][fourthindex_2],temp[min_diff_index][layer_index][fourthindex_1][fourthindex_2]],\
[lats[fifthindex_1][fifthindex_2],lons[fifthindex_1][fifthindex_2],temp[min_diff_index][layer_index][fifthindex_1][fifthindex_2]]]
point_temp = fitting(point, lat, lon)
if np.isnan(point_temp):
continue
if min_diff_time < datetime.timedelta(hours=1):
break
return point_temp, index_1, index_2
def match_tele_raw(
input_dir,
path_save,
telemetry_status,
start_time,
end_time,
telemetry_path='https://www.nefsc.noaa.gov/drifter/emolt.dat',
accept_minutes_diff=20,
acceptable_distance_diff=2,
dpi=300):
"""
match the file and telementy.
we can known how many file send to the satallite and output the figure
"""
#read the file of the telementry_status
telemetrystatus_df = read_telemetrystatus(telemetry_status)
#st the record file use to write minmum maxmum and average of depth and temperature,the numbers of file, telemetry and successfully matched
record_file_df=telemetrystatus_df.loc[:,['Boat','Vessel#']].reindex(columns=['Boat','Vessel#','matched_number','file_number','tele_num','max_diff_depth',\
'min_diff_depth','average_diff_depth','max_diff_temp','min_diff_temp','average_diff_temp','sum_diff_depth','sum_diff_temp',\
'min_lat','max_lat','min_lon','max_lon'],fill_value=None)
#transfer the time format of string to datetime
start_time_local = datetime.strptime(start_time, '%Y-%m-%d')
end_time_local = datetime.strptime(end_time, '%Y-%m-%d')
allfile_lists = zl.list_all_files(input_dir)
######################
file_lists = []
for file in allfile_lists:
if file[len(file) - 4:] == '.csv':
file_lists.append(file)
#download the data of telementry
tele_df = read_telemetry(telemetry_path)
#screen out the data of telemetry in interval
valuable_tele_df = pd.DataFrame(
data=None,
columns=['vessel_n', 'esn', 'time', 'lon', 'lat', 'depth',
'temp']) #use to save the data during start time and end time
for i in range(len(tele_df)):
tele_time=datetime.strptime(str(tele_df['year'].iloc[i])+'-'+str(tele_df['month'].iloc[i])+'-'+str(tele_df['day'].iloc[i])+' '+\
str(tele_df['Hours'].iloc[i])+':'+str(tele_df['minates'].iloc[i])+':'+'00','%Y-%m-%d %H:%M:%S')
if zl.local2utc(start_time_local) <= tele_time < zl.local2utc(
end_time_local):
valuable_tele_df=valuable_tele_df.append(pd.DataFrame(data=[[tele_df['vessel_n'][i],tele_df['esn'][i],tele_time,tele_df['lon'][i],tele_df['lat'][i],tele_df['depth'][i],\
tele_df['temp'][i]]],columns=['vessel_n','esn','time','lon','lat','depth','temp']))
valuable_tele_df.index = range(len(valuable_tele_df))
#whether the data of file and telemetry is exist
if len(valuable_tele_df) == 0 and len(file_lists) == 0:
print(
'please check the data website of telementry and the directory of raw_data is exist!'
)
sys.exit()
elif len(valuable_tele_df) == 0:
print('please check the data website of telementry!')
sys.exit()
elif len(file_lists) == 0:
print('please check the directory raw_data is exist!')
sys.exit()
#match the file
index = telemetrystatus_df['Boat'] #set the index for dictionary
raw_dict = {
} #the dictinary about raw data, use to write the data about 'time','filename','mean_temp','mean_depth'
tele_dict = {
} #the dictionary about telementry data,use to write the data about'time','mean_temp','mean_depth'
for i in range(len(index)): #loop every boat
raw_dict[index[i]] = pd.DataFrame(data=None,
columns=[
'time', 'filename', 'mean_temp',
'mean_depth', 'mean_lat',
'mean_lon'
])
tele_dict[index[i]] = pd.DataFrame(data=None,
columns=[
'time', 'mean_temp',
'mean_depth', 'mean_lat',
'mean_lon'
])
for file in file_lists: # loop raw files
fpath, fname = os.path.split(file) #get the file's path and name
# now, read header and data of every file
header_df = zl.nrows_len_to(file, 2, name=['key',
'value']) #only header
data_df = zl.skip_len_to(file, 2) #only data
#caculate the mean temperature and depth of every file
value_data_df = data_df.ix[(
data_df['Depth(m)'] >
0.85 * mean(data_df['Depth(m)']))] #filter the data
value_data_df = value_data_df.ix[
2:] #delay several minutes to let temperature sensor record the real bottom temp
value_data_df=value_data_df.ix[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \
(value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error
value_data_df.index = range(len(value_data_df)) #reindex
for i in range(len(value_data_df['Lat'])):
value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd(
value_data_df['Lat'][i], value_data_df['Lon'][i])
min_lat = min(value_data_df['Lat'].values)
max_lat = max(value_data_df['Lat'].values)
min_lon = min(value_data_df['Lon'].values)
max_lon = max(value_data_df['Lon'].values)
mean_lat = str(round(mean(value_data_df['Lat'].values), 4))
mean_lon = str(round(mean(value_data_df['Lon'].values),
4)) #caculate the mean depth
mean_temp = str(
round(mean(value_data_df['Temperature(C)'][1:len(value_data_df)]),
2))
mean_depth = str(
abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill(
3) #caculate the mean depth
#get the vessel number of every file
for i in range(len(header_df)):
if header_df['key'][i].lower() == 'vessel number'.lower():
vessel_number = int(header_df['value'][i])
break
#caculate the number of raw files in every vessel,and min,max of lat and lon
for i in range(len(record_file_df)):
if record_file_df['Vessel#'][i] == vessel_number:
if record_file_df['file_number'].isnull()[i]:
record_file_df['min_lat'][i] = min_lat
record_file_df['max_lat'][i] = max_lat
record_file_df['min_lon'][i] = min_lon
record_file_df['max_lon'][i] = max_lon
record_file_df['file_number'][i] = 1
else:
record_file_df['file_number'][i] = int(
record_file_df['file_number'][i] + 1)
if record_file_df['min_lat'][i] > min_lat:
record_file_df['min_lat'][i] = min_lat
if record_file_df['max_lat'][i] < max_lat:
record_file_df['max_lat'][i] = max_lat
if record_file_df['min_lon'][i] > min_lon:
record_file_df['min_lon'][i] = min_lon
if record_file_df['max_lon'][i] < max_lon:
record_file_df['max_lon'][i] = max_lon
#match rawdata and telementry data
time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split(
'.')[0].split('_')[3]
#GMT time to local time of file
time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] +
'-' + time_str[6:8] + ' ' +
time_str[9:11] + ':' + time_str[11:13] +
':' + time_str[13:15])
time_gmt = datetime.strptime(time_str, "%Y%m%d %H%M%S")
#transfer the format latitude and longitude
lat, lon = value_data_df['Lat'][
len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) -
1]
#write the data of raw file to dict
for i in range(len(telemetrystatus_df)):
if telemetrystatus_df['Vessel#'][i] == vessel_number:
raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\
fname,float(mean_temp),float(mean_depth),float(mean_lat),float(mean_lon)]],columns=['time','filename','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True)
#caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file
for i in range(len(valuable_tele_df)):
if valuable_tele_df['vessel_n'][i].split('_')[1] == str(
vessel_number):
if abs(valuable_tele_df['time'][i] - time_gmt) <= timedelta(
minutes=accept_minutes_diff): #time match
if zl.dist(lat1=lat,
lon1=lon,
lat2=float(valuable_tele_df['lat'][i]),
lon2=float(valuable_tele_df['lon'][i])
) <= acceptable_distance_diff: #distance match
for j in range(len(record_file_df)):
if record_file_df['Vessel#'][j] == vessel_number:
diff_temp = round(
(float(mean_temp) -
float(valuable_tele_df['temp'][i])), 4)
diff_depth = round(
(float(mean_depth) -
float(valuable_tele_df['depth'][i])), 4)
if record_file_df['matched_number'].isnull(
)[j]:
record_file_df['matched_number'][j] = 1
record_file_df['sum_diff_temp'][
j] = diff_temp
record_file_df['max_diff_temp'][
j] = diff_temp
record_file_df['min_diff_temp'][
j] = diff_temp
record_file_df['sum_diff_depth'][
j] = diff_depth
record_file_df['max_diff_depth'][
j] = diff_depth
record_file_df['min_diff_depth'][
j] = diff_depth
break
else:
record_file_df['matched_number'][j] = int(
record_file_df['matched_number'][j] +
1)
record_file_df['sum_diff_temp'][
j] = record_file_df['sum_diff_temp'][
j] + diff_temp
record_file_df['sum_diff_depth'][
j] = record_file_df['sum_diff_depth'][
j] + diff_depth
if record_file_df['max_diff_temp'][
j] < diff_temp:
record_file_df['max_diff_temp'][
j] = diff_temp
if record_file_df['min_diff_temp'][
j] > diff_temp:
record_file_df['min_diff_temp'][
j] = diff_temp
if record_file_df['max_diff_depth'][
j] < diff_depth:
record_file_df['max_diff_depth'][
j] = diff_depth
if record_file_df['min_diff_depth'][
j] > diff_depth:
record_file_df['min_diff_depth'][
j] = diff_depth
break
#write 'time','mean_temp','mean_depth' of the telementry to tele_dict
for i in range(
len(valuable_tele_df)
): #valuable_tele_df is the valuable telemetry data during start time and end time
for j in range(len(telemetrystatus_df)):
if int(valuable_tele_df['vessel_n'][i].split('_')
[1]) == telemetrystatus_df['Vessel#'][j]:
#count the numbers by boats
if record_file_df['tele_num'].isnull()[j]:
record_file_df['tele_num'][j] = 1
else:
record_file_df['tele_num'][
j] = record_file_df['tele_num'][j] + 1
if record_file_df['max_lat'].isnull()[j]:
record_file_df['min_lat'][j] = valuable_tele_df['lat'][i]
record_file_df['max_lat'][j] = valuable_tele_df['lat'][i]
record_file_df['min_lon'][j] = valuable_tele_df['lon'][i]
record_file_df['max_lon'][j] = valuable_tele_df['lon'][i]
else:
if record_file_df['min_lat'][j] > valuable_tele_df['lat'][
i]:
record_file_df['min_lat'][j] = valuable_tele_df['lat'][
i]
if record_file_df['max_lat'][j] < valuable_tele_df['lat'][
i]:
record_file_df['max_lat'][j] = valuable_tele_df['lat'][
i]
if record_file_df['min_lon'][j] > valuable_tele_df['lon'][
i]:
record_file_df['min_lon'][j] = valuable_tele_df['lon'][
i]
if record_file_df['max_lon'][j] < valuable_tele_df['lon'][
i]:
record_file_df['max_lon'][j] = valuable_tele_df['lon'][
i]
#write 'time','mean_temp','mean_depth' of the telementry to tele_dict
tele_dict[telemetrystatus_df['Boat'][j]]=tele_dict[telemetrystatus_df['Boat'][j]].append(pd.DataFrame(data=[[valuable_tele_df['time'][i],\
float(valuable_tele_df['temp'][i]),float(valuable_tele_df['depth'][i]),float(valuable_tele_df['lat'][i]),float(valuable_tele_df['lon'][i])]],columns=['time','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True)
print("finish the calculate of min_lat and min_lon!")
for i in range(len(record_file_df)):
if not record_file_df['matched_number'].isnull()[i]:
record_file_df['average_diff_depth'][i] = round(
record_file_df['sum_diff_depth'][i] /
record_file_df['matched_number'][i], 4)
record_file_df['average_diff_temp'][i] = round(
record_file_df['sum_diff_temp'][i] /
record_file_df['matched_number'][i], 4)
else:
record_file_df['matched_number'][i] = 0
if record_file_df['tele_num'].isnull()[i]:
record_file_df['tele_num'][i] = 0
if record_file_df['file_number'].isnull()[i]:
record_file_df['file_number'][i] = 0
for i in index: #loop every boat, i represent the name of boat
raw_dict[i] = raw_dict[i].sort_values(by=['time'])
raw_dict[i].index = range(len(raw_dict[i]))
record_file_df = record_file_df.drop(['sum_diff_depth', 'sum_diff_temp'],
axis=1)
#save the record file
record_file_df.to_csv(path_save + '/' + start_time + '_' + end_time +
' statistics.csv',
index=0)
return raw_dict, tele_dict, record_file_df, index, start_time_local, end_time_local, path_save
