def getclim(lat1=43.,
lon1=-73.0,
yrday=dt.now().strftime('%j'),
var='Bottom_Temperature/BT_'):
# gets climatology of Bottom_Temperature, Surface_Temperature, Bottom_Salinity, or Surface_Salinity
# as calculated by Chris Melrose from 30+ years of NEFSC CTD data on the NE Shelf
# where "lat1", "lon1", and "yrday" are the position and yearday of interest
# where "var" is the variable of interest (usually Bottom_Temperature)
inputdir = '/net/data5/jmanning/clim/' # hardcoded climatology directory name "/home/pi/clim/"
inputdir_csv = '/net/data5/jmanning/li/Matdata/temporary/' # the csv files that have lat/lon "/home/pi/towifi/"
dflat = pd.read_csv(inputdir + 'LatGrid.csv', header=None)
dflon = pd.read_csv(inputdir + 'LonGrid.csv', header=None)
lat = np.array(dflat[0]) # gets the first col (35 to 45)
lon = np.array(dflon.ix[0]) # gets the first row (-75 to -65)
clim = pd.read_csv(inputdir + var + yrday + '.csv',
header=None) # gets bottom temp for this day of year
print lat1, lon1
if lat1 == 43.:
files = sorted(glob.glob(
inputdir_csv +
'*.csv')) # gets all the csv files in the towfi directory
dfcsv = pd.read_csv(files[-1], sep=',', skiprows=7)
[lat1, lon1] = dm2dd(float(dfcsv['lat'][0][0:-1]),
float(dfcsv['lon'][0][0:-1]))
print lat1, lon1
idlat = np.abs(lat - lat1).argmin() # finds the neareast lat to input lat1
idlon = np.abs(lon - lon1).argmin() # finds the neareast lon to input lon1
return clim[idlon][idlat]
def watertemp(self, lon, lat, depth, url):
data = self.get_data(url)
lonc, latc = data['lonc'][:], data['latc'][:]
lonv, latv = data['lon'][:], data['lat'][:]
h = data['h'][:]
siglay = data['siglay'][:]
if lon > 90:
lon, lat = dm2dd(lon, lat)
nodes = dict(lon=[], lat=[])
kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc, num=1)
kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv, num=1)
if h[kv] < 0:
sys.exit(
'Sorry, your position is on land, please try another point')
depth_total = siglay[:, kv] * h[kv]
###############layer###########################
layer = np.argmin(abs(depth_total - depth))
print kv
print 'layer, kv', layer, kv[0][0]
# for i in range(len(data['time'][:])):
temp = data['temp'][:, layer, kv[0][0]]
print 'l'
m = pd.DataFrame(temp, index=data['time'][:])
print 'a'
return m
def get_coors(modelname, lo, la, lonc, latc, lon, lat, siglay, h, depth,startrecord, endrecord):
if lo>90:
[la,lo]=dm2dd(la,lo)
print 'la, lo',la, lo
latd,lond=[la],[lo]
# kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face
# kv,distanceV=nearlonlat(lon,lat,lo,la)
kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1)
kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1)
kf = kf[0][0]
kv = kv[0][0]
print 'kf:', kf
if h[kv] < 0:
print 'Sorry, your position is on land, please try another point'
sys.exit()
depthtotal=siglay[:,kv]*h[kv]
layer=np.argmin(abs(depthtotal-depth))
for i in range(startrecord,endrecord):# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
############read the particular time model from website#########
# print 'la, lo, i', la, lo, i
timeurl='['+str(i)+':1:'+str(i)+']'
uvposition=str([layer])+str([kf])
data_want = ('u'+timeurl+uvposition, 'v'+timeurl+uvposition)
# if urlname=="30yr":
# url='http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?'+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
# elif urlname == "GOM3":
# url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc?"+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
# else:
# url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc?"+\
# 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition
url = url_with_time_position(modelname, data_want)
dataset = open_url(url)
u=np.array(dataset['u'])
v=np.array(dataset['v'])
print 'u, v, i', u[0,0,0], v[0,0,0],i
################get the point according the position###################
par_u=u[0,0,0]
par_v=v[0,0,0]
xdelta=par_u*60*60 #get_coors
ydelta=par_v*60*60
latdelta=ydelta/111111
londelta=(xdelta/(111111*np.cos(la*np.pi/180)))
la=la+latdelta
lo=lo+londelta
latd.append(la)
lond.append(lo)
# kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face
# kv,distanceV=nearlonlat(lon,lat,lo,la)# nearest triangle vertex V - vertex
kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1)
kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1)
kf, kv = kf[0][0], kv[0][0]
depthtotal=siglay[:,kv]*h[kv]
# layer=np.argmin(abs(depthtotal-depth))
if distanceV>=0.3:
if i==startrecord:
print 'Sorry, your start position is NOT in the model domain'
break
return latd ,lond
def __get_track(self, lon, lat, depth, url):
'''
start, end: indices of some period
data: a dict that has 'u' and 'v'
'''
data = self.get_data(url)
lonc, latc = data['lonc'][:], data['latc'][:]
lonv, latv = data['lon'][:], data['lat'][:]
h = data['h'][:]
siglay = data['siglay'][:]
if lon>90:
lon, lat = dm2dd(lon, lat)
nodes = dict(lon=[], lat=[])
kf,distanceF = self.nearest_point_index(lon,lat,lonc,latc,num=1)
kv,distanceV = self.nearest_point_index(lon,lat,lonv,latv,num=1)
if h[kv] < 0:
sys.exit('Sorry, your position is on land, please try another point')
depth_total = siglay[:,kv]*h[kv]
###########################layer#####################################
'''
????? Layer for what?????
'''
layer = np.argmin(abs(depth_total-depth))
for i in range(self.hours):
u_t = data['u'][i, layer, kf[0][0]]
v_t = data['v'][i, layer, kf[0][0]]
print 'u_t, v_t, i', u_t, v_t, i
dx = 60*60*u_t
dy = 60*60*v_t
lon = lon + (dx/(111111*np.cos(lat*np.pi/180)))
lat = lat + dy/111111
nodes['lon'].append(lon)
nodes['lat'].append(lat)
kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc,num=1)
kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv,num=1)
# depth_total = siglay[:][kv]*h[kv]
if distanceV>=.3:
if i==start:
print 'Sorry, your start position is NOT in the model domain'
break
return nodes
def __waternode(self, lon, lat, depth, url):
'''
start, end: indices of some period
data: a dict that has 'u' and 'v'
'''
data = self.get_data(url)
lonc, latc = data['lonc'][:], data['latc'][:]
lonv, latv = data['lon'][:], data['lat'][:]
h = data['h'][:]
siglay = data['siglay'][:]
if lon > 90:
lon, lat = dm2dd(lon, lat)
nodes = dict(lon=[], lat=[])
kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc, num=1)
kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv, num=1)
if h[kv] < 0:
sys.exit(
'Sorry, your position is on land, please try another point')
depth_total = siglay[:, kv] * h[kv]
###############layer###########################
layer = np.argmin(abs(depth_total - depth))
# for i in range(len(data['u'])):
for i in range(self.hours):
u_t = data['u'][i, layer, kf[0][0]]
v_t = data['v'][i, layer, kf[0][0]]
dx = 60 * 60 * u_t
dy = 60 * 60 * v_t
lon = lon + (dx / (111111 * np.cos(lat * np.pi / 180)))
lat = lat + dy / 111111
nodes['lon'].append(lon)
nodes['lat'].append(lat)
kf, distanceF = self.nearest_point_index(lon,
lat,
lonc,
latc,
num=1)
kv, distanceV = self.nearest_point_index(lon,
lat,
lonv,
latv,
num=1)
# depth_total = siglay[:][kv]*h[kv]
if distanceV >= .3:
if i == start:
print 'Sorry, your start position is NOT in the model domain'
break
modtso = pd.DataFrame(nodes, )
return nodes
""" Compares eMOLT with FVCOM bottom temp @author: jmanning, rsignell, yacheng """ import matplotlib.pyplot as plt import netCDF4 from getdata import getemolt_latlon,getemolt_temp from conversions import dm2dd,f2c from utilities import nearxy site='BN01' [lati,loni,on]=getemolt_latlon(site) # extracts lat/lon based on site code [lati,loni]=dm2dd(lati,loni) #converts decimal-minutes to decimal degrees [obs_dt,obs_temp]=getemolt_temp(site) # extracts time series for kk in range(len(obs_temp)): obs_temp[kk]=f2c(obs_temp[kk]) # converts to Celcius # now get the model output urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' nc = netCDF4.Dataset(urlfvcom) nc.variables lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] times = nc.variables['time'] jd = netCDF4.num2date(times[:],times.units) vname = 'temp' var = nc.variables[vname] # find nearest point to desired location and time inode = nearxy(lon,lat,loni,lati) index=netCDF4.date2index([obs_dt[0],obs_dt[-1]],times,select='nearest')#find the model time index at start & end pf obs
def get_w_depth(xi,yi):
url='http://geoport.whoi.edu/thredds/dodsC/bathy/gom03_v1_0'#url='http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol1.nc' ):
if xi[0]>999.: # if it comes in decimal -minutes, conert it
#for kk in range(len(xi)):
(y2,x2)=dm2dd(yi[0],xi[0])
yi[0]=y2
xi[0]=x2
try:
dataset = open_url(url)
except:
print 'Sorry, ' + url + ' is not available'
sys.exit(0)
#read lat, lon,topo from url
xgom_array = dataset['lon']
ygom_array = dataset['lat']
dgom_array = dataset['topo'].topo
#print dgom_array.shape, xgom_array[5:9],dgom_array[5]
#convert the array to a list
xgom, ygom = [], []
for i in xgom_array:
if i > xi[0] - 0.00834 and i < xi[0] + 0.00834:
xgom.append(i)
for i in ygom_array:
if i > yi[0] - 0.00834 and i < yi[0] + 0.00834:
ygom.append(i)
x_index, y_index = [], []
(ys, xs) = dgom_array.shape
for i in range(0, len(xgom)):
x_index.append(int(round(np.interp(xgom[i], xgom_array, range(xs)))))
for i in range(0, len(ygom)):
y_index.append(int(round(np.interp(ygom[i], ygom_array, range(ys)))))
dep, distkm, dist1 = [], [], []
for k in range(len(x_index)):
for j in range(len(y_index)):
dep.append(dgom_array[(y_index[j], x_index[k])])
distkm, b = distance((ygom[j], xgom[k]), (yi[0], xi[0]))
dist1.append(distkm)
#get the nearest,second nearest,third nearest point.
dist_f_nearest = sorted(dist1)[0]
dist_s_nearest = sorted(dist1)[1]
dist_t_nearest = sorted(dist1)[2]
index_dist_nearest = range(len(dist1))
index_dist_nearest.sort(lambda x, y:cmp(dist1[x], dist1[y]))
dep_f_nearest = dep[index_dist_nearest[0]]
dep_s_nearest = dep[index_dist_nearest[1]]
dep_t_nearest = dep[index_dist_nearest[2]]
#compute the finally depth
d1 = dist_f_nearest
d2 = dist_s_nearest
d3 = dist_t_nearest
def1 = dep_f_nearest
def2 = dep_s_nearest
def3 = dep_t_nearest
depth_finally = def1 * d2 * d3 / (d1 * d2 + d2 * d3 + d1 * d3) + def2 * d1 * d3 / (d1 * d2 + d2 * d3 + d1 * d3) + def3 * d2 * d1 / (d1 * d2 + d2 * d3 + d1 * d3)
return depth_finally
return haul,hauls,datetc
# plotting the catch
#..how best to plot as in "skiphole.m" maybe using scikits ...
fig=plt.figure(1,figsize=(8,6))
for k in range(numpeople):
ax1=fig.add_subplot(numpeople,1,k+1)
# Now go get the temperature data associated with this site
# pipe = subprocess.Popen(["perl", "/home3/ocn/jmanning/sql/gettsll_justtemp.plx","sc","depset"
# this subprocess was not possible ... I need to learn how to run Oracle from python
variables=['year','year_day','temp','lat','lon','depth']
df=pd.read_csv(input_directory+'this2.dat', names=variables, sep='\s+')
#t=ml.load('/net/home3/ocn/jmanning/sql/this2.dat') old way of loading
lat,lon,datet=[],[],[]
for j in range(len(df)):
#la,lo=conversions.dm2dd([t[j,3]],[t[j,4]]) #changed this in May 2015
la,lo=conversions.dm2dd(df['lat'].values[0],df['lon'].values[0])
lat.append(la)
lon.append(lo)
if df['year_day'].values[j]+1<366.0:
#datet.append(num2date(t[j,1]+1).replace(year=int(t[j,0])))
datet.append(num2date(df['year_day'].values[j]+1).replace(year=int(df['year'].values[j])))
else:
#datet.append(num2date(t[j,1]+1).replace(year=int(t[j,0])+1))
datet.append(num2date(df['year_day'].values[j]+1).replace(year=int(df['year'].values[j]+1)))
#wd=t[:,5]
#temp=t[:,2]
#temp_smooth=utilities.smooth(temp,24*7,'hanning')
wd=df['depth'].values
temp=df['temp'].values
temp_smooth=utilities.smooth(temp,hrs,smoothing_method)
ax1.plot_date(datet,temp,'k-')
lonsize=[event.xdata-0.6,event.xdata+0.6]
plt.figure(figsize=(7,6))
m = Basemap(projection='cyl',llcrnrlat=min(latsize)-0.01,urcrnrlat=max(latsize)+0.01,\
llcrnrlon=min(lonsize)-0.01,urcrnrlon=max(lonsize)+0.01,resolution='h')#,fix_aspect=False)
m.drawparallels(np.arange(int(min(latsize)),int(max(latsize))+1,1),labels=[1,0,0,0])
m.drawmeridians(np.arange(int(min(lonsize)),int(max(lonsize))+1,1),labels=[0,0,0,1])
m.drawcoastlines()
m.fillcontinents(color='grey')
m.drawmapboundary()
m.plot(lon,lat,'r.',lonc,latc,'b+')
plt.show()
spoint = pylab.ginput(1)
'''
if lo>90:
[la,lo]=dm2dd(la,lo)
latd,lond=[],[]
kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face
kv,distanceV=nearlonlat(lon,lat,lo,la)
if h[kv] < 0:
print 'Sorry, your position is on land, please try another point'
sys.exit()
depthtotal=siglay[:,kv]*h[kv]
layer=np.argmin(abs(depthtotal-depth))
for i in range(startrecord,endrecord):
############read the particular time model from website#########
timeurl='['+str(i)+':1:'+str(i)+']'
2))
mean_depth = str(
abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill(
3) #caculate the mean depth
#match rawdata and telementry data
time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split(
'.')[0].split('_')[3]
#GMT time to local time of file
time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] +
'-' + time_str[6:8] + ' ' +
time_str[9:11] + ':' + time_str[11:13] +
':' + time_str[13:15])
time_gmt = datetime.datetime.strptime(time_str, "%Y%m%d %H%M%S")
#transfer the format latitude and longitude
lat, lon = cv.dm2dd(value_data_df['Lat'][len(value_data_df) - 1],
value_data_df['Lon'][len(value_data_df) - 1])
#write the data of raw file to dict
for i in range(len(telemetrystatus_df)):
if telemetrystatus_df['Vessel#'][i] == vessel_number:
raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\
fname,float(mean_temp),float(mean_depth)]],columns=['time','filename','mean_temp','mean_depth']).iloc[0],ignore_index=True)
#caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file
for i in range(len(valuable_tele_df)):
if valuable_tele_df['vessel_n'][i].split('_')[1] == str(
vessel_number):
if abs(valuable_tele_df['time'][i] -
time_gmt) <= acceptable_time_diff: #time match
if zl.dist(lat1=lat,
lon1=lon,
lat2=float(valuable_tele_df['lat'][i]),
lon2=float(valuable_tele_df['lon'][i])
# now, read header and data of every file
header_df = zl.nrows_len_to(file, 2, name=['key',
'value']) #only header
data_df = zl.skip_len_to(file, 2) #only data
value_data_df = data_df.loc[(
data_df['Depth(m)'] >
0.95 * max(data_df['Depth(m)']))] #filter the data
value_data_df = value_data_df.iloc[
7:] #delay several minutes to let temperature sensor record the real bottom temp
value_data_df=value_data_df.loc[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \
(value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error
value_data_df.index = range(len(value_data_df)) #reindex
value_data_df[
'Datet(Y/m/d)'] = 1 #create a new column for saving another time style of '%Y-%m-%d'
for i in range(len(value_data_df)):
value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd(
value_data_df['Lat'][i], value_data_df['Lon'][i])
#value_data_df['Datet(Y/m/d)'][i]=datetime.strptime(value_data_df['Datet(GMT)'][i],'%Y-%m-%d %H:%M:%S')
Hours_df = pd.DataFrame(
data=None,
columns=['time', 'lat', 'lon', 'depth', 'temp', 'new_time'])
Hours_df['time'] = value_data_df['Datet(GMT)']
Hours_df['lat'] = value_data_df['Lat']
Hours_df['lon'] = value_data_df['Lon']
Hours_df['depth'] = value_data_df['Depth(m)']
Hours_df['temp'] = value_data_df['Temperature(C)']
#value_data_df['Datet(Y/m/d)']=pd.to_datetime(value_data_df['Datet(Y/m/d)'])#change the time style to datetime
Hours_df['new_time'] = value_data_df['Datet(GMT)']
if not os.path.exists(Hours_save + file.split('/')[8]):
os.makedirs(Hours_save + file.split('/')[8])
Hours_df.to_csv(os.path.join(
Hours_save + file.split('/')[8] + '/',
else:
nodes_drifter = drifter.waternode(starttime)
else:
nodes_drifter = drifter.waternode()
lon, lat = nodes_drifter['lon'][0], nodes_drifter['lat'][0]
starttime = nodes_drifter['time'][0]
endtime = nodes_drifter['time'][-1]
'''
lat = 4111.797
lon = 6934.554
depth = -1
days = 3
lat, lon=dm2dd(lat,lon)
starttime = datetime(2014,8,26,12,00)
# endtime = datetime(2014,7,15,19)
endtime = starttime+timedelta(days=days)
water_fvcom = water_fvcom()
url_fvcom = water_fvcom.get_url(starttime, endtime)
nodes_fvcom = water_fvcom.waternode(lon,lat,depth,url_fvcom)
# water_roms = water_roms()
# url_roms = water_roms.get_url(starttime, endtime)
# nodes_roms = water_roms.waternode(lon, lat, depth, url_roms)
'''
water_roms_rk4 = water_roms_rk4()
url_roms_rk4 = water_roms_rk4.get_url(starttime, endtime)
nodes_roms_rk4 = water_roms_rk4.waternode(lon, lat, depth, url_roms_rk4)
lat=[40.,45.]
lon=[-71.,-65.]
comp='p_emoltvsmod'# this is the mode of comparison where p_ referes to percent difference
print comp
m = Basemap(projection='cyl',llcrnrlat=min(lat)-0.01,urcrnrlat=max(lat)+0.01,\
llcrnrlon=min(lon)-0.01,urcrnrlon=max(lon)+0.01,resolution='h')
m.drawcoastlines()
m.fillcontinents(color='grey')
m.drawmapboundary()
ebd,mbd,ubd,hbd=[],[],[],[]
diffd=[]
outf.write('site,lon,lat,ebd,udb,hbd,mbd,perc_diff\n')
for k in range(len(site)):
[elat, elon, original_name,eb]=getemolt_latlon(site[k])
ebd.append(eb)
[elat_dd,elon_dd]=dm2dd(elat,elon)
mbd.append(getFVCOM_depth(elat_dd,elon_dd))
ubd.append(-1*getdepth(elat_dd,elon_dd))#nearset usgs depth
hbd.append(-1*get_w_depth( [elon_dd],[elat_dd]))#huanxins interpolated depth
if comp=='usgsvsmod':
diffd.append(ubd[k]-mbd[k])
elif comp=='p_emoltvsmod':
diffd.append((ebd[k]-mbd[k])/ebd[k]*100)
if diffd[k]<1:
dotcol='magenta'
else:
dotcol='cyan'
plt.scatter(elon_dd,elat_dd,25*abs(diffd[k]),marker='o',color=dotcol)
#print 'ebd='+str(ebd)+' ubd='+str(ubd)+' hbd='+str(hbd)+' mbd='+str(mbd)
outf.write(site[k]+','+'%6.3f' % (elon_dd)+','+'%6.3f' % (elat_dd)+','+'%6.1f' % (ebd[k])+','+'%6.1f' % (mbd[k])+','+'%6.1f' % (ubd[k][0])+','+'%6.1f' % (hbd[k][0])+','+'%6.1f' % (diffd[k])+'\n')
# Forth, the main program ############################################################################
fn='binned_td_test.csv'
direct='/net/data5/jmanning/hoey/'
outputdir='/net/data5/jmanning/modvsobs/sfleet/'
# read in the study fleet data
def parse(datet):
#print datet[0:10],datet[11:13],datet[14:16]
dtt=dt.strptime(datet,'%Y-%m-%d:%H:%M')
return dtt
obstso=pd.read_csv(direct+fn,sep=',',skiprows=0,parse_dates={'datet':[2]},index_col='datet',date_parser=parse,names=['LATITUDE','LONGITUDE','ROUND_DATE_TIME','OBSERVATIONS_TEMP','MEAN_TEMP','MIN_TEMP','MAX_TEMP','STD_DEV_TEMP','OBSERVATIONS_DEPTH','MEAN_DEPTH','MIN_DEPTH','MAX_DEPTH','STD_DEV_DEPTH','nan'])
#convert ddmm.m to dd.ddd
plt.figure(figsize=(16,10))
o=[]
m=[]
for k in range(len(obstso)): #
[la,lo]=dm2dd(obstso['LATITUDE'][k],obstso['LONGITUDE'][k])
if not inmodel_roms([lo], [la]):
print 'point not in roms domain'
continue
else:
romobj = water_roms()
modtso = pd.DataFrame()
st=obstso.index[k]
et=st+td(hours=1)
url_roms = romobj.get_url(st,et )
modtso= romobj.waternode(lo, la,-1*obstso['MEAN_DEPTH'][k], url_roms)
o.append(obstso['MEAN_TEMP'][k])
m.append(modtso.values[0])
plt.plot(modtso.values[0],obstso['MEAN_TEMP'][k],'*', markersize=30)
plt.xlabel('MODEL (degC)')
plt.ylabel('OBSERVATIONS (degC)')
def match_tele_raw(
input_dir,
path_save,
telemetry_status,
start_time,
end_time,
telemetry_path='https://www.nefsc.noaa.gov/drifter/emolt.dat',
accept_minutes_diff=20,
acceptable_distance_diff=2,
dpi=300):
"""
match the file and telementy.
we can known how many file send to the satallite and output the figure
"""
#read the file of the telementry_status
telemetrystatus_df = read_telemetrystatus(telemetry_status)
#st the record file use to write minmum maxmum and average of depth and temperature,the numbers of file, telemetry and successfully matched
record_file_df=telemetrystatus_df.loc[:,['Boat','Vessel#']].reindex(columns=['Boat','Vessel#','matched_number','file_number','tele_num','max_diff_depth',\
'min_diff_depth','average_diff_depth','max_diff_temp','min_diff_temp','average_diff_temp','sum_diff_depth','sum_diff_temp',\
'min_lat','max_lat','min_lon','max_lon'],fill_value=None)
#transfer the time format of string to datetime
start_time_local = datetime.strptime(start_time, '%Y-%m-%d')
end_time_local = datetime.strptime(end_time, '%Y-%m-%d')
allfile_lists = zl.list_all_files(input_dir)
######################
file_lists = []
for file in allfile_lists:
if file[len(file) - 4:] == '.csv':
file_lists.append(file)
#download the data of telementry
tele_df = read_telemetry(telemetry_path)
#screen out the data of telemetry in interval
valuable_tele_df = pd.DataFrame(
data=None,
columns=['vessel_n', 'esn', 'time', 'lon', 'lat', 'depth',
'temp']) #use to save the data during start time and end time
for i in range(len(tele_df)):
tele_time=datetime.strptime(str(tele_df['year'].iloc[i])+'-'+str(tele_df['month'].iloc[i])+'-'+str(tele_df['day'].iloc[i])+' '+\
str(tele_df['Hours'].iloc[i])+':'+str(tele_df['minates'].iloc[i])+':'+'00','%Y-%m-%d %H:%M:%S')
if zl.local2utc(start_time_local) <= tele_time < zl.local2utc(
end_time_local):
valuable_tele_df=valuable_tele_df.append(pd.DataFrame(data=[[tele_df['vessel_n'][i],tele_df['esn'][i],tele_time,tele_df['lon'][i],tele_df['lat'][i],tele_df['depth'][i],\
tele_df['temp'][i]]],columns=['vessel_n','esn','time','lon','lat','depth','temp']))
valuable_tele_df.index = range(len(valuable_tele_df))
#whether the data of file and telemetry is exist
if len(valuable_tele_df) == 0 and len(file_lists) == 0:
print(
'please check the data website of telementry and the directory of raw_data is exist!'
)
sys.exit()
elif len(valuable_tele_df) == 0:
print('please check the data website of telementry!')
sys.exit()
elif len(file_lists) == 0:
print('please check the directory raw_data is exist!')
sys.exit()
#match the file
index = telemetrystatus_df['Boat'] #set the index for dictionary
raw_dict = {
} #the dictinary about raw data, use to write the data about 'time','filename','mean_temp','mean_depth'
tele_dict = {
} #the dictionary about telementry data,use to write the data about'time','mean_temp','mean_depth'
for i in range(len(index)): #loop every boat
raw_dict[index[i]] = pd.DataFrame(data=None,
columns=[
'time', 'filename', 'mean_temp',
'mean_depth', 'mean_lat',
'mean_lon'
])
tele_dict[index[i]] = pd.DataFrame(data=None,
columns=[
'time', 'mean_temp',
'mean_depth', 'mean_lat',
'mean_lon'
])
for file in file_lists: # loop raw files
fpath, fname = os.path.split(file) #get the file's path and name
# now, read header and data of every file
header_df = zl.nrows_len_to(file, 2, name=['key',
'value']) #only header
data_df = zl.skip_len_to(file, 2) #only data
#caculate the mean temperature and depth of every file
value_data_df = data_df.ix[(
data_df['Depth(m)'] >
0.85 * mean(data_df['Depth(m)']))] #filter the data
value_data_df = value_data_df.ix[
2:] #delay several minutes to let temperature sensor record the real bottom temp
value_data_df=value_data_df.ix[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \
(value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error
value_data_df.index = range(len(value_data_df)) #reindex
for i in range(len(value_data_df['Lat'])):
value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd(
value_data_df['Lat'][i], value_data_df['Lon'][i])
min_lat = min(value_data_df['Lat'].values)
max_lat = max(value_data_df['Lat'].values)
min_lon = min(value_data_df['Lon'].values)
max_lon = max(value_data_df['Lon'].values)
mean_lat = str(round(mean(value_data_df['Lat'].values), 4))
mean_lon = str(round(mean(value_data_df['Lon'].values),
4)) #caculate the mean depth
mean_temp = str(
round(mean(value_data_df['Temperature(C)'][1:len(value_data_df)]),
2))
mean_depth = str(
abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill(
3) #caculate the mean depth
#get the vessel number of every file
for i in range(len(header_df)):
if header_df['key'][i].lower() == 'vessel number'.lower():
vessel_number = int(header_df['value'][i])
break
#caculate the number of raw files in every vessel,and min,max of lat and lon
for i in range(len(record_file_df)):
if record_file_df['Vessel#'][i] == vessel_number:
if record_file_df['file_number'].isnull()[i]:
record_file_df['min_lat'][i] = min_lat
record_file_df['max_lat'][i] = max_lat
record_file_df['min_lon'][i] = min_lon
record_file_df['max_lon'][i] = max_lon
record_file_df['file_number'][i] = 1
else:
record_file_df['file_number'][i] = int(
record_file_df['file_number'][i] + 1)
if record_file_df['min_lat'][i] > min_lat:
record_file_df['min_lat'][i] = min_lat
if record_file_df['max_lat'][i] < max_lat:
record_file_df['max_lat'][i] = max_lat
if record_file_df['min_lon'][i] > min_lon:
record_file_df['min_lon'][i] = min_lon
if record_file_df['max_lon'][i] < max_lon:
record_file_df['max_lon'][i] = max_lon
#match rawdata and telementry data
time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split(
'.')[0].split('_')[3]
#GMT time to local time of file
time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] +
'-' + time_str[6:8] + ' ' +
time_str[9:11] + ':' + time_str[11:13] +
':' + time_str[13:15])
time_gmt = datetime.strptime(time_str, "%Y%m%d %H%M%S")
#transfer the format latitude and longitude
lat, lon = value_data_df['Lat'][
len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) -
1]
#write the data of raw file to dict
for i in range(len(telemetrystatus_df)):
if telemetrystatus_df['Vessel#'][i] == vessel_number:
raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\
fname,float(mean_temp),float(mean_depth),float(mean_lat),float(mean_lon)]],columns=['time','filename','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True)
#caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file
for i in range(len(valuable_tele_df)):
if valuable_tele_df['vessel_n'][i].split('_')[1] == str(
vessel_number):
if abs(valuable_tele_df['time'][i] - time_gmt) <= timedelta(
minutes=accept_minutes_diff): #time match
if zl.dist(lat1=lat,
lon1=lon,
lat2=float(valuable_tele_df['lat'][i]),
lon2=float(valuable_tele_df['lon'][i])
) <= acceptable_distance_diff: #distance match
for j in range(len(record_file_df)):
if record_file_df['Vessel#'][j] == vessel_number:
diff_temp = round(
(float(mean_temp) -
float(valuable_tele_df['temp'][i])), 4)
diff_depth = round(
(float(mean_depth) -
float(valuable_tele_df['depth'][i])), 4)
if record_file_df['matched_number'].isnull(
)[j]:
record_file_df['matched_number'][j] = 1
record_file_df['sum_diff_temp'][
j] = diff_temp
record_file_df['max_diff_temp'][
j] = diff_temp
record_file_df['min_diff_temp'][
j] = diff_temp
record_file_df['sum_diff_depth'][
j] = diff_depth
record_file_df['max_diff_depth'][
j] = diff_depth
record_file_df['min_diff_depth'][
j] = diff_depth
break
else:
record_file_df['matched_number'][j] = int(
record_file_df['matched_number'][j] +
1)
record_file_df['sum_diff_temp'][
j] = record_file_df['sum_diff_temp'][
j] + diff_temp
record_file_df['sum_diff_depth'][
j] = record_file_df['sum_diff_depth'][
j] + diff_depth
if record_file_df['max_diff_temp'][
j] < diff_temp:
record_file_df['max_diff_temp'][
j] = diff_temp
if record_file_df['min_diff_temp'][
j] > diff_temp:
record_file_df['min_diff_temp'][
j] = diff_temp
if record_file_df['max_diff_depth'][
j] < diff_depth:
record_file_df['max_diff_depth'][
j] = diff_depth
if record_file_df['min_diff_depth'][
j] > diff_depth:
record_file_df['min_diff_depth'][
j] = diff_depth
break
#write 'time','mean_temp','mean_depth' of the telementry to tele_dict
for i in range(
len(valuable_tele_df)
): #valuable_tele_df is the valuable telemetry data during start time and end time
for j in range(len(telemetrystatus_df)):
if int(valuable_tele_df['vessel_n'][i].split('_')
[1]) == telemetrystatus_df['Vessel#'][j]:
#count the numbers by boats
if record_file_df['tele_num'].isnull()[j]:
record_file_df['tele_num'][j] = 1
else:
record_file_df['tele_num'][
j] = record_file_df['tele_num'][j] + 1
if record_file_df['max_lat'].isnull()[j]:
record_file_df['min_lat'][j] = valuable_tele_df['lat'][i]
record_file_df['max_lat'][j] = valuable_tele_df['lat'][i]
record_file_df['min_lon'][j] = valuable_tele_df['lon'][i]
record_file_df['max_lon'][j] = valuable_tele_df['lon'][i]
else:
if record_file_df['min_lat'][j] > valuable_tele_df['lat'][
i]:
record_file_df['min_lat'][j] = valuable_tele_df['lat'][
i]
if record_file_df['max_lat'][j] < valuable_tele_df['lat'][
i]:
record_file_df['max_lat'][j] = valuable_tele_df['lat'][
i]
if record_file_df['min_lon'][j] > valuable_tele_df['lon'][
i]:
record_file_df['min_lon'][j] = valuable_tele_df['lon'][
i]
if record_file_df['max_lon'][j] < valuable_tele_df['lon'][
i]:
record_file_df['max_lon'][j] = valuable_tele_df['lon'][
i]
#write 'time','mean_temp','mean_depth' of the telementry to tele_dict
tele_dict[telemetrystatus_df['Boat'][j]]=tele_dict[telemetrystatus_df['Boat'][j]].append(pd.DataFrame(data=[[valuable_tele_df['time'][i],\
float(valuable_tele_df['temp'][i]),float(valuable_tele_df['depth'][i]),float(valuable_tele_df['lat'][i]),float(valuable_tele_df['lon'][i])]],columns=['time','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True)
print("finish the calculate of min_lat and min_lon!")
for i in range(len(record_file_df)):
if not record_file_df['matched_number'].isnull()[i]:
record_file_df['average_diff_depth'][i] = round(
record_file_df['sum_diff_depth'][i] /
record_file_df['matched_number'][i], 4)
record_file_df['average_diff_temp'][i] = round(
record_file_df['sum_diff_temp'][i] /
record_file_df['matched_number'][i], 4)
else:
record_file_df['matched_number'][i] = 0
if record_file_df['tele_num'].isnull()[i]:
record_file_df['tele_num'][i] = 0
if record_file_df['file_number'].isnull()[i]:
record_file_df['file_number'][i] = 0
for i in index: #loop every boat, i represent the name of boat
raw_dict[i] = raw_dict[i].sort_values(by=['time'])
raw_dict[i].index = range(len(raw_dict[i]))
record_file_df = record_file_df.drop(['sum_diff_depth', 'sum_diff_temp'],
axis=1)
#save the record file
record_file_df.to_csv(path_save + '/' + start_time + '_' + end_time +
' statistics.csv',
index=0)
return raw_dict, tele_dict, record_file_df, index, start_time_local, end_time_local, path_save
"""
Compares eMOLT with FVCOM bottom temp
@author: jmanning, rsignell, yacheng
"""
import matplotlib.pyplot as plt
import netCDF4
from getdata import getemolt_latlon, getemolt_temp
from conversions import dm2dd, f2c
from utilities import nearxy
site = 'BN01'
[lati, loni, on] = getemolt_latlon(site) # extracts lat/lon based on site code
[lati, loni] = dm2dd(lati, loni) #converts decimal-minutes to decimal degrees
[obs_dt, obs_temp] = getemolt_temp(site) # extracts time series
for kk in range(len(obs_temp)):
obs_temp[kk] = f2c(obs_temp[kk]) # converts to Celcius
# now get the model output
urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3'
nc = netCDF4.Dataset(urlfvcom)
nc.variables
lat = nc.variables['lat'][:]
lon = nc.variables['lon'][:]
times = nc.variables['time']
jd = netCDF4.num2date(times[:], times.units)
vname = 'temp'
var = nc.variables[vname]
# find nearest point to desired location and time
inode = nearxy(lon, lat, loni, lati)
index = netCDF4.date2index(
index_col='datet',
date_parser=parse,
names=[
'LATITUDE', 'LONGITUDE', 'ROUND_DATE_TIME',
'OBSERVATIONS_TEMP', 'MEAN_TEMP', 'MIN_TEMP',
'MAX_TEMP', 'STD_DEV_TEMP', 'OBSERVATIONS_DEPTH',
'MEAN_DEPTH', 'MIN_DEPTH', 'MAX_DEPTH',
'STD_DEV_DEPTH', 'nan'
])
#convert ddmm.m to dd.ddd
plt.figure(figsize=(16, 10))
o = []
m = []
print '1'
for k in range(len(obstso)): #
[la, lo] = dm2dd(obstso['LATITUDE'][k], obstso['LONGITUDE'][k])
if not inmodel_fvcom([lo], [la]):
print 'point not in fvcom domain'
continue
else:
fvcomobj = water_fvcom()
# modtso = pd.DataFrame()
st = obstso.index[k]
et = st + timedelta(hours=1)
url_fvcom = fvcomobj.get_url(st, et)
modtso = fvcomobj.watertemp(lo, la, -1 * obstso['MEAN_DEPTH'][k],
url_fvcom[0])
o.append(obstso['MEAN_TEMP'][k])
m.append(modtso.values[0])
plt.plot(modtso.values[0], obstso['MEAN_TEMP'][k], '.', markersize=30)
plt.xlabel('MODEL (degC)')
def get_w_depth(xi, yi): #xi:lon yi:lat
url = 'http://geoport.whoi.edu/thredds/dodsC/bathy/gom03_v1_0' #url='http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol1.nc' ):
if xi[0] > 999.: # if it comes in decimal -minutes, conert it
#for kk in range(len(xi)):
(y2, x2) = dm2dd(yi[0], xi[0])
yi[0] = y2
xi[0] = x2
try:
dataset = open_url(url)
except:
print 'Sorry, ' + url + ' is not available'
sys.exit(0)
#read lat, lon,topo from url
xgom_array = dataset['lon']
ygom_array = dataset['lat']
dgom_array = dataset['topo'].topo
#print dgom_array.shape, xgom_array[5:9],dgom_array[5]
#convert the array to a list
xgom, ygom = [], []
for i in xgom_array:
if i > xi[0] - 0.00834 and i < xi[0] + 0.00834:
xgom.append(i)
for i in ygom_array:
if i > yi[0] - 0.00834 and i < yi[0] + 0.00834:
ygom.append(i)
x_index, y_index = [], []
(ys, xs) = dgom_array.shape
for i in range(0, len(xgom)):
x_index.append(int(round(np.interp(xgom[i], xgom_array, range(xs)))))
for i in range(0, len(ygom)):
y_index.append(int(round(np.interp(ygom[i], ygom_array, range(ys)))))
dep, distkm, dist1 = [], [], []
for k in range(len(x_index)):
for j in range(len(y_index)):
dep.append(dgom_array[(y_index[j], x_index[k])])
distkm, b = distance((ygom[j], xgom[k]), (yi[0], xi[0]))
dist1.append(distkm)
#print dist1
if len(dist1) >= 3:
#get the nearest,second nearest,third nearest point.
dist_f_nearest = sorted(dist1)[0]
dist_s_nearest = sorted(dist1)[1]
dist_t_nearest = sorted(dist1)[2]
index_dist_nearest = range(len(dist1))
index_dist_nearest.sort(lambda x, y: cmp(dist1[x], dist1[y]))
dep_f_nearest = dep[index_dist_nearest[0]]
dep_s_nearest = dep[index_dist_nearest[1]]
dep_t_nearest = dep[index_dist_nearest[2]]
#compute the finally depth
d1 = dist_f_nearest
d2 = dist_s_nearest
d3 = dist_t_nearest
def1 = dep_f_nearest
def2 = dep_s_nearest
def3 = dep_t_nearest
depth_finally = def1 * d2 * d3 / (
d1 * d2 + d2 * d3 + d1 * d3) + def2 * d1 * d3 / (
d1 * d2 + d2 * d3 +
d1 * d3) + def3 * d2 * d1 / (d1 * d2 + d2 * d3 + d1 * d3)
else:
depth_finally = np.array([[-9999]])
return depth_finally
lonsize=[event.xdata-0.6,event.xdata+0.6]
plt.figure(figsize=(7,6))
m = Basemap(projection='cyl',llcrnrlat=min(latsize)-0.01,urcrnrlat=max(latsize)+0.01,\
llcrnrlon=min(lonsize)-0.01,urcrnrlon=max(lonsize)+0.01,resolution='h')#,fix_aspect=False)
m.drawparallels(np.arange(int(min(latsize)),int(max(latsize))+1,1),labels=[1,0,0,0])
m.drawmeridians(np.arange(int(min(lonsize)),int(max(lonsize))+1,1),labels=[0,0,0,1])
m.drawcoastlines()
m.fillcontinents(color='grey')
m.drawmapboundary()
m.plot(lon,lat,'r.',lonc,latc,'b+')
plt.show()
spoint = pylab.ginput(1)
'''
if lo > 90:
[la, lo] = dm2dd(la, lo)
latd, lond = [], []
kf, distanceF = nearlonlat(lonc, latc, lo,
la) # nearest triangle center F - face
kv, distanceV = nearlonlat(lon, lat, lo, la)
if h[kv] < 0:
print 'Sorry, your position is on land, please try another point'
sys.exit()
depthtotal = siglay[:, kv] * h[kv]
layer = np.argmin(abs(depthtotal - depth))
for i in range(startrecord, endrecord):
############read the particular time model from website#########
save_dir = '/home/zdong/xiaoxu/emolt/figure/' #the path of saving figure
file_catch = 'sqldump_2018_07_BN.csv' # catch data that JiM created & emailed
input_dir = '/home/zdong/xiaoxu/emolt/files/'
file_temp_BN = 'this2.dat' #temp data
file_temp_BD = 'BD01_temp.dat'
file_temp_TS = 'TS01_temp.dat'
variables = ['year', 'year_day', 'temp', 'lat', 'lon', 'depth'] #
df_BN = pd.read_csv(input_dir + file_temp_BN, names=variables,
sep='\s+') # read temperature time series
df_BD = pd.read_csv(input_dir + file_temp_BD, names=variables, sep='\s+')
df_TS = pd.read_csv(input_dir + file_temp_TS, names=variables, sep='\s+')
lat_BD, lon_BD = [], []
lat_BN, lon_BN = [], []
lat_TS, lon_TS = [], []
for i in range(len(df_BN)):
la, lo = conversions.dm2dd(df_BN['lat'].values[i], df_BN['lon'].values[i])
lat_BN.append(la)
lon_BN.append(lo)
for i in range(len(df_BD)):
la, lo = conversions.dm2dd(df_BD['lat'].values[i], df_BD['lon'].values[i])
lat_BD.append(la)
lon_BD.append(lo)
for i in range(len(df_TS)):
la, lo = conversions.dm2dd(df_TS['lat'].values[i], df_TS['lon'].values[i])
lat_TS.append(la)
lon_TS.append(lo)
fig = plt.figure()
a = fig.add_subplot(1, 1, 1)
my_map = Basemap(projection='merc',
resolution='h',
area_thresh=0.3,
