def getclim(lat1=43., lon1=-73.0, yrday=dt.now().strftime('%j'), var='Bottom_Temperature/BT_'): # gets climatology of Bottom_Temperature, Surface_Temperature, Bottom_Salinity, or Surface_Salinity # as calculated by Chris Melrose from 30+ years of NEFSC CTD data on the NE Shelf # where "lat1", "lon1", and "yrday" are the position and yearday of interest # where "var" is the variable of interest (usually Bottom_Temperature) inputdir = '/net/data5/jmanning/clim/' # hardcoded climatology directory name "/home/pi/clim/" inputdir_csv = '/net/data5/jmanning/li/Matdata/temporary/' # the csv files that have lat/lon "/home/pi/towifi/" dflat = pd.read_csv(inputdir + 'LatGrid.csv', header=None) dflon = pd.read_csv(inputdir + 'LonGrid.csv', header=None) lat = np.array(dflat[0]) # gets the first col (35 to 45) lon = np.array(dflon.ix[0]) # gets the first row (-75 to -65) clim = pd.read_csv(inputdir + var + yrday + '.csv', header=None) # gets bottom temp for this day of year print lat1, lon1 if lat1 == 43.: files = sorted(glob.glob( inputdir_csv + '*.csv')) # gets all the csv files in the towfi directory dfcsv = pd.read_csv(files[-1], sep=',', skiprows=7) [lat1, lon1] = dm2dd(float(dfcsv['lat'][0][0:-1]), float(dfcsv['lon'][0][0:-1])) print lat1, lon1 idlat = np.abs(lat - lat1).argmin() # finds the neareast lat to input lat1 idlon = np.abs(lon - lon1).argmin() # finds the neareast lon to input lon1 return clim[idlon][idlat]
def watertemp(self, lon, lat, depth, url): data = self.get_data(url) lonc, latc = data['lonc'][:], data['latc'][:] lonv, latv = data['lon'][:], data['lat'][:] h = data['h'][:] siglay = data['siglay'][:] if lon > 90: lon, lat = dm2dd(lon, lat) nodes = dict(lon=[], lat=[]) kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc, num=1) kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv, num=1) if h[kv] < 0: sys.exit( 'Sorry, your position is on land, please try another point') depth_total = siglay[:, kv] * h[kv] ###############layer########################### layer = np.argmin(abs(depth_total - depth)) print kv print 'layer, kv', layer, kv[0][0] # for i in range(len(data['time'][:])): temp = data['temp'][:, layer, kv[0][0]] print 'l' m = pd.DataFrame(temp, index=data['time'][:]) print 'a' return m
def get_coors(modelname, lo, la, lonc, latc, lon, lat, siglay, h, depth,startrecord, endrecord): if lo>90: [la,lo]=dm2dd(la,lo) print 'la, lo',la, lo latd,lond=[la],[lo] # kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face # kv,distanceV=nearlonlat(lon,lat,lo,la) kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1) kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1) kf = kf[0][0] kv = kv[0][0] print 'kf:', kf if h[kv] < 0: print 'Sorry, your position is on land, please try another point' sys.exit() depthtotal=siglay[:,kv]*h[kv] layer=np.argmin(abs(depthtotal-depth)) for i in range(startrecord,endrecord):# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ############read the particular time model from website######### # print 'la, lo, i', la, lo, i timeurl='['+str(i)+':1:'+str(i)+']' uvposition=str([layer])+str([kf]) data_want = ('u'+timeurl+uvposition, 'v'+timeurl+uvposition) # if urlname=="30yr": # url='http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?'+\ # 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition # elif urlname == "GOM3": # url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc?"+\ # 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition # else: # url="http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc?"+\ # 'Times'+timeurl+',u'+timeurl+uvposition+','+'v'+timeurl+uvposition url = url_with_time_position(modelname, data_want) dataset = open_url(url) u=np.array(dataset['u']) v=np.array(dataset['v']) print 'u, v, i', u[0,0,0], v[0,0,0],i ################get the point according the position################### par_u=u[0,0,0] par_v=v[0,0,0] xdelta=par_u*60*60 #get_coors ydelta=par_v*60*60 latdelta=ydelta/111111 londelta=(xdelta/(111111*np.cos(la*np.pi/180))) la=la+latdelta lo=lo+londelta latd.append(la) lond.append(lo) # kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face # kv,distanceV=nearlonlat(lon,lat,lo,la)# nearest triangle vertex V - vertex kf,distanceF = nearest_point_index(lo,la,lonc,latc,num=1) kv,distanceV = nearest_point_index(lo,la,lon,lat,num=1) kf, kv = kf[0][0], kv[0][0] depthtotal=siglay[:,kv]*h[kv] # layer=np.argmin(abs(depthtotal-depth)) if distanceV>=0.3: if i==startrecord: print 'Sorry, your start position is NOT in the model domain' break return latd ,lond
def __get_track(self, lon, lat, depth, url): ''' start, end: indices of some period data: a dict that has 'u' and 'v' ''' data = self.get_data(url) lonc, latc = data['lonc'][:], data['latc'][:] lonv, latv = data['lon'][:], data['lat'][:] h = data['h'][:] siglay = data['siglay'][:] if lon>90: lon, lat = dm2dd(lon, lat) nodes = dict(lon=[], lat=[]) kf,distanceF = self.nearest_point_index(lon,lat,lonc,latc,num=1) kv,distanceV = self.nearest_point_index(lon,lat,lonv,latv,num=1) if h[kv] < 0: sys.exit('Sorry, your position is on land, please try another point') depth_total = siglay[:,kv]*h[kv] ###########################layer##################################### ''' ????? Layer for what????? ''' layer = np.argmin(abs(depth_total-depth)) for i in range(self.hours): u_t = data['u'][i, layer, kf[0][0]] v_t = data['v'][i, layer, kf[0][0]] print 'u_t, v_t, i', u_t, v_t, i dx = 60*60*u_t dy = 60*60*v_t lon = lon + (dx/(111111*np.cos(lat*np.pi/180))) lat = lat + dy/111111 nodes['lon'].append(lon) nodes['lat'].append(lat) kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc,num=1) kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv,num=1) # depth_total = siglay[:][kv]*h[kv] if distanceV>=.3: if i==start: print 'Sorry, your start position is NOT in the model domain' break return nodes
def __waternode(self, lon, lat, depth, url): ''' start, end: indices of some period data: a dict that has 'u' and 'v' ''' data = self.get_data(url) lonc, latc = data['lonc'][:], data['latc'][:] lonv, latv = data['lon'][:], data['lat'][:] h = data['h'][:] siglay = data['siglay'][:] if lon > 90: lon, lat = dm2dd(lon, lat) nodes = dict(lon=[], lat=[]) kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc, num=1) kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv, num=1) if h[kv] < 0: sys.exit( 'Sorry, your position is on land, please try another point') depth_total = siglay[:, kv] * h[kv] ###############layer########################### layer = np.argmin(abs(depth_total - depth)) # for i in range(len(data['u'])): for i in range(self.hours): u_t = data['u'][i, layer, kf[0][0]] v_t = data['v'][i, layer, kf[0][0]] dx = 60 * 60 * u_t dy = 60 * 60 * v_t lon = lon + (dx / (111111 * np.cos(lat * np.pi / 180))) lat = lat + dy / 111111 nodes['lon'].append(lon) nodes['lat'].append(lat) kf, distanceF = self.nearest_point_index(lon, lat, lonc, latc, num=1) kv, distanceV = self.nearest_point_index(lon, lat, lonv, latv, num=1) # depth_total = siglay[:][kv]*h[kv] if distanceV >= .3: if i == start: print 'Sorry, your start position is NOT in the model domain' break modtso = pd.DataFrame(nodes, ) return nodes
""" Compares eMOLT with FVCOM bottom temp @author: jmanning, rsignell, yacheng """ import matplotlib.pyplot as plt import netCDF4 from getdata import getemolt_latlon,getemolt_temp from conversions import dm2dd,f2c from utilities import nearxy site='BN01' [lati,loni,on]=getemolt_latlon(site) # extracts lat/lon based on site code [lati,loni]=dm2dd(lati,loni) #converts decimal-minutes to decimal degrees [obs_dt,obs_temp]=getemolt_temp(site) # extracts time series for kk in range(len(obs_temp)): obs_temp[kk]=f2c(obs_temp[kk]) # converts to Celcius # now get the model output urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' nc = netCDF4.Dataset(urlfvcom) nc.variables lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] times = nc.variables['time'] jd = netCDF4.num2date(times[:],times.units) vname = 'temp' var = nc.variables[vname] # find nearest point to desired location and time inode = nearxy(lon,lat,loni,lati) index=netCDF4.date2index([obs_dt[0],obs_dt[-1]],times,select='nearest')#find the model time index at start & end pf obs
def get_w_depth(xi,yi): url='http://geoport.whoi.edu/thredds/dodsC/bathy/gom03_v1_0'#url='http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol1.nc' ): if xi[0]>999.: # if it comes in decimal -minutes, conert it #for kk in range(len(xi)): (y2,x2)=dm2dd(yi[0],xi[0]) yi[0]=y2 xi[0]=x2 try: dataset = open_url(url) except: print 'Sorry, ' + url + ' is not available' sys.exit(0) #read lat, lon,topo from url xgom_array = dataset['lon'] ygom_array = dataset['lat'] dgom_array = dataset['topo'].topo #print dgom_array.shape, xgom_array[5:9],dgom_array[5] #convert the array to a list xgom, ygom = [], [] for i in xgom_array: if i > xi[0] - 0.00834 and i < xi[0] + 0.00834: xgom.append(i) for i in ygom_array: if i > yi[0] - 0.00834 and i < yi[0] + 0.00834: ygom.append(i) x_index, y_index = [], [] (ys, xs) = dgom_array.shape for i in range(0, len(xgom)): x_index.append(int(round(np.interp(xgom[i], xgom_array, range(xs))))) for i in range(0, len(ygom)): y_index.append(int(round(np.interp(ygom[i], ygom_array, range(ys))))) dep, distkm, dist1 = [], [], [] for k in range(len(x_index)): for j in range(len(y_index)): dep.append(dgom_array[(y_index[j], x_index[k])]) distkm, b = distance((ygom[j], xgom[k]), (yi[0], xi[0])) dist1.append(distkm) #get the nearest,second nearest,third nearest point. dist_f_nearest = sorted(dist1)[0] dist_s_nearest = sorted(dist1)[1] dist_t_nearest = sorted(dist1)[2] index_dist_nearest = range(len(dist1)) index_dist_nearest.sort(lambda x, y:cmp(dist1[x], dist1[y])) dep_f_nearest = dep[index_dist_nearest[0]] dep_s_nearest = dep[index_dist_nearest[1]] dep_t_nearest = dep[index_dist_nearest[2]] #compute the finally depth d1 = dist_f_nearest d2 = dist_s_nearest d3 = dist_t_nearest def1 = dep_f_nearest def2 = dep_s_nearest def3 = dep_t_nearest depth_finally = def1 * d2 * d3 / (d1 * d2 + d2 * d3 + d1 * d3) + def2 * d1 * d3 / (d1 * d2 + d2 * d3 + d1 * d3) + def3 * d2 * d1 / (d1 * d2 + d2 * d3 + d1 * d3) return depth_finally
return haul,hauls,datetc # plotting the catch #..how best to plot as in "skiphole.m" maybe using scikits ... fig=plt.figure(1,figsize=(8,6)) for k in range(numpeople): ax1=fig.add_subplot(numpeople,1,k+1) # Now go get the temperature data associated with this site # pipe = subprocess.Popen(["perl", "/home3/ocn/jmanning/sql/gettsll_justtemp.plx","sc","depset" # this subprocess was not possible ... I need to learn how to run Oracle from python variables=['year','year_day','temp','lat','lon','depth'] df=pd.read_csv(input_directory+'this2.dat', names=variables, sep='\s+') #t=ml.load('/net/home3/ocn/jmanning/sql/this2.dat') old way of loading lat,lon,datet=[],[],[] for j in range(len(df)): #la,lo=conversions.dm2dd([t[j,3]],[t[j,4]]) #changed this in May 2015 la,lo=conversions.dm2dd(df['lat'].values[0],df['lon'].values[0]) lat.append(la) lon.append(lo) if df['year_day'].values[j]+1<366.0: #datet.append(num2date(t[j,1]+1).replace(year=int(t[j,0]))) datet.append(num2date(df['year_day'].values[j]+1).replace(year=int(df['year'].values[j]))) else: #datet.append(num2date(t[j,1]+1).replace(year=int(t[j,0])+1)) datet.append(num2date(df['year_day'].values[j]+1).replace(year=int(df['year'].values[j]+1))) #wd=t[:,5] #temp=t[:,2] #temp_smooth=utilities.smooth(temp,24*7,'hanning') wd=df['depth'].values temp=df['temp'].values temp_smooth=utilities.smooth(temp,hrs,smoothing_method) ax1.plot_date(datet,temp,'k-')
lonsize=[event.xdata-0.6,event.xdata+0.6] plt.figure(figsize=(7,6)) m = Basemap(projection='cyl',llcrnrlat=min(latsize)-0.01,urcrnrlat=max(latsize)+0.01,\ llcrnrlon=min(lonsize)-0.01,urcrnrlon=max(lonsize)+0.01,resolution='h')#,fix_aspect=False) m.drawparallels(np.arange(int(min(latsize)),int(max(latsize))+1,1),labels=[1,0,0,0]) m.drawmeridians(np.arange(int(min(lonsize)),int(max(lonsize))+1,1),labels=[0,0,0,1]) m.drawcoastlines() m.fillcontinents(color='grey') m.drawmapboundary() m.plot(lon,lat,'r.',lonc,latc,'b+') plt.show() spoint = pylab.ginput(1) ''' if lo>90: [la,lo]=dm2dd(la,lo) latd,lond=[],[] kf,distanceF=nearlonlat(lonc,latc,lo,la) # nearest triangle center F - face kv,distanceV=nearlonlat(lon,lat,lo,la) if h[kv] < 0: print 'Sorry, your position is on land, please try another point' sys.exit() depthtotal=siglay[:,kv]*h[kv] layer=np.argmin(abs(depthtotal-depth)) for i in range(startrecord,endrecord): ############read the particular time model from website######### timeurl='['+str(i)+':1:'+str(i)+']'
2)) mean_depth = str( abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill( 3) #caculate the mean depth #match rawdata and telementry data time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split( '.')[0].split('_')[3] #GMT time to local time of file time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] + '-' + time_str[6:8] + ' ' + time_str[9:11] + ':' + time_str[11:13] + ':' + time_str[13:15]) time_gmt = datetime.datetime.strptime(time_str, "%Y%m%d %H%M%S") #transfer the format latitude and longitude lat, lon = cv.dm2dd(value_data_df['Lat'][len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) - 1]) #write the data of raw file to dict for i in range(len(telemetrystatus_df)): if telemetrystatus_df['Vessel#'][i] == vessel_number: raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\ fname,float(mean_temp),float(mean_depth)]],columns=['time','filename','mean_temp','mean_depth']).iloc[0],ignore_index=True) #caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file for i in range(len(valuable_tele_df)): if valuable_tele_df['vessel_n'][i].split('_')[1] == str( vessel_number): if abs(valuable_tele_df['time'][i] - time_gmt) <= acceptable_time_diff: #time match if zl.dist(lat1=lat, lon1=lon, lat2=float(valuable_tele_df['lat'][i]), lon2=float(valuable_tele_df['lon'][i])
# now, read header and data of every file header_df = zl.nrows_len_to(file, 2, name=['key', 'value']) #only header data_df = zl.skip_len_to(file, 2) #only data value_data_df = data_df.loc[( data_df['Depth(m)'] > 0.95 * max(data_df['Depth(m)']))] #filter the data value_data_df = value_data_df.iloc[ 7:] #delay several minutes to let temperature sensor record the real bottom temp value_data_df=value_data_df.loc[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \ (value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error value_data_df.index = range(len(value_data_df)) #reindex value_data_df[ 'Datet(Y/m/d)'] = 1 #create a new column for saving another time style of '%Y-%m-%d' for i in range(len(value_data_df)): value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd( value_data_df['Lat'][i], value_data_df['Lon'][i]) #value_data_df['Datet(Y/m/d)'][i]=datetime.strptime(value_data_df['Datet(GMT)'][i],'%Y-%m-%d %H:%M:%S') Hours_df = pd.DataFrame( data=None, columns=['time', 'lat', 'lon', 'depth', 'temp', 'new_time']) Hours_df['time'] = value_data_df['Datet(GMT)'] Hours_df['lat'] = value_data_df['Lat'] Hours_df['lon'] = value_data_df['Lon'] Hours_df['depth'] = value_data_df['Depth(m)'] Hours_df['temp'] = value_data_df['Temperature(C)'] #value_data_df['Datet(Y/m/d)']=pd.to_datetime(value_data_df['Datet(Y/m/d)'])#change the time style to datetime Hours_df['new_time'] = value_data_df['Datet(GMT)'] if not os.path.exists(Hours_save + file.split('/')[8]): os.makedirs(Hours_save + file.split('/')[8]) Hours_df.to_csv(os.path.join( Hours_save + file.split('/')[8] + '/',
else: nodes_drifter = drifter.waternode(starttime) else: nodes_drifter = drifter.waternode() lon, lat = nodes_drifter['lon'][0], nodes_drifter['lat'][0] starttime = nodes_drifter['time'][0] endtime = nodes_drifter['time'][-1] ''' lat = 4111.797 lon = 6934.554 depth = -1 days = 3 lat, lon=dm2dd(lat,lon) starttime = datetime(2014,8,26,12,00) # endtime = datetime(2014,7,15,19) endtime = starttime+timedelta(days=days) water_fvcom = water_fvcom() url_fvcom = water_fvcom.get_url(starttime, endtime) nodes_fvcom = water_fvcom.waternode(lon,lat,depth,url_fvcom) # water_roms = water_roms() # url_roms = water_roms.get_url(starttime, endtime) # nodes_roms = water_roms.waternode(lon, lat, depth, url_roms) ''' water_roms_rk4 = water_roms_rk4() url_roms_rk4 = water_roms_rk4.get_url(starttime, endtime) nodes_roms_rk4 = water_roms_rk4.waternode(lon, lat, depth, url_roms_rk4)
lat=[40.,45.] lon=[-71.,-65.] comp='p_emoltvsmod'# this is the mode of comparison where p_ referes to percent difference print comp m = Basemap(projection='cyl',llcrnrlat=min(lat)-0.01,urcrnrlat=max(lat)+0.01,\ llcrnrlon=min(lon)-0.01,urcrnrlon=max(lon)+0.01,resolution='h') m.drawcoastlines() m.fillcontinents(color='grey') m.drawmapboundary() ebd,mbd,ubd,hbd=[],[],[],[] diffd=[] outf.write('site,lon,lat,ebd,udb,hbd,mbd,perc_diff\n') for k in range(len(site)): [elat, elon, original_name,eb]=getemolt_latlon(site[k]) ebd.append(eb) [elat_dd,elon_dd]=dm2dd(elat,elon) mbd.append(getFVCOM_depth(elat_dd,elon_dd)) ubd.append(-1*getdepth(elat_dd,elon_dd))#nearset usgs depth hbd.append(-1*get_w_depth( [elon_dd],[elat_dd]))#huanxins interpolated depth if comp=='usgsvsmod': diffd.append(ubd[k]-mbd[k]) elif comp=='p_emoltvsmod': diffd.append((ebd[k]-mbd[k])/ebd[k]*100) if diffd[k]<1: dotcol='magenta' else: dotcol='cyan' plt.scatter(elon_dd,elat_dd,25*abs(diffd[k]),marker='o',color=dotcol) #print 'ebd='+str(ebd)+' ubd='+str(ubd)+' hbd='+str(hbd)+' mbd='+str(mbd) outf.write(site[k]+','+'%6.3f' % (elon_dd)+','+'%6.3f' % (elat_dd)+','+'%6.1f' % (ebd[k])+','+'%6.1f' % (mbd[k])+','+'%6.1f' % (ubd[k][0])+','+'%6.1f' % (hbd[k][0])+','+'%6.1f' % (diffd[k])+'\n')
# Forth, the main program ############################################################################ fn='binned_td_test.csv' direct='/net/data5/jmanning/hoey/' outputdir='/net/data5/jmanning/modvsobs/sfleet/' # read in the study fleet data def parse(datet): #print datet[0:10],datet[11:13],datet[14:16] dtt=dt.strptime(datet,'%Y-%m-%d:%H:%M') return dtt obstso=pd.read_csv(direct+fn,sep=',',skiprows=0,parse_dates={'datet':[2]},index_col='datet',date_parser=parse,names=['LATITUDE','LONGITUDE','ROUND_DATE_TIME','OBSERVATIONS_TEMP','MEAN_TEMP','MIN_TEMP','MAX_TEMP','STD_DEV_TEMP','OBSERVATIONS_DEPTH','MEAN_DEPTH','MIN_DEPTH','MAX_DEPTH','STD_DEV_DEPTH','nan']) #convert ddmm.m to dd.ddd plt.figure(figsize=(16,10)) o=[] m=[] for k in range(len(obstso)): # [la,lo]=dm2dd(obstso['LATITUDE'][k],obstso['LONGITUDE'][k]) if not inmodel_roms([lo], [la]): print 'point not in roms domain' continue else: romobj = water_roms() modtso = pd.DataFrame() st=obstso.index[k] et=st+td(hours=1) url_roms = romobj.get_url(st,et ) modtso= romobj.waternode(lo, la,-1*obstso['MEAN_DEPTH'][k], url_roms) o.append(obstso['MEAN_TEMP'][k]) m.append(modtso.values[0]) plt.plot(modtso.values[0],obstso['MEAN_TEMP'][k],'*', markersize=30) plt.xlabel('MODEL (degC)') plt.ylabel('OBSERVATIONS (degC)')
def match_tele_raw( input_dir, path_save, telemetry_status, start_time, end_time, telemetry_path='https://www.nefsc.noaa.gov/drifter/emolt.dat', accept_minutes_diff=20, acceptable_distance_diff=2, dpi=300): """ match the file and telementy. we can known how many file send to the satallite and output the figure """ #read the file of the telementry_status telemetrystatus_df = read_telemetrystatus(telemetry_status) #st the record file use to write minmum maxmum and average of depth and temperature,the numbers of file, telemetry and successfully matched record_file_df=telemetrystatus_df.loc[:,['Boat','Vessel#']].reindex(columns=['Boat','Vessel#','matched_number','file_number','tele_num','max_diff_depth',\ 'min_diff_depth','average_diff_depth','max_diff_temp','min_diff_temp','average_diff_temp','sum_diff_depth','sum_diff_temp',\ 'min_lat','max_lat','min_lon','max_lon'],fill_value=None) #transfer the time format of string to datetime start_time_local = datetime.strptime(start_time, '%Y-%m-%d') end_time_local = datetime.strptime(end_time, '%Y-%m-%d') allfile_lists = zl.list_all_files(input_dir) ###################### file_lists = [] for file in allfile_lists: if file[len(file) - 4:] == '.csv': file_lists.append(file) #download the data of telementry tele_df = read_telemetry(telemetry_path) #screen out the data of telemetry in interval valuable_tele_df = pd.DataFrame( data=None, columns=['vessel_n', 'esn', 'time', 'lon', 'lat', 'depth', 'temp']) #use to save the data during start time and end time for i in range(len(tele_df)): tele_time=datetime.strptime(str(tele_df['year'].iloc[i])+'-'+str(tele_df['month'].iloc[i])+'-'+str(tele_df['day'].iloc[i])+' '+\ str(tele_df['Hours'].iloc[i])+':'+str(tele_df['minates'].iloc[i])+':'+'00','%Y-%m-%d %H:%M:%S') if zl.local2utc(start_time_local) <= tele_time < zl.local2utc( end_time_local): valuable_tele_df=valuable_tele_df.append(pd.DataFrame(data=[[tele_df['vessel_n'][i],tele_df['esn'][i],tele_time,tele_df['lon'][i],tele_df['lat'][i],tele_df['depth'][i],\ tele_df['temp'][i]]],columns=['vessel_n','esn','time','lon','lat','depth','temp'])) valuable_tele_df.index = range(len(valuable_tele_df)) #whether the data of file and telemetry is exist if len(valuable_tele_df) == 0 and len(file_lists) == 0: print( 'please check the data website of telementry and the directory of raw_data is exist!' ) sys.exit() elif len(valuable_tele_df) == 0: print('please check the data website of telementry!') sys.exit() elif len(file_lists) == 0: print('please check the directory raw_data is exist!') sys.exit() #match the file index = telemetrystatus_df['Boat'] #set the index for dictionary raw_dict = { } #the dictinary about raw data, use to write the data about 'time','filename','mean_temp','mean_depth' tele_dict = { } #the dictionary about telementry data,use to write the data about'time','mean_temp','mean_depth' for i in range(len(index)): #loop every boat raw_dict[index[i]] = pd.DataFrame(data=None, columns=[ 'time', 'filename', 'mean_temp', 'mean_depth', 'mean_lat', 'mean_lon' ]) tele_dict[index[i]] = pd.DataFrame(data=None, columns=[ 'time', 'mean_temp', 'mean_depth', 'mean_lat', 'mean_lon' ]) for file in file_lists: # loop raw files fpath, fname = os.path.split(file) #get the file's path and name # now, read header and data of every file header_df = zl.nrows_len_to(file, 2, name=['key', 'value']) #only header data_df = zl.skip_len_to(file, 2) #only data #caculate the mean temperature and depth of every file value_data_df = data_df.ix[( data_df['Depth(m)'] > 0.85 * mean(data_df['Depth(m)']))] #filter the data value_data_df = value_data_df.ix[ 2:] #delay several minutes to let temperature sensor record the real bottom temp value_data_df=value_data_df.ix[(value_data_df['Temperature(C)']>mean(value_data_df['Temperature(C)'])-3*std(value_data_df['Temperature(C)'])) & \ (value_data_df['Temperature(C)']<mean(value_data_df['Temperature(C)'])+3*std(value_data_df['Temperature(C)']))] #Excluding gross error value_data_df.index = range(len(value_data_df)) #reindex for i in range(len(value_data_df['Lat'])): value_data_df['Lat'][i], value_data_df['Lon'][i] = cv.dm2dd( value_data_df['Lat'][i], value_data_df['Lon'][i]) min_lat = min(value_data_df['Lat'].values) max_lat = max(value_data_df['Lat'].values) min_lon = min(value_data_df['Lon'].values) max_lon = max(value_data_df['Lon'].values) mean_lat = str(round(mean(value_data_df['Lat'].values), 4)) mean_lon = str(round(mean(value_data_df['Lon'].values), 4)) #caculate the mean depth mean_temp = str( round(mean(value_data_df['Temperature(C)'][1:len(value_data_df)]), 2)) mean_depth = str( abs(int(round(mean(value_data_df['Depth(m)'].values))))).zfill( 3) #caculate the mean depth #get the vessel number of every file for i in range(len(header_df)): if header_df['key'][i].lower() == 'vessel number'.lower(): vessel_number = int(header_df['value'][i]) break #caculate the number of raw files in every vessel,and min,max of lat and lon for i in range(len(record_file_df)): if record_file_df['Vessel#'][i] == vessel_number: if record_file_df['file_number'].isnull()[i]: record_file_df['min_lat'][i] = min_lat record_file_df['max_lat'][i] = max_lat record_file_df['min_lon'][i] = min_lon record_file_df['max_lon'][i] = max_lon record_file_df['file_number'][i] = 1 else: record_file_df['file_number'][i] = int( record_file_df['file_number'][i] + 1) if record_file_df['min_lat'][i] > min_lat: record_file_df['min_lat'][i] = min_lat if record_file_df['max_lat'][i] < max_lat: record_file_df['max_lat'][i] = max_lat if record_file_df['min_lon'][i] > min_lon: record_file_df['min_lon'][i] = min_lon if record_file_df['max_lon'][i] < max_lon: record_file_df['max_lon'][i] = max_lon #match rawdata and telementry data time_str = fname.split('.')[0].split('_')[2] + ' ' + fname.split( '.')[0].split('_')[3] #GMT time to local time of file time_local = zl.gmt_to_eastern(time_str[0:4] + '-' + time_str[4:6] + '-' + time_str[6:8] + ' ' + time_str[9:11] + ':' + time_str[11:13] + ':' + time_str[13:15]) time_gmt = datetime.strptime(time_str, "%Y%m%d %H%M%S") #transfer the format latitude and longitude lat, lon = value_data_df['Lat'][ len(value_data_df) - 1], value_data_df['Lon'][len(value_data_df) - 1] #write the data of raw file to dict for i in range(len(telemetrystatus_df)): if telemetrystatus_df['Vessel#'][i] == vessel_number: raw_dict[telemetrystatus_df['Boat'][i]]=raw_dict[telemetrystatus_df['Boat'][i]].append(pd.DataFrame(data=[[time_local,\ fname,float(mean_temp),float(mean_depth),float(mean_lat),float(mean_lon)]],columns=['time','filename','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True) #caculate the numbers of successful matchs and the minimum,maximum and average different of temperature and depth, and write this data to record file for i in range(len(valuable_tele_df)): if valuable_tele_df['vessel_n'][i].split('_')[1] == str( vessel_number): if abs(valuable_tele_df['time'][i] - time_gmt) <= timedelta( minutes=accept_minutes_diff): #time match if zl.dist(lat1=lat, lon1=lon, lat2=float(valuable_tele_df['lat'][i]), lon2=float(valuable_tele_df['lon'][i]) ) <= acceptable_distance_diff: #distance match for j in range(len(record_file_df)): if record_file_df['Vessel#'][j] == vessel_number: diff_temp = round( (float(mean_temp) - float(valuable_tele_df['temp'][i])), 4) diff_depth = round( (float(mean_depth) - float(valuable_tele_df['depth'][i])), 4) if record_file_df['matched_number'].isnull( )[j]: record_file_df['matched_number'][j] = 1 record_file_df['sum_diff_temp'][ j] = diff_temp record_file_df['max_diff_temp'][ j] = diff_temp record_file_df['min_diff_temp'][ j] = diff_temp record_file_df['sum_diff_depth'][ j] = diff_depth record_file_df['max_diff_depth'][ j] = diff_depth record_file_df['min_diff_depth'][ j] = diff_depth break else: record_file_df['matched_number'][j] = int( record_file_df['matched_number'][j] + 1) record_file_df['sum_diff_temp'][ j] = record_file_df['sum_diff_temp'][ j] + diff_temp record_file_df['sum_diff_depth'][ j] = record_file_df['sum_diff_depth'][ j] + diff_depth if record_file_df['max_diff_temp'][ j] < diff_temp: record_file_df['max_diff_temp'][ j] = diff_temp if record_file_df['min_diff_temp'][ j] > diff_temp: record_file_df['min_diff_temp'][ j] = diff_temp if record_file_df['max_diff_depth'][ j] < diff_depth: record_file_df['max_diff_depth'][ j] = diff_depth if record_file_df['min_diff_depth'][ j] > diff_depth: record_file_df['min_diff_depth'][ j] = diff_depth break #write 'time','mean_temp','mean_depth' of the telementry to tele_dict for i in range( len(valuable_tele_df) ): #valuable_tele_df is the valuable telemetry data during start time and end time for j in range(len(telemetrystatus_df)): if int(valuable_tele_df['vessel_n'][i].split('_') [1]) == telemetrystatus_df['Vessel#'][j]: #count the numbers by boats if record_file_df['tele_num'].isnull()[j]: record_file_df['tele_num'][j] = 1 else: record_file_df['tele_num'][ j] = record_file_df['tele_num'][j] + 1 if record_file_df['max_lat'].isnull()[j]: record_file_df['min_lat'][j] = valuable_tele_df['lat'][i] record_file_df['max_lat'][j] = valuable_tele_df['lat'][i] record_file_df['min_lon'][j] = valuable_tele_df['lon'][i] record_file_df['max_lon'][j] = valuable_tele_df['lon'][i] else: if record_file_df['min_lat'][j] > valuable_tele_df['lat'][ i]: record_file_df['min_lat'][j] = valuable_tele_df['lat'][ i] if record_file_df['max_lat'][j] < valuable_tele_df['lat'][ i]: record_file_df['max_lat'][j] = valuable_tele_df['lat'][ i] if record_file_df['min_lon'][j] > valuable_tele_df['lon'][ i]: record_file_df['min_lon'][j] = valuable_tele_df['lon'][ i] if record_file_df['max_lon'][j] < valuable_tele_df['lon'][ i]: record_file_df['max_lon'][j] = valuable_tele_df['lon'][ i] #write 'time','mean_temp','mean_depth' of the telementry to tele_dict tele_dict[telemetrystatus_df['Boat'][j]]=tele_dict[telemetrystatus_df['Boat'][j]].append(pd.DataFrame(data=[[valuable_tele_df['time'][i],\ float(valuable_tele_df['temp'][i]),float(valuable_tele_df['depth'][i]),float(valuable_tele_df['lat'][i]),float(valuable_tele_df['lon'][i])]],columns=['time','mean_temp','mean_depth','mean_lat','mean_lon']).iloc[0],ignore_index=True) print("finish the calculate of min_lat and min_lon!") for i in range(len(record_file_df)): if not record_file_df['matched_number'].isnull()[i]: record_file_df['average_diff_depth'][i] = round( record_file_df['sum_diff_depth'][i] / record_file_df['matched_number'][i], 4) record_file_df['average_diff_temp'][i] = round( record_file_df['sum_diff_temp'][i] / record_file_df['matched_number'][i], 4) else: record_file_df['matched_number'][i] = 0 if record_file_df['tele_num'].isnull()[i]: record_file_df['tele_num'][i] = 0 if record_file_df['file_number'].isnull()[i]: record_file_df['file_number'][i] = 0 for i in index: #loop every boat, i represent the name of boat raw_dict[i] = raw_dict[i].sort_values(by=['time']) raw_dict[i].index = range(len(raw_dict[i])) record_file_df = record_file_df.drop(['sum_diff_depth', 'sum_diff_temp'], axis=1) #save the record file record_file_df.to_csv(path_save + '/' + start_time + '_' + end_time + ' statistics.csv', index=0) return raw_dict, tele_dict, record_file_df, index, start_time_local, end_time_local, path_save
""" Compares eMOLT with FVCOM bottom temp @author: jmanning, rsignell, yacheng """ import matplotlib.pyplot as plt import netCDF4 from getdata import getemolt_latlon, getemolt_temp from conversions import dm2dd, f2c from utilities import nearxy site = 'BN01' [lati, loni, on] = getemolt_latlon(site) # extracts lat/lon based on site code [lati, loni] = dm2dd(lati, loni) #converts decimal-minutes to decimal degrees [obs_dt, obs_temp] = getemolt_temp(site) # extracts time series for kk in range(len(obs_temp)): obs_temp[kk] = f2c(obs_temp[kk]) # converts to Celcius # now get the model output urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3' nc = netCDF4.Dataset(urlfvcom) nc.variables lat = nc.variables['lat'][:] lon = nc.variables['lon'][:] times = nc.variables['time'] jd = netCDF4.num2date(times[:], times.units) vname = 'temp' var = nc.variables[vname] # find nearest point to desired location and time inode = nearxy(lon, lat, loni, lati) index = netCDF4.date2index(
index_col='datet', date_parser=parse, names=[ 'LATITUDE', 'LONGITUDE', 'ROUND_DATE_TIME', 'OBSERVATIONS_TEMP', 'MEAN_TEMP', 'MIN_TEMP', 'MAX_TEMP', 'STD_DEV_TEMP', 'OBSERVATIONS_DEPTH', 'MEAN_DEPTH', 'MIN_DEPTH', 'MAX_DEPTH', 'STD_DEV_DEPTH', 'nan' ]) #convert ddmm.m to dd.ddd plt.figure(figsize=(16, 10)) o = [] m = [] print '1' for k in range(len(obstso)): # [la, lo] = dm2dd(obstso['LATITUDE'][k], obstso['LONGITUDE'][k]) if not inmodel_fvcom([lo], [la]): print 'point not in fvcom domain' continue else: fvcomobj = water_fvcom() # modtso = pd.DataFrame() st = obstso.index[k] et = st + timedelta(hours=1) url_fvcom = fvcomobj.get_url(st, et) modtso = fvcomobj.watertemp(lo, la, -1 * obstso['MEAN_DEPTH'][k], url_fvcom[0]) o.append(obstso['MEAN_TEMP'][k]) m.append(modtso.values[0]) plt.plot(modtso.values[0], obstso['MEAN_TEMP'][k], '.', markersize=30) plt.xlabel('MODEL (degC)')
def get_w_depth(xi, yi): #xi:lon yi:lat url = 'http://geoport.whoi.edu/thredds/dodsC/bathy/gom03_v1_0' #url='http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol1.nc' ): if xi[0] > 999.: # if it comes in decimal -minutes, conert it #for kk in range(len(xi)): (y2, x2) = dm2dd(yi[0], xi[0]) yi[0] = y2 xi[0] = x2 try: dataset = open_url(url) except: print 'Sorry, ' + url + ' is not available' sys.exit(0) #read lat, lon,topo from url xgom_array = dataset['lon'] ygom_array = dataset['lat'] dgom_array = dataset['topo'].topo #print dgom_array.shape, xgom_array[5:9],dgom_array[5] #convert the array to a list xgom, ygom = [], [] for i in xgom_array: if i > xi[0] - 0.00834 and i < xi[0] + 0.00834: xgom.append(i) for i in ygom_array: if i > yi[0] - 0.00834 and i < yi[0] + 0.00834: ygom.append(i) x_index, y_index = [], [] (ys, xs) = dgom_array.shape for i in range(0, len(xgom)): x_index.append(int(round(np.interp(xgom[i], xgom_array, range(xs))))) for i in range(0, len(ygom)): y_index.append(int(round(np.interp(ygom[i], ygom_array, range(ys))))) dep, distkm, dist1 = [], [], [] for k in range(len(x_index)): for j in range(len(y_index)): dep.append(dgom_array[(y_index[j], x_index[k])]) distkm, b = distance((ygom[j], xgom[k]), (yi[0], xi[0])) dist1.append(distkm) #print dist1 if len(dist1) >= 3: #get the nearest,second nearest,third nearest point. dist_f_nearest = sorted(dist1)[0] dist_s_nearest = sorted(dist1)[1] dist_t_nearest = sorted(dist1)[2] index_dist_nearest = range(len(dist1)) index_dist_nearest.sort(lambda x, y: cmp(dist1[x], dist1[y])) dep_f_nearest = dep[index_dist_nearest[0]] dep_s_nearest = dep[index_dist_nearest[1]] dep_t_nearest = dep[index_dist_nearest[2]] #compute the finally depth d1 = dist_f_nearest d2 = dist_s_nearest d3 = dist_t_nearest def1 = dep_f_nearest def2 = dep_s_nearest def3 = dep_t_nearest depth_finally = def1 * d2 * d3 / ( d1 * d2 + d2 * d3 + d1 * d3) + def2 * d1 * d3 / ( d1 * d2 + d2 * d3 + d1 * d3) + def3 * d2 * d1 / (d1 * d2 + d2 * d3 + d1 * d3) else: depth_finally = np.array([[-9999]]) return depth_finally
lonsize=[event.xdata-0.6,event.xdata+0.6] plt.figure(figsize=(7,6)) m = Basemap(projection='cyl',llcrnrlat=min(latsize)-0.01,urcrnrlat=max(latsize)+0.01,\ llcrnrlon=min(lonsize)-0.01,urcrnrlon=max(lonsize)+0.01,resolution='h')#,fix_aspect=False) m.drawparallels(np.arange(int(min(latsize)),int(max(latsize))+1,1),labels=[1,0,0,0]) m.drawmeridians(np.arange(int(min(lonsize)),int(max(lonsize))+1,1),labels=[0,0,0,1]) m.drawcoastlines() m.fillcontinents(color='grey') m.drawmapboundary() m.plot(lon,lat,'r.',lonc,latc,'b+') plt.show() spoint = pylab.ginput(1) ''' if lo > 90: [la, lo] = dm2dd(la, lo) latd, lond = [], [] kf, distanceF = nearlonlat(lonc, latc, lo, la) # nearest triangle center F - face kv, distanceV = nearlonlat(lon, lat, lo, la) if h[kv] < 0: print 'Sorry, your position is on land, please try another point' sys.exit() depthtotal = siglay[:, kv] * h[kv] layer = np.argmin(abs(depthtotal - depth)) for i in range(startrecord, endrecord): ############read the particular time model from website#########
save_dir = '/home/zdong/xiaoxu/emolt/figure/' #the path of saving figure file_catch = 'sqldump_2018_07_BN.csv' # catch data that JiM created & emailed input_dir = '/home/zdong/xiaoxu/emolt/files/' file_temp_BN = 'this2.dat' #temp data file_temp_BD = 'BD01_temp.dat' file_temp_TS = 'TS01_temp.dat' variables = ['year', 'year_day', 'temp', 'lat', 'lon', 'depth'] # df_BN = pd.read_csv(input_dir + file_temp_BN, names=variables, sep='\s+') # read temperature time series df_BD = pd.read_csv(input_dir + file_temp_BD, names=variables, sep='\s+') df_TS = pd.read_csv(input_dir + file_temp_TS, names=variables, sep='\s+') lat_BD, lon_BD = [], [] lat_BN, lon_BN = [], [] lat_TS, lon_TS = [], [] for i in range(len(df_BN)): la, lo = conversions.dm2dd(df_BN['lat'].values[i], df_BN['lon'].values[i]) lat_BN.append(la) lon_BN.append(lo) for i in range(len(df_BD)): la, lo = conversions.dm2dd(df_BD['lat'].values[i], df_BD['lon'].values[i]) lat_BD.append(la) lon_BD.append(lo) for i in range(len(df_TS)): la, lo = conversions.dm2dd(df_TS['lat'].values[i], df_TS['lon'].values[i]) lat_TS.append(la) lon_TS.append(lo) fig = plt.figure() a = fig.add_subplot(1, 1, 1) my_map = Basemap(projection='merc', resolution='h', area_thresh=0.3,