Created on Fri Dec 12 09:21:16 2014 @author: zhaobin """ 'draw temp change of all turtle data and model data and calculate ‘bad’ turtles.' import numpy as np import pandas as pd import matplotlib.pyplot as plt import matplotlib as mpl from math import sqrt from datetime import datetime, timedelta from turtleModule import str2ndlist, np_datetime ###################################main code################################### criteria=3 # criteria for rms obsData=pd.read_csv('ctdWithdepthofbottom_roms.csv') obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR']),index=obsData.index) modDepth=pd.Series(obsData['depth_bottom'],index=obsData.index) indx=[] for i in obsData.index: if abs(obsDepth[i][-1]-modDepth[i])<10: #diff<10m,we can use the data indx.append(i) obsturtle_id=pd.Series(obsData['PTT'][indx],index=indx) obsTemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]),index=indx) modTemp=pd.Series(str2ndlist(obsData['modTempByDepth'][indx],bracket=True),index=indx) obsTime=pd.Series(np_datetime(obsData['END_DATE'][indx]),index=indx) obsturtle_ids=obsturtle_id.unique() ids=[] for i in range(len(obsturtle_ids)): ids.append([]) for i in range(len(obsturtle_ids)): for j in indx:
Extract new data file named "06-08_indexANDlayer.csv "with new column "modNearestIndex" and "modDepthLayer" ''' import pandas as pd import numpy as np import netCDF4 from datetime import datetime, timedelta from turtleModule import str2ndlist, dist import watertempModule as wtm # a module that has all classes using ROMS and FVCOm model. def nearest_point_index2(lon, lat, lons, lats): d = dist(lon, lat, lons ,lats) min_dist = np.min(d) index = np.where(d==min_dist) return index #################################################################################33 obsData = pd.read_csv('06-08_ch.csv', index_col=0) obsLon = pd.Series(str2ndlist(obsData['lon'],bracket=True)) obsLat = pd.Series(str2ndlist(obsData['lat'],bracket=True)) url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his' modData = netCDF4.Dataset(url) modLons = modData.variables['lon_rho'][:] modLats = modData.variables['lat_rho'][:] s_rho = modData.variables['s_rho'][:] h = modData.variables['h'][:] indexNotNull = obsLon[obsLon.isnull()==False].index # some obslat and obslon of point are empty, get rid of them. # or this line can be the indices of TF which is less. # indexTF = np.where(obsData['TF'].notnull())[0] loc = [] for i in indexNotNull: ind = [] lon = obsLon[i]
cbar.ax.set_ylabel('Quantity', fontsize=FONTSIZE) cbar.ax.tick_params(labelsize=20) plt.text(x=1, y=27, s=text[j], fontsize=15) plt.text(x=1, y=23, s=r'$\mathregular{R^2}$=' + str(round(r_squared, 3)), fontsize=15) plt.savefig('obsVSmodelDeepShallow' + str(i) + '.png', dpi=200) #########################################MAIN CODE##################################################################################################### FONTSIZE = 25 ##################################roms######################################### obs = pd.read_csv('ctdWithModTempByDepth.csv') tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data. obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index) tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index], bracket=True), index=tf_index) tempObs1_roms, tempObs2_roms, tempObs3_roms, tempObs4_roms, tempObs5_roms, tempObs6_roms = [],[],[],[],[],[] tempMod1_roms, tempMod2_roms, tempMod3_roms, tempMod4_roms, tempMod5_roms, tempMod6_roms = [],[],[],[],[],[] for i in range(len(obsTemp.values)): for j in range(len(obsDepth.values[i])): d = obsDepth.values[i][j] if tempMod.values[i][j] > 100: continue if d < 25.0: tempObs1_roms.append(obsTemp.values[i][j]) tempMod1_roms.append(tempMod.values[i][j]) if d >= 25.0 and d < 50.0: tempObs2_roms.append(obsTemp.values[i][j]) tempMod2_roms.append(tempMod.values[i][j])
import watertempModule as wtm ############################################################################### ''' starttime = datetime(2013,07,10) endtime = starttime + timedelta(hours=1) # starttime and endtime can be any time that included by model, we just want a url to get "lon_rho", "lat_rho" in model. tempObj = wtm.water_roms() url = tempObj.get_url(starttime, endtime) ''' url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his' modData = netCDF4.Dataset(url) modLons = modData.variables['lon_rho'][:] modLats = modData.variables['lat_rho'][:] #the lon and lat are ROMS`s obsData = pd.read_csv('ctdWithdepthofbottom_roms.csv') modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'],bracket=True), index=obsData.index) # if str has '[' and ']', bracket should be True obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS']), index=obsData.index) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR']), index=obsData.index) depthBottom = pd.Series(obsData['depth_bottom'],index=obsData.index) for i in obsData.index: if depthBottom[i]>200: depthBottom[i]=200 #Just a little points are deeper than 200m.Removing them can plot depth better obsLon, obsLat = obsData['LON'], obsData['LAT'] #use for plotting depth line modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'], bracket=True), index=obsData.index) data = pd.DataFrame({'obstemp': obsTemp,'modtemp':modTemp, 'nearestIndex': modNearestIndex, 'obsdepth':obsDepth,'moddepth':depthBottom},index=obsData.index) lon_m=[] lat_n=[] dataNum = []
(depth-lyrDepth[0]) + temp[0,a,b] else: ind = closest_num(depth, lyrDepth) d = (temp[ind,a,b]-temp[ind-1,a,b])/(lyrDepth[ind]-lyrDepth[ind-1]) * \ (depth-lyrDepth[ind-1]) + temp[ind-1,a,b] t.append(d) modTemp.append(t) modTemp = np.array(modTemp) return modTemp #########################################MAIN CODE##################################################################################################### FONTSIZE = 25 obs = pd.read_csv('ctdWithModTempByDepth.csv') # Extracted from "errorMap.py" tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data. obsLon, obsLat = obs['LON'][tf_index], obs['LAT'][tf_index] obsTime = pd.Series(np_datetime(obs['END_DATE'][tf_index]), index=tf_index) obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index) # obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index) # obsDepth = obs['MAX_DBAR'][tf_index] modLayer = pd.Series(str2ndlist(obs['modDepthLayer'][tf_index],bracket=True), index=tf_index) # bracket is to get rid of symbol "[" and "]" in string modNearestIndex = pd.Series(str2ndlist(obs['modNearestIndex'][tf_index], bracket=True), index=tf_index) starttime = datetime(2009, 8, 24) endtime = datetime(2013, 12, 13) # for depth tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index],bracket=True), index=tf_index) ''' # for layers tempObj = wtm.waterCTD() url = tempObj.get_url(starttime, endtime) # tempMod = tempObj.watertemp(obsLon.values, obsLat.values, obsDepth.values, obsTime.values, url)
Smooth the data, get rid of data that changes too much. ''' for i in range(2, len(v)-1): a, b, c = v[i-1], v[i], v[i+1] diff1 = abs(b - c) #diff1 is not used diff2 = abs(b - a) if diff2>e: v[i] = a return v ######################################################### obsData = pd.read_csv('ctdWithModTempByDepth.csv', index_col=0) tf_index = np.where(obsData['TF'].notnull())[0] obsData = obsData.ix[tf_index] id = obsData['PTT'].drop_duplicates().values tID = id[3] # 0~4, 6,7,8,9, turtle ID. layers = pd.Series(str2ndlist(obsData['modDepthLayer'], bracket=True), index=obsData.index) # If str has '[' and ']', bracket should be True. modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'].values, bracket=True), index=obsData.index) obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'].values), index=obsData.index) obsTime = pd.Series(np_datetime(obsData['END_DATE'].values), index=obsData.index) modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'].values,bracket=True), index=obsData.index) layers = layers[obsData['PTT']==tID] modNearestIndex = modNearestIndex[obsData['PTT']==tID] time = obsTime[obsData['PTT']==tID] temp = obsTemp[obsData['PTT']==tID] #modTemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][temp.index],bracket=True), index=temp.index) ''' starttime, endtime=np.amin(time), np.amax(time)+timedelta(hours=1) modObj = wtm.waterCTD() url = modObj.get_url(starttime, endtime) #something wrong with ROMS website oceantime = netCDF4.Dataset(url).variables['ocean_time'] modTempAll = netCDF4.Dataset(url).variables['temp']
var=[] for i in layer: t = nc.variables['water_temp'][TIME,i,1400+indlat[id],1250+indlon[id]] #creates a "netCDF4 object" if t<100: #change bad data to float. var.append(t) else: var.append(-100) nc.close() return var ############################################################## #turtle data obsData = pd.read_csv('ctd_good.csv') tf_index = np.where(obsData['TF'].notnull())[0] obsLon, obsLat = obsData['LON'][tf_index], obsData['LAT'][tf_index] obsTime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index) MODtemp=[] for i in tf_index: t=getHYcom(obsLat[i],obsLon[i],obsTime[i],obsDepth[i]) MODtemp.append(t) print i obsData['modtemp_HYCOM']=pd.Series(MODtemp,index=tf_index) obsData.to_csv('ctd_withHYCOMtemp.csv') ''' #all ships data shipData=pd.read_csv('ship06-08_ch.csv') shipLon=pd.Series(str2ndlist(shipData['lon'],bracket=True)) shipLat=pd.Series(str2ndlist(shipData['lat'],bracket=True)) shipTime=pd.Series(shipData['time']) shipDepth=pd.Series(str2ndlist(shipData['depth'],bracket=True))
''' return ratio of the deepest depth ''' import numpy as np import matplotlib.pyplot as plt import pandas as pd import netCDF4 from datetime import datetime, timedelta from turtleModule import str2ndlist import watertempModule as wtm # A module of classes that using ROMS, FVCOM. obsData = pd.read_csv('ctd_good.csv') # From nearestIndexInMod.py tf_index = np.where(obsData['TF'].notnull())[0] # Get index of good data. obsLat, obsLon = obsData['LAT'][tf_index], obsData['LON'][tf_index] obsDeepest = obsData['MAX_DBAR'][tf_index] # get deepest data file depth obsID = obsData['PTT'][tf_index] # Get ID of turtle. modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'][tf_index], bracket=True), index=tf_index) #starttime = datetime(2009, 8, 24) starttime = datetime(2013,05,20) # starttime and endtime here is just to get 'h', the model depth endtime = datetime(2013, 12, 13) tempObj = wtm.waterCTD() url = tempObj.get_url(starttime, endtime) modData = netCDF4.Dataset(url) # modTempAll = modData.variables['temp'] h = modData.variables['h'] newH=[] for i in tf_index: m, n = int(modNearestIndex[i][0]), int(modNearestIndex[i][1]) newH.append(h[m][n]) fig = plt.figure()
'compare temperature of turtle and roms model before and after a storm' @author: zdong,yifan """ import numpy as np import pandas as pd import matplotlib as mpl from datetime import datetime,timedelta import matplotlib.pyplot as plt from turtleModule import str2ndlist,dist,np_datetime ################################################################## obsData=pd.read_csv('tu94wModel.csv',index_col=0) turtle_id=obsData['PTT'] obslat = obsData['LAT'] obslon = obsData['LON'] obstime = pd.Series(np_datetime(obsData['date']),index=obsData.index) obsDepth=pd.Series(str2ndlist(obsData['obstemp']),index=obsData.index) obstemp=pd.Series(str2ndlist(obsData['depth']),index=obsData.index) buoyData=pd.read_csv('ndbc_2011.csv') #data of wind speed and wave height in 2011 tf_index=np.where(buoyData['TF'].notnull())[0] buoylat = pd.Series(buoyData['lat'],index=tf_index) buoylon = pd.Series(buoyData['lon'],index=tf_index) buoyspeed=pd.Series(buoyData['wind_speed'],index=tf_index) buoywave=pd.Series(buoyData['wave_height'],index=tf_index) buoytime = pd.Series(buoyData['Date'],index=tf_index) indx=[] for i in tf_index: buoytime[i]=datetime.strptime(buoytime[i], "%Y-%m-%d %H:%M") # change str to datatime if 17.2<buoyspeed[i]: if buoywave[i]>5.5: indx.append(i) #choose which day has storm
ax.set_ylabel('Model temperature($^\circ$C)', fontsize=FONTSIZE) plt.xticks(fontsize=20) plt.yticks(fontsize=20) cbar = plt.colorbar(c) if j==1: cbar.ax.set_ylabel('Quantity', fontsize=FONTSIZE) cbar.ax.tick_params(labelsize=20) plt.text(x=1,y=27,s=text[j],fontsize=15) plt.text(x=1,y=23,s=r'$\mathregular{R^2}$='+str(round(r_squared,3)),fontsize=15) plt.savefig('obsVSmodelDeepShallow'+str(i)+'.png', dpi=200) #########################################MAIN CODE##################################################################################################### FONTSIZE = 25 ##################################roms######################################### obs = pd.read_csv('ctdWithModTempByDepth.csv') tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data. obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index) tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index],bracket=True), index=tf_index) tempObs1_roms, tempObs2_roms, tempObs3_roms, tempObs4_roms, tempObs5_roms, tempObs6_roms = [],[],[],[],[],[] tempMod1_roms, tempMod2_roms, tempMod3_roms, tempMod4_roms, tempMod5_roms, tempMod6_roms = [],[],[],[],[],[] for i in range(len(obsTemp.values)): for j in range(len(obsDepth.values[i])): d = obsDepth.values[i][j] if tempMod.values[i][j] > 100: continue if d<25.0: tempObs1_roms.append(obsTemp.values[i][j]) tempMod1_roms.append(tempMod.values[i][j]) if d>=25.0 and d<50.0: tempObs2_roms.append(obsTemp.values[i][j]) tempMod2_roms.append(tempMod.values[i][j]) if d>=50.0 and d<75:
""" 'plot temperature of turtles and shipboard ' import pandas as pd import numpy as np import matplotlib.pyplot as plt from datetime import datetime, timedelta from turtleModule import mon_alpha2num, np_datetime, dist,str2ndlist,colors,str2list ################################################################################### r = 10 # the obs position that has shipboard position within (r) kilometers might be considered as good data. day = 3 # the obs time that has shipboard time within (day) days might be considered as good data. obsData=pd.read_csv('ctdWithModTempByDepth.csv',index_col=0) tf_index=np.where(obsData['TF'].notnull())[0] obslat = pd.Series(obsData['LAT'][tf_index],index=tf_index) obslon = pd.Series(obsData['LON'][tf_index],index=tf_index) obstime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]),index=tf_index) obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]),index=tf_index) obstemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]),index=tf_index) modtemp=pd.Series(str2ndlist(obsData['modTempByDepth'][tf_index],bracket=True),index=tf_index) obsData1=pd.read_csv('ctd_FVcom_temp.csv') #this ctd`s FVCOM temperature tf_index_FVCOM=np.where(obsData1['modtempBYdepth'].notnull())[0] modtemp_FVCOM=pd.Series(obsData1['modtempBYdepth'],index=tf_index_FVCOM) for i in tf_index_FVCOM: modtemp_FVCOM[i]=str2list(modtemp_FVCOM[i],bracket=True) obsData2=pd.read_csv('ctd_withHYCOMtemp.csv') #this ctd`s HYCOM temperature tf_index_HYCOM=np.where(obsData2['modtemp_HYCOM'].notnull())[0] modtemp_HYCOM=pd.Series(obsData2['modtemp_HYCOM'],index=tf_index_HYCOM) for i in tf_index_HYCOM: modtemp_HYCOM[i]=str2list(modtemp_HYCOM[i],bracket=True) shipData=pd.read_csv('ship06-08_MODELtemp.csv',index_col=0)
import pandas as pd import numpy as np import matplotlib.pyplot as plt from datetime import datetime from turtleModule import str2ndlist ############################################################ second_data = pd.read_csv('turtle_12487_tdr.csv') depth = pd.Series(second_data['Depth']) temp = pd.Series(second_data['temp']) time = pd.Series(second_data['Date']) for i in second_data.index: time[i] = datetime.strptime(time[i], '%Y/%m/%d %H:%M:%S') data = pd.read_csv('12487up_down.csv') up = pd.Series(str2ndlist(data['up'], bracket=True)) down = pd.Series(str2ndlist(data['down'], bracket=True)) indx = [] for i in data.index: if depth[down[i][-1]] > 20: #depth>20m if depth[up[i][0]] > 20: indx.append(i) Down_mean_temp,Up_mean_temp,Down_min_temp,Up_min_temp,Down_max_temp,Up_max_temp=[],[],[],[],[],[] diff_temp_up, diff_temp_down = [], [] Down_speed, Up_speed = [], [] for i in indx[1:len(indx)]: if down[i][-1] - down[i][0] > 0: a = temp[int(down[i][0]):int(down[i][-1])] b = temp[int(up[i][0]):int(up[i][-1])] Down_mean_temp.append(np.mean(a)) Up_mean_temp.append(np.mean(b))
@author: zhobin """ 'compare temperature before and after a storm' import numpy as np import pandas as pd import matplotlib as mpl from datetime import datetime,timedelta import matplotlib.pyplot as plt from turtleModule import str2ndlist,dist,np_datetime ################################################################## obsData=pd.read_csv('ctdwithoutbad_roms.csv',index_col=0) turtle_id=pd.Series(obsData['PTT'],index=obsData.index) obslat = pd.Series(obsData['LAT'],index=obsData.index) obslon = pd.Series(obsData['LON'],index=obsData.index) obstime = pd.Series(np_datetime(obsData['END_DATE']),index=obsData.index) obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR']),index=obsData.index) obstemp=pd.Series(str2ndlist(obsData['TEMP_VALS']),index=obsData.index) buoyData=pd.read_csv('ndbc_2011.csv') #data of wind speed and wave height in 2011 tf_index=np.where(buoyData['TF'].notnull())[0] buoylat = pd.Series(buoyData['lat'],index=tf_index) buoylon = pd.Series(buoyData['lon'],index=tf_index) buoyspeed=pd.Series(buoyData['wind_speed'],index=tf_index) buoywave=pd.Series(buoyData['wave_height'],index=tf_index) buoytime = pd.Series(buoyData['Date'],index=tf_index) indx=[] for i in tf_index: buoytime[i]=datetime.strptime(buoytime[i], "%Y-%m-%d %H:%M") # change str to datatime if 17.2<buoyspeed[i]: if buoywave[i]>5.5: indx.append(i) #choose which day has storm
ids=[] for i in range(length): ids.append([]) for i in range(length): for j in obsData.index: if obsturtle_id[j]==obsturtle_ids[i]: ids[i].append(j) #collect each turtle`s index obsmins=[] obsmaxs=[] obsmeans=[] modmins=[] modmaxs=[] modmeans=[] # collect min,max and mean of model and observe temperature for i in range(len(ids)): obsdata=obsData.ix[ids[i]] goodobstemp=pd.Series(str2ndlist(obsdata['TEMP_VALS']), index=ids[i]) goodmodtemp=pd.Series(str2ndlist(obsdata['modTempByDepth'],bracket=True), index=ids[i]) #each turtle`s modtemp and obstemp obsmin=[] obsmax=[] obsmean=[] modmin=[] modmax=[] modmean=[] for j in ids[i]: obsmin.append(np.min(np.array(goodobstemp[j]))),modmin.append(np.min(np.array(goodmodtemp[j]))) obsmax.append(np.max(np.array(goodobstemp[j]))),modmax.append(np.max(np.array(goodmodtemp[j]))) obsmean.append(np.mean(np.array(goodobstemp[j]))),modmean.append(np.mean(np.array(goodmodtemp[j]))) #each turtle in one location`s max,min,mean temperature obsmins.append(np.min(np.array(obsmin))) obsmaxs.append(np.max(np.array(obsmax))) obsmeans.append(np.mean(np.array(obsmean)))
@author: zhaobin """ 'get temperature from fvcom`s website and create ctd_FVcom_temp.csv or ship_FVcom_temp.csv' import numpy as np import pandas as pd import netCDF4 from turtleModule import str2ndlist ############################################################## url='http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?temp[0:1:316008][0:1:44][0:1:48450]' modData=netCDF4.Dataset(url) modtemp=modData.variables['temp'] obsData=pd.read_csv('ctd_FVcom_node_time_layer.csv') #ctd #obsData=pd.read_csv('ship_FVcom_node_time_layer.csv') #ship tf_index=np.where(obsData['in FVcom range'].notnull())[0] obsnode=pd.Series(obsData['modnode'][tf_index],index=tf_index) obstime=pd.Series(obsData['modtime'][tf_index],index=tf_index) obslayer=pd.Series(str2ndlist(obsData['modlayer'][tf_index],bracket=True),index=tf_index) TEMP=[] for i in tf_index: temp=[] for j in obslayer[i]: t=modtemp[int(obstime[i])][int(j)][int(obsnode[i])] temp.append(t) TEMP.append(temp) print i obsData['modtempBYdepth']=pd.Series(TEMP,index=tf_index) obsData.to_csv('ctd_FVcom_temp.csv') #ctd #obsData.to_csv('ship_FVcom_temp.csv') #ship
Otemp = data2['obs_temp'] Dtemp = data2['doppio_temp'] obsTime = pd.Series( (datetime.strptime(x, '%Y-%m-%d %H:%M:%S') for x in data2['argos_date'])) for i in range(len(t_id)): e = t_id[i] indx = [] for i in data2.index: if data2['PTT'][i] == e: indx.append(i) Data_e = data2.loc[indx] Data_e.index = range(len(indx)) Time_e = pd.Series((datetime.strptime(x, '%Y-%m-%d %H:%M:%S') for x in Data_e['argos_date'])) Otemp = pd.Series(str2ndlist(Data_e['obs_temp'])) Dtemp = pd.Series(str2ndlist(Data_e['doppio_temp'])) Depth_e = pd.Series(str2ndlist(Data_e['depth'])) fig = plt.figure() ax1 = fig.add_subplot(1, 1, 1) l = len(Data_e) / 40.0 for j in Data_e.index: for c in range(len(t_id)): if e == t_id[c]: ax1.plot(np.array(Otemp[j]) + shift * j, Depth_e[j], color='r', linewidth=1) ax1.plot(np.array(Dtemp[j]) + shift * j, Depth_e[j],
plt.plot(x, x, 'r-', linewidth=2) plt.plot(x2, y2, 'y-', linewidth=2) plt.text(x=1,y=27,s=text[j],fontsize=15) plt.text(x=1,y=23,s=r'$\mathregular{R^2}$='+str(round(r_squared[j],3)),fontsize=15) if j==0: plt.title('%s'%tp,fontsize=FONTSIZE) plt.savefig('obsVSmodel_%s.png'%tp,dpi=200) return r_squared ####################################roms####################################### FONTSIZE = 25 obs = pd.read_csv('ctdWithModTempByDepth.csv') tf_index = np.where(obs['TF'].notnull())[0] obsdepth = pd.Series(obs['MAX_DBAR'][tf_index],index=tf_index) obstemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]),index=tf_index) obsdata_roms= pd.DataFrame({'depth':obsdepth, 'temp':obstemp}).sort_index(by='depth') Temp_roms=pd.Series(str2ndlist(obs['modTempByDepth'][obsdata_roms['temp'].index],bracket=True),index=obsdata_roms['temp'].index) temp_roms=[] for i in obsdata_roms['temp'].index: if Temp_roms[i][-1]>100: #get rid of some bad data temp_roms.append(obsdata_roms['temp'][i]) else: temp_roms.append(Temp_roms[i][-1]) temp_roms=pd.Series(temp_roms,index=obsdata_roms['temp'].index) index_roms=index_by_depth(obsdata_roms['depth'], 50) ###################################fvcom####################################### obs = pd.read_csv('ctdWithdepthofbottom_fvcom.csv') tf_index = np.where(obs['in FVcom range'].notnull())[0] obsdepth = pd.Series(obs['MAX_DBAR'][tf_index],index=tf_index) obstemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]),index=tf_index) obsdata_fvcom = pd.DataFrame({'depth':obsdepth, 'temp':obstemp}).sort_index(by='depth') Temp_fvcom =pd.Series(str2ndlist(obs['modtempBYdepth'][obsdata_fvcom['temp'].index],bracket=True),index=obsdata_fvcom['temp'].index)
Created on Thu Nov 20 10:57:43 2014 @author: zhaobin """ 'plot contour of observe and model temperature in four seasons' from matplotlib.mlab import griddata import matplotlib.pyplot as plt import numpy as np import pandas as pd import netCDF4 from turtleModule import str2ndlist,draw_basemap,np_datetime ##############################main code######################################## lonsize = [-79.5, -72.5] latsize = [34.5, 41] # range of basemap obsData = pd.read_csv('ctdWithdepthofbottom_roms.csv') modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'],bracket=True), index=obsData.index) # if str has '[' and ']', bracket should be True obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS']), index=obsData.index) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR']), index=obsData.index) obsTime = pd.Series(np_datetime(obsData['END_DATE']),index=obsData.index) depthBottom = pd.Series(obsData['depth_bottom'],index=obsData.index) for i in obsData.index: if depthBottom[i]>200: depthBottom[i]=200 #Just a little points are deeper than 200m.Removing them can plot depth betterex) obsLons, obsLats = obsData['LON'], obsData['LAT'] index_inside=[] for i in obsLons.index: if lonsize[0]<obsLons[i]<lonsize[1]: if abs(obsDepth[i][-1]-depthBottom[i])<10: #diff<10m,wo can use the data index_inside.append(i) #Just use data in this area. obsLon, obsLat = obsData['LON'][index_inside], obsData['LAT'][index_inside]
output two files:ctdwithoutbad_roms.csv and ctdWithdepthofbottom_roms.csv @author: zhaobin """ from matplotlib.mlab import griddata import matplotlib.pyplot as plt import pandas as pd import numpy as np import netCDF4 from datetime import datetime, timedelta import math from turtleModule import str2ndlist,draw_basemap import watertempModule as wtm ############################################################################################ obsData = pd.read_csv('ctdWithModTempByDepth.csv') tf_index = np.where(obsData['TF'].notnull())[0] modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'][tf_index],bracket=True), index=tf_index) # if str has '[' and ']', bracket should be True indx=[] for j in tf_index: if modTemp[j][-1]<100: indx.append(j) obs=obsData.ix[indx] obs=obs.drop(['Unnamed: 0','Unnamed: 0.1'],axis=1) obs.to_csv('ctdwithoutbad_roms.csv') ############################################################################### starttime = datetime(2013,07,10) # starttime and endtime can be any time that included by model, we just want a url to get "lon_rho", "lat_rho","h"in model. endtime = starttime + timedelta(hours=1) tempObj = wtm.water_roms() url = tempObj.get_url(starttime, endtime) modData = netCDF4.Dataset(url) modLons = modData.variables['lon_rho'][:] modLats = modData.variables['lat_rho'][:]
import pandas as pd from datetime import datetime, timedelta from mpl_toolkits.basemap import Basemap import matplotlib.pyplot as plt import netCDF4 import watertempModule as wtm # A module of classes that using ROMS, FVCOM from turtleModule import str2ndlist, np_datetime, bottom_value, dist FONTSIZE = 25 criteria = 10 # error criteria depth = -10 # The first depth map wanted to plot depth_interval = 25 # the interval depth of each map. obsData = pd.read_csv('ctdWithModTempByDepth.csv') # extracted from ctdWithModTempByDepth.py tf_index = np.where(obsData['TF'].notnull())[0] # get the index of good data obsLon, obsLat = obsData['LON'][tf_index], obsData['LAT'][tf_index] obsTime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]), index=tf_index) obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]), index=tf_index) # obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index) modLayer = pd.Series(np.array(str2ndlist(obsData['modDepthLayer'][tf_index],bracket=True)), index=tf_index) # bracket is to get rid of symbol "[" and "]" in string modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'][tf_index], bracket=True), index=tf_index) modTemp = pd.Series(np.array(str2ndlist(obsData['modTempByDepth'][tf_index], bracket=True)), index=tf_index) data = pd.DataFrame({'lon': obsLon, 'lat': obsLat, 'obstemp': obsTemp.values,'modtemp':modTemp, 'depth': obsDepth, 'time': obsTime.values, 'layer': modLayer}, index=tf_index) ind,Ind = [],[] # the indices needed obst,Obst = [],[] modt,Modt = [],[] lyr,Lyr = [],[] dep,Dep=[],[]
import pandas as pd import numpy as np import matplotlib.pyplot as plt from datetime import datetime from turtleModule import str2ndlist ############################################################ second_data=pd.read_csv('turtle_12487_tdr.csv') depth=pd.Series(second_data['Depth']) temp=pd.Series(second_data['temp']) time=pd.Series(second_data['Date']) for i in second_data.index: time[i]=datetime.strptime(time[i],'%Y/%m/%d %H:%M:%S') data=pd.read_csv('12487up_down.csv') up=pd.Series(str2ndlist(data['up'],bracket=True)) down=pd.Series(str2ndlist(data['down'],bracket=True)) indx=[] for i in data.index: if depth[down[i][-1]]>20: #depth>20m if depth[up[i][0]]>20: indx.append(i) Down_mean_temp,Up_mean_temp,Down_min_temp,Up_min_temp,Down_max_temp,Up_max_temp=[],[],[],[],[],[] diff_temp_up,diff_temp_down=[],[] Down_speed,Up_speed=[],[] for i in indx[1:len(indx)]: if down[i][-1]-down[i][0]>0: a=temp[int(down[i][0]):int(down[i][-1])] b=temp[int(up[i][0]):int(up[i][-1])] Down_mean_temp.append(np.mean(a)) Up_mean_temp.append(np.mean(b))
depthLayers = h[index[0][0]][index[1][0]] * s_rho # layerDepth = [depthLayers[-layer+1], depthLayers[-layer]] l = np.argmin(abs(depthLayers + vDepth)) return l ###################################MAIN CODE########################################### FONTSIZE = 25 criteria = 10 # error criteria layer = 15 # the first layer you want to plot obs = pd.read_csv("ctd_good.csv") # From nearestIndexInMod.py tf_index = np.where(obs["TF"].notnull())[0] obsLon, obsLat = obs["LON"][tf_index], obs["LAT"][tf_index] obsTime = pd.Series(np_datetime(obs["END_DATE"][tf_index]), index=tf_index) obsTemp = pd.Series(str2ndlist(obs["TEMP_VALS"][tf_index]), index=tf_index) # obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obs["TEMP_DBAR"][tf_index]), index=tf_index) modDepthLayer = pd.Series(str2ndlist(obs["modDepthLayer"][tf_index], bracket=True), index=tf_index) starttime = datetime(2009, 8, 24) # Choose the starttime and endtime that cover the whole observation time range. endtime = datetime(2013, 12, 13) tempObj = wtm.waterCTD() url = tempObj.get_url(starttime, endtime) modTemp = tempObj.watertemp(obsLon.values, obsLat.values, obsDepth.values, obsTime.values, url) d = { "lon": obsLon, "lat": obsLat, "time": obsTime.values, "obstemp": obsTemp.values,
secondData=pd.read_csv('12487_location.csv') tf_index1 = np.where(secondData['index'].notnull())[0] time=pd.Series(secondData['time'],index=tf_index1) depth=pd.Series(secondData['depth'],index=tf_index1) temp=pd.Series(secondData['temp'],index=tf_index1) lon=pd.Series(secondData['lon'],index=tf_index1) lat=pd.Series(secondData['lat'],index=tf_index1) inde=pd.Series(secondData['index'],index=tf_index1) indx=[] for i in tf_index: if obsturtle_id[i]==118905: #this turtle is same turtle with 4-second turtle indx.append(i) obsLon, obsLat = obsData['LON'][indx], obsData['LAT'][indx] obsTime = pd.Series(np_datetime(obsData['END_DATE'][indx]), index=indx) obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]), index=indx) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][indx]), index=indx) modTemp = pd.Series(obsData['modTempByDepth'],index=indx) for i in indx: modTemp[i]=str2list(modTemp[i],bracket=True) INDX=[] for j in tf_index1: if i==inde[j]: INDX.append(j) #find indices which are in same area fig=plt.figure() ax=fig.add_subplot(111) ax.plot(temp[INDX],depth[INDX],'ro-', label='raw',linewidth=1) ax.plot(modTemp[i],obsDepth[i],'go--' ,label='roms',linewidth=3) ax.plot(obsTemp[i],obsDepth[i],'bo--', label='telemetered',linewidth=3) ax.set_xlim([0, 30]) ax.set_ylim([max(obsDepth[i])+3, -1])
from turtleModule import str2ndlist, np_datetime from utilities import smooth, get_wind_fvcom from get_ncep_wind_test import get_weather_wind import math example = [117170, 129775, 118905, 129779] # those turtle's id is random T = [] # get all information of six example turtle for e in range(len(example)): obsData = pd.read_csv('ctdWithModTempByDepth.csv') obs_data = pd.read_csv('ctd_good_new.csv') tf_index = np.where(obsData['TF'].notnull())[0] obsturtle_id = pd.Series(obsData['PTT'][tf_index], index=tf_index) indx = [] for i in tf_index: if obsturtle_id[i] == example[e]: # we can get each turtle's ILD indx.append(i) Temp = pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]), index=indx) Indx = [] for i in indx: if len(Temp[i]) == 10: Indx.append(i) Data = obsData.ix[Indx] obs_data = obs_data.ix[Indx] obsTime = pd.Series(np_datetime(Data['END_DATE'].values), index=Indx) obsTemp = pd.Series(str2ndlist(Data['TEMP_VALS'].values), index=Indx) obsDepth = pd.Series(str2ndlist(Data['TEMP_DBAR'].values), index=Indx) obsLon = pd.Series(Data['LON'], index=Indx) obsLat = pd.Series(Data['LAT'], index=Indx) modDepth = pd.Series( str2ndlist(obs_data['LayerDepth'], bracket=True), index=Indx) # If str has '[' and ']', bracket should be True. modTemp = pd.Series(str2ndlist(Data['modTempByDepth'].values,
''' plot 4 maps in 1 figure to show which depth has the most errors. Also plot the errorbar and ratio Plot error bar and ratio of error. ''' import numpy as np import pandas as pd from datetime import datetime, timedelta from mpl_toolkits.basemap import Basemap import matplotlib.pyplot as plt import netCDF4 import watertempModule as wtm # A module of classes that using ROMS, FVCOM from turtleModule import str2ndlist, np_datetime, bottom_value, dist #####################################roms###################################### obsData = pd.read_csv('ctdWithModTempByDepth.csv') # extracted from ctdWithModTempByDepth.py tf_index = np.where(obsData['TF'].notnull())[0] # get the index of good data obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]), index=tf_index) obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index) modTemp = pd.Series(np.array(str2ndlist(obsData['modTempByDepth'][tf_index], bracket=True)), index=tf_index) data_roms = pd.DataFrame({'obstemp': obsTemp.values,'modtemp':modTemp,'depth': obsDepth}, index=tf_index) TEMP_roms_all=[] TEMP_roms=[] TEMP_roms_negivate=[] for i in range(50): #depth 0~50m TEMP_roms.append(0) TEMP_roms_negivate.append(0) TEMP_roms_all.append(0) for j in data_roms.index: for q in range(len(data_roms['depth'][j])): if int(data_roms['depth'][j][q])==i: TEMP_roms_all[i]=TEMP_roms_all[i]+1 if data_roms['modtemp'][j][q]<100: #some bad data>100 degC
Temp = (temp[-2,a,b]-temp[-1,a,b])/(lyrDepth[-2]-lyrDepth[-1]) * \ (depth-lyrDepth[-1]) + temp[-1,a,b] elif depth < lyrDepth[0]: # Obs is deeper than first layer which is the bottom. Temp = (temp[1,a,b]-temp[0,a,b])/(lyrDepth[1]-lyrDepth[0]) * \ (depth-lyrDepth[0]) + temp[0,a,b] else: ind = closest_num(depth, lyrDepth) Temp = (temp[ind,a,b]-temp[ind-1,a,b])/(lyrDepth[ind]-lyrDepth[ind-1]) * \ (depth-lyrDepth[ind-1]) + temp[ind-1,a,b] t.append(Temp) modTemp.append(t) modTemp = np.array(modTemp) return modTemp ##################################################################################### obsData = pd.read_csv('ship06-08_indexANDlayer.csv') obsLon = pd.Series(str2ndlist(obsData['lon'],bracket=True)) obsLat = pd.Series(str2ndlist(obsData['lat'],bracket=True)) obsTime = pd.Series(obsData['time']) obsTemp = pd.Series(str2ndlist(obsData['temperature'],bracket=True)) obsDepth = pd.Series(str2ndlist(obsData['depth'],bracket=True)) modLayer = pd.Series(str2ndlist(obsData['modDepthLayer'],bracket=True)) modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'], bracket=True)) for i in obsTime.index: obsTime[i]=datetime.strptime(obsTime[i], "%Y-%m-%d %H:%M:%S") # change str to datatime url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his' modDataAll = netCDF4.Dataset(url) oceantime = modDataAll['ocean_time'] modTempAll = modDataAll['temp'] s_rho = modDataAll['s_rho'] waterDepth = modDataAll['h'] modTemp = getModTemp(modTempAll, obsTime, modLayer, modNearestIndex, s_rho, waterDepth, starttime, oceantime)