Created on Fri Dec 12 09:21:16 2014
@author: zhaobin
"""
'draw temp change of all turtle data and model data and calculate ‘bad’ turtles.'
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl
from math import sqrt
from datetime import datetime, timedelta
from turtleModule import str2ndlist, np_datetime
###################################main code###################################
criteria=3 # criteria for rms
obsData=pd.read_csv('ctdWithdepthofbottom_roms.csv')
obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR']),index=obsData.index)
modDepth=pd.Series(obsData['depth_bottom'],index=obsData.index)
indx=[]
for i in obsData.index:
if abs(obsDepth[i][-1]-modDepth[i])<10: #diff<10m,we can use the data
indx.append(i)
obsturtle_id=pd.Series(obsData['PTT'][indx],index=indx)
obsTemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]),index=indx)
modTemp=pd.Series(str2ndlist(obsData['modTempByDepth'][indx],bracket=True),index=indx)
obsTime=pd.Series(np_datetime(obsData['END_DATE'][indx]),index=indx)
obsturtle_ids=obsturtle_id.unique()
ids=[]
for i in range(len(obsturtle_ids)):
ids.append([])
for i in range(len(obsturtle_ids)):
for j in indx:
Extract new data file named "06-08_indexANDlayer.csv "with new column "modNearestIndex" and "modDepthLayer"
'''
import pandas as pd
import numpy as np
import netCDF4
from datetime import datetime, timedelta
from turtleModule import str2ndlist, dist
import watertempModule as wtm # a module that has all classes using ROMS and FVCOm model.
def nearest_point_index2(lon, lat, lons, lats):
d = dist(lon, lat, lons ,lats)
min_dist = np.min(d)
index = np.where(d==min_dist)
return index
#################################################################################33
obsData = pd.read_csv('06-08_ch.csv', index_col=0)
obsLon = pd.Series(str2ndlist(obsData['lon'],bracket=True))
obsLat = pd.Series(str2ndlist(obsData['lat'],bracket=True))
url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his'
modData = netCDF4.Dataset(url)
modLons = modData.variables['lon_rho'][:]
modLats = modData.variables['lat_rho'][:]
s_rho = modData.variables['s_rho'][:]
h = modData.variables['h'][:]
indexNotNull = obsLon[obsLon.isnull()==False].index # some obslat and obslon of point are empty, get rid of them.
# or this line can be the indices of TF which is less.
# indexTF = np.where(obsData['TF'].notnull())[0]
loc = []
for i in indexNotNull:
ind = []
lon = obsLon[i]
cbar.ax.set_ylabel('Quantity', fontsize=FONTSIZE)
cbar.ax.tick_params(labelsize=20)
plt.text(x=1, y=27, s=text[j], fontsize=15)
plt.text(x=1,
y=23,
s=r'$\mathregular{R^2}$=' + str(round(r_squared, 3)),
fontsize=15)
plt.savefig('obsVSmodelDeepShallow' + str(i) + '.png', dpi=200)
#########################################MAIN CODE#####################################################################################################
FONTSIZE = 25
##################################roms#########################################
obs = pd.read_csv('ctdWithModTempByDepth.csv')
tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data.
obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index)
tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index], bracket=True),
index=tf_index)
tempObs1_roms, tempObs2_roms, tempObs3_roms, tempObs4_roms, tempObs5_roms, tempObs6_roms = [],[],[],[],[],[]
tempMod1_roms, tempMod2_roms, tempMod3_roms, tempMod4_roms, tempMod5_roms, tempMod6_roms = [],[],[],[],[],[]
for i in range(len(obsTemp.values)):
for j in range(len(obsDepth.values[i])):
d = obsDepth.values[i][j]
if tempMod.values[i][j] > 100: continue
if d < 25.0:
tempObs1_roms.append(obsTemp.values[i][j])
tempMod1_roms.append(tempMod.values[i][j])
if d >= 25.0 and d < 50.0:
tempObs2_roms.append(obsTemp.values[i][j])
tempMod2_roms.append(tempMod.values[i][j])
import watertempModule as wtm
###############################################################################
'''
starttime = datetime(2013,07,10)
endtime = starttime + timedelta(hours=1)
# starttime and endtime can be any time that included by model, we just want a url to get "lon_rho", "lat_rho" in model.
tempObj = wtm.water_roms()
url = tempObj.get_url(starttime, endtime)
'''
url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his'
modData = netCDF4.Dataset(url)
modLons = modData.variables['lon_rho'][:]
modLats = modData.variables['lat_rho'][:] #the lon and lat are ROMS`s
obsData = pd.read_csv('ctdWithdepthofbottom_roms.csv')
modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'],bracket=True), index=obsData.index) # if str has '[' and ']', bracket should be True
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS']), index=obsData.index)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR']), index=obsData.index)
depthBottom = pd.Series(obsData['depth_bottom'],index=obsData.index)
for i in obsData.index:
if depthBottom[i]>200:
depthBottom[i]=200 #Just a little points are deeper than 200m.Removing them can plot depth better
obsLon, obsLat = obsData['LON'], obsData['LAT'] #use for plotting depth line
modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'], bracket=True), index=obsData.index)
data = pd.DataFrame({'obstemp': obsTemp,'modtemp':modTemp,
'nearestIndex': modNearestIndex,
'obsdepth':obsDepth,'moddepth':depthBottom},index=obsData.index)
lon_m=[]
lat_n=[]
dataNum = []
(depth-lyrDepth[0]) + temp[0,a,b]
else:
ind = closest_num(depth, lyrDepth)
d = (temp[ind,a,b]-temp[ind-1,a,b])/(lyrDepth[ind]-lyrDepth[ind-1]) * \
(depth-lyrDepth[ind-1]) + temp[ind-1,a,b]
t.append(d)
modTemp.append(t)
modTemp = np.array(modTemp)
return modTemp
#########################################MAIN CODE#####################################################################################################
FONTSIZE = 25
obs = pd.read_csv('ctdWithModTempByDepth.csv') # Extracted from "errorMap.py"
tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data.
obsLon, obsLat = obs['LON'][tf_index], obs['LAT'][tf_index]
obsTime = pd.Series(np_datetime(obs['END_DATE'][tf_index]), index=tf_index)
obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index)
# obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index)
# obsDepth = obs['MAX_DBAR'][tf_index]
modLayer = pd.Series(str2ndlist(obs['modDepthLayer'][tf_index],bracket=True), index=tf_index) # bracket is to get rid of symbol "[" and "]" in string
modNearestIndex = pd.Series(str2ndlist(obs['modNearestIndex'][tf_index], bracket=True), index=tf_index)
starttime = datetime(2009, 8, 24)
endtime = datetime(2013, 12, 13)
# for depth
tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index],bracket=True), index=tf_index)
'''
# for layers
tempObj = wtm.waterCTD()
url = tempObj.get_url(starttime, endtime)
# tempMod = tempObj.watertemp(obsLon.values, obsLat.values, obsDepth.values, obsTime.values, url)
Smooth the data, get rid of data that changes too much.
'''
for i in range(2, len(v)-1):
a, b, c = v[i-1], v[i], v[i+1]
diff1 = abs(b - c) #diff1 is not used
diff2 = abs(b - a)
if diff2>e:
v[i] = a
return v
#########################################################
obsData = pd.read_csv('ctdWithModTempByDepth.csv', index_col=0)
tf_index = np.where(obsData['TF'].notnull())[0]
obsData = obsData.ix[tf_index]
id = obsData['PTT'].drop_duplicates().values
tID = id[3] # 0~4, 6,7,8,9, turtle ID.
layers = pd.Series(str2ndlist(obsData['modDepthLayer'], bracket=True), index=obsData.index) # If str has '[' and ']', bracket should be True.
modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'].values, bracket=True), index=obsData.index)
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'].values), index=obsData.index)
obsTime = pd.Series(np_datetime(obsData['END_DATE'].values), index=obsData.index)
modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'].values,bracket=True), index=obsData.index)
layers = layers[obsData['PTT']==tID]
modNearestIndex = modNearestIndex[obsData['PTT']==tID]
time = obsTime[obsData['PTT']==tID]
temp = obsTemp[obsData['PTT']==tID]
#modTemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][temp.index],bracket=True), index=temp.index)
'''
starttime, endtime=np.amin(time), np.amax(time)+timedelta(hours=1)
modObj = wtm.waterCTD()
url = modObj.get_url(starttime, endtime) #something wrong with ROMS website
oceantime = netCDF4.Dataset(url).variables['ocean_time']
modTempAll = netCDF4.Dataset(url).variables['temp']
var=[]
for i in layer:
t = nc.variables['water_temp'][TIME,i,1400+indlat[id],1250+indlon[id]] #creates a "netCDF4 object"
if t<100: #change bad data to float.
var.append(t)
else:
var.append(-100)
nc.close()
return var
##############################################################
#turtle data
obsData = pd.read_csv('ctd_good.csv')
tf_index = np.where(obsData['TF'].notnull())[0]
obsLon, obsLat = obsData['LON'][tf_index], obsData['LAT'][tf_index]
obsTime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index)
MODtemp=[]
for i in tf_index:
t=getHYcom(obsLat[i],obsLon[i],obsTime[i],obsDepth[i])
MODtemp.append(t)
print i
obsData['modtemp_HYCOM']=pd.Series(MODtemp,index=tf_index)
obsData.to_csv('ctd_withHYCOMtemp.csv')
'''
#all ships data
shipData=pd.read_csv('ship06-08_ch.csv')
shipLon=pd.Series(str2ndlist(shipData['lon'],bracket=True))
shipLat=pd.Series(str2ndlist(shipData['lat'],bracket=True))
shipTime=pd.Series(shipData['time'])
shipDepth=pd.Series(str2ndlist(shipData['depth'],bracket=True))
'''
return ratio of the deepest depth
'''
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import netCDF4
from datetime import datetime, timedelta
from turtleModule import str2ndlist
import watertempModule as wtm # A module of classes that using ROMS, FVCOM.
obsData = pd.read_csv('ctd_good.csv') # From nearestIndexInMod.py
tf_index = np.where(obsData['TF'].notnull())[0] # Get index of good data.
obsLat, obsLon = obsData['LAT'][tf_index], obsData['LON'][tf_index]
obsDeepest = obsData['MAX_DBAR'][tf_index] # get deepest data file depth
obsID = obsData['PTT'][tf_index] # Get ID of turtle.
modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'][tf_index], bracket=True), index=tf_index)
#starttime = datetime(2009, 8, 24)
starttime = datetime(2013,05,20) # starttime and endtime here is just to get 'h', the model depth
endtime = datetime(2013, 12, 13)
tempObj = wtm.waterCTD()
url = tempObj.get_url(starttime, endtime)
modData = netCDF4.Dataset(url)
# modTempAll = modData.variables['temp']
h = modData.variables['h']
newH=[]
for i in tf_index:
m, n = int(modNearestIndex[i][0]), int(modNearestIndex[i][1])
newH.append(h[m][n])
fig = plt.figure()
'compare temperature of turtle and roms model before and after a storm'
@author: zdong,yifan
"""
import numpy as np
import pandas as pd
import matplotlib as mpl
from datetime import datetime,timedelta
import matplotlib.pyplot as plt
from turtleModule import str2ndlist,dist,np_datetime
##################################################################
obsData=pd.read_csv('tu94wModel.csv',index_col=0)
turtle_id=obsData['PTT']
obslat = obsData['LAT']
obslon = obsData['LON']
obstime = pd.Series(np_datetime(obsData['date']),index=obsData.index)
obsDepth=pd.Series(str2ndlist(obsData['obstemp']),index=obsData.index)
obstemp=pd.Series(str2ndlist(obsData['depth']),index=obsData.index)
buoyData=pd.read_csv('ndbc_2011.csv') #data of wind speed and wave height in 2011
tf_index=np.where(buoyData['TF'].notnull())[0]
buoylat = pd.Series(buoyData['lat'],index=tf_index)
buoylon = pd.Series(buoyData['lon'],index=tf_index)
buoyspeed=pd.Series(buoyData['wind_speed'],index=tf_index)
buoywave=pd.Series(buoyData['wave_height'],index=tf_index)
buoytime = pd.Series(buoyData['Date'],index=tf_index)
indx=[]
for i in tf_index:
buoytime[i]=datetime.strptime(buoytime[i], "%Y-%m-%d %H:%M") # change str to datatime
if 17.2<buoyspeed[i]:
if buoywave[i]>5.5:
indx.append(i) #choose which day has storm
ax.set_ylabel('Model temperature($^\circ$C)', fontsize=FONTSIZE)
plt.xticks(fontsize=20)
plt.yticks(fontsize=20)
cbar = plt.colorbar(c)
if j==1:
cbar.ax.set_ylabel('Quantity', fontsize=FONTSIZE)
cbar.ax.tick_params(labelsize=20)
plt.text(x=1,y=27,s=text[j],fontsize=15)
plt.text(x=1,y=23,s=r'$\mathregular{R^2}$='+str(round(r_squared,3)),fontsize=15)
plt.savefig('obsVSmodelDeepShallow'+str(i)+'.png', dpi=200)
#########################################MAIN CODE#####################################################################################################
FONTSIZE = 25
##################################roms#########################################
obs = pd.read_csv('ctdWithModTempByDepth.csv')
tf_index = np.where(obs['TF'].notnull())[0] # Get the index of good data.
obsTemp = pd.Series(str2ndlist(obs['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obs['TEMP_DBAR'][tf_index]), index=tf_index)
tempMod = pd.Series(str2ndlist(obs['modTempByDepth'][tf_index],bracket=True), index=tf_index)
tempObs1_roms, tempObs2_roms, tempObs3_roms, tempObs4_roms, tempObs5_roms, tempObs6_roms = [],[],[],[],[],[]
tempMod1_roms, tempMod2_roms, tempMod3_roms, tempMod4_roms, tempMod5_roms, tempMod6_roms = [],[],[],[],[],[]
for i in range(len(obsTemp.values)):
for j in range(len(obsDepth.values[i])):
d = obsDepth.values[i][j]
if tempMod.values[i][j] > 100: continue
if d<25.0:
tempObs1_roms.append(obsTemp.values[i][j])
tempMod1_roms.append(tempMod.values[i][j])
if d>=25.0 and d<50.0:
tempObs2_roms.append(obsTemp.values[i][j])
tempMod2_roms.append(tempMod.values[i][j])
if d>=50.0 and d<75:
"""
'plot temperature of turtles and shipboard '
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
from turtleModule import mon_alpha2num, np_datetime, dist,str2ndlist,colors,str2list
###################################################################################
r = 10 # the obs position that has shipboard position within (r) kilometers might be considered as good data.
day = 3 # the obs time that has shipboard time within (day) days might be considered as good data.
obsData=pd.read_csv('ctdWithModTempByDepth.csv',index_col=0)
tf_index=np.where(obsData['TF'].notnull())[0]
obslat = pd.Series(obsData['LAT'][tf_index],index=tf_index)
obslon = pd.Series(obsData['LON'][tf_index],index=tf_index)
obstime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]),index=tf_index)
obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]),index=tf_index)
obstemp=pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]),index=tf_index)
modtemp=pd.Series(str2ndlist(obsData['modTempByDepth'][tf_index],bracket=True),index=tf_index)
obsData1=pd.read_csv('ctd_FVcom_temp.csv') #this ctd`s FVCOM temperature
tf_index_FVCOM=np.where(obsData1['modtempBYdepth'].notnull())[0]
modtemp_FVCOM=pd.Series(obsData1['modtempBYdepth'],index=tf_index_FVCOM)
for i in tf_index_FVCOM:
modtemp_FVCOM[i]=str2list(modtemp_FVCOM[i],bracket=True)
obsData2=pd.read_csv('ctd_withHYCOMtemp.csv') #this ctd`s HYCOM temperature
tf_index_HYCOM=np.where(obsData2['modtemp_HYCOM'].notnull())[0]
modtemp_HYCOM=pd.Series(obsData2['modtemp_HYCOM'],index=tf_index_HYCOM)
for i in tf_index_HYCOM:
modtemp_HYCOM[i]=str2list(modtemp_HYCOM[i],bracket=True)
shipData=pd.read_csv('ship06-08_MODELtemp.csv',index_col=0)
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
from turtleModule import str2ndlist
############################################################
second_data = pd.read_csv('turtle_12487_tdr.csv')
depth = pd.Series(second_data['Depth'])
temp = pd.Series(second_data['temp'])
time = pd.Series(second_data['Date'])
for i in second_data.index:
time[i] = datetime.strptime(time[i], '%Y/%m/%d %H:%M:%S')
data = pd.read_csv('12487up_down.csv')
up = pd.Series(str2ndlist(data['up'], bracket=True))
down = pd.Series(str2ndlist(data['down'], bracket=True))
indx = []
for i in data.index:
if depth[down[i][-1]] > 20: #depth>20m
if depth[up[i][0]] > 20:
indx.append(i)
Down_mean_temp,Up_mean_temp,Down_min_temp,Up_min_temp,Down_max_temp,Up_max_temp=[],[],[],[],[],[]
diff_temp_up, diff_temp_down = [], []
Down_speed, Up_speed = [], []
for i in indx[1:len(indx)]:
if down[i][-1] - down[i][0] > 0:
a = temp[int(down[i][0]):int(down[i][-1])]
b = temp[int(up[i][0]):int(up[i][-1])]
Down_mean_temp.append(np.mean(a))
Up_mean_temp.append(np.mean(b))
@author: zhobin
"""
'compare temperature before and after a storm'
import numpy as np
import pandas as pd
import matplotlib as mpl
from datetime import datetime,timedelta
import matplotlib.pyplot as plt
from turtleModule import str2ndlist,dist,np_datetime
##################################################################
obsData=pd.read_csv('ctdwithoutbad_roms.csv',index_col=0)
turtle_id=pd.Series(obsData['PTT'],index=obsData.index)
obslat = pd.Series(obsData['LAT'],index=obsData.index)
obslon = pd.Series(obsData['LON'],index=obsData.index)
obstime = pd.Series(np_datetime(obsData['END_DATE']),index=obsData.index)
obsDepth=pd.Series(str2ndlist(obsData['TEMP_DBAR']),index=obsData.index)
obstemp=pd.Series(str2ndlist(obsData['TEMP_VALS']),index=obsData.index)
buoyData=pd.read_csv('ndbc_2011.csv') #data of wind speed and wave height in 2011
tf_index=np.where(buoyData['TF'].notnull())[0]
buoylat = pd.Series(buoyData['lat'],index=tf_index)
buoylon = pd.Series(buoyData['lon'],index=tf_index)
buoyspeed=pd.Series(buoyData['wind_speed'],index=tf_index)
buoywave=pd.Series(buoyData['wave_height'],index=tf_index)
buoytime = pd.Series(buoyData['Date'],index=tf_index)
indx=[]
for i in tf_index:
buoytime[i]=datetime.strptime(buoytime[i], "%Y-%m-%d %H:%M") # change str to datatime
if 17.2<buoyspeed[i]:
if buoywave[i]>5.5:
indx.append(i) #choose which day has storm
ids=[]
for i in range(length):
ids.append([])
for i in range(length):
for j in obsData.index:
if obsturtle_id[j]==obsturtle_ids[i]:
ids[i].append(j) #collect each turtle`s index
obsmins=[]
obsmaxs=[]
obsmeans=[]
modmins=[]
modmaxs=[]
modmeans=[] # collect min,max and mean of model and observe temperature
for i in range(len(ids)):
obsdata=obsData.ix[ids[i]]
goodobstemp=pd.Series(str2ndlist(obsdata['TEMP_VALS']), index=ids[i])
goodmodtemp=pd.Series(str2ndlist(obsdata['modTempByDepth'],bracket=True), index=ids[i]) #each turtle`s modtemp and obstemp
obsmin=[]
obsmax=[]
obsmean=[]
modmin=[]
modmax=[]
modmean=[]
for j in ids[i]:
obsmin.append(np.min(np.array(goodobstemp[j]))),modmin.append(np.min(np.array(goodmodtemp[j])))
obsmax.append(np.max(np.array(goodobstemp[j]))),modmax.append(np.max(np.array(goodmodtemp[j])))
obsmean.append(np.mean(np.array(goodobstemp[j]))),modmean.append(np.mean(np.array(goodmodtemp[j])))
#each turtle in one location`s max,min,mean temperature
obsmins.append(np.min(np.array(obsmin)))
obsmaxs.append(np.max(np.array(obsmax)))
obsmeans.append(np.mean(np.array(obsmean)))
@author: zhaobin
"""
'get temperature from fvcom`s website and create ctd_FVcom_temp.csv or ship_FVcom_temp.csv'
import numpy as np
import pandas as pd
import netCDF4
from turtleModule import str2ndlist
##############################################################
url='http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3?temp[0:1:316008][0:1:44][0:1:48450]'
modData=netCDF4.Dataset(url)
modtemp=modData.variables['temp']
obsData=pd.read_csv('ctd_FVcom_node_time_layer.csv') #ctd
#obsData=pd.read_csv('ship_FVcom_node_time_layer.csv') #ship
tf_index=np.where(obsData['in FVcom range'].notnull())[0]
obsnode=pd.Series(obsData['modnode'][tf_index],index=tf_index)
obstime=pd.Series(obsData['modtime'][tf_index],index=tf_index)
obslayer=pd.Series(str2ndlist(obsData['modlayer'][tf_index],bracket=True),index=tf_index)
TEMP=[]
for i in tf_index:
temp=[]
for j in obslayer[i]:
t=modtemp[int(obstime[i])][int(j)][int(obsnode[i])]
temp.append(t)
TEMP.append(temp)
print i
obsData['modtempBYdepth']=pd.Series(TEMP,index=tf_index)
obsData.to_csv('ctd_FVcom_temp.csv') #ctd
#obsData.to_csv('ship_FVcom_temp.csv') #ship
Otemp = data2['obs_temp']
Dtemp = data2['doppio_temp']
obsTime = pd.Series(
(datetime.strptime(x, '%Y-%m-%d %H:%M:%S') for x in data2['argos_date']))
for i in range(len(t_id)):
e = t_id[i]
indx = []
for i in data2.index:
if data2['PTT'][i] == e:
indx.append(i)
Data_e = data2.loc[indx]
Data_e.index = range(len(indx))
Time_e = pd.Series((datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
for x in Data_e['argos_date']))
Otemp = pd.Series(str2ndlist(Data_e['obs_temp']))
Dtemp = pd.Series(str2ndlist(Data_e['doppio_temp']))
Depth_e = pd.Series(str2ndlist(Data_e['depth']))
fig = plt.figure()
ax1 = fig.add_subplot(1, 1, 1)
l = len(Data_e) / 40.0
for j in Data_e.index:
for c in range(len(t_id)):
if e == t_id[c]:
ax1.plot(np.array(Otemp[j]) + shift * j,
Depth_e[j],
color='r',
linewidth=1)
ax1.plot(np.array(Dtemp[j]) + shift * j,
Depth_e[j],
plt.plot(x, x, 'r-', linewidth=2)
plt.plot(x2, y2, 'y-', linewidth=2)
plt.text(x=1,y=27,s=text[j],fontsize=15)
plt.text(x=1,y=23,s=r'$\mathregular{R^2}$='+str(round(r_squared[j],3)),fontsize=15)
if j==0:
plt.title('%s'%tp,fontsize=FONTSIZE)
plt.savefig('obsVSmodel_%s.png'%tp,dpi=200)
return r_squared
####################################roms#######################################
FONTSIZE = 25
obs = pd.read_csv('ctdWithModTempByDepth.csv')
tf_index = np.where(obs['TF'].notnull())[0]
obsdepth = pd.Series(obs['MAX_DBAR'][tf_index],index=tf_index)
obstemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]),index=tf_index)
obsdata_roms= pd.DataFrame({'depth':obsdepth, 'temp':obstemp}).sort_index(by='depth')
Temp_roms=pd.Series(str2ndlist(obs['modTempByDepth'][obsdata_roms['temp'].index],bracket=True),index=obsdata_roms['temp'].index)
temp_roms=[]
for i in obsdata_roms['temp'].index:
if Temp_roms[i][-1]>100: #get rid of some bad data
temp_roms.append(obsdata_roms['temp'][i])
else:
temp_roms.append(Temp_roms[i][-1])
temp_roms=pd.Series(temp_roms,index=obsdata_roms['temp'].index)
index_roms=index_by_depth(obsdata_roms['depth'], 50)
###################################fvcom#######################################
obs = pd.read_csv('ctdWithdepthofbottom_fvcom.csv')
tf_index = np.where(obs['in FVcom range'].notnull())[0]
obsdepth = pd.Series(obs['MAX_DBAR'][tf_index],index=tf_index)
obstemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]),index=tf_index)
obsdata_fvcom = pd.DataFrame({'depth':obsdepth, 'temp':obstemp}).sort_index(by='depth')
Temp_fvcom =pd.Series(str2ndlist(obs['modtempBYdepth'][obsdata_fvcom['temp'].index],bracket=True),index=obsdata_fvcom['temp'].index)
Created on Thu Nov 20 10:57:43 2014
@author: zhaobin
"""
'plot contour of observe and model temperature in four seasons'
from matplotlib.mlab import griddata
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import netCDF4
from turtleModule import str2ndlist,draw_basemap,np_datetime
##############################main code########################################
lonsize = [-79.5, -72.5]
latsize = [34.5, 41] # range of basemap
obsData = pd.read_csv('ctdWithdepthofbottom_roms.csv')
modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'],bracket=True), index=obsData.index) # if str has '[' and ']', bracket should be True
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS']), index=obsData.index)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR']), index=obsData.index)
obsTime = pd.Series(np_datetime(obsData['END_DATE']),index=obsData.index)
depthBottom = pd.Series(obsData['depth_bottom'],index=obsData.index)
for i in obsData.index:
if depthBottom[i]>200:
depthBottom[i]=200 #Just a little points are deeper than 200m.Removing them can plot depth betterex)
obsLons, obsLats = obsData['LON'], obsData['LAT']
index_inside=[]
for i in obsLons.index:
if lonsize[0]<obsLons[i]<lonsize[1]:
if abs(obsDepth[i][-1]-depthBottom[i])<10: #diff<10m,wo can use the data
index_inside.append(i) #Just use data in this area.
obsLon, obsLat = obsData['LON'][index_inside], obsData['LAT'][index_inside]
output two files:ctdwithoutbad_roms.csv and ctdWithdepthofbottom_roms.csv
@author: zhaobin
"""
from matplotlib.mlab import griddata
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import netCDF4
from datetime import datetime, timedelta
import math
from turtleModule import str2ndlist,draw_basemap
import watertempModule as wtm
############################################################################################
obsData = pd.read_csv('ctdWithModTempByDepth.csv')
tf_index = np.where(obsData['TF'].notnull())[0]
modTemp = pd.Series(str2ndlist(obsData['modTempByDepth'][tf_index],bracket=True), index=tf_index) # if str has '[' and ']', bracket should be True
indx=[]
for j in tf_index:
if modTemp[j][-1]<100:
indx.append(j)
obs=obsData.ix[indx]
obs=obs.drop(['Unnamed: 0','Unnamed: 0.1'],axis=1)
obs.to_csv('ctdwithoutbad_roms.csv')
###############################################################################
starttime = datetime(2013,07,10) # starttime and endtime can be any time that included by model, we just want a url to get "lon_rho", "lat_rho","h"in model.
endtime = starttime + timedelta(hours=1)
tempObj = wtm.water_roms()
url = tempObj.get_url(starttime, endtime)
modData = netCDF4.Dataset(url)
modLons = modData.variables['lon_rho'][:]
modLats = modData.variables['lat_rho'][:]
import pandas as pd
from datetime import datetime, timedelta
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import netCDF4
import watertempModule as wtm # A module of classes that using ROMS, FVCOM
from turtleModule import str2ndlist, np_datetime, bottom_value, dist
FONTSIZE = 25
criteria = 10 # error criteria
depth = -10 # The first depth map wanted to plot
depth_interval = 25 # the interval depth of each map.
obsData = pd.read_csv('ctdWithModTempByDepth.csv') # extracted from ctdWithModTempByDepth.py
tf_index = np.where(obsData['TF'].notnull())[0] # get the index of good data
obsLon, obsLat = obsData['LON'][tf_index], obsData['LAT'][tf_index]
obsTime = pd.Series(np_datetime(obsData['END_DATE'][tf_index]), index=tf_index)
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]), index=tf_index)
# obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index)
modLayer = pd.Series(np.array(str2ndlist(obsData['modDepthLayer'][tf_index],bracket=True)), index=tf_index) # bracket is to get rid of symbol "[" and "]" in string
modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'][tf_index], bracket=True), index=tf_index)
modTemp = pd.Series(np.array(str2ndlist(obsData['modTempByDepth'][tf_index], bracket=True)), index=tf_index)
data = pd.DataFrame({'lon': obsLon, 'lat': obsLat,
'obstemp': obsTemp.values,'modtemp':modTemp,
'depth': obsDepth, 'time': obsTime.values,
'layer': modLayer}, index=tf_index)
ind,Ind = [],[] # the indices needed
obst,Obst = [],[]
modt,Modt = [],[]
lyr,Lyr = [],[]
dep,Dep=[],[]
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
from turtleModule import str2ndlist
############################################################
second_data=pd.read_csv('turtle_12487_tdr.csv')
depth=pd.Series(second_data['Depth'])
temp=pd.Series(second_data['temp'])
time=pd.Series(second_data['Date'])
for i in second_data.index:
time[i]=datetime.strptime(time[i],'%Y/%m/%d %H:%M:%S')
data=pd.read_csv('12487up_down.csv')
up=pd.Series(str2ndlist(data['up'],bracket=True))
down=pd.Series(str2ndlist(data['down'],bracket=True))
indx=[]
for i in data.index:
if depth[down[i][-1]]>20: #depth>20m
if depth[up[i][0]]>20:
indx.append(i)
Down_mean_temp,Up_mean_temp,Down_min_temp,Up_min_temp,Down_max_temp,Up_max_temp=[],[],[],[],[],[]
diff_temp_up,diff_temp_down=[],[]
Down_speed,Up_speed=[],[]
for i in indx[1:len(indx)]:
if down[i][-1]-down[i][0]>0:
a=temp[int(down[i][0]):int(down[i][-1])]
b=temp[int(up[i][0]):int(up[i][-1])]
Down_mean_temp.append(np.mean(a))
Up_mean_temp.append(np.mean(b))
depthLayers = h[index[0][0]][index[1][0]] * s_rho
# layerDepth = [depthLayers[-layer+1], depthLayers[-layer]]
l = np.argmin(abs(depthLayers + vDepth))
return l
###################################MAIN CODE###########################################
FONTSIZE = 25
criteria = 10 # error criteria
layer = 15 # the first layer you want to plot
obs = pd.read_csv("ctd_good.csv") # From nearestIndexInMod.py
tf_index = np.where(obs["TF"].notnull())[0]
obsLon, obsLat = obs["LON"][tf_index], obs["LAT"][tf_index]
obsTime = pd.Series(np_datetime(obs["END_DATE"][tf_index]), index=tf_index)
obsTemp = pd.Series(str2ndlist(obs["TEMP_VALS"][tf_index]), index=tf_index)
# obsTemp = pd.Series(bottom_value(obs['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obs["TEMP_DBAR"][tf_index]), index=tf_index)
modDepthLayer = pd.Series(str2ndlist(obs["modDepthLayer"][tf_index], bracket=True), index=tf_index)
starttime = datetime(2009, 8, 24) # Choose the starttime and endtime that cover the whole observation time range.
endtime = datetime(2013, 12, 13)
tempObj = wtm.waterCTD()
url = tempObj.get_url(starttime, endtime)
modTemp = tempObj.watertemp(obsLon.values, obsLat.values, obsDepth.values, obsTime.values, url)
d = {
"lon": obsLon,
"lat": obsLat,
"time": obsTime.values,
"obstemp": obsTemp.values,
secondData=pd.read_csv('12487_location.csv')
tf_index1 = np.where(secondData['index'].notnull())[0]
time=pd.Series(secondData['time'],index=tf_index1)
depth=pd.Series(secondData['depth'],index=tf_index1)
temp=pd.Series(secondData['temp'],index=tf_index1)
lon=pd.Series(secondData['lon'],index=tf_index1)
lat=pd.Series(secondData['lat'],index=tf_index1)
inde=pd.Series(secondData['index'],index=tf_index1)
indx=[]
for i in tf_index:
if obsturtle_id[i]==118905: #this turtle is same turtle with 4-second turtle
indx.append(i)
obsLon, obsLat = obsData['LON'][indx], obsData['LAT'][indx]
obsTime = pd.Series(np_datetime(obsData['END_DATE'][indx]), index=indx)
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]), index=indx)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][indx]), index=indx)
modTemp = pd.Series(obsData['modTempByDepth'],index=indx)
for i in indx:
modTemp[i]=str2list(modTemp[i],bracket=True)
INDX=[]
for j in tf_index1:
if i==inde[j]:
INDX.append(j) #find indices which are in same area
fig=plt.figure()
ax=fig.add_subplot(111)
ax.plot(temp[INDX],depth[INDX],'ro-', label='raw',linewidth=1)
ax.plot(modTemp[i],obsDepth[i],'go--' ,label='roms',linewidth=3)
ax.plot(obsTemp[i],obsDepth[i],'bo--', label='telemetered',linewidth=3)
ax.set_xlim([0, 30])
ax.set_ylim([max(obsDepth[i])+3, -1])
from turtleModule import str2ndlist, np_datetime
from utilities import smooth, get_wind_fvcom
from get_ncep_wind_test import get_weather_wind
import math
example = [117170, 129775, 118905, 129779] # those turtle's id is random
T = [] # get all information of six example turtle
for e in range(len(example)):
obsData = pd.read_csv('ctdWithModTempByDepth.csv')
obs_data = pd.read_csv('ctd_good_new.csv')
tf_index = np.where(obsData['TF'].notnull())[0]
obsturtle_id = pd.Series(obsData['PTT'][tf_index], index=tf_index)
indx = []
for i in tf_index:
if obsturtle_id[i] == example[e]: # we can get each turtle's ILD
indx.append(i)
Temp = pd.Series(str2ndlist(obsData['TEMP_VALS'][indx]), index=indx)
Indx = []
for i in indx:
if len(Temp[i]) == 10:
Indx.append(i)
Data = obsData.ix[Indx]
obs_data = obs_data.ix[Indx]
obsTime = pd.Series(np_datetime(Data['END_DATE'].values), index=Indx)
obsTemp = pd.Series(str2ndlist(Data['TEMP_VALS'].values), index=Indx)
obsDepth = pd.Series(str2ndlist(Data['TEMP_DBAR'].values), index=Indx)
obsLon = pd.Series(Data['LON'], index=Indx)
obsLat = pd.Series(Data['LAT'], index=Indx)
modDepth = pd.Series(
str2ndlist(obs_data['LayerDepth'], bracket=True),
index=Indx) # If str has '[' and ']', bracket should be True.
modTemp = pd.Series(str2ndlist(Data['modTempByDepth'].values,
'''
plot 4 maps in 1 figure to show which depth has the most errors. Also plot the errorbar and ratio
Plot error bar and ratio of error.
'''
import numpy as np
import pandas as pd
from datetime import datetime, timedelta
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import netCDF4
import watertempModule as wtm # A module of classes that using ROMS, FVCOM
from turtleModule import str2ndlist, np_datetime, bottom_value, dist
#####################################roms######################################
obsData = pd.read_csv('ctdWithModTempByDepth.csv') # extracted from ctdWithModTempByDepth.py
tf_index = np.where(obsData['TF'].notnull())[0] # get the index of good data
obsTemp = pd.Series(str2ndlist(obsData['TEMP_VALS'][tf_index]), index=tf_index)
obsDepth = pd.Series(str2ndlist(obsData['TEMP_DBAR'][tf_index]), index=tf_index)
modTemp = pd.Series(np.array(str2ndlist(obsData['modTempByDepth'][tf_index], bracket=True)), index=tf_index)
data_roms = pd.DataFrame({'obstemp': obsTemp.values,'modtemp':modTemp,'depth': obsDepth}, index=tf_index)
TEMP_roms_all=[]
TEMP_roms=[]
TEMP_roms_negivate=[]
for i in range(50): #depth 0~50m
TEMP_roms.append(0)
TEMP_roms_negivate.append(0)
TEMP_roms_all.append(0)
for j in data_roms.index:
for q in range(len(data_roms['depth'][j])):
if int(data_roms['depth'][j][q])==i:
TEMP_roms_all[i]=TEMP_roms_all[i]+1
if data_roms['modtemp'][j][q]<100: #some bad data>100 degC
Temp = (temp[-2,a,b]-temp[-1,a,b])/(lyrDepth[-2]-lyrDepth[-1]) * \
(depth-lyrDepth[-1]) + temp[-1,a,b]
elif depth < lyrDepth[0]: # Obs is deeper than first layer which is the bottom.
Temp = (temp[1,a,b]-temp[0,a,b])/(lyrDepth[1]-lyrDepth[0]) * \
(depth-lyrDepth[0]) + temp[0,a,b]
else:
ind = closest_num(depth, lyrDepth)
Temp = (temp[ind,a,b]-temp[ind-1,a,b])/(lyrDepth[ind]-lyrDepth[ind-1]) * \
(depth-lyrDepth[ind-1]) + temp[ind-1,a,b]
t.append(Temp)
modTemp.append(t)
modTemp = np.array(modTemp)
return modTemp
#####################################################################################
obsData = pd.read_csv('ship06-08_indexANDlayer.csv')
obsLon = pd.Series(str2ndlist(obsData['lon'],bracket=True))
obsLat = pd.Series(str2ndlist(obsData['lat'],bracket=True))
obsTime = pd.Series(obsData['time'])
obsTemp = pd.Series(str2ndlist(obsData['temperature'],bracket=True))
obsDepth = pd.Series(str2ndlist(obsData['depth'],bracket=True))
modLayer = pd.Series(str2ndlist(obsData['modDepthLayer'],bracket=True))
modNearestIndex = pd.Series(str2ndlist(obsData['modNearestIndex'], bracket=True))
for i in obsTime.index:
obsTime[i]=datetime.strptime(obsTime[i], "%Y-%m-%d %H:%M:%S") # change str to datatime
url='http://tds.marine.rutgers.edu:8080/thredds/dodsC/roms/espresso/2009_da/his'
modDataAll = netCDF4.Dataset(url)
oceantime = modDataAll['ocean_time']
modTempAll = modDataAll['temp']
s_rho = modDataAll['s_rho']
waterDepth = modDataAll['h']
modTemp = getModTemp(modTempAll, obsTime, modLayer, modNearestIndex, s_rho, waterDepth, starttime, oceantime)
