def exportData(t1, y1, yErr1, t2, y2, yErr2, table1, variable1, table2,
variable2, lat1, lat2, lon1, lon2, depth1, depth2):
df = pd.DataFrame()
df['time_X'] = t1
df[variable1] = y1
df[variable1 + '_std_X'] = yErr1
df['time_Y'] = t2
df[variable2] = y2
df[variable2 + '_std_Y'] = yErr2
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
if db.hasField(table1, 'depth') or db.hasField(table2, 'depth'):
df['depth1'] = depth1
df['depth2'] = depth2
# dirPath = 'data/'
# if not os.path.exists(dirPath):
# os.makedirs(dirPath)
# path = dirPath + 'XY_' + table1 + '_' + variable1 + '_vs_' + table2 + '_' + variable2 + '.csv'
# df.to_csv(path, index=False)
export.dump(df, table1, variable1, prefix='Mutual', fmt='.csv')
export.dump(df, table2, variable2, prefix='Mutual', fmt='.csv')
return
def plotDepthProfile(tables, variables, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag, marker='-', msize=25, clr='orangered'):
p = []
lw = 2
w = 800
h = 400
TOOLS = 'pan,wheel_zoom,zoom_in,zoom_out,box_zoom, undo,redo,reset,tap,save,box_select,poly_select,lasso_select'
for i in range(len(tables)):
if not db.hasField(tables[i], 'depth'):
continue
if not iterative(tables[i]):
depths, y, yErr = depthProfile(tables[i], variables[i], dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag)
else:
depths, y, yErr = depthProfile_iterative(tables[i], variables[i], dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag)
p1 = figure(tools=TOOLS, toolbar_location="above", plot_width=w, plot_height=h)
#p1.xaxis.axis_label = 'Depth'
p1.yaxis.axis_label = variables[i] + ' [' + db.getVar(tables[i], variables[i]).iloc[0]['Unit'] + ']'
leg = variables[i]
cr = p1.circle(depths, y, fill_color="grey", hover_fill_color="firebrick", fill_alpha=0.25, hover_alpha=0.3, line_color=None, hover_line_color="white", legend=leg, size=msize)
p1.line(depths, y, line_color=clr, line_width=lw, legend=leg)
p1.add_tools(HoverTool(tooltips=None, renderers=[cr], mode='hline'))
p.append(p1)
dirPath = 'embed/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
if not inline: ## if jupyter is not the caller
output_file(dirPath + fname + ".html", title="Depth Profile")
show(column(p))
return
def depthProfile_iterative(table, field, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag):
if db.isClimatology(table) and db.hasField(table, 'month'):
m1 = clim.timeToMonth(dt1)
m2 = clim.timeToMonth(dt2)
if m2>m1:
timesteps = range(m1, m2+1)
else:
timesteps = range(m2, m1+1)
timesteps = ['2016-%2.2d-01' % m for m in timesteps]
else:
delta = datetime.strptime(dt2, '%Y-%m-%d') - datetime.strptime(dt1, '%Y-%m-%d')
timesteps = [(datetime.strptime(dt1, '%Y-%m-%d') + timedelta(days=x)).strftime('%Y-%m-%d') for x in range(delta.days+1)]
zs, ys, y_stds = [], [], []
for dt in timesteps:
df = subset.depthProfile(table, field, dt, dt, lat1, lat2, lon1, lon2, depth1, depth2)
if len(df[field]) < 1:
continue
zs.append(df['depth'])
ys.append(df[field])
y_stds.append(df[field + '_std'])
depth = np.mean( np.stack(zs, axis=0), axis=0 )
y = np.mean( np.stack(ys, axis=0), axis=0 )
y_std = np.mean( np.stack(y_stds, axis=0), axis=0 )
if exportDataFlag:
exportData(depth, y, y_std, table, field, dt1, dt2, lat1, lat2, lon1, lon2, fname)
return depth, y, y_std
def depthProfile_iterative(table, field, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag):
if db.isClimatology(table) and db.hasField(table, 'month'):
m1 = clim.timeToMonth(dt1)
m2 = clim.timeToMonth(dt2)
if m2>m1:
timesteps = range(m1, m2+1)
else:
timesteps = range(m2, m1+1)
timesteps = ['2016-%2.2d-01' % m for m in timesteps]
elif table.lower().find('tblPisces'.lower()) != -1: # currently (Nov 2018) only Pisces table has a calendar table. all datasets have to have a calendar table. we can then remove thw else: clause below
calTable = table+'_Calendar'
timesteps = com.timesBetween(calTable, dt1, dt2)
else:
delta = datetime.strptime(dt2, '%Y-%m-%d') - datetime.strptime(dt1, '%Y-%m-%d')
timesteps = [(datetime.strptime(dt1, '%Y-%m-%d') + timedelta(days=x)).strftime('%Y-%m-%d') for x in range(delta.days+1)]
zs, ys, y_stds = [], [], []
for dt in timesteps:
df = subset.depthProfile(table, field, dt, dt, lat1, lat2, lon1, lon2, depth1, depth2)
if len(df[field]) < 1:
continue
zs.append(df['depth'])
ys.append(df[field])
y_stds.append(df[field + '_std'])
depth = np.mean( np.stack(zs, axis=0), axis=0 )
y = np.mean( np.stack(ys, axis=0), axis=0 )
y_std = np.mean( np.stack(y_stds, axis=0), axis=0 )
if exportDataFlag:
exportData(depth, y, y_std, table, field, dt1, dt2, lat1, lat2, lon1, lon2, fname)
return depth, y, y_std
def sectionMap(tables, variabels, dt1, dt2, lat1, lat2, lon1, lon2, depth1,
depth2, fname, exportDataFlag):
data, lats, lons, subs, frameVars = [], [], [], [], []
for i in range(len(tables)):
if not db.hasField(tables[i], 'depth'):
continue
#query = prepareSectionQuery(tables[i], variabels[i], dt, lat1, lat2, lon1, lon2, depth1, depth2)
#df = db.dbFetch(query)
df = subset.section(tables[i], variabels[i], dt1, dt2, lat1, lat2,
lon1, lon2, depth1, depth2)
if len(df) < 1:
continue
############### export retrieved data ###############
if exportDataFlag: # export data
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
exportData(df,
path=dirPath + fname + '_' + tables[i] + '_' +
variabels[i] + '.csv')
#####################################################
times = df[df.columns[0]].unique()
lats = df.lat.unique()
lons = df.lon.unique()
depths = df.depth.unique()
shape = (len(lats), len(lons), len(depths))
hours = [None]
if 'hour' in df.columns:
hours = df.hour.unique()
unit = ' [' + db.getVar(tables[i], variabels[i]).iloc[0]['Unit'] + ']'
for t in times:
for h in hours:
frame = df[df[df.columns[0]] == t]
sub = variabels[i] + unit + ', ' + df.columns[0] + ': ' + str(
t)
if h != None:
frame = frame[frame['hour'] == h]
sub = sub + ', hour: ' + str(h) + 'hr'
try:
shot = frame[variabels[i]].values.reshape(shape)
except Exception as e:
continue
data.append(shot)
frameVars.append(variabels[i])
subs.append(sub)
bokehSec(data=data,
subject=subs,
fname=fname,
lat=lats,
lon=lons,
depth=depths,
variabels=frameVars)
return
def exportData(y, table, variable, startDate, endDate, lat1, lat2, lon1, lon2,
depth1, depth2):
df = pd.DataFrame()
df[variable] = y
timeField = 'time'
if db.isClimatology(table) and db.hasField(table, 'month'):
timeField = 'month'
df['start_' + timeField] = startDate
df['end_' + timeField] = endDate
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
if db.hasField(table, 'depth'):
df['depth1'] = depth1
df['depth2'] = depth2
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
path = dirPath + 'Hist_' + table + '_' + variable + '.csv'
df.to_csv(path, index=False)
return
def exportData(t, y, yErr, table, variable, lat1, lat2, lon1, lon2, depth1,
depth2):
df = pd.DataFrame()
timeField = 'time'
if db.isClimatology(table) and db.hasField(table, 'month'):
timeField = 'month'
df[timeField] = t
df[variable] = y
df[variable + '_std'] = yErr
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
if db.hasField(table, 'depth'):
df['depth1'] = depth1
df['depth2'] = depth2
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
path = dirPath + 'TS_' + table + '_' + variable + '.csv'
df.to_csv(path, index=False)
return
def sectionMap(tables, variables, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag):
data, lats, lons, subs, frameVars, units = [], [], [], [], [], []
xs, ys, zs = [], [], []
for i in tqdm(range(len(tables)), desc='overall'):
if not db.hasField(tables[i], 'depth'):
continue
df = subset.section(tables[i], variables[i], dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2)
if len(df) < 1:
com.printTQDM('%d: No matching entry found: Table: %s, Variable: %s ' % (i+1, tables[i], variables[i]), err=True )
continue
com.printTQDM('%d: %s retrieved (%s).' % (i+1, variables[i], tables[i]), err=False)
if exportDataFlag:
export.dump(df, tables[i], variables[i], prefix='Section', fmt='.csv')
times = df[df.columns[0]].unique()
lats = df.lat.unique()
lons = df.lon.unique()
depths = df.depth.unique()
shape = (len(lats), len(lons), len(depths))
hours = [None]
if 'hour' in df.columns:
hours = df.hour.unique()
unit = com.getUnit(tables[i], variables[i])
for t in times:
for h in hours:
frame = df[df[df.columns[0]] == t]
sub = variables[i] + unit + ', ' + df.columns[0] + ': ' + str(t)
if h != None:
frame = frame[frame['hour'] == h]
sub = sub + ', hour: ' + str(h) + 'hr'
try:
shot = frame[variables[i]].values.reshape(shape)
except Exception as e:
continue
data.append(shot)
xs.append(lons)
ys.append(lats)
zs.append(depths)
frameVars.append(variables[i])
units.append(unit)
subs.append(sub)
bokehSec(data=data, subject=subs, fname=fname, ys=ys, xs=xs, zs=zs, units=units, variables=frameVars)
return