def timeSeries(table,
field,
startDate,
endDate,
lat1,
lat2,
lon1,
lon2,
depth1,
depth2,
fmt='%Y-%m-%d',
dt=24 * 60):
if iterative(table, field, dt):
ts, y, y_std = timeSeries_iterative(table, field, startDate, endDate,
lat1, lat2, lon1, lon2, depth1,
depth2, fmt, dt)
else:
df = subset.timeSeries(table, field, startDate, endDate, lat1, lat2,
lon1, lon2, depth1, depth2)
if not db.isClimatology(table):
ts, y, y_std = pd.to_datetime(
df[df.columns[0]]), df[field], df[field + '_std']
ts, y, y_std = fillGaps(ts, y, y_std, startDate, endDate, fmt, dt)
else:
ts, y, y_std = df[df.columns[0]], df[field], df[field + '_std']
return ts, y, y_std
def depthProfile_iterative(table, field, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag):
if db.isClimatology(table) and db.hasField(table, 'month'):
m1 = clim.timeToMonth(dt1)
m2 = clim.timeToMonth(dt2)
if m2>m1:
timesteps = range(m1, m2+1)
else:
timesteps = range(m2, m1+1)
timesteps = ['2016-%2.2d-01' % m for m in timesteps]
else:
delta = datetime.strptime(dt2, '%Y-%m-%d') - datetime.strptime(dt1, '%Y-%m-%d')
timesteps = [(datetime.strptime(dt1, '%Y-%m-%d') + timedelta(days=x)).strftime('%Y-%m-%d') for x in range(delta.days+1)]
zs, ys, y_stds = [], [], []
for dt in timesteps:
df = subset.depthProfile(table, field, dt, dt, lat1, lat2, lon1, lon2, depth1, depth2)
if len(df[field]) < 1:
continue
zs.append(df['depth'])
ys.append(df[field])
y_stds.append(df[field + '_std'])
depth = np.mean( np.stack(zs, axis=0), axis=0 )
y = np.mean( np.stack(ys, axis=0), axis=0 )
y_std = np.mean( np.stack(y_stds, axis=0), axis=0 )
if exportDataFlag:
exportData(depth, y, y_std, table, field, dt1, dt2, lat1, lat2, lon1, lon2, fname)
return depth, y, y_std
def depthProfile_iterative(table, field, dt1, dt2, lat1, lat2, lon1, lon2, depth1, depth2, fname, exportDataFlag):
if db.isClimatology(table) and db.hasField(table, 'month'):
m1 = clim.timeToMonth(dt1)
m2 = clim.timeToMonth(dt2)
if m2>m1:
timesteps = range(m1, m2+1)
else:
timesteps = range(m2, m1+1)
timesteps = ['2016-%2.2d-01' % m for m in timesteps]
elif table.lower().find('tblPisces'.lower()) != -1: # currently (Nov 2018) only Pisces table has a calendar table. all datasets have to have a calendar table. we can then remove thw else: clause below
calTable = table+'_Calendar'
timesteps = com.timesBetween(calTable, dt1, dt2)
else:
delta = datetime.strptime(dt2, '%Y-%m-%d') - datetime.strptime(dt1, '%Y-%m-%d')
timesteps = [(datetime.strptime(dt1, '%Y-%m-%d') + timedelta(days=x)).strftime('%Y-%m-%d') for x in range(delta.days+1)]
zs, ys, y_stds = [], [], []
for dt in timesteps:
df = subset.depthProfile(table, field, dt, dt, lat1, lat2, lon1, lon2, depth1, depth2)
if len(df[field]) < 1:
continue
zs.append(df['depth'])
ys.append(df[field])
y_stds.append(df[field + '_std'])
depth = np.mean( np.stack(zs, axis=0), axis=0 )
y = np.mean( np.stack(ys, axis=0), axis=0 )
y_std = np.mean( np.stack(y_stds, axis=0), axis=0 )
if exportDataFlag:
exportData(depth, y, y_std, table, field, dt1, dt2, lat1, lat2, lon1, lon2, fname)
return depth, y, y_std
def isClimatology(tableName, varName=None):
res = db.isClimatology(tableName)
'''
query = "SELECT Climatology FROM tblVariables "
query = query + "JOIN tblDatasets ON [tblVariables].Dataset_ID=[tblDatasets].ID "
query = query + "WHERE Table_Name='%s' AND Short_Name='%s' "
query = query % (tableName, varName)
df = db.dbFetch(query)
res = df.Climatology[0]
'''
return
def timeSeries(table, field, startDate, endDate, lat1, lat2, lon1, lon2, depth1, depth2, fmt='%Y-%m-%d', dt=24*60):
if iterative(table, field, dt):
ts, y, y_std = timeSeries_iterative(table, field, startDate, endDate, lat1, lat2, lon1, lon2, depth1, depth2, fmt, dt)
else:
df = subset.timeSeries(table, field, startDate, endDate, lat1, lat2, lon1, lon2, depth1, depth2)
if table.lower().find('tblseaflow') != -1:
from plotCruise import resample
df = resample(df, 'D', removeNAs=False)
df[field+'_std'] = None
if not db.isClimatology(table):
ts, y, y_std = pd.to_datetime(df[df.columns[0]]), df[field], df[field+'_std']
ts, y, y_std = fillGaps(ts, y, y_std, startDate, endDate, fmt, dt)
else:
ts, y, y_std = df[df.columns[0]], df[field], df[field+'_std']
return ts, y, y_std
def exportData(z, y, yErr, table, variable, date1, date2, lat1, lat2, lon1, lon2, fname):
df = pd.DataFrame()
df['depth'] = z
df[variable] = y
df[variable+'_std'] = yErr
if db.isClimatology(table):
df['month1'] = clim.timeToMonth(date1)
df['month2'] = clim.timeToMonth(date2)
else:
df['time1'] = date1
df['time2'] = date2
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
path = dirPath + fname + '_' + table + '_' + variable + '.csv'
df.to_csv(path, index=False)
return
def exportData(y, table, variable, startDate, endDate, lat1, lat2, lon1, lon2,
depth1, depth2):
df = pd.DataFrame()
df[variable] = y
timeField = 'time'
if db.isClimatology(table) and db.hasField(table, 'month'):
timeField = 'month'
df['start_' + timeField] = startDate
df['end_' + timeField] = endDate
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
if db.hasField(table, 'depth'):
df['depth1'] = depth1
df['depth2'] = depth2
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
path = dirPath + 'Hist_' + table + '_' + variable + '.csv'
df.to_csv(path, index=False)
return
def exportData(t, y, yErr, table, variable, lat1, lat2, lon1, lon2, depth1,
depth2):
df = pd.DataFrame()
timeField = 'time'
if db.isClimatology(table) and db.hasField(table, 'month'):
timeField = 'month'
df[timeField] = t
df[variable] = y
df[variable + '_std'] = yErr
df['lat1'] = lat1
df['lat2'] = lat2
df['lon1'] = lon1
df['lon2'] = lon2
if db.hasField(table, 'depth'):
df['depth1'] = depth1
df['depth2'] = depth2
dirPath = 'data/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
path = dirPath + 'TS_' + table + '_' + variable + '.csv'
df.to_csv(path, index=False)
return
def plotTS(tables,
variables,
startDate,
endDate,
lat1,
lat2,
lon1,
lon2,
depth1,
depth2,
fname,
exportDataFlag,
marker='-',
msize=20,
clr='purple'):
p = []
lw = 2
w = 800
h = 400
TOOLS = 'pan,wheel_zoom,zoom_in,zoom_out,box_zoom, undo,redo,reset,tap,save,box_select,poly_select,lasso_select'
for i in range(len(tables)):
t, y, yErr = TS.timeSeries(tables[i], variables[i], startDate, endDate,
lat1, lat2, lon1, lon2, depth1, depth2)
if exportDataFlag:
exportData(t, y, yErr, tables[i], variables[i], lat1, lat2, lon1,
lon2, depth1, depth2)
p1 = figure(tools=TOOLS,
toolbar_location="above",
plot_width=w,
plot_height=h)
#p1.xaxis.axis_label = 'Time'
p1.yaxis.axis_label = variables[i] + ' [' + db.getVar(
tables[i], variables[i]).iloc[0]['Unit'] + ']'
leg = variables[i]
fill_alpha = 0.3
cr = p1.circle(t,
y,
fill_color="grey",
hover_fill_color="firebrick",
fill_alpha=fill_alpha,
hover_alpha=0.3,
line_color=None,
hover_line_color="white",
legend=leg,
size=msize)
p1.line(t, y, line_color=clr, line_width=lw, legend=leg)
p1.add_tools(HoverTool(tooltips=None, renderers=[cr], mode='hline'))
if not db.isClimatology(tables[i]):
p1.xaxis.formatter = DatetimeTickFormatter(
hours=["%d %B %Y"],
days=["%d %B %Y"],
months=["%d %B %Y"],
years=["%d %B %Y"],
)
p1.xaxis.major_label_orientation = pi / 4
#p1.xaxis.visible = False
p.append(p1)
dirPath = 'embed/'
if not os.path.exists(dirPath):
os.makedirs(dirPath)
if not inline: ## if jupyter is not the caller
output_file(dirPath + fname + ".html", title="TimeSeries")
show(column(p))
return