def get_user_activity(session):
"""Create a plot showing the inline usage statistics."""
def running_window(session, subquery):
# Create a running window which sums all users up to this point for the current millennium ;P
users = session.query(
subquery.c.creation_date,
cast(func.sum(subquery.c.count).over(
partition_by=func.extract('millennium', subquery.c.creation_date),
order_by=subquery.c.creation_date.asc(),
), Integer).label('running_total'),
) \
.order_by(subquery.c.creation_date) \
.all()
return users
# Grid style
plt.style.use('seaborn-whitegrid')
creation_date = func.cast(User.created_at, Date).label('creation_date')
# Group the started users by date
started_users_subquery = session.query(creation_date, func.count(User.id).label('count')) \
.filter(User.started.is_(True)) \
.group_by(creation_date) \
.order_by(creation_date) \
.subquery()
started_users = running_window(session, started_users_subquery)
started_users = [('started', q[0], q[1]) for q in started_users]
# Group the started users by date
all_users_subquery = session.query(creation_date, func.count(User.id).label('count')) \
.group_by(creation_date) \
.order_by(creation_date) \
.subquery()
all_users = running_window(session, all_users_subquery)
all_users = [('all', q[0], q[1]) for q in all_users]
# Group the started users by date
voting_users_subquery = session.query(creation_date, func.count(User.id).label('count')) \
.filter(User.votes.any()) \
.group_by(creation_date) \
.order_by(creation_date) \
.subquery()
voting_users = running_window(session, voting_users_subquery)
voting_users = [('voted', q[0], q[1]) for q in voting_users]
# Group the started users by date
owning_users_subquery = session.query(creation_date, func.count(User.id).label('count')) \
.filter(User.polls.any()) \
.group_by(creation_date) \
.order_by(creation_date) \
.subquery()
owning_users = running_window(session, owning_users_subquery)
owning_users = [('currently owning poll', q[0], q[1]) for q in owning_users]
# Combine the results in a single dataframe and name the columns
user_statistics = started_users + all_users + voting_users + owning_users
dataframe = pandas.DataFrame(user_statistics, columns=['type', 'date', 'users'])
months = mdates.MonthLocator() # every month
months_fmt = mdates.DateFormatter('%Y-%m')
max_number = all_users[len(all_users) - 1][2]
# Plot each result set
fig, ax = plt.subplots(figsize=(30, 15), dpi=120)
for key, group in dataframe.groupby(['type']):
ax = group.plot(ax=ax, kind='line', x='date', y='users', label=key)
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(months_fmt)
ax.yaxis.set_ticks(np.arange(0, max_number, 5000))
image = image_from_figure(fig)
image.name = 'user_statistics.png'
return image
dates.remove('')
x = [dt.datetime.strptime(d, '%Y-%m-%d').date() for d in dates]
fig, ax1 = plt.subplots()
color = 'red'
ax1.set_xlabel('time')
ax1.set_ylabel('number of users infected')
ax1.plot(x, stillinfected.values(), color=color)
ax1.tick_params(axis='y', labelcolor=color)
ax2 = ax1.twinx()
color = 'green'
ax2.set_ylabel('number of users recoverd')
ax2.plot(x, totalrecoved_by_day, color=color)
ax2.tick_params(axis='y', labelcolor=color)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator())
plt.gcf().autofmt_xdate()
fig.tight_layout() # otherwise the right y-label is slightly clipped
plt.show()
# dates.remove('')
# x = [dt.datetime.strptime(d,'%Y-%m-%d').date() for d in dates]
# print(len(x))
# plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b'))
# plt.gca().xaxis.set_major_locator(mdates.MonthLocator())
# plt.plot(x, stillinfected.values())
# plt.xlabel("time")
# plt.ylabel("number of users still infected")
# plt.show()
var_time.append(v)
bm = pd.DataFrame({'Value': var_time}, index=time)
bm128 = bm.resample(delta).mean()
## Get Merge observations
df = CV.get_from_merge(d[variable])
df = df[merra.index[0]:merra.index[-1]]
df = df.resample(delta).mean()
new_index = []
for dt in df.index:
dtt = dt.replace(day=1)
new_index.append(dtt)
df.index = new_index
f, ax = plt.subplots(figsize=(16, 4))
ax.plot(df.index, df.Value, 'k', label='CVAO')
#ax.plot(df.index, merra.Value , '#2ca25f', label='GC Merra' )
#ax.plot(df.index, geosfp.Value, '#e5f5f9', label='GC Geos-FP')
ax.plot(df.index, bm124.Value, '#fef0d9', label='GC 12.4')
ax.plot(df.index, bm126.Value, '#fdcc8a', label='GC 12.6')
ax.plot(df.index, bm128.Value, '#fc8d59', label='GC 12.8')
ax.plot(df.index, bm129.Value, '#d7301f', label='GC 12.9')
plt.ylabel('%s (%s)' % (d[variable]['abbr'], d[variable]['unit']))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator(interval=1))
plt.legend()
plt.savefig('/users/mjr583/GC/eval_12.9.3/plots/benchmarks_%s.png' %
(variable))
plt.close()
def send_temperature(self): # pour afficher les informations d'une station
# Connexion à la BDD
conn = sqlite3.connect('Temperatures.sqlite')
c = conn.cursor()
stations_selectionnes_classifiees=[]
for i in stations_selectionnes: # récréer une liste des stations classifiées
for s in station_list:
if s.get_nom()==i:
stations_selectionnes_classifiees.append(s)
# Intervalle de temps considéré
deb,fin = self.path_info[3],self.path_info[4]
# Passage du format AAAA-MM-JJ à AAAAMMJJ
deb=deb[:4]+deb[5:7]+deb[8:10]
fin=fin[:4]+fin[5:7]+fin[8:10]
pas = self.path_info[5]
if self.path_info[2]=='Moyenne': sheet= 'TG_1978-2018'
elif self.path_info[2]=='Maximale': sheet= 'TX_1978-2018'
elif self.path_info[2]=='Minimale': sheet= 'TN_1978-2018'
# Valeur par defaut de r pour le if not
r = 0
# Le chache est uniquement utilisé si on a que une station a montrer
if len(stations_selectionnes_classifiees) == 1:
# On voit si l'entrée est déjà dans le cache
c.execute("SELECT * FROM cache WHERE station='{}' AND type='{}' AND debut='{}' AND fin='{}' AND pas='******'".format(self.path_info[1],self.path_info[2],deb,fin,pas))
r = c.fetchall()
#Creation du schema du ficher qu'on veut
fichier = 'courbes/temperature_'+self.path_info[1]+self.path_info[2] +deb+fin+pas+'.png'
else:
#Creation du schema du ficher qu'on veut
fichier = 'courbes/temperature_multiple.png'
# Selon la reponse que cette requete, on crée la courbe ou on la cherche
# La courbe n'est pas dans le cache
if not r:
if len(stations_selectionnes_classifiees) == 1:
# On ajoute la ligne a la base des données
c.execute("INSERT INTO cache VALUES(?,?,?,?,?)",[self.path_info[1],self.path_info[2],deb,fin,pas])
conn.commit()
# Creation d'un figure auquel on ajoute le traces
# configuration du tracé
fig = plt.figure(figsize=(18,6))
ax = fig.add_subplot(111)
ax.set_ylim(bottom=-10,top=40)
ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')
# Configuration de l'axe des absisses en fonction de la longueur de l'intervalle
if int(fin[:4])-int(deb[:4]) < 3:
ax.xaxis.set_major_locator(mdates.MonthLocator())
ax.xaxis.set_major_formatter(mdates.DateFormatter('%m-%Y'))
ax.xaxis.set_minor_locator(mdates.MonthLocator())
else:
ax.xaxis.set_major_locator(mdates.YearLocator())
ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y'))
ax.xaxis.set_minor_locator(mdates.YearLocator())
ax.xaxis.set_tick_params(labelsize=12)
ax.xaxis.set_label_text("Années")
ax.yaxis.set_label_text("Température °C")
colorArr = ['1','2','3','4','5','6','7','8','9','A','B','C','D','E','F']
for region in stations_selectionnes_classifiees: # pour chaque station qu'on a choisie
# Requête SQL
c.execute("SELECT * FROM '{}' WHERE Date > {} AND Date < {} AND STAID={} ORDER BY Date".format(sheet,deb,fin,region.get_num()))
r = c.fetchall()
# prise en compte du pas, intervalle de temps minimale considéré
r_pas = [r[i] for i in range(0,len(r),int(pas))]
# recupération de la date (colonne 2) et transformation dans le format de pyplot
x = [mdates.date2num(dt.date(int(str(a[2])[:4]),int(str(a[2])[4:6]),int(str(a[2])[6:8]))) for a in r_pas]
# récupération de les températures (colonne 4)
y = [float(a[3])/10 for a in r_pas]
Color = "#"
for i in range(6):
Color += colorArr[random.randint(0,14)] #randommer une couleur pour chaque affichage
# tracé de la courbe
plt.plot(x,y,linewidth=1, linestyle='-', marker='o', color=Color, label=region.get_nom())
fig.autofmt_xdate()
ax.grid(True)
# légendes
plt.legend(loc='lower left')
plt.title('Température {} de {} en ºC'.format(self.path_info[2],self.path_info[1]),fontsize=16)
# génération des courbes dans un fichier PNG
plt.savefig('client/{}'.format(fichier))
body = json.dumps({
'title': 'Température {} de'.format(self.path_info[2])+self.path_info[1], \
'img': '/'+fichier \
});
# Envoie de la requête
headers = [('Content-Type','application/json')];
self.send(body,headers)
def main(sDir, url_list, preferred_only):
rd_list = []
ms_list = []
for uu in url_list:
elements = uu.split('/')[-2].split('-')
rd = '-'.join((elements[1], elements[2], elements[3], elements[4]))
ms = uu.split(rd + '-')[1].split('/')[0]
if rd not in rd_list:
rd_list.append(rd)
if ms not in ms_list:
ms_list.append(ms)
for r in rd_list:
print('\n{}'.format(r))
subsite = r.split('-')[0]
array = subsite[0:2]
# filter datasets
datasets = []
for u in url_list:
print(u)
splitter = u.split('/')[-2].split('-')
rd_check = '-'.join((splitter[1], splitter[2], splitter[3], splitter[4]))
if rd_check == r:
udatasets = cf.get_nc_urls([u])
datasets.append(udatasets)
datasets = list(itertools.chain(*datasets))
fdatasets = []
if preferred_only == 'yes':
# get the preferred stream information
ps_df, n_streams = cf.get_preferred_stream_info(r)
for index, row in ps_df.iterrows():
for ii in range(n_streams):
try:
rms = '-'.join((r, row[ii]))
except TypeError:
continue
for dd in datasets:
spl = dd.split('/')[-2].split('-')
catalog_rms = '-'.join((spl[1], spl[2], spl[3], spl[4], spl[5], spl[6]))
fdeploy = dd.split('/')[-1].split('_')[0]
if rms == catalog_rms and fdeploy == row['deployment']:
fdatasets.append(dd)
else:
fdatasets = datasets
main_sensor = r.split('-')[-1]
fdatasets = cf.filter_collocated_instruments(main_sensor, fdatasets)
# ps_df, n_streams = cf.get_preferred_stream_info(r)
# get end times of deployments
dr_data = cf.refdes_datareview_json(r)
deployments = []
end_times = []
for index, row in ps_df.iterrows():
deploy = row['deployment']
deploy_info = get_deployment_information(dr_data, int(deploy[-4:]))
deployments.append(int(deploy[-4:]))
end_times.append(pd.to_datetime(deploy_info['stop_date']))
# filter datasets
# datasets = []
# for u in url_list:
# splitter = u.split('/')[-2].split('-')
# rd_check = '-'.join((splitter[1], splitter[2], splitter[3], splitter[4]))
# if rd_check == r:
# udatasets = cf.get_nc_urls([u])
# datasets.append(udatasets)
# datasets = list(itertools.chain(*datasets))
# main_sensor = r.split('-')[-1]
# fdatasets = cf.filter_collocated_instruments(main_sensor, datasets)
# fdatasets = cf.filter_other_streams(r, ms_list, fdatasets)
methodstream = []
for f in fdatasets:
methodstream.append('-'.join((f.split('/')[-2].split('-')[-2], f.split('/')[-2].split('-')[-1])))
for ms in np.unique(methodstream):
fdatasets_sel = [x for x in fdatasets if ms in x]
save_dir = os.path.join(sDir, array, subsite, r, 'timeseries_yearly_plot', ms.split('-')[0])
cf.create_dir(save_dir)
stream_sci_vars_dict = dict()
for x in dr_data['instrument']['data_streams']:
dr_ms = '-'.join((x['method'], x['stream_name']))
if ms == dr_ms:
stream_sci_vars_dict[dr_ms] = dict(vars=dict())
sci_vars = dict()
for y in x['stream']['parameters']:
if 'light' in y['name']:
sci_vars.update({y['name']: dict(db_units=y['unit'])})
# if y['data_product_type'] == 'Science Data':
# sci_vars.update({y['name']: dict(db_units=y['unit'])})
if len(sci_vars) > 0:
stream_sci_vars_dict[dr_ms]['vars'] = sci_vars
sci_vars_dict = cd.initialize_empty_arrays(stream_sci_vars_dict, ms)
print('\nAppending data from files: {}'.format(ms))
fcount = 0
for fd in fdatasets_sel:
ds = xr.open_dataset(fd, mask_and_scale=False)
for var in list(sci_vars_dict[ms]['vars'].keys()):
sh = sci_vars_dict[ms]['vars'][var]
if ds[var].units == sh['db_units']:
if ds[var]._FillValue not in sh['fv']:
sh['fv'].append(ds[var]._FillValue)
if ds[var].units not in sh['units']:
sh['units'].append(ds[var].units)
tD = ds['time'].values
varD = ds[var].values
sh['t'] = np.append(sh['t'], tD) # put deployments time series together
if fcount == 0:
if varD.ndim > 1:
sh['values'] = np.zeros(shape=[varD.shape[0], varD.shape[1]])
sh['values'][:] = varD
else:
sh['values'] = np.append(sh['values'], varD)
else:
if varD.ndim > 1:
sh['values'] = np.vstack([sh['values'], varD])
else:
sh['values'] = np.append(sh['values'], varD)
print(fcount, var, sh['values'].shape, sh['t'].shape)
fcount += 1
print('\nPlotting data')
for m, n in sci_vars_dict.items():
for sv, vinfo in n['vars'].items():
print(sv)
if len(vinfo['t']) < 1:
print('no variable data to plot')
else:
sv_units = vinfo['units'][0]
t0 = pd.to_datetime(min(vinfo['t'])).strftime('%Y-%m-%dT%H:%M:%S')
t1 = pd.to_datetime(max(vinfo['t'])).strftime('%Y-%m-%dT%H:%M:%S')
x = vinfo['t']
y = vinfo['values']
if len(y) > 0:
if m == 'common_stream_placeholder':
sname = '-'.join((r, sv))
else:
sname = '-'.join((r, m, sv))
if y.ndim == 1:
num_col = 1
else:
num_col = y.shape[1]
col = [str(x) for x in list(range(1, num_col + 1))]
col.insert(0, 'time')
groups, d_groups = gt.group_by_time_frequency(x, y, col, 'A')
if len(d_groups.columns) == len(col):
time_ind = [1]
data_ind0 = [2]
data_ind1 = [len(col)]
else:
time_ind = list(range(1, len(d_groups.columns), len(col)))
data_ind0 = list(range(2, len(d_groups.columns), len(col)))
if data_ind0[-1] == d_groups.columns.values[-(len(col)+1)]:
data_ind0.insert(len(groups) - 1, len(d_groups.columns))
data_ind1 = list(range(len(col), len(d_groups.columns), len(col)))
data_ind1.insert(len(groups)-1, len(d_groups.columns))
save_file = os.path.join(save_dir, sname)
print('\ncreating images')
fig, ax = pyplot.subplots(nrows=len(groups), ncols=1)
if len(groups) == 1:
ax = [ax]
colors = color_names[:len(groups)]
images = []
for group in range(len(groups)):
nan_ind = d_groups[time_ind[group]].notnull()
xtime = d_groups[time_ind[group]][nan_ind]
ycol = list(range(data_ind0[group], data_ind1[group] + 1))
ydata = d_groups[ycol][nan_ind]
if len(xtime) != 0 and len(ydata) != 0:
n_year = xtime[0].year
print(n_year)
ydata = ydata.set_index(xtime)
if len(ydata.columns) > 1:
print('more than one col: ', len(ydata.columns))
b = fsplt.split_by_timegap(ydata, 86400)
if b:
print('gaps exist, splitting data')
for ib in (range(len(b))):
iydata = b[ib][b[ib].columns.values[0:-2]]
Y = iydata.columns.values
X = iydata.index.values
Z = iydata.values
if Z.shape[0] == 1:
X = np.repeat(X[0], len(Y), axis=0)
df = pd.DataFrame(dict(a=list(X), b=list(Y), c=list(Z[0])))
images.append(ax[group].scatter(df['a'].values, df['b'].values, c=df['c'].values, cmap='Blues', s=1))
else:
Z = Z.T
x, y = np.meshgrid(X, Y)
images.append(ax[group].contourf(x, y, Z, alpha=0.7, cmap='Blues'))
else:
print('no gaps exist, not splitting data')
iydata = ydata[ydata.columns.values[0:-2]]
Y = iydata.columns.values
X = iydata.index.values
Z = iydata.values
if Z.shape[0] == 1:
X = np.repeat(X[0], len(Y), axis=0)
df = pd.DataFrame(dict(a=list(X), b=list(Y), c=list(Z[0])))
images.append(
ax[group].scatter(df['a'].values, df['b'].values, c=df['c'].values,
cmap='Blues', s=1))
elif Z.shape[1] < 2:
Y = np.repeat(Y[0], len(X), axis=1)
df = pd.DataFrame(dict(a=list(X), b=list(Y), c=list(Z[0])))
images.append(
ax[group].scatter(df['a'].values, df['b'].values, c=df['c'].values,
cmap='Blues', s=1))
else:
Z = Z.T
x, y = np.meshgrid(X, Y)
im = ax[group].contourf(x, y, Z, alpha=0.7, cmap='Blues') #pyplot.cm.jet
images.append(ax[group].contourf(x, y, Z, alpha=0.7, cmap='Blues'))
else:
print('with one column:', len(ydata.columns))
ax[group].contourf(x, y, Z, alpha=0.7, cmap='Blues')(ydata.plot(ax=ax[group],
linestyle='None',
marker='.',
markersize=0.5,
color=colors[group]))
ax[group].legend().set_visible(False)
# plot Mean and Standard deviation
ma = ydata.rolling('86400s').mean()
mstd = ydata.rolling('86400s').std()
m_mstd_min = ma[ycol].values - 2 * mstd[ycol].values
m_mstd_max = ma[ycol].values + 2 * mstd[ycol].values
ax[group].plot(ma.index.values, ma[ycol].values, 'k', linewidth=0.15)
ax[group].fill_between(mstd.index.values, m_mstd_min, m_mstd_max, color='b', alpha=0.2)
# flag deployments end-time for reference
ymin, ymax = ax[group].get_ylim()
dep = 1
for etimes in end_times:
if etimes.year == n_year:
ax[group].axvline(x=etimes, color='b', linestyle='--', linewidth=.6)
ax[group].text(etimes, ymin, 'End' + str(dep), fontsize=6, style='italic',
bbox=dict(boxstyle='round', ec=(0., 0.5, 0.5), fc=(1., 1., 1.)))
dep += 1
# prepare the time axis parameters
datemin = datetime.date(n_year, 1, 1)
datemax = datetime.date(n_year, 12, 31)
ax[group].set_xlim(datemin, datemax)
xlocator = mdates.MonthLocator() # every month
myFmt = mdates.DateFormatter('%m')
ax[group].xaxis.set_minor_locator(xlocator)
ax[group].xaxis.set_major_formatter(myFmt)
# prepare the y axis parameters
ax[group].set_ylabel(n_year, rotation=0, fontsize=8, color='b', labelpad=20)
ax[group].yaxis.set_label_position("right")
ylocator = MaxNLocator(prune='both', nbins=3)
ax[group].yaxis.set_major_locator(ylocator)
ax[group].yaxis.set_ticklabels([]) #range(1, len(col), 1)
# format figure
ax[group].tick_params(axis='both', color='r', labelsize=7, labelcolor='m')
if group == 0:
ax[group].set_title(sv + '( ' + sv_units + ')', fontsize=8)
if group < len(groups) - 1:
ax[group].tick_params(which='both', pad=0.1, length=1, labelbottom=False)
ax[group].set_xlabel(' ')
else:
ax[group].tick_params(which='both', color='r', labelsize=7, labelcolor='m',
pad=0.1, length=1, rotation=0)
ax[group].set_xlabel('Months', rotation=0, fontsize=8, color='b')
vmin = min(image.get_array().min() for image in images)
vmax = max(image.get_array().max() for image in images)
norm = mcolors.Normalize(vmin=vmin, vmax=vmax)
for im in images:
im.set_norm(norm)
fig.colorbar(images[0], ax=ax, orientation='horizontal', fraction=.1, spacing='proportional')
fig.savefig(str(save_file), dpi=150)
pyplot.close()
def scatterPlot(**kwargs):
fig, ax = plt.subplots(figsize=(8, 5))
number_of_locations = len(kwargs['locations'])
number_of_samples = len(kwargs['a_df'])
if (number_of_locations > 1):
plural_or_not = 'locations'
else:
plural_or_not = 'location'
kwargs['the_title'][
"label"] = "Total number of samples: {} from {} {}.".format(
number_of_samples, number_of_locations, plural_or_not)
kwargs['the_sup_title']["label"] = '{}, {} - {}'.format(
kwargs['the_sup_title']["label"], kwargs['min_date'],
kwargs['max_date'])
color_map = plt.cm.get_cmap(kwargs['color_map'], 100)
color = iter(color_map(np.linspace(.2, .75, number_of_locations)))
a_df = kwargs['a_df']
for location in kwargs['locations']:
new_color = next(color)
new_df = a_df[a_df['location'] == location]
x = new_df['py_date']
y = new_df['total']
plt.scatter(x,
y,
color=new_color,
label=location,
s=kwargs['point_size'],
edgecolor=kwargs['edge_c'])
plt.ylabel(kwargs['y_axis']['label'],
fontfamily=kwargs['y_axis']['fontfamily'],
labelpad=kwargs['y_axis']['lablepad'],
color=kwargs['y_axis']['color'],
size=kwargs['y_axis']['size'])
plt.xlabel(kwargs['x_axis']['label'],
fontfamily=kwargs['x_axis']['fontfamily'],
labelpad=kwargs['x_axis']['lablepad'],
color=kwargs['x_axis']['color'],
size=kwargs['x_axis']['size'],
ha='left',
x=0)
plt.subplots_adjust(**kwargs['subplot_params'])
plt.title(
kwargs['the_title']['label'],
fontdict=kwargs['title_style'],
pad=kwargs['the_title_position']['pad'],
loc=kwargs['the_title_position']['loc'],
)
plt.suptitle(
kwargs['the_sup_title']['label'],
fontdict=kwargs['sup_title_style'],
# color=kwargs['sup_title_style']['color'],
x=kwargs['sup_title_position']['x'],
y=kwargs['sup_title_position']['y'],
va=kwargs['sup_title_position']['va'],
ha=kwargs['sup_title_position']['ha'])
plt.grid(b=True, which='major', axis='both')
years = mdates.YearLocator()
months = mdates.MonthLocator()
days = mdates.DayLocator()
weeks = mdates.WeekdayLocator(byweekday=1, interval=1, tz=None)
years_fmt = mdates.DateFormatter(kwargs['x_tick_date']['years'])
months_fmt = mdates.DateFormatter(kwargs['x_tick_date']['months'])
days_fmt = mdates.DateFormatter(kwargs['x_tick_date']['days'])
if (kwargs['ticks'] == 'years'):
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(years_fmt)
ax.xaxis.set_minor_locator(months)
elif (kwargs['ticks'] == 'months'):
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(years_fmt)
ax.xaxis.set_minor_locator(months)
ax.xaxis.set_minor_formatter(months_fmt)
elif (kwargs['ticks'] == 'days'):
ax.xaxis.set_major_locator(weeks)
ax.xaxis.set_major_formatter(days_fmt)
ax.xaxis.set_minor_locator(days)
save_the_figure(**kwargs['save_this'])
plt.show()
plt.close()
df.DATE = pd.to_datetime(df.DATE)
df.year = df.DATE.apply(lambda r: r.year)
df.crd = df['DISCHARGE'].apply(lambda x: x**(1. / 3))
numb_year_bins = 7
start, stop = float(df.year.min()), float(df.year.max())
df.year_bin = ((df.year - start) / (stop - start) *
(numb_year_bins - 1)).apply(math.floor)
def rgb(r, g, b):
return (r / 255., g / 255., b / 255.)
ys = mdates.YearLocator()
ys_fmt = mdates.DateFormatter('%Y')
ms_fmt = mdates.MonthLocator()
fig, ax = plt.subplots(figsize=(10, 5))
ax.plot('DATE', 'STREAM TEMP', data=df, linewidth=1)
ax.xaxis.set_major_locator(ys)
ax.xaxis.set_major_formatter(ys_fmt)
# ax.xaxis.set_minor_locator(ms_fmt)
plt.ylabel('Stream Temperature ($^oC$)', fontsize=14)
plt.title('Plot C', fontsize=16)
dates = df.DATE
ax.set_xlim(dates.iloc[0], dates.iloc[-1])
ax.format_xdata = mdates.DateFormatter('%m/%d/%y')
ax.grid(True)
def main():
# read data
df = read_data()
df = df.set_index('centroid_day')
# first datapoint was skipped (by accident)
df = df.iloc[1:]
# plot
fig, ax = plt.subplots(1, 1, figsize=(3, 1.8))
# compute baselines
df['b1_train'] = 0
df['b1_test'] = 0
df['train'] = 0
df['test'] = 0
# b1 train
df.loc[df.iloc[1:5].index, 'b1_train'] = num_train_samples_per_bin
# b1 test
df.loc[df.iloc[4:].index, 'b1_test'] = num_test_samples_per_bin
# train
df.loc[df.iloc[:1].index, 'train'] = num_train_samples_per_bin
df.loc[df.iloc[5:].index, 'train'] = num_train_samples_per_bin
# test
df.loc[df.iloc[3:4].index, 'test'] = num_test_samples_per_bin
df['other'] = df['b1_train'] + df['b1_test'] + df['train'] + df['test']
df['total'] = df['all'].copy()
df['all'] -= df['other']
# plot
width = 40
ax.bar(df.index.to_pydatetime(),
df['train'].values.tolist(),
width=width,
color='C0',
label=f'Train')
ax.bar(df.index.to_pydatetime(),
df['test'].values.tolist(),
bottom=df['train'] + df['b1_train'],
color='C3',
width=width,
label=f'Eval')
ax.bar(df.index.to_pydatetime(),
df['b1_train'].values.tolist(),
width=width,
color='C0',
hatch=6 * '/',
label='Training data $b_1$',
ec='white')
ax.bar(df.index.to_pydatetime(),
df['b1_test'].values.tolist(),
bottom=df['b1_train'] + df['train'],
width=width,
color='C3',
hatch=6 * '/',
label='Eval datasets $b_1$',
ec='white')
ax.bar(df.index.to_pydatetime(),
df['all'].values.tolist(),
bottom=df['other'].values.tolist(),
color='.8',
width=width,
label='Unused')
# annotate
ax.annotate('$b_0$', (df.index[3], df.iloc[3].total),
ha='center',
va='bottom',
xytext=(0, 1),
textcoords='offset points')
ax.annotate('$b_1$', (df.index[4], df.iloc[4].total),
ha='center',
va='bottom',
xytext=(0, 1),
textcoords='offset points')
ax.annotate('$b_{8}$', (df.index[11], df.iloc[11].total),
ha='center',
va='bottom',
xytext=(0, 1),
textcoords='offset points')
# legend
leg = plt.legend(loc='center left',
title='Annotation type',
bbox_to_anchor=(1.05, .5),
frameon=False,
handleheight=.4,
handlelength=1.2)
leg._legend_box.align = "left"
# tick frequency
ax.xaxis.set_minor_locator(mdates.MonthLocator())
ax.xaxis.set_major_locator(mdates.MonthLocator(bymonth=[1]))
ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m'))
ax.yaxis.set_major_locator(matplotlib.ticker.MultipleLocator(base=400))
# tick direction
ax.tick_params(axis='x', direction='out', which='minor', zorder=2, size=2)
ax.tick_params(axis='x', direction='out', which='major', zorder=2, size=4)
ax.set_ylim((0, 1500))
ax.set_xlim((datetime(2017, 10, 1), datetime(2020, 9, 1)))
ax.grid(True)
# annotations
ts = ax.transAxes
coords = ts.transform([-0.092, -0.065])
tr = mpl.transforms.Affine2D().rotate_deg_around(*coords, 90)
t = ts + tr
brace = curly_brace(x=.1,
y=.1,
width=.03,
height=.54,
lw=.5,
pointing='right',
transform=t,
color='.15')
ax.add_artist(brace)
ax.text(.25,
.85,
'training window\nfor $b_1$',
ha='center',
va='bottom',
transform=ts,
fontsize=7)
# labels
ax.set_ylabel('Number of annotations')
# cosmetics
sns.despine()
# save
save_fig(fig, 'fig1', version=1, plot_formats=['png', 'pdf'])
def hurst_graph(stocks, maximum_chunk, minimum_chunk, r):
def hurst(data_set, minimum_chunk, r):
def chunk_calc(data_set):
for num in range(len(data_set)):
data_set_adjusted = data_set[num:]
n = (math.log(len(data_set_adjusted), minimum_chunk))
if (n % int(n)) == 0.0:
return int(n), data_set_adjusted
break
def chunk_pad(it, size, padval=None):
it = chain(iter(it), repeat(padval))
return list(
iter(lambda: tuple(islice(it, size)), (padval, ) * size))
expo, data_set_a = chunk_calc(data_set)
def chunks(data_set_a):
def first_chunk(data_set_a):
mean = round(np.mean(data_set), r)
sd = round(std(data_set), r)
mean_centered_series = []
cumulative_deviation = []
for num in data_set_a:
mean_centered_series.append(round(num - mean, r))
for num in range(len(mean_centered_series)):
cumulative_deviation.append(
round(sum(mean_centered_series[:num]), r))
Range = round(
max(cumulative_deviation) - min(cumulative_deviation), r)
rescaled_range = round((Range / sd), r)
log_of_rs = round(log(rescaled_range), r)
log_of_size = round(log(len(data_set_a)), r)
return log_of_rs, log_of_size
list_of_log_of_rs = []
list_of_log_of_size = []
list_of_log_of_rs.append(first_chunk(data_set_a)[0])
list_of_log_of_size.append(first_chunk(data_set_a)[1])
for num in (list(range(expo + 1))[1:-1]):
Ranges = []
rescaled_ranges = []
denominater = minimum_chunk**num
num_of_chunks = (int(int(len(data_set_a)) / denominater))
for chunk in (chunk_pad(data_set_a, num_of_chunks)):
mean = round(np.mean(chunk), r)
sd = round(std(chunk), r)
mean_centered_series = []
cumulative_deviation = []
for n in chunk:
mean_centered_series.append(round(n - mean, r))
for n in range(len(chunk)):
cumulative_deviation.append(
round(sum(mean_centered_series[:n]), r))
Range = round(
max(cumulative_deviation) - min(cumulative_deviation),
r)
rescaled_ranges.append(round(Range / sd, r))
avg_rescaled_range = round(
sum(rescaled_ranges) / denominater, r)
list_of_log_of_rs.append(round(log(avg_rescaled_range), r))
list_of_log_of_size.append(
round(log(len((chunk_pad(data_set_a, num_of_chunks))[0])),
r))
#plt.scatter(list_of_log_of_size,list_of_log_of_rs)
#plt.show()
#return list_of_log_of_rs,list_of_log_of_size
def invertList(input_list):
for item in range(len(input_list) // 2):
input_list[item], input_list[len(input_list) - 1 -
item] = input_list[
len(input_list) - 1 -
item], input_list[item]
return input_list
Y = invertList(list_of_log_of_rs)
X = invertList(list_of_log_of_size)
def best_fit(X, Y):
xbar = sum(X) / len(X)
ybar = sum(Y) / len(Y)
n = len(X) # or len(Y)
numer = sum([xi * yi
for xi, yi in zip(X, Y)]) - n * xbar * ybar
denum = sum([xi**2 for xi in X]) - n * xbar**2
b = numer / denum
a = ybar - b * xbar
return a
return best_fit(X, Y)
return np.absolute(chunks(data_set_a))
def hurst_alt(ts):
H, c, val = compute_Hc(ts)
return H
hurst_values = []
time_adjusted_dates = []
counter1 = 0
counter2 = maximum_chunk
skipped = 0
for period in list(range(0, len(stocks[0].getValuesClose()),
maximum_chunk))[1:]:
difference = []
for i in range(counter1, counter2):
difference.append(stocks[0].percent_change[i] -
stocks[1].percent_change[i])
time_adjusted_dates.append(stocks[0].getDates()[period])
if maximum_chunk < 100:
hurst_values.append(hurst(difference, minimum_chunk, r))
else:
hurst_values.append(hurst_alt(difference))
counter1 += maximum_chunk
counter2 += maximum_chunk
plt.plot(time_adjusted_dates, hurst_values)
plt.xlabel('Dates')
plt.ylabel('Hurst Values')
plt.title(str(maximum_chunk) + '-Day Hurst Chart')
#dealing with the x-axis labels
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
years_fmt = mdates.DateFormatter('%Y')
plt.gca().xaxis.set_major_locator(years)
plt.gca().xaxis.set_major_formatter(years_fmt)
plt.gca().xaxis.set_minor_locator(months)
plt.gcf().autofmt_xdate()
plt.show()
def plot_launches(filename, type):
f = open(filename, 'r')
dates = []
times = []
launches = []
distances = []
for line in f:
data = line.rstrip("\n").split(" ")
dates.append(data[0])
launches.append(float(data[1]))
times.append(float(data[2]))
distance = float(data[3]) - MARS_RADIUS
if distance < 0:
distance = 0
distances.append(distance)
f.close()
min_index = np.argmin(distances)
print("Minimum distance to Mars achieved:")
print("\tLaunch Day:",
(INITIAL_DATE + datetime.timedelta(seconds=launches[min_index])
).strftime('%Y-%m-%d %H:%M:%S'),
"(" + str(launches[min_index]) + "[s])")
print("\tArrival day:",
(INITIAL_DATE + datetime.timedelta(seconds=times[min_index])
).strftime('%Y-%m-%d %H:%M:%S') + "[s]")
print("\tTime of Flight:", str(times[min_index]) + "[s]")
print("\tDistance to Mars:", str(distances[min_index]) + "[km]")
if type == TYPE_MINUTE or type == TYPE_SECOND or type == TYPE_HOUR:
date_index = 0
x_values = []
for i in range(len(launches)):
sub_date = INITIAL_DATE + datetime.timedelta(seconds=launches[i])
x_values.append(sub_date)
y_values = distances
elif type == TYPE_DAY:
x_values = [
datetime.datetime.strptime(d, "%Y-%m-%d").date() for d in dates
]
y_values = distances
elif type == TYPE_WEEK:
x_values = [
datetime.datetime.strptime(d, "%Y-%m-%d").date() for d in dates
]
y_values = distances
ax = plt.gca()
formatter = mdates.DateFormatter("%d-%m-%Y")
ax.xaxis.set_major_formatter(formatter)
locator = mdates.DayLocator()
ax.xaxis.set_major_locator(locator)
plt.scatter(x_values, y_values, color="red")
# Removes the scientific notation on top
# https://stackoverflow.com/questions/28371674/prevent-scientific-notation-in-matplotlib-pyplot
ax.ticklabel_format(style='plain', axis='y')
# Format the date into months & days
# Change the tick interval
if type == TYPE_SECOND:
interval = 15
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S'))
plt.gca().xaxis.set_major_locator(
mdates.SecondLocator(interval=interval))
plt.gca().set_ylabel("Distancia a Superficie de Marte [km]")
plt.gca().set_xlabel("Fecha de Despegue [" + dates[0] + "]")
elif type == TYPE_MINUTE:
interval = 10
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%H:%M:%S'))
plt.gca().xaxis.set_major_locator(
mdates.MinuteLocator(interval=interval))
plt.gca().set_ylabel("Distancia a Superficie de Marte [km]")
plt.gca().set_xlabel("Fecha de Despegue [" + dates[0] + "]")
elif type == TYPE_HOUR:
interval = 4
plt.gca().xaxis.set_major_formatter(
mdates.DateFormatter('%m-%d %H:%M'))
plt.gca().xaxis.set_major_locator(
mdates.HourLocator(interval=interval))
plt.gca().set_ylabel("Distancia a Superficie de Marte [km]")
plt.gca().set_xlabel("Fecha de Despegue [Año " +
dates[0].split("-")[0] + "]")
elif type == TYPE_DAY:
interval = 2
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%d-%m'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval=interval))
plt.gca().set_ylabel("Distancia a Superficie de Marte [km]")
plt.gca().set_xlabel("Fecha de Despegue [Año " +
dates[0].split("-")[0] + "]")
elif type == TYPE_WEEK:
# https://stackoverflow.com/questions/46555819/months-as-axis-ticks
interval = 2
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%Y'))
plt.gca().xaxis.set_major_locator(
mdates.MonthLocator(interval=interval))
plt.gca().set_ylabel("Distancia a Superficie de Marte [km]")
plt.gca().set_xlabel("Fecha de Despegue")
# Puts x-axis labels on an angle
plt.gca().xaxis.set_tick_params(rotation=30)
plt.show()
model_plot_name = model[1]
model_stat_values_array = stat_values_array[model_index, :, :]
ax = plt.subplot(gs[model_index])
ax.grid(True)
ax.set_xticks(leads)
ax.set_xlim([leads[0], leads[-1]])
if ax.is_last_row() or (nmodels % 2 != 0 and model_num == nmodels - 1):
ax.set_xlabel("Forecast Hour")
else:
plt.setp(ax.get_xticklabels(), visible=False)
ax.set_ylim([plot_time_dates[0], plot_time_dates[-1]])
day_interval = int(len(plot_time_dates) / nticks)
ax.set_yticks(plot_time_dates[::day_interval])
ax.yaxis.set_major_formatter(md.DateFormatter('%d%b%Y'))
if len(plot_time_dates) > 60:
ax.yaxis.set_minor_locator(md.MonthLocator())
else:
ax.yaxis.set_minor_locator(md.DayLocator())
if ax.is_first_col():
ax.set_ylabel(plot_time.title() + " Date")
else:
plt.setp(ax.get_yticklabels(), visible=False)
if stat == "bias":
logger.debug("Plotting model " + str(model_num) + " " +
model_name + " with name on plot " + model_plot_name)
ax.set_title(model_plot_name, loc='left')
if model_num == 1:
clevels_bias = plot_util.get_clevels(model_stat_values_array)
CF1 = ax.contourf(
xx,
yy,
def _generate_burn_down_chart(self, name, min_y, max_y, step_y,
burn_down_show, burn_down_begin_date,
burn_down_begin_value, burn_down_end_date,
burn_down_end_value, data, graph_filename):
date_format = "%d-%b-%Y"
plt.clf()
# set range
min_x = datetime.strptime(self.chart_begin_date, date_format)
max_x = datetime.strptime(self.chart_end_date, date_format)
#plt.xlim(min_x, max_x)
# sort values by datetime
dates = []
values = []
for key, value in sorted(data.items()):
dates.append(key)
values.append(value)
min_val = min(values) - 10
max_val = max(values) + 10
print(':::::::::::::::::::::', min_val)
print(dates)
print(dates[0])
print(dates[-1])
print(values)
print(':::::::::::::::::::::', max_val)
plt.xlim(dates[0], dates[-1])
history_interval_in_days = (dates[-1] - dates[0]).days
# plot history data
plt.plot_date(dates, values, "-")
# plot burn down
if burn_down_show:
burn_down_begin_x = datetime.strptime(burn_down_begin_date,
date_format)
burn_down_end_x = datetime.strptime(burn_down_end_date,
date_format)
plt.plot([burn_down_begin_x, burn_down_end_x],
[burn_down_begin_value, burn_down_end_value],
'--r',
label="Burndown")
# plot labels
plt.xlabel('Date')
plt.title(name)
# plot ticks
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
day = mdates.DayLocator()
years_fmt = mdates.DateFormatter('%Y')
dt_fmt = mdates.DateFormatter('%d-%m-%y')
if (history_interval_in_days > 365):
plt.gca().xaxis.set_major_locator(years)
plt.gca().xaxis.set_minor_locator(months)
elif (history_interval_in_days > 90):
plt.gca().xaxis.set_major_locator(months)
plt.gca().xaxis.set_minor_locator(
mdates.WeekdayLocator(interval=2))
elif (history_interval_in_days > 30):
plt.gca().xaxis.set_major_locator(
mdates.WeekdayLocator(interval=2))
plt.gca().xaxis.set_minor_locator(
mdates.WeekdayLocator(interval=1))
elif (history_interval_in_days < 13):
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
#plt.gca().xaxis.set_minor_locator(mdates.WeekdayLocator(interval=1))
else:
plt.gca().xaxis.set_major_locator(
mdates.WeekdayLocator(interval=1))
plt.gca().xaxis.set_minor_locator(mdates.DayLocator(interval=2))
#plt.gca().xaxis.set_major_locator(mdates.WeekdayLocator(interval=1))
plt.gca().xaxis.set_major_formatter(
mdates.DateFormatter('%d-%m-%y')) #dt_fmt)#years_fmt)
#plt.gca().xaxis.set_minor_locator(day)#months)
plt.gca().xaxis.set_minor_formatter(DateFormatter('%d'))
#stp=8#int( (max_val-min_val)/8 )
#plt.yticks(np.arange(min_val, max_val, step=stp))
# plt.yticks(np.arange(min_y, max_y, step=step_y))
# save image
plt.savefig(graph_filename, bbox_inches='tight', dpi=70)
fig, ax = plt.subplots(figsize=fig_dims)
g = sns.scatterplot(data=peer,
x='Days',
y='Double',
hue='Data from Asia',
palette="colorblind",
style='Estimate',
size=peer['Citations'],
sizes=(20, 1000),
alpha=alpha,
legend='brief')
ax.set_ylabel("Doubling time (days)", fontsize=8)
ax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))
ax.xaxis.set_major_formatter(mdates.DateFormatter('%d-%m'))
plt.gcf().autofmt_xdate()
ax.tick_params(labelsize=8)
plt.ylim(0, 14)
sns.despine()
g.set(xlabel=None)
g1 = sns.scatterplot(data=summary_uom_sep,
x='Days',
# Shift the markers to the baseline by replacing the y-data by zeros.
markerline.set_ydata(np.zeros(len(dates)))
# annotate lines
vert = np.array(['top', 'bottom'])[(levels > 0).astype(int)]
for d, l, r, va in zip(dates, levels, names, vert):
ax.annotate(r,
xy=(d, l),
xytext=(-3, np.sign(l) * 3),
textcoords="offset points",
va=va,
ha="right")
# format xaxis with 4 month intervals
ax.get_xaxis().set_major_locator(mdates.MonthLocator(interval=4))
ax.get_xaxis().set_major_formatter(mdates.DateFormatter("%b %Y"))
plt.setp(ax.get_xticklabels(), rotation=30, ha="right")
# remove y axis and spines
ax.get_yaxis().set_visible(False)
for spine in ["left", "top", "right"]:
ax.spines[spine].set_visible(False)
ax.margins(y=0.1)
plt.show()
#############################################################################
#
# ------------
#
def checkVisPA(ra, dec, targetName=None, ephFileName=pkg_resources.resource_filename('ExoCTK', 'data/contam_visibility/JWST_ephem_short.txt'), save=False, fig=''):
if ra.find(':')>-1: #format is hh:mm:ss.s or dd:mm:ss.s
ra = convert_ddmmss_to_float(ra) * 15. * D2R
dec = convert_ddmmss_to_float(dec) * D2R
else: #format is decimal
ra = float(ra) * D2R
dec = float(dec) * D2R
#load ephemeris
eclFlag = False
eph = EPH.Ephemeris(ephFileName, eclFlag)
#convert dates from MJD to Gregorian calendar dates
mjd = np.array(eph.datelist)
d = mdates.julian2num(mjd+2400000.5)
gd = mdates.num2date(d)
#loop through dates and determine VIS and PAs (nominal, min, max)
vis = np.empty(mjd.size,dtype=bool)
paNom, paMin, paMax = np.empty(mjd.size), np.empty(mjd.size), np.empty(mjd.size)
for i in range(mjd.size):
#is it visible?
vis[i] = eph.in_FOR(mjd[i],ra,dec)
#nominal PA at this date
pa = eph.normal_pa(mjd[i],ra,dec)
#search for minimum PA allowed by roll
pa0 = pa
while eph.is_valid(mjd[i],ra,dec,pa0-0.002):
pa0 -= 0.002
#search for maximum PA allowed by roll
pa1 = pa
while eph.is_valid(mjd[i],ra,dec,pa1+0.002):
pa1 += 0.002
paNom[i] = (pa*R2D)%360
paMin[i] = (pa0*R2D)%360
paMax[i] = (pa1*R2D)%360
#does PA go through 360 deg?
wrap = np.any(np.abs(np.diff(paNom[np.where(vis)[0]])) > 350)
#Determine good and bad PA ranges
#Good PAs
i, = np.where(vis)
pa = np.concatenate((paNom[i],paMin[i],paMax[i]))
if wrap:
pa = np.append(pa,(0.,360.))
pa.sort()
i1, = np.where(np.diff(pa)>10)
i0 = np.insert(i1+1,0,0)
i1 = np.append(i1,-1)
paGood = np.dstack((pa[i0],pa[i1])).round(1).reshape(-1,2).tolist()
#bad PAs (complement of the good PAs)
paBad = []
if paGood[0][0]>0:
paBad.append([0.,paGood[0][0]])
for i in range(1,len(paGood)):
paBad.append([paGood[i-1][1],paGood[i][0]])
if paGood[-1][1]<360.:
paBad.append([paGood[-1][1],360.])
#print results to file
"""
if save:
fName='visibilityPA-'+targetName+'.txt'
fic=open(fName,'w')
fic.write('#Date MJD VIS? PAnom PArange\n')
for i in range(vis.size):
tmp1='{:7.3f}'.format(paNom[i]) if vis[i] else 7*'-'
tmp2='{:7.3f}--{:7.3f}'.format(paMin[i],paMax[i]) if vis[i] else 16*'-'
#fic.write(gd[i].strftime("%y-%m-%d")+' {:f} {:5s} {:7.3f} {:7.3f}--{:7.3f} \n'.format(mjd[i],str(vis[i]),paNom[i],paMin[i],paMax[i]))
fic.write(gd[i].strftime("%y-%m-%d")+' {:f} {:5s} {} {} \n'.format(mjd[i],str(vis[i]),tmp1,tmp2))
fic.write("\n")
fic.write("Accessible PA ranges: ")
fic.write(','.join([str(x) for x in paGood]))
fic.write("\n")
fic.write("Non-accessible PA ranges: ")
fic.write(','.join([str(x) for x in paBad]))
fic.write("\n")
fic.close()
"""
# Make a figure
if not fig or fig==True:
fig = plt.gcf()
# Do all figure calculations
iBad, = np.where(vis==False)
paMasked = np.copy(paNom)
paMasked[iBad] = np.nan
gdMasked = np.copy(gd)
i = np.argmax(paNom)
if paNom[i+1]<10:
i+=1
paMasked = np.insert(paMasked,i,np.nan)
gdMasked = np.insert(gdMasked,i,gdMasked[i])
i = np.argmax(paMin)
goUp = paMin[i-2]<paMin[i-1] #PA going up at wrap point?
# Top part
i0_top = 0 if goUp else i
i1_top = i if goUp else paMin.size-1
paMaxTmp = np.copy(paMax)
paMaxTmp[np.where(paMin>paMax)[0]] = 360
# Bottom part
i = np.argmin(paMax)
i0_bot = i if goUp else 0
i1_bot = paMin.size-1 if goUp else i
paMinTmp = np.copy(paMin)
paMinTmp[np.where(paMin>paMax)[0]] = 0
# Add fits to matplotlib
if isinstance(fig, matplotlib.figure.Figure):
# Make axes
ax = plt.axes()
plt.title(targetName)
#plot nominal PA
plt.plot(gdMasked,paMasked,color='k')
#plot ranges allowed through roll
if wrap:
i = np.argmax(paMin)
goUp = paMin[i-2]<paMin[i-1] #PA going up at wrap point?
#top part
plt.fill_between(gd[i0_top:i1_top+1],paMin[i0_top:i1_top+1],paMaxTmp[i0_top:i1_top+1],where=vis[i0_top:i1_top+1],lw=0,facecolor='k',alpha=0.5)
#bottom part
plt.fill_between(gd[i0_bot:i1_bot+1],paMinTmp[i0_bot:i1_bot+1],paMax[i0_bot:i1_bot+1],where=vis[i0_bot:i1_bot+1],lw=0,facecolor='k',alpha=0.5)
else:
plt.fill_between(gd,paMin,paMax,where=vis,lw=0,facecolor='k',alpha=0.5)
plt.ylabel('Position Angle (degrees)')
plt.xlim(min(gd),max(gd))
ax.xaxis.set_major_locator(mdates.MonthLocator())
ax.xaxis.set_major_formatter(mdates.DateFormatter("%b '%y"))
ax.xaxis.set_minor_locator(mdates.DayLocator(list(range(1,32,5))))
plt.ylim(0,360)
ax.yaxis.set_major_locator(MultipleLocator(25))
ax.yaxis.set_minor_locator(MultipleLocator(5))
plt.grid()
for label in ax.get_xticklabels():
label.set_rotation(45)
# Or to bokeh!
else:
# Convert datetime to a number for Bokeh
gdMaskednum = [datetime.date(2019, 6, 1)+datetime.timedelta(days=n) for n,d in enumerate(gdMasked)]
color = 'green'
# Draw the curve and error
fig.line(gdMaskednum, paMasked, legend='cutoff', line_color=color)
# Top
err_y = np.concatenate([paMin[i0_top:i1_top+1],paMaxTmp[i0_top:i1_top+1][::-1]])
# err_x = np.concatenate([[d.timestamp() for d in gd[i0_top:i1_top+1]],[d.timestamp() for d in gd[i0_top:i1_top+1]][::-1]])
err_x = np.concatenate([gdMaskednum[i0_top:i1_top+1],gdMaskednum[i0_top:i1_top+1][::-1]])
fig.patch(err_x, err_y, color=color, fill_alpha=0.2, line_alpha=0)
# Bottom
err_y = np.concatenate([paMinTmp[i0_bot:i1_bot+1],paMax[i0_bot:i1_bot+1][::-1]])
# err_x = np.concatenate([[d.timestamp() for d in gd[i0_bot:i1_bot+1]],[d.timestamp() for d in gd[i0_bot:i1_bot+1]][::-1]])
err_x = np.concatenate([gdMaskednum[i0_bot:i1_bot+1],gdMaskednum[i0_bot:i1_bot+1][::-1]])
fig.patch(err_x, err_y, color=color, fill_alpha=0.2, line_alpha=0)
# Plot formatting
fig.xaxis.axis_label = 'Date'
fig.yaxis.axis_label = 'Position Angle (degrees)'
return paGood, paBad, gd, fig
# load some financial data; apple's stock price
fh = cbook.get_sample_data('aapl.npy.gz')
try:
# Python3 cannot load python2 .npy files with datetime(object) arrays
# unless the encoding is set to bytes. However this option was
# not added until numpy 1.10 so this example will only work with
# python 2 or with numpy 1.10 and later.
r = np.load(fh, encoding='bytes')
except TypeError:
r = np.load(fh)
fh.close()
r = r[-250:] # get the last 250 days
fig, ax = plt.subplots()
ax.plot(r.date, r.adj_close)
ax.xaxis.set_major_locator(dates.MonthLocator())
ax.xaxis.set_minor_locator(dates.MonthLocator(bymonthday=15))
ax.xaxis.set_major_formatter(ticker.NullFormatter())
ax.xaxis.set_minor_formatter(dates.DateFormatter('%b'))
for tick in ax.xaxis.get_minor_ticks():
tick.tick1line.set_markersize(0)
tick.tick2line.set_markersize(0)
tick.label1.set_horizontalalignment('center')
imid = len(r) // 2
ax.set_xlabel(str(r.date[imid].year))
plt.show()
dates.append(current_date)
highs.append(high)
lows.append(low)
# Draw figure from data
fig = plt.figure(dpi=128, figsize=(10, 6))
plt.plot(dates, highs, c='red', alpha=0.5, label='high')
plt.plot(dates, lows, c='blue', alpha=0.5, label='low')
plt.fill_between(dates, highs, lows, facecolor='blue', alpha=0.1)
# Set figure's format
plt.title("Daily high and low temperatures, 2014", fontsize=24)
plt.xlim([datetime(2014, 1, 1), datetime(2014, 12, 22)])
plt.gca().xaxis.set_major_formatter(
mdates.DateFormatter('%b %Y')) # datetime format, b is month in short
plt.gca().xaxis.set_major_locator(mdates.MonthLocator()) # datetime interval
plt.xlabel('', fontsize=16)
fig.autofmt_xdate()
plt.yticks(range(10, 80, 10)) # temperature limits and interval
plt.ylabel("Temperature (F)", fontsize=16)
plt.tick_params(axis='both', which='major', direction='in', labelsize=16)
xlm = plt.gca().get_xlim()
xtk = plt.gca().get_xticks()
ylm = plt.gca().get_ylim()
ytk = plt.gca().get_yticks()
x2 = plt.gca().twiny()
x2.xaxis_date()
x2.set_xlim(xlm)
x2.set_xticks(xtk)
ax.plot(time, baseline, 'o', alpha=0.7, c='g')
year_min = datetime.datetime(min(time).year, 1, 1)
year_max = datetime.datetime(max(time).year + 1, 1, 1)
for x in pairs:
rdate, sdate = x.split('-')
rtime = datetime.datetime.strptime(rdate, "%y%m%d")
stime = datetime.datetime.strptime(sdate, "%y%m%d")
time = [rtime, stime]
baseline = [baseline_dict[rdate], baseline_dict[sdate]]
ax.plot(time, baseline, '-', lw=.5, c='b')
ax.xaxis.set_major_locator(mdates.YearLocator())
ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y'))
ax.xaxis.set_minor_locator(mdates.MonthLocator())
ax.minorticks_on()
ax.tick_params('both', length=7, which='major', width=1)
ax.tick_params('both', length=4, which='minor', width=0.5)
ax.set_xlim(year_min, year_max)
ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')
# rotates and right aligns the x labels, and moves the bottom of the
# axes up to make room for them
#fig.autofmt_xdate()
ax.set_xlabel('Time [years]')
ax.set_ylabel('Perpendicular Baseline [meters]')
fig = plt.figure(figsize=(12,12))
e = to_pivot_table(tr.trans,'Category','Year','Amount', np.sum)
ax = sns.heatmap(data=e, vmin=e.min().min(), vmax=e.max().max(),
annot=True, fmt='.2f', linewidths=.5,
cbar_kws={"shrink": .80}, cmap='PuBuGn')
ax.set_title('Sum Of Categories Per Year', fontsize=11)
ax.set_xlabel('Year', fontsize=11)
ax.set_ylabel('Category', fontsize=11)
fig = ax.get_figure()
fig.savefig('graphs/heatmap_e.svg')
################################################################################
# Line charts
years = mdates.YearLocator()
months = mdates.MonthLocator()
yearsFmt = mdates.DateFormatter('%Y')
# Chart on balance
def save_line_chart_a():
bal = tr.trans[['Balance','Date_Time']][::-1]
bal.set_index('Date_Time')
ax = bal.plot(x='Date_Time', y='Balance', figsize=(16, 6))
ax.set_title('Balance over Time', fontsize=11)
ax.set_xlabel('Year', fontsize=11)
ax.set_ylabel('Balance', fontsize=11)
ax.xaxis.set_major_locator(years)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
# pickle_file = open('/home/nealzc1991/PycharmProjects/Py4Invst/Market_Analysis/Market_Analysis_Tools/df_spread.pkl', 'wb')
# pickle.dump(df_spread, pickle_file)
# pickle_file.close()
#
# pickle_file = open('/home/nealzc1991/PycharmProjects/Py4Invst/Market_Analysis/Market_Analysis_Tools/df_cc0.pkl', 'wb')
# pickle.dump(df_cc0, pickle_file)
# pickle_file.close()
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.ticker as ticker
### Plotting
months = mdates.MonthLocator(bymonth=range(1, 13),
bymonthday=1,
interval=1)
monthsFmt = mdates.DateFormatter('%m-%b')
fig, ax1 = plt.subplots(figsize=(15, 8))
ax1.plot_date(df_cc0.index,
df_cc0.loc[:, 'RBcc0_p'],
'-',
color='#FFCC99',
alpha=0.8)
ax1.plot_date(df_adjusted_cc0.index,
df_adjusted_cc0.loc[:, 'RBcc0_p'],
'-',
color='#99CC99',
alpha=0.8)
def _set_xax_locators(self):
# set xaxis date locators
self.xaxyears = self.xlim[1].year-self.xlim[0].year+1
for ax in self.axeslist:
self.axeslist[ax].set_xlim(self.xlim)
years = mdates.YearLocator() # every year
months = mdates.MonthLocator() # every month
years_fmt = mdates.DateFormatter('%Y')
self.axeslist[ax].xaxis.set_major_formatter(years_fmt)
##print(f'self.xaxyears is {self.xaxyears}')
##plt.rcParams['dates.epoch'] = '0000-12-31'
##dt.set_epoch = '0000-12-31'
if self.xaxyears in range(0,3):
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(1, month=1,day=1))
self.axeslist[ax].xaxis.set_major_formatter(years_fmt)
#YearLocator(1, month=1,day=1))
#YearLocator(byyear=1))
self.axeslist[ax].xaxis.set_minor_locator(
MonthLocator(bymonth=[1,4,7,10]))
labels = ['jan','apr','jul','okt']*3
myfontdic = {'fontsize': 4}
self.axeslist[ax].set_xticklabels(labels,
fontdict=myfontdic, minor=True)
if self.xaxyears in range(3,5):
#pass
#"""
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(1, month=1,day=1))
self.axeslist[ax].xaxis.set_major_formatter(years_fmt)
self.axeslist[ax].xaxis.set_minor_locator(
MonthLocator(bymonth=[4,7,10]))
labels = ['apr','jul','okt']*5
myfontdic = {'fontsize': 8}
self.axeslist[ax].set_xticklabels(labels,
fontdict=myfontdic, minor=True)
#"""
#self.axeslist[ax].xaxis.set_major_formatter(
# DateFormatter('%y')) #%m-%d'))
if self.xaxyears in range(5,11):
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(1, month=1,day=1))
self.axeslist[ax].xaxis.set_major_formatter(years_fmt)
self.axeslist[ax].xaxis.set_minor_locator(
YearLocator(1, month=1,day=1))
if self.xaxyears in range(11,25):
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(5, month=1,day=1))
self.axeslist[ax].xaxis.set_minor_locator(
YearLocator(1, month=1,day=1))
if self.xaxyears in range(25,50):
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(5, month=1,day=1))
if self.xaxyears in range(50,1000):
self.axeslist[ax].xaxis.set_major_locator(
YearLocator(10, month=1,day=1))
def plot_single_historical_subplot(subplot, df, near, far):
#https://xkcd.com/color/rgb/
xkcd20 = [
"windows blue", "amber", "ruby", "reddy brown", "carnation", "spruce",
"cool green", "swamp", "dusk blue", "indian red", "purple/blue",
"petrol", "royal", "tan brown", "lightish red"
]
sns.set_palette(sns.xkcd_palette(xkcd20))
# show the color palette
#sns.palplot(sns.color_palette())
subplot.set_prop_cycle(
cycler('color', sns.color_palette()) +
(3 * cycler('marker', ['^', 'H', 'p', 'v', '*'])))
###########
# plot shifted data with the mean
###########
shiftedCols = [col for col in df.columns]
subplot.xaxis.set_major_locator(dates.MonthLocator(interval=1))
#subplot.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x,pos: get_month_label(x)))
subplot.xaxis.set_major_formatter(dates.DateFormatter('%b-01'))
subplot.xaxis.set_minor_locator(
dates.DayLocator(bymonthday=[15], interval=1))
subplot.xaxis.set_minor_formatter(dates.DateFormatter('%d'))
currYearColName = get_most_recent_year_column_name(df)
for i, c in enumerate(shiftedCols):
l = None
if (c == currYearColName):
subplot.plot(df[c],
label=c,
linewidth=2.5,
color='black',
markevery=100)
subplot.axvline(df[c].last_valid_index(),
color='blue',
linewidth=1.75)
subplot.axhline(df[c][df[c].last_valid_index()],
color='blue',
linewidth=1.75)
else:
subplot.plot(df[c], label=c, linewidth=1, markevery=14 + (2 * i))
# calculate the mean for all years (minus the current contract) and plot it
cols = [col for col in shiftedCols if col not in [currYearColName]]
df['mean'] = df[cols].mean(axis=1)
df['99per'] = df[cols].apply(lambda x: np.nanpercentile(x, 99), axis=1)
df['75per'] = df[cols].apply(lambda x: np.nanpercentile(x, 75), axis=1)
df['25per'] = df[cols].apply(lambda x: np.nanpercentile(x, 25), axis=1)
df['1per'] = df[cols].apply(lambda x: np.nanpercentile(x, 1), axis=1)
subplot.fill_between(
df.index,
df['75per'].rolling(window=1, min_periods=1).mean(),
df['25per'].rolling(window=1, min_periods=1).mean(),
#color='#FFF717',
color='#DEE2E3',
alpha=0.65)
subplot.plot(df['mean'],
color='red',
linewidth=2.5,
label='AVG',
markevery=100,
linestyle='dashed')
shiftedCols = [
col for col in df.columns if col.startswith('orig_') == False
]
legend = subplot.legend(loc='lower left', ncol=2, frameon=True)
frame = legend.get_frame()
frame.set_facecolor('white')
# Shrink the x-axis and place the legend on the outside
# http://stackoverflow.com/a/4701285
#box = subplot.get_position()
#subplot.set_position([box.x0, box.y0, box.width * 0.95, box.height * 0.90])
#legend = subplot.legend(frameon=True, loc='center left', bbox_to_anchor=(1,0.5),
# fancybox=True, shadow=True, ncol=2)
#frame = legend.get_frame()
#frame.set_facecolor('white')
subplot.set_title('{0} - {1}'.format(near, far))
subplot.grid(b=True,
which='major',
color='k',
linewidth=0.5,
linestyle='dashed')
subplot.grid(b=True,
which='minor',
color='k',
linewidth=0.5,
linestyle='dotted')
# rotate major tick labels to avoid overlapping
# http://stackoverflow.com/a/28063506
plt.setp(subplot.get_xticklabels(),
rotation=30,
horizontalalignment='right')
marker='o',
linestyle='')
plt_price.set_title("{0}:{1}:{2} {3} {4}".format(
issuer, bbg_id,
trades.TICKER.unique()[0], trades.COUPON.max(),
trades.MATURITY_DATE.max().strftime("%Y/%m/%d")))
plt_price.xaxis_date()
plt_price.grid(b=True,
which='major',
color='black',
linestyle='--')
plt_price.set_ylabel("Closing Price")
plt_price.set_yticklabels(
['{:3.2f}'.format(x) for x in plt_price.get_yticks()])
plt_price.xaxis.set_major_locator(dates.MonthLocator())
#plt.xlim(args.start_date, args.end_date)
plt_trd_par.stem(trades['TRD_TRADE_DATE'], trades['TRD_ORIG_FACE'])
plt_trd_par.plot(positions['AS_OF_DATE'],
positions['CURRENT_FACE'],
color='g',
marker='',
linestyle='-')
plt_trd_par.xaxis_date()
plt_trd_par.axhline(0, color='black')
plt_trd_par.set_ylabel(
"Total Face Amount \nFace Amount Traded \n(Millions)")
plt_trd_par.set_yticklabels([
"{:,}".format(x / pow(10, 6))
def plot_individual_against_current(dfList, near, selected_far_contracts):
for pair in dfList:
farContractName = pair[0]
if (farContractName[2] not in selected_far_contracts):
continue
df = pair[1]
dfCleaned = pd.DataFrame(df)
df = dfCleaned.dropna(thresh=1)
currYearColName = get_most_recent_year_column_name(df)
#http://stackoverflow.com/questions/14888473/python-pandas-dataframe-subplot-in-columns-and-rows
#http://stackoverflow.com/questions/14770735/changing-figure-size-with-subplots
f2, axes = plt.subplots(ncols=2,
nrows=math.ceil(df.columns.size / 2),
figsize=(20, 10),
dpi=80)
title = '{0} - {1}'.format(near, farContractName)
for i, c in enumerate(df.columns):
if (i == 0):
df[c].plot(ax=axes[0, 0], color='magenta', lw=1.0)
df[currYearColName].plot(ax=axes[0, 0], color='black', lw=1.8)
axes[0, 0].set_title('{0} {1}'.format(title, c))
axes[0,
0].xaxis.set_minor_locator(dates.MonthLocator(interval=2))
axes[0, 0].xaxis.set_minor_formatter(dates.DateFormatter('%b'))
axes[0, 0].xaxis.set_major_locator(dates.YearLocator())
axes[0, 0].xaxis.set_major_formatter(dates.DateFormatter(''))
axes[0, 0].grid(b=True,
which='major',
color='k',
linewidth=0.5,
linestyle='dotted')
axes[0, 0].grid(b=True,
which='minor',
color='k',
linewidth=0.5,
linestyle='dotted')
else:
axis = axes[math.floor(i / 2), i % 2]
df[c].plot(ax=axis, color='magenta', lw=1.0)
df[currYearColName].plot(ax=axis, color='black', lw=1.8)
axis.set_title('{0} {1}'.format(title, c))
axis.xaxis.set_minor_locator(dates.MonthLocator(interval=2))
axis.xaxis.set_minor_formatter(dates.DateFormatter('%b'))
axis.xaxis.set_major_locator(dates.YearLocator())
axis.xaxis.set_major_formatter(dates.DateFormatter(''))
axis.grid(b=True,
which='major',
color='k',
linewidth=0.5,
linestyle='dotted')
axis.grid(b=True,
which='minor',
color='k',
linewidth=0.5,
linestyle='dotted')
f2.set_tight_layout(True)
f2.suptitle = title
plt.show()
def time_graph(author, table, date, session):
"""creates the time graph for given user
Uploads the picture and returns the image link"""
# data for activity per hour in a day
logger.debug("Author: %s - Preparing time_graph", author)
result_day = session.query(func.date_part('hour', table.datum).label("time"), func.count(table.id).label("count"))\
.filter(and_(table.autor == author, table.datum >= date))\
.group_by(func.date_part('hour', table.datum))
times_day = []
counts_day = []
for item in result_day:
times_day.append(datetime.datetime.combine(datetime.date(1900, 1, 1), datetime.time(int(item.time), 0, 0))) #200,1,1 is a random day. won't be show in the end
counts_day.append(item.count)
# data for activity graph all time
times_all = []
counts_all = []
result_all_time = session.query(func.date_trunc('week', table.datum).label("time"), func.count(table.id).label("count"))\
.filter(and_(table.autor == author, table.datum >= date))\
.group_by(func.date_trunc('week', table.datum))\
.order_by(func.date_trunc('week', table.datum))
for item in result_all_time:
times_all.append(item.time)
counts_all.append(item.count)
# Matplotlib part
plt.style.use('ggplot')
plt.rcParams.update({'figure.autolayout': True})
plt.figure(figsize=(8, 5))
plt.tight_layout()
dates = mdates.DateFormatter('%H:%M')
weeks = mdates.MonthLocator()
plt1 = plt.subplot(2, 1, 1)
plt1.set_title("Average daily activity")
plt1.bar(times_day, counts_day, 0.03)
plt1.xaxis.set_tick_params(labelrotation=55)
plt1.xaxis.set_major_formatter(dates)
plt1.set_xticks(times_day)
plt1.set_ylabel('Number of comments\n per hour')
plt2 = plt.subplot(2,1,2)
plt2.set_title("Activity progress")
plt2.plot(times_all, counts_all)
plt2.xaxis.set_tick_params(labelrotation=55)
plt2.xaxis.set_minor_locator(weeks)
plt2.set_ylabel('Number of comments\n per week')
timeimagepath = "output/time_graph.png"
try:
plt.savefig(timeimagepath)
except:
logger.warn("Author: %s - Could not save image time_graph", author)
logger.debug("Author: %s - Saving time_graph image", author)
return(timeimagepath)
#Draw the graph
gret.index = pd.to_datetime(gret.index)
date = mpl.dates.date2num(gret.index.to_pydatetime())
retur = gret["bad"].values
price = gret["price"].values
stars = gret["rol_mean"].values
sent = gret["rol_sent"].values
reviews = gret["reviews"].values
t = date
objects = gret.index
y_pos = np.arange(len(objects))
fig, axs = plt.subplots(5, 1, figsize=(16.5, 10))
axs[0].bar(t, retur, align='center', color="gray")
axs[0].set_ylabel('Returns')
axs[0].grid(True)
axs[0].xaxis.set_major_locator(mdates.MonthLocator())
axs[0].xaxis.set_major_formatter(mdates.DateFormatter('%B'))
axs[1].plot(t, stars, color="gray")
axs[1].set_ylabel('Avg. rating')
axs[1].grid(True)
axs[1].xaxis.set_major_locator(mdates.MonthLocator())
axs[1].xaxis.set_major_formatter(mdates.DateFormatter('%B'))
axs[2].plot(t, price, color="gray")
axs[2].set_ylabel('Price in [$]')
axs[2].grid(True)
axs[2].xaxis.set_major_locator(mdates.MonthLocator())
axs[2].xaxis.set_major_formatter(mdates.DateFormatter('%B'))
axs[3].plot(t, sent, color="gray")
axs[3].set_ylabel('Avg. sentiment')
axs[3].grid(True)
axs[3].xaxis.set_major_locator(mdates.MonthLocator())
from netCDF4 import Dataset
import pickle
from scipy.interpolate import interp1d
from matplotlib import animation
from scipy import signal
import matplotlib.gridspec as gridspec
import matplotlib.dates as mdates
from datetime import datetime
import sys
sys.path.insert(1, '../sar/')
from coord_conversion import *
golden_ratio = 1.618
onepltsize = (golden_ratio * 2.5, 2.5)
date_initial_str = '2018-07-01'
date_final_str = '2018-08-05'
months = mdates.MonthLocator() # every month
days = mdates.DayLocator() # every day
date_fmt = mdates.DateFormatter('%m/%d')
date_initial_plot = mdates.date2num(datetime.strptime('2018-07-01',
'%Y-%m-%d'))
date_initial_volumes = mdates.date2num(
datetime.strptime('2018-05-01', '%Y-%m-%d')) #for GLISTEN volume
date_final_plot = mdates.date2num(datetime.strptime('2018-08-05', '%Y-%m-%d'))
date_eruption = mdates.date2num(datetime.strptime('2018-05-04', '%Y-%m-%d'))
date_eruption_end = mdates.date2num(datetime.strptime('2018-08-04',
'%Y-%m-%d'))
date_explosion1 = mdates.date2num(datetime.strptime('2018-05-16', '%Y-%m-%d'))
date_explosion2 = mdates.date2num(datetime.strptime('2018-05-26', '%Y-%m-%d'))
date_intensity_eruption = mdates.date2num(
datetime.strptime('2018-05-18', '%Y-%m-%d'))
date_caldera_start = mdates.date2num(
if (v1 is None):
return v2
if (v2 is None):
return v1
else:
return (v1[0],v1[1]+v2[1])
conf = SparkConf().setAppName("Spark Count")
sc = SparkContext(conf=conf)
map3 =sc.textFile('/Users/licheng5625/PythonCode/masterarbeit/data/webpagefortwitter/Thailand_Snake_Girl.txt').map(lambda line:(line.split(" ")[0][:7],1)).filter(lambda v1:len(v1[0])==7 and v1[0][:2]=="20").reduceByKey(lambda v1,v2:v1+v2).sortByKey(True,1)
dates=map3.map(lambda v1:datetime.datetime.strptime(v1[0],"%Y-%m")).collect()
times=map3.map(lambda v1:v1[1]).collect()
years = mdates.YearLocator() # every year
months = mdates.MonthLocator(interval=4) # every month
daysFmt = mdates.DateFormatter('%Y-%m-%d')
yearsFmt = mdates.DateFormatter('%Y-%m')
fig, ax = plt.subplots()
dates = matplotlib.dates.date2num(dates)
ax.plot_date(dates, times,'-' )
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(yearsFmt)
ax.xaxis.set_minor_locator(months)
ax.autoscale_view()
def price(x):
return x
def power_imshow(self, shading=None, downscale_fac=None, anomalie=False,
downscale_type=None, fig_size=None , nopower=False, ax=None, cbar=True):
import matplotlib.pyplot as plt
import datetime as DT
import matplotlib.colors as colors
from matplotlib import dates
import time
import scipy.signal as signal
import matplotlib.ticker as ticker
import numpy as np
import sys
sys.path.append('/home/lbaratgin/work/modules/stormy_forerunners/')
#from . import tools as MT
shading='gouraud' if shading is True else 'flat'
fig_size=[10,4] if fig_size is None else fig_size
if ax:
assert type(ax) is tuple, "put ax as tuple ax=(ax,F)"
self.F=ax[1]
ax_local=ax[0]
else:
self.F=figure_axis_xy(fig_size[0], fig_size[1], fig_scale=2)
ax_local=self.F.ax
if nopower is True:
dd=self.data
else:
dd=10*np.log10(self.data[:-1,:])
if anomalie is True:
dd_tmp=dd.mean(axis=0).repeat(self.time.size-1)
dd=dd- dd_tmp.reshape(self.fs.size,self.time.size-1).T
self.clevs=self.clevs if self.clevs is not None else clevels(dd)
norm = colors.BoundaryNorm(boundaries=self.clevs, ncolors=256)
tt = self.time
#tvec=np.arange(0,tt.size,1)
ax_local.set_yscale("log", nonposy='clip')
if downscale_fac is not None:
if downscale_type =='inter':
fn=[]
for yr in np.arange(0,self.fs.size,downscale_fac):
fn.append(np.mean(self.fs[yr:yr+downscale_fac]))
else:
ddn=np.empty((self.time.size-1))
fsn_p=gen_log_space(self.fs.size,int(np.round(self.fs.size/downscale_fac)))
fsn_p_run=np.append(fsn_p,fsn_p[-1])
dd=dd.T
#print(ddn.shape, fsn_p.shape)
for fr in np.arange(0,fsn_p.size,1):
#print(np.mean(dd[fsn_p[fr]:fsn_p[fr+1], :],axis=0).shape)
ddn=np.vstack((ddn, np.mean(dd[fsn_p_run[fr]:fsn_p_run[fr+1], :],axis=0)))
ddn=np.delete(ddn, 0,0)
#print(ddn.shape)
dd2=ddn
fn=self.fs[fsn_p]
if nopower is True:
tt=tt
else:
tt=tt[:-1]
#print(dd2.shape, fn.shape, tt.shape)
else:
if nopower is True:
tt=tt
else:
tt=tt[:-1]
dd2=dd.T
fn=self.fs
if isinstance(tt[0], np.datetime64):
print('time axis is numpy.datetime64, converted to number for plotting')
ttt=dates.date2num(tt.astype(DT.datetime))
#print(ttt)
elif isinstance(tt[0], np.timedelta64):
print('time axis is numpy.timedelta64, converted to number for plotting')
#print(tt)
ttt=tt
else:
#print(type(tt[0]))
#print(tt)
print('time axis is not converted')
ttt=tt
MT.stats_format(dd2)
self.cs=plt.pcolormesh(ttt,fn ,dd2,cmap=self.cmap , norm=norm,
shading=shading)#, origin='lower',aspect='auto',
#interpolation='none',
#extent=[tvec.min(),tvec.max(),self.fs.min(),self.fs.max()])
#self.F.ax.set_yscale("log", nonposy='clip')
#self.cs=plt.pcolormesh(self.time[:-2], self.fs[:],dd,cmap=self.cmap,shading='gouraud')
#print(self.clevs)
plt.ylabel(('f (' + self.sample_unit+ ')'))
if cbar is True:
self.cbar= plt.colorbar(self.cs,pad=0.01)#, Location='right')#
self.cbar.ax.aspect=100
self.cbar.outline.set_linewidth(0)
self.cbar.set_label('Power db(' + self.data_unit + '^2/f )')
ax =ax_local#plt.gca()
if isinstance(tt[0], np.datetime64):
plt.xlabel('Time')
#Set y-lim
ax.set_ylim(self.ylim[0], self.ylim[1])
ax.set_xlim(ttt[0], ttt[-1])
#format X-Axis
#ax.xaxis_date()
Month = dates.MonthLocator()
Day = dates.DayLocator(interval=5)#bymonthday=range(1,32)
dfmt = dates.DateFormatter('%b/%y')
ax.xaxis.set_major_locator(Month)
ax.xaxis.set_major_formatter(dfmt)
ax.xaxis.set_minor_locator(Day)
elif isinstance(tt[0], np.float64):
plt.xlabel('Time')
#Set y-lim
ax.set_ylim(self.ylim[0], self.ylim[1])
ax.set_xlim(ttt[0], ttt[-1])
#format X-Axis
#ax.xaxis_date()
Month = dates.MonthLocator()
Day = dates.DayLocator(interval=5)#bymonthday=range(1,32)
dfmt = dates.DateFormatter('%b/%y')
ax.xaxis.set_major_locator(Month)
ax.xaxis.set_major_formatter(dfmt)
ax.xaxis.set_minor_locator(Day)
else:
plt.xlabel('Time (' + self.time_unit+ ')')
ax.set_ylim(self.ylim[0], self.ylim[1])
ax.xaxis.set_major_locator(ticker.MultipleLocator(5))
ax.xaxis.set_minor_locator(ticker.MultipleLocator(1))
#ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f'))
# Set both ticks to be outside
ax.tick_params(which = 'both', direction = 'out')
ax.tick_params('both', length=6, width=1, which='major')
ax.tick_params('both', length=3, width=1, which='minor')
# Make grid white
ax.grid()
self.ax=ax
gridlines = ax.get_xgridlines() + ax.get_ygridlines()
for line in gridlines:
line.set_color('white')
#line.set_linestyle('-')
self.x=ttt
def format_x_axis(ax):
formatter = mdates.DateFormatter("%m-%d")
ax.xaxis.set_major_formatter(formatter)
ax.xaxis.set_major_locator(mdates.MonthLocator())
