def run(input_files, output_files):
# Read reaction sequences
sequs = gazelib.io.load_json(input_files[0])
# Collect figures here.
figs = []
for sequence in sequs:
srts = []
# Trials
for trial in sequence:
if trial['heuristic_saccade_validity']:
srts.append(trial['srt'])
else:
srts.append(0)
# To milliseconds
srts = list(map(to_milliseconds, srts))
d = {
'SRT': srts,
'Trial': list(range(1,13))
}
p = Bar(d, label='Trial', values='SRT', color='black')
figs.append(p)
# Visualization with bokeh. Combine figures in vertical layout.
p = plotting.vplot(*figs)
plotting.output_file(output_files[0],
'Saccadic reaction time variability')
plotting.save(p)
def Main():
parser = argparse.ArgumentParser()
parser.add_argument("-st", dest="stationId", nargs = '*', help="Please provide a list of station Id.")
parser.add_argument("-ct", dest="cityName", nargs = '*', help="Please provide a list of city names corresponding to stations.")
args = parser.parse_args()
CurrentPath = os.getcwd()
for i in range(len(args.stationId)):
FilePath= (CurrentPath+'/DataFiles/GDD_Data_'+args.cityName[i]+'.csv')
Data, Date, MaxTemp, MinTemp = extract_data_from_csv(FilePath)
Data['date'] = pd.to_datetime(Date)
Data['left'] = Data.date - pd.DateOffset(days=0.5)
Data['right'] = Data.date + pd.DateOffset(days=0.5)
PlotDate =[]
plotDate = Data['right']
Data = Data.set_index(['date'])
Data.sort_index(inplace=True)
source = ColumnDataSource(data=Data)
AverageTemp = []
for index in range(len(MinTemp)):
Average=(MinTemp[index]+MaxTemp[index])/2
AverageTemp.append(Average)
percent = 5
Min_5_95, Max_5_95 =percentile_Calculation(MinTemp,MaxTemp,percent)
percent = 25
Min_25_75, Max_25_75 =percentile_Calculation(MinTemp,MaxTemp,percent)
plot = make_plot(source,AverageTemp,Min_5_95,Max_5_95,Min_25_75,Max_25_75, MinTemp,MaxTemp,plotDate,args.cityName[i])
output_file("./Plots/Op1_"+args.cityName[i]+".html", title="Optional Task # 1 ("+args.cityName[i]+")")
save(plot)
def render_table(sym):
company_info = scrape_page(sym)
columns = ['name', 'date', 'open', 'high', 'low', 'close', 'volume']
df = pd.DataFrame(company_info, columns=columns)
df['date'] = pd.to_datetime(df['date']).apply(pd.datetools.normalize_date)
text_input = TextInput(value=sym.upper(), title="Label:")
mids = (df['open'] + df['close'])/2
spans = abs(df['open'] - df['close'])
inc = df['close'] > df['open']
dec = df['close'] < df['open']
w = 12*60*60*1000 # half day in ms
TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
p = figure(x_axis_type="datetime", tools=TOOLS, plot_width=750, plot_height=300, toolbar_location="left")
p.title = sym.upper() + " Candlestick"
p.xaxis.major_label_orientation = pi/4
p.grid.grid_line_alpha=0.3
p.segment(df.date, df.high, df.date, df.low, color="black")
p.rect(df.date[inc], mids[inc], w, spans[inc], fill_color="#D5E1DD", line_color="black")
p.rect(df.date[dec], mids[dec], w, spans[dec], fill_color="#F2583E", line_color="black")
output_file("templates/candlestick.html", title="candlestick.py example")
save(p)
return "candlestick.html"
def plot_interactive(mergepkl, noisepkl=None, thresh=6.0, thresh_link=7.0, ignoret=None, savehtml=True, url_path='plots'):
""" Backwards compatible function for making interactive candidate summary plot """
data = readdata(mergepkl)
circleinds = calcinds(data, thresh, ignoret)
crossinds = calcinds(data, -1*thresh, ignoret)
edgeinds = calcinds(data, thresh_link, ignoret)
workdir = os.path.dirname(mergepkl)
fileroot = os.path.basename(mergepkl).rstrip('_merge.pkl').lstrip('cands_')
logger.info('Total on target time: {} s'.format(calcontime(data, inds=circleinds+crossinds+edgeinds)))
if noisepkl:
noiseplot = plotnoisecum(noisepkl)
else:
noiseplot = None
combined = plotall(data, circleinds=circleinds, crossinds=crossinds, edgeinds=edgeinds,
htmlname=None, noiseplot=noiseplot, url_path=url_path, fileroot=fileroot)
if savehtml:
output_file(mergepkl.rstrip('.pkl') + '.html')
save(combined)
else:
return combined
def plot(output_directory, normalized_counts, chromosome, cell_types):
bp.output_file(output_directory + "/" + chromosome + ".html", mode="cdn")
plots = []
summary = plot_summary(normalized_counts, chromosome)
if summary:
plots.append(summary)
else:
# if summary can't be plotted, then rest probably can't be plotted either,
# so return without even saving the file (the file is never created on disk if not saved)
return
for cell_type in cell_types:
logging.debug(" - plotting %s", cell_type)
bin_number = []
count = []
condition = "(file_key == 0) & (chromosome == '%s') & (cell_type == '%s')" % (chromosome, cell_type)
for row in normalized_counts.where(condition):
bin_number.append(row["bin_number"])
count.append(row["count"])
plot = bp.figure()
plot.title = "%s counts per bin" % cell_type
plot.scatter(bin_number, count)
plots.append(plot)
bp.save(bp.vplot(*plots))
def gap_test():
from math import pi
import pandas as pd
from bokeh.plotting import figure, show, output_file
from bokeh.sampledata.stocks import MSFT
df = pd.DataFrame(MSFT)[:50]
df["date"] = pd.to_datetime(df["date"])
mids = (df.open + df.close)/2
spans = abs(df.close-df.open)
inc = df.close > df.open
dec = df.open > df.close
w = 12*60*60*1000 # half day in ms
TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
p = figure(x_axis_type="datetime", tools=TOOLS, plot_width=1000, title="MSFT Candlestick")
p.xaxis.major_label_orientation = pi/4
p.grid.grid_line_alpha = 0.3
p.segment(df.date, df.high, df.date, df.low, color="black")
p.rect(df.date[inc], mids[inc], w, spans[inc], fill_color="#D5E1DD", line_color="black")
p.rect(df.date[dec], mids[dec], w, spans[dec], fill_color="#F2583E", line_color="black")
output_file(config.bokeh_output_dir + 'candlestick.html', title='candlestick.py example')
save(p)
return send_from_directory(config.static_files_path,
config.bokeh_output_dir + 'candlestick.html')
def index():
if request.method == 'GET':
return render_template('index.html')
else:
#request was a POST
app.vars['abbrev'] = request.form['abbrev']
abbrev = app.vars['abbrev']
f = open('%s.txt'%(app.vars['abbrev']),'w')
f.write('Stock abbreviation: %s'%(app.vars['abbrev']))
f.close()
#Get the closing value of the stock over the last month into a pandas Series:
StockURL = 'https://www.quandl.com/api/v3/datasets/WIKI/' + abbrev + '.csv'
dataSet = pd.read_csv(StockURL, parse_dates=['Date'])
output_file('templates/datetime_'+abbrev+'.html')
p = figure(width=800, height=250, x_axis_type="datetime", title=abbrev, y_axis_label='Price (in dollars)')
#p.grid.bounds(dataSet['Date'][0], dataSet['Date'][29])
#print dataSet['Date'][0], dataSet['Date'][29]
p.line(dataSet[0:30]['Date'], dataSet[0:30]['Close'], color='navy', alpha=0.5, legend = 'Closing price')
if 'adjustedClose' in request.form:
p.line(dataSet[0:30]['Date'], dataSet[0:30]['Adj. Close'], color='purple', alpha=0.5, legend = 'Adjusted Closing price')
if 'openValue' in request.form:
p.line(dataSet[0:30]['Date'], dataSet[0:30]['Open'], color='red', alpha=0.5, legend = 'Opening price')
if 'adjustedOpen' in request.form:
p.line(dataSet[0:30]['Date'], dataSet[0:30]['Adj. Open'], color='black', alpha=0.5, legend = 'Adjusted opening price')
p.legend.orientation = "top_left"
save(p)
return render_template('datetime_'+abbrev+'.html')
def render(df, xlabel, ylabel, city):
output_file('./html/'+city+'_'+xlabel+'_'+ylabel+'.html', ylabel+'/'+xlabel)
hover = HoverTool(
tooltips=[
('站名', '@desc'),
(xlabel, 'exp(@x)'),
(ylabel, 'exp(@y)')
]
)
source = ColumnDataSource(
data=dict(
x = df[xlabel].map(np.log),
y = df[ylabel].map(np.log),
desc = df['index']
)
)
p = figure(plot_width=1000, plot_height=680,
tools=['box_zoom, wheel_zoom, reset', hover],
title=ylabel+'/'+xlabel)
p.xaxis[0].formatter = PrintfTickFormatter(format="exp(%0.1f )")
p.yaxis[0].formatter = PrintfTickFormatter(format="exp(%0.1f )")
p.xaxis.axis_label = xlabel
p.yaxis.axis_label = ylabel
p.circle('x', 'y', source=source, fill_color='navy', hover_fill_color="firebrick",
fill_alpha=0.05, hover_alpha=0.3, size=5)
p.background_fill_color = "beige"
p.background_fill_alpha = 0.5
save(p)
def show_map(country_xs, country_ys, country_colours, country_names, country_users, outputname, plot_title):
print("Plotting values...")
source = ColumnDataSource(
data = dict(
x = country_xs,
y = country_ys,
colour = country_colours,
name = country_names,
users = country_users
)
)
# print(source)
#output_notebook #outputfile instead
output_file(outputname)
tools = 'pan,wheel_zoom,box_zoom,reset,hover,save'
p = figure(
title=plot_title,
tools=tools,
plot_width=800
)
p.patches('x','y',
fill_color = 'colour',
fill_alpha = 0.7,
line_color='white',
line_width=0.5,
source=source)
hover = p.select(dict(type=HoverTool))
hover.point_policy = 'follow_mouse'
hover.tooltips = OrderedDict([
('Name','@name'),
('Number of Users','@users')
])
save(p)
def plotNegativeCorrelations(stockList, plotType='m'):
#stockList is a list of the stocks to have their correlations compared
#containing the names of thes secondary stocks, plotType = 'm' (matplotlib) or 'b'(Bokeh)
fracNeg=[]
names =[]
#Calculate the fraction of years that have a negative correlation for all pairs
# of stocks in stockList
for i in xrange(len(stockList)-1):
stockA = stockList[i]
for stockB in stockList[i+1:]:
#Calculates negative correlation fraction
fracNeg.append(negativeCorrelation(stockA,stockB))
#Gives names of stock pairs. For x tick labels
names.append(stockA.stockName + '--' + stockB.stockName)
#Plot as matplotlib
if plotType=='m':
plt.figure()
plt.bar(range(len(fracNeg)), fracNeg, width=0.9)
plt.ylabel('Proportion of years')
plt.xlabel('Stock pairing')
plt.title("Proportion of Years the Stock Correlation Coefficient is Less than 0.")
plt.xticks(np.arange(len(fracNeg))+.45, names, rotation='horizontal')
plt.show()
#Plots data using Bokeh
elif plotType=='b':
output_file("correlationsBar.html", title = "Proportion of Years the Stock Correlation Coefficient is Less than 0.")
df = pd.DataFrame({'values':fracNeg, 'names':names})
p = figure(plot_width=800, plot_height=350, title="Correlations with S&P500")
p = Bar(df, 'names', values='values', title = "Proportion of Years the Stock Correlation Coefficient is Less than 0.",
xlabel="Stock pairing", ylabel="Proportion of years")
#show(p)
save(p)
def plotData2(df, df2):
dates = list(zip(*df['data'])[0])
for i in range(0,len(dates)):
dates[i] = datetime.strptime(dates[i],'%Y-%m-%d')
values = list(zip(*df['data'])[1])
dates2 = list(zip(*df2['data'])[0])
for i in range(0,len(dates2)):
dates2[i] = datetime.strptime(dates2[i],'%Y-%m-%d')
values2 = list(zip(*df2['data'])[1])
output_file("templates/datetime.html")
# data = pandas.DataFrame(dict(stock1=values, Date=dates, stock2=values2))
# col_names = [app.vars['name_stock'], 'Date', app.vars['name_stock_second']]
# data.columns[0] = app.vars['name_stock']
# data.columns[2] = app.vars['names_stock_second']
# p = TimeSeries(data, index='Date', title=app.vars['name_stock']+' and '+app.vars['name_stock_second'], ylabel=app.vars[app.vars['button']], legend=True, xlabel='date', width=800, height=400)
p = figure(width=800, height=500, x_axis_type="datetime", title="Stock Ticker Information")
p.xaxis.axis_label = 'date'
p.yaxis.axis_label = app.vars[app.vars['button']]
p.line(dates, values, color='navy', alpha=0.5)
p.circle(dates, values, color='navy', alpha=0.5, legend=app.vars['name_stock'])
p.line(dates2, values2, color='red', alpha=0.5)
p.circle(dates2, values2, color='red', alpha=0.5, legend=app.vars['name_stock_second'])
# p.circle(dates2, values2, color='red', alpha=0.5)
save(p)
def plot_summaries(output_directory, normalized_counts, chromosomes):
bp.output_file(output_directory + "/summary.html")
for chromosome in chromosomes:
plot_summary(normalized_counts, chromosome)
bp.save()
def submit():
ticker = request.form['ticker']
ticker = ticker.upper()
stock = yf.Share(ticker)
begin_year = int(request.form['begin_year'])
begin_month = int(request.form['begin_month'])
begin_day = int(request.form['begin_day'])
end_year = int(request.form['end_year'])
end_month = int(request.form['end_month'])
end_day = int(request.form['end_day'])
begin = datetime.date(begin_year, begin_month, begin_day)
end = datetime.date(end_year, end_month, end_day)
hist = stock.get_historical(str(begin), str(end))
years = [int(x['Date'].split("-")[0]) for x in hist]
months = [int(x['Date'].split("-")[1]) for x in hist]
days = [int(x['Date'].split("-")[2]) for x in hist]
x = [datetime.date(y, m, d) for y, m, d in zip(years, months, days)]
price = request.form['features']
if price == 'Open':
y = [y['Open'] for y in hist]
if price == 'High':
y = [y['High'] for y in hist]
if price == 'Close':
y = [y['Close'] for y in hist]
if price == 'Adj. Close':
y = [y['Adj_Close'] for y in hist]
bp.output_file("templates/results.html", title = "Stock Results")
p = bp.figure(tools="pan,box_zoom,reset,save", title = ticker + ' (' + price + ")",
x_axis_label='date', x_axis_type = "datetime", y_axis_label='price ($)')
p.line(x, y, line_width = 2)
bp.save(p)
return render_template('results.html')
def write_tour_to_img(coords,tour,title,img_file_name):
from bokeh.plotting import figure, output_file, show, save
import numpy as np
# from bokeh.models.mappers import LinearColorMapper
# output to static HTML file
output_file(img_file_name)
p = figure(plot_width=400, plot_height=400)
num_cities=len(tour)
color_list = ["#%02x%02x%02x" % (r, g, 255./num_cities) for r, g in zip(np.arange(0,255,255./num_cities), np.arange(0,255,255./num_cities))]
# print(color_list)
# color_list = ["#%02x%02x%02x" % (r, g, num_cities) for r, g in zip(np.floor(50+2*x), np.floor(30+2*y))]
for i in range(num_cities):
# print(color_list[i])
j=(i+1)%num_cities
city_i=tour[i]
city_j=tour[j]
x1,y1=coords[city_i]
x2,y2=coords[city_j]
p.line([x1,x2],[y1,y2],color=color_list[i])
p.circle([x1,x2],[y1,y2],color=color_list[i])
# show the results
show(p)
save(p)
def run(input_files, output_files):
# Read reaction sequences
stats = gazelib.io.load_json(input_files[0])
# Collect figures here.
figs = []
# Sequence completeness distribution
d = {
'num saccades captured': stats['total']['completeness'],
'sequences': [1] * len(stats['total']['completeness'])
}
p = Bar(d, agg='count', label='num saccades captured', values='sequences', color='black')
figs.append(p)
# Saccademodel MSE distribution
mses = stats['total']['all_mses']
d = { 'MSE': list(filter(lambda x: x < 0.1, mses)) }
p = Histogram(d, bins=40, values='MSE', color='black')
figs.append(p)
# Visualization with bokeh. Combine figures in vertical layout.
p = plotting.vplot(*figs)
plotting.output_file(output_files[0],
'Saccadic reaction time variability')
plotting.save(p)
def plot_log_data(sa_log_data, enable_plotting):
if enable_plotting:
from bokeh.plotting import figure, output_file, show, save
import bokeh.plotting as bp
import numpy as np
# output to static HTML file
IMG_FILE_NAME = 'way_down.html'
bp.output_file(IMG_FILE_NAME)
p = figure(plot_width=400, plot_height=400)
points_num = len(sa_log_data)
color_list = ["#%02x%02x%02x" % (r, g, 255./points_num) for r, g in zip(np.arange(0,255,255./points_num), np.arange(0,255,255./points_num))]
# print(color_list)
# color_list = ["#%02x%02x%02x" % (r, g, points_num) for r, g in zip(np.floor(50+2*x), np.floor(30+2*y))]
import numpy as np
nparr = np.array(sa_log_data)
x_list = list(range(points_num))
p.circle(x_list,nparr[:,1],color=color_list)
t_np_arr = np.array(x_list)+nparr[:,0]
p.circle(t_np_arr,nparr[:,1],color=color_list)
# show the results
show(p)
save(p)
def Main():
# Taking the arguments from command line.
parser = argparse.ArgumentParser()
parser.add_argument("-st", dest="stationId", nargs = '*', help="Please provide a list of station Id.")
parser.add_argument("-ct", dest="cityName", nargs = '*', help="Please provide a list of city names corresponding to stations.")
args = parser.parse_args()
cityData = {}
cities = {
args.cityName[0] : {'ID':args.stationId[0]},
args.cityName[1] : {'ID':args.stationId[1]},
args.cityName[2] : {'ID':args.stationId[2]}
}
for c in cities.keys():
cityData[c] = DataSet(c)
plot = make_plot(cityData)
output_file("./Plots/Op4.html", title="Optional Task # 4")
save(plot)
scr, div = components(plot)
fs = open("./Plots/Op4.scr", 'w')
fs.write(scr)
fd = open("./Plots/Op4.div", 'w')
fd.write(div)
print(div)
def visualize_data(data_output_file):
from bokeh.plotting import figure, output_file, save, vplot
from bokeh.models import ColumnDataSource, FactorRange
plots = []
# output to static HTML file
output_file(data_output_file, title='Eye Tracking Data', mode='cdn')
for file_name, file_data in data.iteritems():
# prepare the data
line_types = [line_data[0] for line_data in file_data]
#line_types_top = [line_data[0] + '2.0' for line_data in file_data]
#line_types_bot = [line_data[0] + '1.0' for line_data in file_data]
times = [line_data[1] for line_data in file_data]
start_times = [line_data[2] for line_data in file_data]
end_times = [line_data[3] for line_data in file_data]
descs = [line_data[4] for line_data in file_data]
#x0 = [0 for type_ in line_types]
'''source = ColumnDataSource({
'time': times,
'start': start_times,
'end': end_times,
'line_types_top': line_types_top,
'line_types_bot': line_types_bot,
})'''
source = ColumnDataSource({
'x': [(t1 + t2)/2 for t1, t2 in zip(start_times, end_times)],
'y': line_types,
'width': times,
'height': [0.7 for _ in line_types],
'fill_color': ['red' if 'Incorrect' in desc \
else 'green' for desc in descs]
})
# create a new plot
plot = figure(
tools='pan,reset,save',
title=file_name, y_range=FactorRange(factors=line_types[::-1]),
x_range=[0, sum(times)],
x_axis_label='Time (ms)', y_axis_label='Line Type'
)
plot.rect(x='x', y='y', width='width', height='height', \
fill_color='fill_color', source=source)
#plot.quad(left='start', right='end', \
# top='line_types_top', bottom='line_types_bot', source=source)
# add some renderers
#plot.segment(x0, line_types, times, line_types, \
# line_width=2, line_color="green")
plots.append(plot)
# show the results
save(vplot(*plots))
def main(target, regulator=None, fasta=None, output_dir=None,
assembly='hg38', datadir=None, n_proc=1, ensembl_gtf=None):
if output_dir is None:
output_dir = Path.cwd()
bt = pybedtools.BedTool(target)
if regulator:
bt = bt.filter(lambda x: regulator in x.name).saveas()
df = bt.to_dataframe()
if fasta:
df['seq'] = get_sequences(bt, fasta=fasta)
output_file(str(output_dir / 'score_dist.html'))
save(plot_hist(df['score'], logx=True, title='Score'))
output_file(str(output_dir / 'peak_length.html'))
save(plot_hist(df['end'] - df['start'], title='Peak length'))
output_file(str(output_dir / 'count_per_chr.html'))
save(plot_chr_counts(assembly, df))
output_file(str(output_dir / 'count_per_feature.html'))
save(plot_feat_counts(bt, datadir, n_proc=n_proc))
output_file(str(output_dir / 'count_per_biotype.html'))
save(plot_biotype_counts(bt, ensembl_gtf))
return 0
def createChart(t,d):
ticker = t
trading_days = int(d)
try:
df = Quandl.get('WIKI/' + ticker, authtoken=q_key)[-trading_days:]
ptitle = ticker + ' Closing Price: last ' + str(trading_days) + ' trading days'
max_split = max(df['Split Ratio'])
if max_split > 1:
split_date = np.argmax(df['Split Ratio'])
df['Multiply'] = [split_date > d for d in df.index]
df['AdjClose'] = [df.Close[x]*(1 - (max_split - 1)/max_split*df.Multiply[x].sum()) for x in np.arange(len(df))]
df['Close'] = df['AdjClose']
output_file("stockplot.html", title='Close Plot')
p = figure(x_axis_type = 'datetime')
p.line(df.index, df['Close'], color='black')
p.title = ptitle
p.xaxis.axis_label = 'Date'
p.yaxis.axis_label = 'Price'
save(p)
st = df['Close'][0]
ed = df['Close'][-1]
cdgr = round((ed-st)/st*100.,2)
script, div = components(p)
p_dict = {'script':script, 'div':div, 'cdgr':cdgr}
except Quandl.Quandl.DatasetNotFound:
p_dict = {'script':None, 'div':None, 'cdgr':None}
return p_dict
def compute(A, b, w, T, resolution=500):
"""Return filename of plot of the damped_vibration function."""
t = linspace(0, T, resolution+1)
u = damped_vibrations(t, A, b, w)
# output to static HTML file
outname = 'tmp.html'
plt.output_file(outname, title="Damped vibrations")
# create a new plot with a title and axis labels
TOOLS = "pan,wheel_zoom,box_zoom,reset,save,box_select,lasso_select"
p = plt.figure(title="simple line example", tools=TOOLS,
x_axis_label='t', y_axis_label='y')
# add a line renderer with legend and line thickness
p.line(t, u, legend="u(t)", line_width=2)
# show the results
#plt.show(p)
plt.save(p)
with open(outname, 'r') as f:
text = f.read()
# Grab head and body
m = re.search('<head>(.+)</head>.+<body>(.+)</body>', text,
flags=re.DOTALL)
if m:
head = m.group(1)
body = m.group(2)
else:
raise ValueError('Could not parse the bokeh output in tmp.html')
#os.remove(outname)
return head, body
def plotCorrelations(primaryStock, secondaryStocks, plotType='m'):
#primaryStock is the name of the primary Stock, secondaryStock is a list
#containing the names of thes secondary stocks, plotType = 'm' (matplotlib) or 'b'(Bokeh)
if plotType=='m':
plt.figure()
elif plotType=='b':
output_file(primaryStock.stockName+"correlations.html", title = "Correlations with " + primaryStock.stockName)
colors = ['green','red','mediumblue','orange','purple','teal','gold']
i=0
p = figure(plot_width=1200, plot_height=600, title="Correlations with " + primaryStock.stockName)
for stock in secondaryStocks:
yearData, corrAB = primaryStock.stockCorrelation(stock)
if plotType=='m':
plt.plot(yearData,corrAB,'-o')
elif plotType=='b':
p.line(yearData, corrAB, line_width=2, line_color=colors[i])
p.circle(yearData, corrAB, line_width=2, line_color=colors[i], legend = stock.stockName, fill_color="white", size=10)
i+=1
if plotType=='m':
plt.xlabel('Year')
plt.ylabel('Correlation coefficient')
plt.title("Correlations with " + primaryStock.stockName)
plt.legend([n.stockName for n in secondaryStocks], loc=3)
plt.show()
elif plotType=='b':
p.xaxis.axis_label = "Year"
p.yaxis.axis_label = 'Correlation coefficient'
p.legend.location = "bottom_left"
#show(p)
save(p)
def Main():
# parser = argparse.ArgumentParser()
# parser.add_argument("-st", dest="stationId", nargs = '*', help="Please provide a list of station Id.")
# parser.add_argument("-ct", dest="cityName", nargs = '*', help="Please provide a list of city names corresponding to stations.")
# args = parser.parse_args()
#city = args.cityName[0]
#cities = {
# args.cityName[0] : {'ID':args.stationId[0]},
# args.cityName[1] : {'ID':args.stationId[1]},
# args.cityName[2] : {'ID':args.stationId[2]}
#}
# for c in cities.keys():
global source
source = DataSet('St_Johns')
plot = make_plot(source, city)
city_select.on_change('value', update_plot)
# add to document
output_file("./Plots/Op5.html", title="Optional Task # 5")
save(HBox(city_select, plot))
curdoc().add_root(HBox(city_select, plot))
def plot_summaries(output_directory, normalized_counts, chromosomes):
bp.output_file(output_directory + "/summary.html", mode="cdn")
plots = []
for chromosome in chromosomes:
plot = plot_summary(normalized_counts, chromosome)
if plot:
plots.append(plot)
bp.save(bp.vplot(*plots))
def plot_summaries(chromosomes):
bp.output_file("summary.html")
for chromosome in chromosomes:
common_clause = " counter_version = %s AND chromosome = '%s' " % (version, chromosome)
plot_summary(chromosome, common_clause)
bp.save()
def plot(self, identifier, chart, data, width):
chart_title = chart + " - " + identifier;
y_label = "Social Cost (2007 US $)";
if(chart == "emissions"):
y_label = "Total emissions (1E12 g)";
output_file("plots/tables/" + chart + "/plots/" + identifier + "_" + chart + "_visualization.html", title = chart_title);
p = Bar(data, width=width, height=700, title=chart_title, xlabel="Source", ylabel=y_label, legend="top_left", stacked=True, palette=self.colors, tools=self.TOOLS);
save(p);
def plot(self):
"""
Create the performance profile using matplotlib.
"""
if not self.force:
try:
file_ = open(self.output, 'r')
file_.close()
raise ValueError(_('ERROR: File {} exists.\nUse `-f` to overwrite').format(self.output))
except FileNotFoundError:
pass
if not self.already_scaled:
self.scale()
try:
self.ppsbt
except AttributeError:
self.set_percent_problems_solved_by_time()
plt.output_file(self.output, title=self.plot_lang(self.title))
# Axis
try:
maxt = min(max(self.times), self.tau)
except (AttributeError, TypeError):
maxt = max(self.times)
boken_plot_options = {"x_range":[1,maxt], "y_range":[0,1]}
# Change the xscale to log scale
if self.semilog:
boken_plot_options["x_axis_type"] = "log"
p = plt.figure(title=self.plot_lang(self.title),
x_axis_label=self.plot_lang(self.ylabel),
y_axis_label=self.plot_lang(self.xlabel),
**boken_plot_options)
for idx, solver in enumerate(self.solvers):
p.line(self.times,
self.ppsbt[solver],
legend=solver,
line_width=2,
line_color=BOKEH_COLOR_LIST[idx % len(BOKEH_COLOR_LIST)])
# Legend
p.legend.orientation = "bottom_right"
# Help lines
p.grid.grid_line_color = "black"
p.grid.grid_line_alpha = 0.5
# Save the plot
plt.save(p)
def save(self):
"""Save the figure at the specified location."""
if self.interactive:
interactive_figure = self.interactive_figure()
save(interactive_figure, filename=SAVE_FOLDER + self.plotName)
else:
self.plot_figure()
save(obj=self.fig, filename=SAVE_FOLDER + self.plotName)
logging.debug('Figure saved')
def main(plot_fname="gmap_example_bokeh.html"):
output_file(plot_fname)
tver_coords = {u'lat':56.8583600, u'lng':35.9005700}
plot_created = create_plot(tver_coords,zoom_level = 13)
lats=[56.8583600, 56.8583600*1.0001, 56.8583600*1.0002]
lngs=[35.9005700, 35.9005700*1.0001, 35.9005700*1.0002]
coords_dict_list = [{u'lat':la, u'lng':ln} for la,ln in zip(lats,lngs)]
add_line(plot_created,coords_dict_list,circle_size=10)
save(plot_created)
def produce_plot(mean_window=1):
'''
'''
try:
timestamps, moistures, voltages = [], {'moisture': []}, {'voltage': []}
for data_point in data_store.find():
moisture = float(data_point.get('soil moisture', 0))
moistures['moisture'].append(moisture)
avg_moist_df = pd.DataFrame(moistures)
avg_moist = avg_moist_df.rolling(center=False,
window=mean_window).mean()
voltage = float(data_point.get('voltage', 0))
voltages['voltage'].append(voltage)
avg_voltage_df = pd.DataFrame(voltages)
avg_voltage = avg_voltage_df.rolling(center=False,
window=mean_window).mean()
# UTC -> PDT, server is 3 hours fast
timestamp = data_point.get('timestamp') - 43200
timestamps.append(timestamp * 1000)
output_file(config.bokeh_output_dir + 'lines.html')
p = figure(title='ESP8266 Sensor #1',
x_axis_label='time (UTC-8:00)',
x_axis_type='datetime',
y_axis_label='voltage',
tools='',
y_range=(3, 4.3))
p.line(timestamps,
avg_voltage,
legend="voltage",
line_width=2,
color='red')
# moisture
p.extra_y_ranges = {'moisture': Range1d(start=-5, end=100)}
p.line(timestamps,
avg_moist,
legend="moisture",
line_width=2,
color='blue',
y_range_name='moisture')
p.add_layout(LinearAxis(y_range_name='moisture',
axis_label='moisture'),
'right')
save(p)
except:
sentry.captureMessage('produce plot failure')
sentry.captureException()
def save(self):
self.activateHoverTools()
plotting.save(self.plot)
Created on Wed Jan 17 14:35:34 2018 @author: cscuser """ import geopandas as gpd from bokeh.plotting import save, figure from bokeh.models import GeoJSONDataSource #Filepaths address_fp = r"C:\Users\cscuser\Desktop\TS\INTERACTIVEMAPS\addresses.shp" #Read the data address = gpd.read_file(address_fp) #Reproject address = address.to_crs(epsg=3067) #Convert the GeoDataFrame point_src = GeoJSONDataSource(geojson=address.to_json()) #Initialize p = figure(title='Station map') p.circle(x='x', y='y', source=point_src, color='green', size=8) #Save to file outfp = r"C:\Users\cscuser\Desktop\TS\INTERACTIVEMAPS\address_map.html" save(p, outfp)
def train(batch_size, num_train_steps, model_dir, beat_type):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
ode_params = ODEParams(device)
#
# Support for tensorboard:
#
writer = SummaryWriter(model_dir)
#
# 1. create the ECG dataset:
#
composed = transforms.Compose([ecg_dataset_pytorch.Scale(), ecg_dataset_pytorch.ToTensor()])
dataset = ecg_dataset_pytorch.EcgHearBeatsDataset(transform=composed, beat_type=beat_type, lstm_setting=False)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
shuffle=True, num_workers=1)
print("Size of real dataset is {}".format(len(dataset)))
#
# 2. Create the models:
# netG = ode_gan_aaai.DCGenerator(0).to(device)
# netD = ode_gan_aaai.DCDiscriminator(0).to(device)
# netD.apply(weights_init)
# netG.apply(weights_init)
netG = vanila_gan.VGenerator(0).to(device)
netD = vanila_gan.VDiscriminator(0).to(device)
#
# Define loss functions:
#
cross_entropy_loss = nn.BCELoss()
mse_loss = nn.MSELoss()
#
# Optimizers:
#
lr = 0.0002
beta1 = 0.5
writer.add_scalar('Learning_Rate', lr)
optimizer_d = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.999))
optimizer_g = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.999))
#
# Noise for validation:
#
val_noise = torch.Tensor(np.random.uniform(0, 1, (4, 100))).to(device)
# val_noise = torch.Tensor(np.random.normal(0, 1, (4, 100))).to(device)
#
# Training loop"
#
epoch = 0
iters = 0
while True:
num_of_beats_seen = 0
if iters == num_train_steps:
break
for i, data in enumerate(dataloader):
if iters == num_train_steps:
break
netD.zero_grad()
#
# Discriminator from real beats:
#
ecg_batch = data['cardiac_cycle'].float().to(device)
b_size = ecg_batch.shape[0]
num_of_beats_seen += ecg_batch.shape[0]
output = netD(ecg_batch)
labels = torch.full((b_size,), 1, device=device)
ce_loss_d_real = cross_entropy_loss(output, labels)
writer.add_scalar('Discriminator/cross_entropy_on_real_batch', ce_loss_d_real.item(), global_step=iters)
writer.add_scalars('Merged/losses', {'d_cross_entropy_on_real_batch': ce_loss_d_real.item()},
global_step=iters)
ce_loss_d_real.backward()
mean_d_real_output = output.mean().item()
#
# Discriminator from fake beats:
#
noise_input = torch.Tensor(np.random.uniform(0, 1, (b_size, 100))).to(device)
# noise_input = torch.Tensor(np.random.normal(0, 1, (b_size, 100))).to(device)
output_g_fake = netG(noise_input)
output = netD(output_g_fake.detach()).to(device)
labels.fill_(0)
ce_loss_d_fake = cross_entropy_loss(output, labels)
writer.add_scalar('Discriminator/cross_entropy_on_fake_batch', ce_loss_d_fake.item(), iters)
writer.add_scalars('Merged/losses', {'d_cross_entropy_on_fake_batch': ce_loss_d_fake.item()},
global_step=iters)
ce_loss_d_fake.backward()
mean_d_fake_output = output.mean().item()
total_loss_d = ce_loss_d_fake + ce_loss_d_real
writer.add_scalar(tag='Discriminator/total_loss', scalar_value=total_loss_d.item(),
global_step=iters)
optimizer_d.step()
netG.zero_grad()
labels.fill_(1)
output = netD(output_g_fake)
#
# Add euler loss:
#
delta_hb_signal, f_ode_z_signal = ode_loss(output_g_fake, ode_params, device, beat_type)
mse_loss_euler = mse_loss(delta_hb_signal, f_ode_z_signal)
logging.info("MSE ODE loss: {}".format(mse_loss_euler.item()))
ce_loss_g_fake = cross_entropy_loss(output, labels)
total_g_loss = mse_loss_euler + ce_loss_g_fake
# total_g_loss = mse_loss_euler
total_g_loss.backward()
writer.add_scalar(tag='Generator/mse_ode', scalar_value=mse_loss_euler.item(), global_step=iters)
writer.add_scalar(tag='Generator/cross_entropy_on_fake_batch', scalar_value=ce_loss_g_fake.item(),
global_step=iters)
writer.add_scalars('Merged/losses', {'g_cross_entropy_on_fake_batch': ce_loss_g_fake.item()},
global_step=iters)
mean_d_fake_output_2 = output.mean().item()
optimizer_g.step()
if iters % 50 == 0:
print("{}/{}: Epoch #{}: Iteration #{}: Mean D(real_hb_batch) = {}, mean D(G(z)) = {}."
.format(num_of_beats_seen, len(dataset), epoch, iters, mean_d_real_output, mean_d_fake_output),
end=" ")
print("mean D(G(z)) = {} After backprop of D".format(mean_d_fake_output_2))
print("Loss D from real beats = {}. Loss D from Fake beats = {}. Total Loss D = {}".
format(ce_loss_d_real, ce_loss_d_fake, total_loss_d), end=" ")
print("Loss G = {}".format(ce_loss_g_fake))
#
# Norma of gradients:
#
gNormGrad = get_gradient_norm_l2(netG)
dNormGrad = get_gradient_norm_l2(netD)
writer.add_scalar('Generator/gradients_norm', gNormGrad, iters)
writer.add_scalar('Discriminator/gradients_norm', dNormGrad, iters)
print(
"Generator Norm of gradients = {}. Discriminator Norm of gradients = {}.".format(gNormGrad, dNormGrad))
if iters % 25 == 0:
with torch.no_grad():
with torch.no_grad():
output_g = netG(val_noise)
fig = plt.figure()
plt.title("Fake beats from Generator. iteration {}".format(i))
for p in range(4):
plt.subplot(2, 2, p + 1)
plt.plot(output_g[p].cpu().detach().numpy(), label="fake beat")
plt.plot(ecg_batch[p].cpu().detach().numpy(), label="real beat")
plt.legend()
writer.add_figure('Generator/output_example', fig, iters)
plt.close()
#
# Add bokeh plot:
#
p = figure(x_axis_label='Sample number (360 Hz)', y_axis_label='Voltage[mV]')
time = np.arange(0, 216)
fake_beat = output_g[0].cpu().detach().numpy()
w = 'hanning'
smoothed_beat = smooth_signal.smooth(fake_beat, 10, w)
p.line(time, smoothed_beat, line_width=2, line_color="blue")
output_file("N_{}_ODE.html".format(iters))
save(p)
if iters % 50 == 0:
torch.save({
'epoch': epoch,
'generator_state_dict': netG.state_dict(),
}, model_dir + '/checkpoint_epoch_{}_iters_{}'.format(epoch, iters))
iters += 1
epoch += 1
writer.close()
def new_time_series(start,
end,
y_range=None,
country='world',
variable='new',
use_top_n=False,
log=False):
""" Show the time series of new cases.
variable = 'new_7_days' means we are using the last cases in the last week for each day """
# Explanatory text for each variable
variable_str = {
'new': '',
# 'new_7_days': 'en los últimos 7 días'
'new_7_days': '(promedio de 7 días)'
}
# Start and end indices
start_index, end_index = get_start_end(start, end)
# Choose title
title = "Casos nuevos %s en " % variable_str[variable]
title = "Casos nuevos (%s) en " % variable_str[variable]
if country == 'world':
addstr = 'el mundo'
elif country == 'Spain':
addstr = 'España'
elif country == 'Colombia':
addstr = 'Colombia'
title = title + addstr
data = get_new_7_days(start_index,
end_index,
variable,
country=country,
avg=True)
# Bokeh figure for the country
# Figure
if log:
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
y_axis_type="log",
)
else:
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
)
if y_range is not None:
p.y_range = Range1d(*y_range)
# Add a circle renderer with a size, color and alpha
p.circle('date_obj',
variable,
source=data,
size=5,
color="black",
alpha=0.5)
p.line('date_obj',
variable,
source=data,
line_width=2,
color="black",
alpha=0.5)
# Arrange figure
# p.x_range = Range1d(data['date_obj'][0], data['date_obj'][-1])
p.xaxis.axis_label = 'Fecha'
# p.yaxis.axis_label = 'Casos nuevos en los últimos 7 días'
p.yaxis.axis_label = 'Casos nuevos (promedio de 7 días)'
p.xaxis.formatter = DatetimeTickFormatter(days="%d %B")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# Output to static HTML file
output_file(
os.path.join(ws.folders["website/static/images"],
"%s_vs_date_%s.html" % (variable, country.lower())))
# show(p)
save(p)
######################################
# Bokeh figure for all the countries
if country == 'world':
# Hover tool indicating the country
# hover2 = HoverTool(tooltips=[('País', '@country'), ('Fecha', '@date'), ('Casos nuevos (últ. 7d)', '@new_7_days')])
hover2 = HoverTool(tooltips=[('País', '@country'), (
'Fecha', '@date'), ('Casos nuevos (prom. 7 días)', '@new_7_days')])
# Figure
if log:
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=
"Casos nuevos %s en los %i países con más casos confirmados" %
(variable_str[variable], ws.ntop),
# tools = [hover2],
y_axis_type="log",
)
else:
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=
"Casos nuevos %s en los %i países con más casos confirmados" %
(variable_str[variable], ws.ntop),
# tools = [hover2],
)
p.add_tools(hover2)
if y_range is not None:
p.y_range = Range1d(*y_range)
df = ws.data_countries_only
countries = set(df.country_region)
if use_top_n:
countries = ws.top_ten
for i, country in enumerate(countries):
data = get_new_7_days(start_index,
end_index,
variable,
country=country,
avg=True)
# Add a circle renderer with a size, color and alpha
p.circle('date_obj',
variable,
source=data,
color=Category10[10][i],
size=5,
alpha=1.,
legend_label=country)
p.line('date_obj',
variable,
source=data,
color=Category10[10][i],
line_width=2,
alpha=1.,
legend_label=country)
# Arrange figure
# p.x_range = Range1d(data['date_obj'][0], data['date_obj'][-1])
p.xaxis.axis_label = 'Fecha'
p.yaxis.axis_label = 'Casos nuevos %s' % variable_str[variable]
p.xaxis.formatter = DatetimeTickFormatter(days="%d %B")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.legend.location = 'top_left'
p.toolbar.logo = None
# Output to static HTML file
use_log_str = 'logscale' if log else ''
if use_top_n:
output_file(
os.path.join(
ws.folders["website/static/images"],
"%s_vs_date_top_n%01i_%s.html" %
(variable, ws.ntop, use_log_str)))
else:
output_file(
os.path.join(
ws.folders["website/static/images"],
" %s_vs_date_whole_world_%s.html" %
(variable_str[variable], use_log_str)))
# show(p)
save(p)
def compare_countries(start,
end,
variable='confirmed',
countries=None,
label='',
title_add=''):
""" Show the time series of the world in a HTML graph """
# Get data
df = ws.data_countries_only
# Date range
start_index, end_index = get_start_end(start, end)
# Choose title
title = "Casos %s%s" % (ws.trans[variable], title_add)
# Hover tool indicating the country
hover = HoverTool(tooltips=[('País', '@country'), (
'Fecha',
'@date_key'), ('Casos %s' % ws.trans[variable], '@%s' % variable)])
# Figure
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
# tools=[hover],
)
p.add_tools(hover)
if countries is None:
countries = []
# Choose graph options depending on the number of countries
ncountries = len(countries)
# category20 = list(Category20.values())
category20 = Category20[20]
category10 = Category10[10]
many_countries = ncountries > len(category20)
alpha = 1.
legend = True
add_text = False
if many_countries:
colors = ncountries * ['black']
alpha = 0.2
legend = False
add_text = True
elif ncountries > 10:
colors = category20[:]
else:
colors = category10[:]
# Iterate over countries
for i, country in enumerate(countries):
# Get data for the country
df_ = df[df['country_region'] == country]
data = {}
data['date_obj'], data[variable] = get_single_time_series(
df_, variable, start_index, end_index)
# data['date_key'] = [x.strftime("%d/%m/%Y") for x in data['date_obj']]
data['date_key'] = ws.dates_keys[start_index:end_index + 1]
data['country'] = len(data[variable]) * [country]
# Plot data
if not many_countries:
p.circle('date_obj',
variable,
source=data,
color=colors[i],
size=5,
alpha=alpha,
legend_label=country)
p.line('date_obj',
variable,
source=data,
color=colors[i],
line_width=2,
alpha=alpha,
legend_label=country)
else:
p.circle('date_obj',
variable,
source=data,
color=colors[i],
size=5,
alpha=alpha)
p.line('date_obj',
variable,
source=data,
color=colors[i],
line_width=2,
alpha=alpha)
# p.text(x=data['date_obj'], y=data[variable], text=[country],text_baseline="middle", text_align="left")
p.text(x=data['date_obj'][-1],
y=data[variable][-1],
text=[country])
# Arrange figure
p.xaxis.axis_label = 'Fecha'
p.yaxis.axis_label = 'Número de casos'
if legend:
p.legend.location = 'top_left'
p.xaxis.formatter = DatetimeTickFormatter(days="%d %B")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# Output to static HTML file
output_file(
os.path.join(ws.folders["website/static/images"],
"%s_%s_time_series.html" % (label, variable)))
# show(p)
save(p)
def visualize(self):
"""
Generates a plot using bokeh, which displays the initial trajectory and
the optimized trajectory of the cutting tool.
"""
# Tools that will be displayed on the plots
tools = "pan,wheel_zoom,reset,save"
# Plot displaying the optimized path
result_plot = figure(plot_width=1000,
plot_height=500,
tools=tools,
active_scroll='wheel_zoom')
result_plot.title.text = "Optimized Path"
# Plot displaying the non optimized path
initial_plot = figure(plot_width=1000,
plot_height=500,
tools=tools,
active_scroll='wheel_zoom')
initial_plot.title.text = "Initial Path"
# Add the data to the result plot
result_plot = self.populate_plot(result_plot, self.result)
result_plot.legend.location = "bottom_right"
# Add the data to the initial plot
initial_plot = self.populate_plot(initial_plot, self.initial)
initial_plot.legend.location = "bottom_right"
# Add cutting tool to plots
# Generate the points on which the triangle should move on
result_lines_x, result_lines_y = self.generate_tool_path(
self.result, 1)
initial_lines_x, initial_lines_y = self.generate_tool_path(
self.initial, 1)
# Add cutting tool triangle to optimized path
result_triangle_position = ColumnDataSource(
data=dict(x=[result_lines_x[0]], y=[result_lines_y[0]]))
result_triangle = Triangle(x='x',
y='y',
line_color=Category10_4[3],
line_width=3,
size=20,
fill_alpha=0)
result_plot.add_glyph(result_triangle_position, result_triangle)
# Add cutting tool triangle to initial path
initial_triangle_position = ColumnDataSource(
data=dict(x=[initial_lines_x[0]], y=[initial_lines_y[0]]))
initial_triangle = Triangle(x='x',
y='y',
line_color=Category10_4[3],
line_width=3,
size=20,
fill_alpha=0)
initial_plot.add_glyph(initial_triangle_position, initial_triangle)
# Add button to start moving the triangle
button = Button(label='Start')
result_num_steps = result_lines_x.shape[0]
initial_num_steps = initial_lines_x.shape[0]
num_steps = max(result_num_steps, initial_num_steps)
# JavaScript callback which will be called once the button is pressed
callback = CustomJS(args=dict(
result_triangle_position=result_triangle_position,
result_lines_x=result_lines_x,
result_lines_y=result_lines_y,
result_num_steps=result_num_steps,
initial_triangle_position=initial_triangle_position,
initial_lines_x=initial_lines_x,
initial_lines_y=initial_lines_y,
initial_num_steps=initial_num_steps,
num_steps=num_steps),
code="""
// Animate optimal path plot
for(let i = 0; i < num_steps; i += 50) {
setTimeout(function() {
if (i < result_num_steps) {
result_triangle_position.data['x'][0] = result_lines_x[i]
result_triangle_position.data['y'][0] = result_lines_y[i]
}
if (i < initial_num_steps) {
initial_triangle_position.data['x'][0] = initial_lines_x[i]
initial_triangle_position.data['y'][0] = initial_lines_y[i]
}
result_triangle_position.change.emit()
initial_triangle_position.change.emit()
}, i)
}
""")
# Add callback function to button, which starts the whole animation
button.js_on_click(callback)
# Save the plot
result_plot = row([result_plot, button])
plot = column([result_plot, initial_plot])
output_file("visualization.html", title="CNC Path Optimization")
save(plot)
def save_outputs(results_dir, user_prediction_dir, epoch, perf, outputs, labels_tensor, original_boundaries,
training):
"""Save predictions and labels for one shift of every song in batch"""
try:
f_start = "training" if training is True else "testing"
ts = str(calendar.timegm(time.gmtime()))
bplt.output_file(os.path.join(
user_prediction_dir, f_start + "_epoch_" + str(epoch) + "_" + ts + "_" + perf + ".html"))
# load the original, in-tune pitch track
pitch_track = np.load(os.path.join(pyin_directory, perf + ".npy"))
frames = len(pitch_track)
# convert to shape < notes, shifts >
outputs = np.squeeze(outputs)
labels = np.squeeze(labels_tensor)
# plot the shifts after applying them to the original frame indices
frame_outputs_0 = np.zeros(frames)
frame_labels_0 = np.zeros(frames)
frame_outputs_5 = np.zeros(frames)
frame_labels_5 = np.zeros(frames)
for i in range(len(labels[:, 0])):
frame_outputs_0[original_boundaries[i, 0]: original_boundaries[i, 1]] += outputs[i, 0]
frame_labels_0[original_boundaries[i, 0]: original_boundaries[i, 1]] += labels[i, 0]
frame_outputs_5[original_boundaries[i, 0]: original_boundaries[i, 1]] += outputs[i, 5]
frame_labels_5[original_boundaries[i, 0]: original_boundaries[i, 1]] += labels[i, 5]
s1 = bplt.figure(title="Pitch shifts: ground truth versus predictions shift 0")
s1.line(np.arange(len(frame_labels_0)), frame_labels_0, color="red")
s1.line(np.arange(len(frame_outputs_0)), frame_outputs_0, color="blue")
s2 = bplt.figure(title="Pitch shifts: ground truth versus predictions shift 5")
s2.line(np.arange(len(frame_labels_5)), frame_labels_5, color="red")
s2.line(np.arange(len(frame_outputs_5)), frame_outputs_5, color="blue")
# shift pitch to get the de-tuned input to the neural net, then apply correction (negative of learned shift)
shifted_pitch_track_0 = np.copy(pitch_track)
corrected_pitch_track_0 = np.copy(pitch_track)
shifted_pitch_track_5 = np.copy(pitch_track)
corrected_pitch_track_5 = np.copy(pitch_track)
for i in range(len(labels[:, 0])):
shifted_pitch_track_0[original_boundaries[i, 0]: original_boundaries[i, 1]] *= \
np.power(2, max_semitone * labels[i, 0] / 12.0)
corrected_pitch_track_0[original_boundaries[i, 0]: original_boundaries[i, 1]] *= \
np.power(2, max_semitone * (labels[i, 0] - outputs[i, 0]) / 12.0)
shifted_pitch_track_5[original_boundaries[i, 0]: original_boundaries[i, 1]] *= \
np.power(2, max_semitone * labels[i, 5] / 12.0)
corrected_pitch_track_5[original_boundaries[i, 0]: original_boundaries[i, 1]] *= \
np.power(2, max_semitone * (labels[i, 5] - outputs[i, 5]) / 12.0)
s3 = bplt.figure(title="Original input versus de-tuned input before and after pitch correction shift 0",
y_range=(np.min(pitch_track[pitch_track > 10] - 50), np.max(pitch_track) + 50))
s3.line(np.arange(frames), pitch_track, color='black')
s3.line(np.arange(frames), shifted_pitch_track_0, color='red')
s3.line(np.arange(frames), corrected_pitch_track_0, color='green')
s4 = bplt.figure(title="Original input versus de-tuned input before and after pitch correction shift 5",
y_range=(np.min(pitch_track[pitch_track > 10] - 50), np.max(pitch_track) + 50))
s4.line(np.arange(frames), pitch_track, color='black')
s4.line(np.arange(frames), shifted_pitch_track_5, color='red')
s4.line(np.arange(frames), corrected_pitch_track_5, color='green')
bplt.save(bplt.gridplot([s1], [s2], [s3], [s4]))
np.save(os.path.join(
results_dir, f_start + "_epoch_" + str(epoch) + "_outputs" + "_" + perf), outputs)
np.save(os.path.join(
results_dir, f_start + "_epoch_" + str(epoch) + "_labels" + "_" + perf), labels)
except Exception as e:
logger.info("exception in save_outputs {0} skipping song {1}".format(e, perf))
return
#CheckboxGroup(labels=["Checkbox Option 7", "Checkbox Option 8", "Checkbox Option 9"], active=[0, 2]),
#CheckboxGroup(labels=["Checkbox Option 1", "Checkbox Option 2", "Checkbox Option 3"], active=[0, 1], inline=True),
#CheckboxGroup(labels=["Checkbox Option 4", "Checkbox Option 5", "Checkbox Option 6"], active=[1, 2], inline=True),
#CheckboxGroup(labels=["Checkbox Option 7", "Checkbox Option 8", "Checkbox Option 9"], active=[0, 2], inline=True),
#RadioGroup(labels=["Radio Option 1", "Radio Option 2", "Radio Option 3"], active=0),
#RadioGroup(labels=["Radio Option 4", "Radio Option 5", "Radio Option 6"], active=1),
#RadioGroup(labels=["Radio Option 7", "Radio Option 8", "Radio Option 9"], active=2),
#RadioGroup(labels=["Radio Option 1", "Radio Option 2", "Radio Option 3"], active=0, inline=True),
#RadioGroup(labels=["Radio Option 4", "Radio Option 5", "Radio Option 6"], active=1, inline=True),
#RadioGroup(labels=["Radio Option 7", "Radio Option 8", "Radio Option 9"], active=2, inline=True),
#Select(options=["Select Option 1", "Select Option 2", "Select Option 3"]),
#MultiSelect(options=["MultiSelect Option %d" % (i+1) for i in range(16)], size=6),
#Paragraph(text="Paragraph 1"),
#Paragraph(text="Paragraph 2"),
#Paragraph(text="Paragraph 3"),
#Div(text="Div 1"),
#Div(text="Div 2"),
#Div(text="Div 3"),
#PreText(text="PreText 1"),
#PreText(text="PreText 2"),
#PreText(text="PreText 3"),
save(layout)
def make_heatmap(samples, variant_dict, fig_name):
"""Create a heatmap of Samples x Variants."""
# Prepare data: keep only variants shared by at least one sample:
shared_variants = {}
for var_names, var_samples in variant_dict.items():
# There may be many variants on same position in same sample, so count *distinct* samples for each variant
if len(set([sample_name for (sample_name, _) in var_samples])) > 1:
shared_variants[var_names] = var_samples
if not shared_variants:
with open("{}.txt".format(fig_name.replace(" ", "")), "wt") as ofile:
ofile.write("No shared variants for selected samples.\n")
return
# Prepare data: make sample and variant list and NumPy array of AF.
x_names = sorted(shared_variants.keys())
y_names = sorted(samples)
data = np.zeros((len(x_names), len(y_names)))
for k, v in shared_variants.items():
for item in v:
variant_index = x_names.index(k)
sample_index = y_names.index(item[0])
data[variant_index, sample_index] = "{0:g}".format(
round(float(item[1]) * 100, DECIMALS))
width = min(len(x_names) * 100 + 200, 1100)
height = max(min(len(y_names) * 100, 580), 300)
# Create DataFrame and ColumnDataSource
df1 = pd.DataFrame(data=data, index=x_names, columns=y_names)
df1.index.name = "Variant"
df1.columns.name = "Sample"
df = pd.DataFrame(df1.stack(), columns=["af"]).reset_index()
source = ColumnDataSource(df)
p = figure(
title="Shared {} across samples".format(fig_name),
x_axis_location="above",
tools="hover,save,pan,box_zoom,wheel_zoom,reset",
plot_width=width,
plot_height=height,
x_range=x_names,
y_range=y_names,
toolbar_location="below",
)
p.grid.grid_line_color = None
p.axis.axis_line_color = None
p.axis.major_tick_line_color = None
p.axis.major_label_text_font_size = "5pt"
p.axis.major_label_standoff = 0
p.xaxis.major_label_orientation = None
p.xaxis.major_label_orientation = np.pi / 2
palette = list(reversed(palettes.YlGnBu[9]))
low, high = np.amin(data), np.amax(data)
if low == high:
low, high = 0.0, 100.0
mapper = LinearColorMapper(palette=palette, low=low, high=high)
p.rect(
"Variant",
"Sample",
1,
1,
source=source,
fill_color={
"field": "af",
"transform": mapper
},
line_color=None,
hover_line_color="black",
)
p.select_one(HoverTool).tooltips = [
("Sample", "@Sample"),
("Variant", "@Variant"),
("Allele Frequency (%)", "@af"),
]
color_bar = ColorBar(
color_mapper=mapper,
ticker=BasicTicker(desired_num_ticks=len(palette) + 1),
formatter=PrintfTickFormatter(format="%d%%"),
major_tick_line_color="black",
major_tick_out=5,
major_tick_in=0,
label_standoff=10,
border_line_color=None,
location=(0, 0),
title="Allele Frequency",
title_text_font_size="7pt",
title_standoff=5,
)
p.add_layout(color_bar, "right")
output_file("{}.html".format(fig_name.replace(" ", "")), title=fig_name)
save(p)
hover = HoverTool() hover.tooltips = """ <div> <h3>@Server</h3> <div><strong>Seated: </strong>@Seated</div> <div><strong>Greated: </strong>@Greated</div> <div><strong>Ordered: </strong>@Ordered</div> <div><strong>Drinks: </strong>@Drinks</div> <div><strong>Appetizer: </strong>@Appetizer</div> <div><strong>Food: </strong>@Food</div> <div><strong>Cleaned: </strong>@Cleaned</div> <div><strong>Satisfied: </strong>@Satisfied</div> <div><strong>Friendly: </strong>@Friendly</div> <div><strong>Bill: </strong>@Bill</div> <div><strong>Tips: </strong>@Tips</div> <div><strong>Total: </strong>@Total</div> <div><img src="@Image" alt="" width="200" /></div> </div> """ p.add_tools(hover) # Show results show(p) #Save File save(p) # Print out div and script #script, div = components(p) #print(div) #print (script)
def create_tabs(config_file,
title='',
start_date=(),
end_date=(),
verbose=True):
if not start_date:
lookback = 7
now = datetime.now()
start = now - timedelta(days=lookback)
start_date = (start.year, start.month, start.day)
end_date = (now.year, now.month, now.day)
dictionary = ordered_load(open(config_file), yaml.SafeLoader)
units = yaml.safe_load(open('units.yml'))
if not title: title = 'tabs'
output_file(title + ".html", title=title.replace('_', ' '))
#get all the data for plots
for key, value in dictionary.items():
for k, v in value.items():
for label, path in v.items():
if verbose:
sys.stdout.write('\nCollecting %s\n' % path)
df = get_cwms(path,
start_date=start_date,
end_date=end_date,
fill=False)
df.columns = [label]
dictionary[key][k][label] = df
#consolidate data to feed checkbox.py
for key, value in dictionary.items():
for k, v in value.items():
dictionary[key][k] = [df for label, df in v.items()]
#create all plots
for key, value in dictionary.items():
for k, v in value.items():
unit = units[k]
if verbose:
sys.stdout.write('\nCreating %s plots.\n' % k)
dictionary[key][k] = (create_plot(unit, dictionary[key][k]))
#link the plots on the first plot so share the same x axis
for key, value in dictionary.items():
children = []
link = []
i = 0
for k, v in value.items():
checkbox, p = v
if i == 0:
link.append(p)
i = 1
else:
p.x_range = link[0].x_range
count = 0
# check to see how many lines are on the plot
#it doesn't make sense to turn line off on single line
#so will add a blank checkboxgroup instead
for renderer in p.to_json(False)['renderers']:
if renderer['type'] == 'GlyphRenderer':
count += 1
if count < 2: children.append([CheckboxGroup(width=200), p])
else: children.append([checkbox, p])
p = layout(children)
dictionary[key] = p
#create and save tabs as html doc
tabs = []
for k, v in dictionary.items():
t = Panel(child=v, title=k)
tabs.append(t)
tabs = Tabs(tabs=tabs)
doc = curdoc()
doc.theme = theme
doc.add_root(tabs)
if verbose:
sys.stdout.write('\nSaving html document')
save(tabs)
# import pandas and bokeh
from bokeh.plotting import figure, output_file, save
import numpy as np
import pandas as pd
# create set of values for x and y
x = [n for n in range(10)]
y = [np.random.randint(10) for n in x]
# create html file
output_file('bokeh_graph_1.html')
# make figure
figure_1 = figure(width=400, height=400)
figure_1.circle(x, y)
save(figure_1)
# make another figure
y2 = np.random.rand(10) * 10
figure_1.asterisk(x, y2, line_color='black', size=10)
save(figure_1)
# make yet another figure
y3 = np.random.randint(10)
figure_1.line(x, y3, line_color='red')
# Add markers representing each y value
figure_1.square(x,
y3,
line_color='black',
fill_color='purple',
fill_alpha=.5,
def train_iters(self):
logger.info("number of parameters {0}".format(utils.get_n_params(self.model)))
for epoch in range(self.start_epoch, self.epochs + 1):
start_time = time.time()
total_training_loss = 0.0
training_batch_count = 0
counter = 0
# training
for i, data_dict in enumerate(self.training_dataset):
if i < 2 or i % 30 == 0 or counter < 1 or counter % 30 == 0:
logger.info("epoch {0} step {1} counter {2}".format(epoch, i, counter))
if data_dict is None:
continue
save_song = True if i % self.report_step < 5 else False # return the predictions from train() if true
# train on one batch, interating through the segments
outputs, loss = self.train(data_dict, save_song_outputs=save_song,
plot=True if i == 0 or i == 1 else False, register_hooks=False)
training_batch_count += 1
total_training_loss += loss
# save sample outputs
if save_song:
save_outputs(self.results_directory, self.user_prediction_directory, epoch,
data_dict['perf_id'], outputs, data_dict['shifts_gt'].detach().cpu().numpy(),
data_dict['original_boundaries'], training=True)
del data_dict; del outputs # clear memory
# validate and report losses every report_step and at end of epoch. Save the checkpoint if it is better
if counter % self.report_step == 0:
# iterate through the full validation set
mean_validation_loss = self.validate_iters(epoch, self.validation_dataset)
mean_training_loss = total_training_loss / training_batch_count # running average of training loss
# report current losses and print list of losses so far
logger.info("***********************************************************************************")
logger.info("{0}: Training and validation loss epoch {1} step {2} : {3} {4}".format(
self.extension, epoch, i, mean_training_loss, mean_validation_loss))
self.training_losses.append(mean_training_loss)
self.validation_losses.append(mean_validation_loss)
logger.info("***********************************************************************************")
logger.info("Training and validation losses so far:\n{0}\n{1}".format(
self.training_losses, self.validation_losses))
# plot the loss curves
bplt.output_file(os.path.join(self.plot_directory, "rnn_losses.html"))
fig_tr = bplt.figure(title="Training losses")
fig_ev = bplt.figure(title="Evaluation losses")
fig_cb = bplt.figure(title="Training and evaluation losses")
fig_fx = bplt.figure(title="Losses with fixed y-axis range", y_range=[0, 6.0e-4])
fig_tr.circle(np.arange(len(self.training_losses)), self.training_losses, color="red")
fig_ev.circle(np.arange(len(self.validation_losses)), self.validation_losses, color="red")
fig_cb.circle(np.arange(len(self.training_losses)), self.training_losses, color="green")
fig_cb.circle(np.arange(len(self.validation_losses)), self.validation_losses, color="orange")
fig_fx.circle(np.arange(len(self.training_losses)), self.training_losses, color="green")
fig_fx.circle(np.arange(len(self.validation_losses)), self.validation_losses, color="orange")
bplt.save(bplt.gridplot([fig_tr, fig_ev], [fig_cb, fig_fx]))
# save model and replace best if necessary
logger.info("is_best before {0} mean loss {1} best prec {2}".format(self.is_best, mean_validation_loss, self.best_prec1))
self.is_best = True if mean_validation_loss < self.best_prec1 else False
self.best_prec1 = min(mean_validation_loss, self.best_prec1)
logger.info("is_best after {0} mean loss {1} best prec {2}".format(self.is_best, mean_validation_loss, self.best_prec1))
if self.sandbox is False:
save_checkpoint({'epoch': epoch + 1, 'state_dict': self.model.state_dict(),
'best_prec1': self.best_prec1, 'optimizer': self.optimizer.state_dict(),
'training_losses': self.training_losses,
'validation_losses': self.validation_losses}, self.is_best,
latest_filename=self.latest_checkpoint_file,
best_filename=self.best_checkpoint_file)
counter += 1
# simulated annealing
# for param_group in self.optimizer.param_groups:
# param_group['lr'] *= 0.998
logger.info("--- {0} time elapsed for one epoch ---".format(time.time() - start_time))
def __getitem__(self, idx):
"""
:param keep: number of songs to keep, less or equal to the number of pitch shifts
:return: a list of length num_performance_shifts.
Each item is a tuple with the labels <seq len, shifts>, input <seq len, shifts, fdim>, key (str)
"""
fpath = os.path.join(autotune_preprocessed_directory, self.performance_list[idx] + ".pkl")
if not os.path.exists(fpath):
try:
pyin = np.load(os.path.join(pyin_directory, self.performance_list[idx] + ".npy"))
# load stft of vocals, keep complex values in order to use istft later for pitch shifting
stft_v = dataset_analysis.get_stft(
os.path.join(vocals_directory, self.performance_list[idx] + ".wav")).T
# load cqt of backing track
cqt_b = np.abs(dataset_analysis.get_cqt(os.path.join(backing_tracks_directory,
self.arr_keys[self.performance_list[idx]] + ".wav"))).T
# truncate pitch features to same length
frames = min(cqt_b.shape[0], stft_v.shape[0], len(pyin))
pyin = pyin[:frames]
stft_v = stft_v[:frames, :]
cqt_b = cqt_b[:frames, :]
original_boundaries = np.arange(frames).astype(np.int64) # store the original indices of the notes here
# find locations of note onsets using pYIN
min_note_frames = 24 # half second
audio_beginnings = np.array([i for i in range(frames - min_note_frames) # first nonzero frames
if i == 0 and pyin[i] > 0 or i > 0 and pyin[i] > 0 and pyin[i - 1] == 0])
if self.plot is True:
utils.reset_directory("./plots")
bplt.output_file(os.path.join("./plots", "note_parse_" + self.performance_list[idx]) + ".html")
s1 = bplt.figure(title="note parse")
s1.line(np.arange(len(pyin)), pyin)
for i, ab in enumerate(audio_beginnings):
loc = Span(location=ab, dimension='height', line_color='green')
s1.add_layout(loc)
# discard silent frames
silent_frames = np.ones(frames)
silent_frames[np.where(pyin < 1)[0]] *= 0
pyin = pyin[silent_frames.astype(bool)]
stft_v = stft_v[silent_frames.astype(bool), :]
cqt_b = cqt_b[silent_frames.astype(bool), :]
original_boundaries = original_boundaries[silent_frames.astype(bool)]
audio_beginnings = [n - np.sum(silent_frames[:n] == 0) for _, n in enumerate(audio_beginnings)]
frames = len(pyin)
audio_endings = np.hstack((audio_beginnings[1:], frames - 1))
# merge notes that are too short
note_beginnings = []
note_endings = []
min_note_frames = 24
start_note = end_note = 0
while start_note < len(audio_beginnings):
note_beginnings.append(audio_beginnings[start_note])
while (audio_endings[end_note] - audio_beginnings[start_note] < min_note_frames and
end_note < len(audio_endings) - 1):
end_note += 1
note_endings.append(audio_endings[end_note])
start_note = end_note + 1
end_note = start_note
# check that the last note is long enough
while note_endings[-1] - note_beginnings[-1] < min_note_frames:
del note_beginnings[-1]
del note_endings[-2]
notes = np.array([note_beginnings, note_endings]).T
# one minor issue
if notes[-1, 1] > frames - 1:
notes[-1, 1] = frames - 1
if self.plot is True:
s2 = bplt.figure(title="note parse of active frames")
s2.line(np.arange(len(pyin)), pyin)
for i, ab in enumerate(note_beginnings):
loc = Span(location=ab, dimension='height', line_color='green')
s2.add_layout(loc)
for i, ab in enumerate(note_endings):
loc = Span(location=ab+1, dimension='height', line_color='red', line_dash='dotted')
s2.add_layout(loc)
bplt.save(bplt.gridplot([[s1, s2]], toolbar_location=None))
# store the original indices of the notes
original_boundaries = np.array([original_boundaries[notes[:, 0]], original_boundaries[notes[:, 1]]]).T
# compute shifts for every note in every version in the batch (num_shifts)
note_shifts = np.random.rand(self.num_shifts, notes.shape[0]) * 2 - 1 # all shift combinations
if self.freeze is True:
note_shifts[:3, :] = self.frozen_shifts[:3, :note_shifts.shape[1]]
# compute the framewise shifts
frame_shifts = np.zeros((self.num_shifts, frames)) # this will be truncated later
for i in range(self.num_shifts):
for j in range(len(notes)):
# only shift the non-silent frames between the note onset and note offset
frame_shifts[i, notes[j][0]:notes[j][1]] = note_shifts[i][j]
# de-tune the pYIN pitch tracks and STFT of vocals
shifted_pyin = np.vstack([pyin] * self.num_shifts) * np.power(2, max_semitone * frame_shifts / 12)
# de-tune the vocals stft and vocals cqt
stacked_cqt_v = np.zeros((frames, self.num_shifts, cqt_params['total_bins']))
for i, note in enumerate(notes):
note_stft = np.array(stft_v[note[0]:note[1], :]).T
note_rt = librosa.istft(note_stft, hop_length=hopSize, center=False)
for j in range(self.num_shifts):
shifted_note_rt = librosa.effects.pitch_shift(note_rt, sr=global_fs, n_steps=note_shifts[j, i])
stacked_cqt_v[note[0]:note[1], j, :] = np.abs(librosa.core.cqt(
shifted_note_rt, sr=global_fs, hop_length=hopSize, n_bins=cqt_params['total_bins'],
bins_per_octave=cqt_params['bins_per_8va'], fmin=cqt_params['fmin']))[:, 4:-4].T
# get the data into the proper format and shape for tensors
cqt_b_binary = np.copy(cqt_b) # copy single-channel CQT for binarization
# need to repeat the backing track for the batch
cqt_b = np.stack([cqt_b] * self.num_shifts, axis=1)
# third channel
stacked_cqt_v_binary = np.copy(stacked_cqt_v)
for i in range(self.num_shifts):
thresh = threshold_mean(stacked_cqt_v_binary[:, i, :])
stacked_cqt_v_binary[:, i, :] = (stacked_cqt_v_binary[:, i, :] > thresh).astype(np.float)
thresh = threshold_mean(cqt_b_binary)
cqt_b_binary = (cqt_b_binary > thresh).astype(np.float)
stacked_cqt_b_binary = np.stack([cqt_b_binary] * self.num_shifts, axis=1)
stacked_cqt_combined = np.abs(stacked_cqt_v_binary - stacked_cqt_b_binary)
data_dict = dict()
data_dict['notes'] = notes
data_dict['spect_v'] = stacked_cqt_v
data_dict['spect_b'] = cqt_b
data_dict['spect_c'] = stacked_cqt_combined
data_dict['shifted_pyin'] = shifted_pyin
data_dict['shifts_gt'] = note_shifts
data_dict['original_boundaries'] = original_boundaries
data_dict['perf_id'] = self.performance_list[idx]
with open(fpath, "wb") as f:
pickle.dump(data_dict, f) # save for future epochs
except Exception as e:
logger.info("exception in dataset {0} skipping song {1}".format(e, self.performance_list[idx]))
return None
else:
# pre-processing has already been computed: load from file
try:
data_dict = self.loaditem(fpath)
except Exception as e:
logger.info("exception in dataset {0} skipping song {1}".format(e, self.performance_list[idx]))
return None
try:
# now format the numpy arrays into torch tensors with note-wise splits
data_dict['spect_v'] = torch.Tensor(data_dict['spect_v'])
data_dict['spect_b'] = torch.Tensor(data_dict['spect_b'])
data_dict['spect_c'] = torch.Tensor(data_dict['spect_c'])
data_dict['shifted_pyin'] = torch.Tensor(data_dict['shifted_pyin'].T)
data_dict['shifts_gt'] = torch.Tensor(data_dict['shifts_gt'].T)
# adjust dimension of note shifts
data_dict['shifts_gt'].unsqueeze_(1)
# split full songs into sequences
split_sizes = tuple(np.append(
np.diff(data_dict['notes'][:, 0]), data_dict['notes'][-1, 1] - data_dict['notes'][-1, 0] + 1))
data_dict['spect_v'] = torch.split(data_dict['spect_v'], split_size_or_sections=split_sizes, dim=0)
data_dict['spect_b'] = torch.split(data_dict['spect_b'], split_size_or_sections=split_sizes, dim=0)
data_dict['spect_c'] = torch.split(data_dict['spect_c'], split_size_or_sections=split_sizes, dim=0)
data_dict['shifted_pyin'] = torch.split(data_dict['shifted_pyin'], split_size_or_sections=split_sizes,
dim=0)
except Exception as e:
logger.info("exception in dataset {0} skipping song {1}".format(e, self.performance_list[idx]))
return None
return data_dict
def plot_embeddings_bokeh(emb,
emb_method=None,
classes=None,
labels=None,
color=None,
color_category=None,
cmap=None,
cmap_reverse=False,
colorbar=False,
colorbar_ticks=None,
outfile=None,
title=None,
scatter_labels=False,
**circle_kwargs):
"""
Creates an interactive scatterplot of the embeddings contained in emb,
using the bokeh library.
emb: an array with dim (n_embeddings x 2) or (n_embeddings x emb_dim).
In the latter case an embedding method emb_method should be supplied
to project from emb_dim to dim=2.
emb_method: "UMAP", "TSNE", or any other algorithm in sklearn.manifold
labels: Optional text labels for each embedding
color: Optional color for each embedding, according to which it will be
colored in the plot.
classes: Optional class for each embedding, according to which it will
be colored in the plot.
outfile: If provided, save plot to this file instead of showing it
cmap: colormap
title: optional title of the plot
"""
from bokeh.plotting import figure, output_file, show, save
from bokeh.models import (ColumnDataSource, CategoricalColorMapper,
LinearColorMapper, ColorBar, FixedTicker, Text)
from bokeh.palettes import Category20, Viridis256, viridis
if emb_method:
emb = embed_2d(emb, emb_method)
if outfile:
output_file(outfile)
source_dict = dict(x=emb[:, 0], y=emb[:, 1])
if classes is not None:
source_dict["cls"] = classes
if labels is not None:
source_dict["label"] = labels
if color is not None:
source_dict["color"] = color
source = ColumnDataSource(source_dict)
if classes is not None and color is None:
n_classes = len(set(classes))
if n_classes <= 20:
if n_classes <= 2:
palette = Category20[3]
palette = [palette[0], palette[-1]]
else:
palette = Category20[n_classes]
else:
palette = viridis(n_classes)
color_conf = {
"field":
"cls",
"transform":
CategoricalColorMapper(factors=list(set(classes)), palette=palette)
}
elif color is not None:
if cmap is not None:
if isinstance(cmap, str):
import bokeh.palettes
# matplotib suffix for reverse color maps
if cmap.endswith("_r"):
cmap_reverse = True
cmap = cmap[:-2]
cmap = getattr(bokeh.palettes, cmap)
elif isinstance(cmap, dict):
cmap = cmap[max(cmap.keys())]
else:
cmap = Viridis256
if cmap_reverse:
cmap.reverse()
color_mapper = LinearColorMapper(cmap)
color_conf = {"field": "color", "transform": color_mapper}
if colorbar:
if colorbar_ticks:
ticker = FixedTicker(ticks=colorbar_ticks)
else:
ticker = None
colorbar = ColorBar(color_mapper=color_mapper, ticker=ticker)
else:
color_conf = "red"
tools = "crosshair,pan,wheel_zoom,box_zoom,reset,hover,previewsave"
p = figure(tools=tools, sizing_mode='scale_both')
if title:
p.title.text = title
if labels is not None and scatter_labels:
glyph = Text(x="x",
y="y",
text="label",
angle=0.0,
text_color=color_conf,
text_alpha=0.95,
text_font_size="8pt")
p.add_glyph(source, glyph)
else:
p.circle(x='x',
y='y',
source=source,
color=color_conf,
legend='cls' if classes is not None else None,
**circle_kwargs)
if labels is not None:
from bokeh.models import HoverTool
from collections import OrderedDict
hover = p.select(dict(type=HoverTool))
hover_entries = [
("label", "@label"),
("(x, y)", "(@x, @y)"),
]
if color is not None and color_category:
hover_entries.append((color_category, "@color"))
hover.tooltips = OrderedDict(hover_entries)
if colorbar:
assert color is not None
p.add_layout(colorbar, 'right')
if outfile:
save(p)
else:
show(p)
def timeline_days_hours(interval_frequency, all_co2_dataframe,
sensors_with_anomalies, data, upper_bound,
destination_path):
times = pd.date_range(start='00:00:00',
end='23:55:00',
freq=str(interval_frequency) +
'Min').strftime('%H:%M:%S')
days = all_co2_dataframe['timestamp'].dt.strftime('%Y-%m-%d').to_list()
days = list(dict.fromkeys(days))
timestamp = list(times)
for sensor_name in sensors_with_anomalies:
all_durations = []
for k in data['anomalies']['anomaly_co2_values'][sensor_name][0]:
k = data['anomalies']['anomaly_co2_values'][sensor_name][0].index(
k)
all_durations.append(
int(data['anomalies']['anomaly_co2_values'][sensor_name][0][k]
['duration']))
data['anomalies']['anomaly_co2_values'][sensor_name][0]
p = figure(plot_height=500,
plot_width=2000,
x_range=timestamp,
y_range=days,
title='Timeline of periods with CO2 levels higher than ' +
str(upper_bound) + ' ppm in ' + sensor_name +
'\n Based on data for last ' + str(len(days)) + ' days',
active_drag=None,
toolbar_location=None)
p.x_range.range_padding = 0
p.y_range.range_padding = 0
p.title.text_font_size = '15pt'
p.xaxis.axis_label_text_font_size = "15pt"
p.yaxis.axis_label_text_font_size = "15pt"
p.yaxis.major_label_text_font_size = '9pt'
p.xaxis.major_label_text_font_size = '5pt'
# set x axis to invisible
p.xaxis.visible = False
# Add custom axis with tickers labels only every 1 hour
labels = np.arange(0, 288, 12).tolist()
ticker = FixedTicker()
ticker.ticks = labels
xaxis = LinearAxis(ticker=ticker)
xaxis.major_label_orientation = math.pi / 3
p.add_layout(xaxis, 'below')
xaxis.major_label_overrides = {
0: '00:00',
12: '01:00',
24: '02:00',
36: '3:00',
48: '04:00',
60: '05:00',
72: '06:00',
84: '07:00',
96: '08:00',
108: '09:00',
120: '10:00',
132: '11:00',
144: '12:00',
156: '13:00',
168: '14:00',
180: '15:00',
192: '16:00',
204: '17:00',
216: '18:00',
228: '19:00',
240: '20:00',
252: '21:00',
264: '22:00',
276: '23:00'
}
#add anomalies recorded for each day in considered period
for i in data['anomalies']['anomaly_co2_values'][sensor_name][0]:
i = data['anomalies']['anomaly_co2_values'][sensor_name][0].index(
i)
x = []
y = []
x.append(data['anomalies']['anomaly_co2_values'][sensor_name][0][i]
['anomalies_details'][0][0][11:])
x.append(data['anomalies']['anomaly_co2_values'][sensor_name][0][i]
['anomalies_details'][-1][0][11:])
y.append(data['anomalies']['anomaly_co2_values'][sensor_name][0][i]
['anomalies_details'][0][0][0:10])
y.append(data['anomalies']['anomaly_co2_values'][sensor_name][0][i]
['anomalies_details'][-1][0][0:10])
if data['anomalies']['anomaly_co2_values'][sensor_name][0][i][
'anomalies_details'][0][0][0:10] == data['anomalies'][
'anomaly_co2_values'][sensor_name][0][i][
'anomalies_details'][-1][0][0:10]:
# print('yes')
p.line(x,
y,
line_width=2,
color='blue',
legend_label='CO2 above critical value')
p.circle(x, y, fill_color="blue", line_color='blue', size=5)
else:
x1 = x.copy()
y1 = y.copy()
x2 = x.copy()
y2 = y.copy()
x1[-1] = times[-1]
y1[-1] = y1[0]
p.line(x1, y1, line_width=2, color='blue')
x2[0] = times[0]
y2[0] = y2[-1]
p.line(x2, y2, line_width=2, color='blue')
x3 = [x1[0], x2[-1]]
y3 = [y1[0], y2[-1]]
p.circle(x3, y3, fill_color="blue", line_color='blue', size=5)
#save graph in output location
output_file(destination_path + '/anomalies_timeline_' + sensor_name +
'.html')
save(p)
return p
miny, maxy = 500, 1000
if pars[id]["type"] == "CRS 1000-B":
miny, maxy = 900, 1500
if pars[id]["type"] == "CRS 2000":
miny, maxy = 1000, 1400
if miny is not None:
p.y_range = Range1d(miny, maxy)
p.legend.location = "center_left"
p.legend.background_fill_alpha = 0.6
#p.legend.location = "center_left"
plts.append(p)
grid = gridplot(plts, merge_tools=False, ncols=1)
output_file(htmlfile_merge, title="Time series of counts per hour")
save(grid)
# Add correct header to html
f = open(htmlfile_merge, "r")
html = f.read()
f.close()
html = html.strip()
replacement = '''<html>
<div class="blurb">
<h1>JFC Fendt: Processed neutron counts </h1>
</div><!-- /.blurb -->
<p> Data merged from SD and telemetry...counts per hour...filtered...resampled.
'''
html = html.replace('<html lang="en">', replacement)
f = open(htmlfile_merge, "w")
f.write("---\nlayout: default\ntitle: cosmic pages\n---\n")
def time_series_bokeh(start, end, country='world'):
""" Show the time series of the world in a HTML graph """
# Get data
df = ws.data_countries_only
if country != 'world':
df = df[df['country_region'] == country]
start_index, end_index = get_start_end(start, end)
data = {}
variables = ['confirmed', 'recovered', 'deaths']
for variable in variables:
data['dates'], data[variable] = get_single_time_series(
df, variable, start_index, end_index)
# Existing cases
data['resolved'] = data['recovered'] + data['deaths']
data['active'] = data['confirmed'] - data['resolved']
# New in 7 days (average)
data_n7d = get_new_7_days(start_index,
end_index,
variable,
country=country,
avg=True)
data['new_7_days'] = data_n7d['new_7_days']
data['date_keys'] = data_n7d['date']
variables = variables + ['active']
# Choose title
title = "Casos confirmados, activos y nuevos en "
if country == 'world':
addstr = 'el mundo'
elif country == 'Spain':
addstr = 'España'
elif country == 'Colombia':
addstr = 'Colombia'
title = title + addstr
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
# x_range=(dates_[0], dates_[-1]),
# y_range=(data['confirmed'].min(), data['confirmed'].max()),
)
# Add a circle renderer with a size, color and alpha
colors = {
'confirmed': 'navy',
'active': 'orange',
'new_7_days': 'red',
}
labels = {
'confirmed': 'Confirmados',
'active': 'Activos',
'new_7_days': 'Nuevos (prom. 7 días)',
}
plots = {}
for v, c in colors.items():
plots[v] = p.circle('dates',
v,
source=data,
size=5,
color=c,
alpha=0.5)
p.line('dates',
v,
source=data,
line_width=2,
color=c,
alpha=0.5,
legend_label=labels[v])
# Hover tools
hover1 = HoverTool(renderers=[plots['confirmed']],
tooltips=[('Fecha', '@date_keys'),
('Confirmados', '@confirmed')])
hover2 = HoverTool(renderers=[plots['active']],
tooltips=[('Fecha', '@date_keys'),
('Activos', '@active')])
hover3 = HoverTool(renderers=[plots['new_7_days']],
tooltips=[('Fecha', '@date_keys'),
('Nuevos (prom. 7 días)', '@new_7_days')])
p.add_tools(hover1)
p.add_tools(hover2)
p.add_tools(hover3)
# Arrange figure
p.xaxis.axis_label = 'Fecha'
p.yaxis.axis_label = 'Número de casos'
p.legend.location = 'top_left'
p.xaxis.formatter = DatetimeTickFormatter(days="%d %B")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# Output to static HTML file
output_file(
os.path.join(ws.folders["website/static/images"],
"%s_graph.html" % country.lower()))
# show(p)
save(p)
def make_training_history_report(predictor, filename=None):
plot_width, plot_height = 600, 300
history = predictor.history.copy()
history['epoch'] = history.index
targets = predictor.regressors + predictor.classifiers
# sort targets in mae and accuracy evaluated variables: regressors and classifiers
mae_targets, accuracy_targets = [], []
for target in targets:
if target + '_mae' in history.columns:
mae_targets.append(target)
if target + '_accuracy' in history.columns:
accuracy_targets.append(target)
def plot_metric_and_loss(history, target, metric='mae'):
ykey = target + '_' + metric
p1 = bpl.figure(plot_height=plot_height, plot_width=plot_width, title=target)
p1.line(history['epoch'], history[ykey], legend_label='training')
if 'val_' + ykey in history.columns:
p1.line(history['epoch'], history['val_' + ykey], color='orange', legend_label='validation')
# p1.xaxis.axis_label = 'Epoch'
p1.yaxis.axis_label = 'MAE' if metric == 'mae' else 'Accuracy'
p1.legend.location = "top_right" if metric == 'mae' else "bottom_right"
p1.title.align = 'center'
p1.title.text_font_size = '14pt'
ykey = target + '_loss'
p2 = bpl.figure(plot_height=plot_height, plot_width=plot_width)
p2.line(history['epoch'], history[ykey])
if 'val_' + ykey in history.columns:
p2.line(history['epoch'], history['val_' + ykey], color='orange')
p2.xaxis.axis_label = 'Epoch'
p2.yaxis.axis_label = 'Loss'
return gridplot([[p1], [p2]], toolbar_location='right')
# make the figure
bpl.output_file(filename, title='Training history')
title_div = Div(text="""<h1>Training History</h1>""", width=1000, height=50)
section_plots = [[title_div]]
section_div = Div(text="""<h2>Regressors</h2>""", width=400, height=40)
section_plots.append([section_div])
mae_plots = []
for target in mae_targets:
p = plot_metric_and_loss(history, target, metric='mae')
mae_plots.append(p)
section_plots.append(mae_plots)
section_div = Div(text="""<h2>Classifiers</h2>""", width=400, height=40)
section_plots.append([section_div])
acc_plots = []
for target in accuracy_targets:
p = plot_metric_and_loss(history, target, metric='accuracy')
acc_plots.append(p)
section_plots.append(acc_plots)
cm_plots = []
y_true = predictor.train_data[predictor.classifiers]
y_pred = predictor.predict(predictor.train_data)[predictor.classifiers]
for target in accuracy_targets:
p_ = bokeh_ext.confusion_matrix(y_true[target], y_pred[target])
p_.title.text = target
p_.title.align = 'center'
p_.title.text_font_size = '14pt'
cm_plots.append([p_])
section_plots.append(cm_plots)
p = layout(section_plots)
bpl.save(p)
def fortney_grid(args, write_plot=False, write_table=False):
"""
Function to grab a Fortney Grid model, plot it, and make a table.
Parameters
----------
args : dict
Dictionary of arguments for the Fortney Grid. Must include :
temp
chem
cloud
pmass
m_unit
reference_radius
r_unit
rstar
rstar_unit
write_plot : bool, optional
Whether or not to save the bokeh plot, defaults to False.
write_table : bool, optional
Whether or not to save the ascii table, defaults to False.
Returns
-------
fig : bokeh object
The unsaved bokeh plot.
fh : ascii table object
The unsaved ascii table.
temp_out : list of str of int
The list of temperatures in the model grid.
"""
# Check for Fortney Grid database
print(
os.path.join(get_env_variables()['exoctk_data'],
'fortney/fortney_models.db'))
try:
db = create_engine('sqlite:///' + os.path.join(
get_env_variables()['exoctk_data'], 'fortney/fortney_models.db'))
header = pd.read_sql_table('header', db)
except:
raise Exception(
'Fortney Grid File Path is incorrect, or not initialized')
if args:
rstar = float(args['rstar'])
rstar = (rstar * u.Unit(args['rstar_unit'])).to(u.km)
reference_radius = float(args['reference_radius'])
rplan = (reference_radius * u.Unit(args['r_unit'])).to(u.km)
temp = float(args['temp'])
# clouds
cloud = args['cloud']
if cloud.find('flat') != -1:
flat = int(cloud[4:])
ray = 0
elif cloud.find('ray') != -1:
ray = int(cloud[3:])
flat = 0
elif int(cloud) == 0:
flat = 0
ray = 0
else:
flat = 0
ray = 0
print('No cloud parameter not specified, default no clouds added')
# chemistry
chem = args['chem']
if chem == 'noTiO':
noTiO = True
if chem == 'eqchem':
noTiO = False
# grid does not allow clouds for cases with TiO
flat = 0
ray = 0
fort_grav = 25.0 * u.m / u.s**2
df = header.loc[(header.gravity == fort_grav) & (header.temp == temp) &
(header.noTiO == noTiO) & (header.ray == ray) &
(header.flat == flat)]
wave_planet = np.array(
pd.read_sql_table(df['name'].values[0], db)['wavelength'])[::-1]
r_lambda = np.array(
pd.read_sql_table(df['name'].values[0], db)['radius']) * u.km
# All fortney models have fixed 1.25 radii
z_lambda = r_lambda - (1.25 * u.R_jup).to(u.km)
# Scale with planetary mass
pmass = float(args['pmass'])
mass = (pmass * u.Unit(args['m_unit'])).to(u.kg)
# Convert radius to m for gravity units
gravity = constants.G * (mass) / (rplan.to(u.m))**2.0
# Scale lambbda (this technically ignores the fact that scaleheight
# is altitude dependent) therefore, it will not be valide for very
# very low gravities
z_lambda = z_lambda * fort_grav / gravity
# Create new wavelength dependent R based on scaled ravity
r_lambda = z_lambda + rplan
# Finally compute (rp/r*)^2
flux_planet = np.array(r_lambda**2 / rstar**2)
x = wave_planet
y = flux_planet[::-1]
else:
df = pd.read_sql_table('t1000g25_noTiO', db)
x, y = df['wavelength'], df['radius']**2.0 / 7e5**2.0
tab = at.Table(data=[x, y])
fh = StringIO()
tab.write(fh, format='ascii.no_header')
if write_table:
tab.write('fortney.dat', format='ascii.no_header')
fig = figure(plot_width=1100, plot_height=400)
fig.line(x, 1e6 * (y - np.mean(y)), color='Black', line_width=0.5)
fig.xaxis.axis_label = 'Wavelength (um)'
fig.yaxis.axis_label = 'Rel. Transit Depth (ppm)'
if write_plot:
output_file('fortney.html')
save(fig)
# Return temperature list for the fortney grid page
temp_out = list(map(str, header.temp.unique()))
return fig, fh, temp_out
def top_n_time_series(df,
n=10,
key_groupby='country_region',
dates='date',
variable='confirmed',
label='country_region',
title='',
region_label='world',
logscale=False):
""" Show time series of the top 5 countries/regions/provinces in a dataset """
# Show the top 10 provinces
top_5 = top_n(df, n=5, groupby=[key_groupby])
# Show graph
plt.close('all')
width = 8
fig, ax = plt.subplots(figsize=(width, width * 0.61803398874988))
data = {}
for key in top_5:
# Select the part of the dataframe I want
data_ = df[df[key_groupby] == key]
ax.plot(data_[dates], data_[variable], label=data_[label].tolist()[0])
data[key] = data_
ax.set_xlabel('Fecha')
ax.set_ylabel('Casos %s' % ws.trans[variable])
ax.legend()
ax.set_title(title)
plt.xticks(rotation=45)
if logscale:
ax.set_yscale('log')
# plt.show()
logstrings = {True: 'logscale', False: ''}
filename = '%s_time_series_%s_v1.png' % (region_label.lower(),
logstrings[logscale])
fig.savefig(os.path.join(ws.folders['website/static/images'], filename),
bbox_inches='tight',
dpi=300)
# Figure in bokeh
# print(list(data.values())[0][dates].tolist())
dates_ = list(data.values())[0][dates].tolist()
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
# x_range=(dates_[0], dates_[-1]),
)
colors = Category10[5]
# Iterate over top_5
for i, key in enumerate(top_5):
data_ = data[key]
# print(data_[dates])
region_name = data_[label].tolist()[0]
p.circle(dates,
variable,
source=data_,
color=colors[i],
size=5,
alpha=0.5,
legend_label=region_name)
p.line(dates,
variable,
source=data_,
color=colors[i],
line_width=2,
alpha=0.5,
legend_label=region_name)
# p.circle(x=data_[dates], y=data_[variable], color=colors[i], size=5, alpha=0.5, legend_label=region_name)
# p.line(x=data_[dates], y=data_[variable], color=colors[i], line_width=2, alpha=0.5, legend_label=region_name)
# Arrange figure
p.xaxis.axis_label = 'Fecha'
p.yaxis.axis_label = 'Casos %s' % ws.trans[variable]
p.legend.location = 'top_left'
p.xaxis.formatter = DatetimeTickFormatter(days="%d %B")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# It seems I need to put this, otherwise the x ranges go crazy
dates_ = data_[dates].tolist()
# p.x_range = Range1d(dates_[0], dates_[-1])
# Output to static HTML file
output_file(
os.path.join(ws.folders['website/static/images'],
filename.replace('.png', '.html')))
# show(p)
save(p)
def visualize_documents(
self,
texts=None,
doc_topics=None,
width=700,
height=700,
point_size=5,
title='Document Visualization',
extra_info={},
colors=None,
filepath=None,
):
"""
Generates a visualization of a set of documents based on model.
If <texts> is supplied, raw documents will be first transformed into document-topic
matrix. If <doc_topics> is supplied, then this will be used for visualization instead.
Args:
texts(list of str): list of document texts. Mutually-exclusive with <doc_topics>
doc_topics(ndarray): pre-computed topic distribution for each document in texts.
Mutually-exclusive with <texts>.
width(int): width of image
height(int): height of image
point_size(int): size of circles in plot
title(str): title of visualization
extra_info(dict of lists): A user-supplied information for each datapoint (attributes of the datapoint).
The keys are field names. The values are lists - each of which must
be the same number of elements as <texts> or <doc_topics>. These fields are displayed
when hovering over datapoints in the visualization.
colors(list of str): list of Hex color codes for each datapoint.
Length of list must match either len(texts) or doc_topics.shape[0]
filepath(str): Optional filepath to save the interactive visualization
"""
# error-checking
if texts is not None: length = len(texts)
else: length = doc_topics.shape[0]
if colors is not None and len(colors) != length:
raise ValueError(
'length of colors is not consistent with length of texts or doctopics'
)
if texts is not None and doc_topics is not None:
raise ValueError('texts is mutually-exclusive with doc_topics')
if texts is None and doc_topics is None:
raise ValueError('One of texts or doc_topics is required.')
if extra_info:
invalid_keys = ['x', 'y', 'topic', 'fill_color']
for k in extra_info.keys():
if k in invalid_keys:
raise ValueError('cannot use "%s" as key in extra_info' %
(k))
lst = extra_info[k]
if len(lst) != length:
raise ValueError(
'texts and extra_info lists must be same size')
# check fo bokeh
try:
import bokeh.plotting as bp
from bokeh.plotting import save
from bokeh.models import HoverTool
from bokeh.io import output_notebook
except:
warnings.warn(
'visualize_documents method requires bokeh package: pip3 install bokeh'
)
return
# prepare data
if doc_topics is not None:
X_topics = doc_topics
else:
if self.verbose: print('transforming texts...', end='')
X_topics = self.predict(texts, harden=False)
if self.verbose: print('done.')
# reduce to 2-D
if self.verbose: print('reducing to 2 dimensions...', end='')
tsne_model = TSNE(n_components=2,
verbose=self.verbose,
random_state=0,
angle=.99,
init='pca')
tsne_lda = tsne_model.fit_transform(X_topics)
print('done.')
# get random colormap
colormap = U.get_random_colors(self.n_topics)
# generate inline visualization in Jupyter notebook
lda_keys = self._harden_topics(X_topics)
if colors is None: colors = colormap[lda_keys]
topic_summaries = self.get_topics(n_words=5)
os.environ["BOKEH_RESOURCES"] = "inline"
output_notebook()
dct = {
'x': tsne_lda[:, 0],
'y': tsne_lda[:, 1],
'topic': [topic_summaries[tid] for tid in lda_keys],
'fill_color': colors,
}
tool_tups = [('index', '$index'), ('(x,y)', '($x,$y)'),
('topic', '@topic')]
for k in extra_info.keys():
dct[k] = extra_info[k]
tool_tups.append((k, '@' + k))
source = bp.ColumnDataSource(data=dct)
hover = HoverTool(tooltips=tool_tups)
p = bp.figure(
plot_width=width,
plot_height=height,
tools=[
hover, 'previewsave', 'pan', 'wheel_zoom', 'box_zoom', 'reset'
],
#tools="pan,wheel_zoom,box_zoom,reset,hover,previewsave",
title=title)
#plot_lda = bp.figure(plot_width=1400, plot_height=1100,
#title=title,
#tools="pan,wheel_zoom,box_zoom,reset,hover,previewsave",
#x_axis_type=None, y_axis_type=None, min_border=1)
p.circle('x',
'y',
size=point_size,
source=source,
fill_color='fill_color')
bp.show(p)
if filepath is not None:
bp.output_file(filepath)
bp.save(p)
return
def current_waveform_plot(samples, voltage, dest_path, plot_title):
"""Plot the current data using bokeh interactive plotting tool.
Plotting power measurement data with bokeh to generate interactive plots.
You can do interactive data analysis on the plot after generating with the
provided widgets, which make the debugging much easier. To realize that,
bokeh callback java scripting is used. View a sample html output file:
https://drive.google.com/open?id=0Bwp8Cq841VnpT2dGUUxLYWZvVjA
Args:
samples: a list of tuples in which the first element is a timestamp and
the second element is the sampled current in milli amps at that time.
voltage: the voltage that was used during the measurement.
dest_path: destination path.
plot_title: a filename and title for the plot.
Returns:
plot: the plotting object of bokeh, optional, will be needed if multiple
plots will be combined to one html file.
dt: the datatable object of bokeh, optional, will be needed if multiple
datatables will be combined to one html file.
"""
logging.info('Plotting the power measurement data.')
time_relative = [sample[0] for sample in samples]
duration = time_relative[-1] - time_relative[0]
current_data = [sample[1] * 1000 for sample in samples]
avg_current = sum(current_data) / len(current_data)
color = ['navy'] * len(samples)
# Preparing the data and source link for bokehn java callback
source = ColumnDataSource(
data=dict(x=time_relative, y=current_data, color=color))
s2 = ColumnDataSource(
data=dict(a=[duration],
b=[round(avg_current, 2)],
c=[round(avg_current * voltage, 2)],
d=[round(avg_current * voltage * duration, 2)],
e=[round(avg_current * duration, 2)]))
# Setting up data table for the output
columns = [
TableColumn(field='a', title='Total Duration (s)'),
TableColumn(field='b', title='Average Current (mA)'),
TableColumn(field='c', title='Average Power (4.2v) (mW)'),
TableColumn(field='d', title='Average Energy (mW*s)'),
TableColumn(field='e', title='Normalized Average Energy (mA*s)')
]
dt = DataTable(source=s2,
columns=columns,
width=1300,
height=60,
editable=True)
output_file(os.path.join(dest_path, plot_title + '.html'))
tools = 'box_zoom,box_select,pan,crosshair,redo,undo,reset,hover,save'
# Create a new plot with the datatable above
plot = figure(plot_width=1300,
plot_height=700,
title=plot_title,
tools=tools)
plot.add_tools(bokeh_tools.WheelZoomTool(dimensions='width'))
plot.add_tools(bokeh_tools.WheelZoomTool(dimensions='height'))
plot.line('x', 'y', source=source, line_width=2)
plot.circle('x', 'y', source=source, size=0.5, fill_color='color')
plot.xaxis.axis_label = 'Time (s)'
plot.yaxis.axis_label = 'Current (mA)'
plot.title.text_font_size = {'value': '15pt'}
# Callback JavaScript
source.selected.js_on_change(
"indices",
CustomJS(args=dict(source=source, mytable=dt),
code="""
const inds = source.selected.indices;
const d1 = source.data;
const d2 = mytable.source.data;
var ym = 0
var ts = 0
var min=d1['x'][inds[0]]
var max=d1['x'][inds[0]]
d2['a'] = []
d2['b'] = []
d2['c'] = []
d2['d'] = []
d2['e'] = []
if (inds.length==0) {return;}
for (var i = 0; i < inds.length; i++) {
ym += d1['y'][inds[i]]
d1['color'][inds[i]] = "red"
if (d1['x'][inds[i]] < min) {
min = d1['x'][inds[i]]}
if (d1['x'][inds[i]] > max) {
max = d1['x'][inds[i]]}
}
ym /= inds.length
ts = max - min
d2['a'].push(Math.round(ts*1000.0)/1000.0)
d2['b'].push(Math.round(ym*100.0)/100.0)
d2['c'].push(Math.round(ym*4.2*100.0)/100.0)
d2['d'].push(Math.round(ym*4.2*ts*100.0)/100.0)
d2['e'].push(Math.round(ym*ts*100.0)/100.0)
source.change.emit();
mytable.change.emit();
"""))
# Layout the plot and the datatable bar
save(layout([[dt], [plot]]))
return plot, dt
def generic_grid(input_args, write_plot=False, write_table=False):
"""
Build a plot and table from the generic grid results.
Parameters
----------
input_args : dict
A dictionary of the form output from the generic grid form.
If manual input must include :
r_star : The radius of the star.
r_planet : The radius of the planet.
gravity : The gravity.
temperature : The temperature.
condensation : local or rainout
metallicity
c_o : carbon/oxygen ratio
haze
cloud
write_plot : bool, optional
Whether to write the plot out. Defaults to False.
write_table : bool, optional
Whether to write the table out. Defaults to Fals.
Returns
-------
plot : bokeh object
Unsaved bokeh plot.
table : ascii table object
Unsaved ascii table.
closest_match : dict
A dictionary with the parameters/model name of the closest
match in the grid.
error_message : str
An error message, or lack therof.
"""
# Find path to the database.
database_path = os.path.join(get_env_variables()['exoctk_data'],
'generic/generic_grid_db.hdf5')
# Build rescaled model
solution, inputs, closest_match, error_message = rescale_generic_grid(
input_args, database_path)
# Build file out
tab = at.Table(data=[solution['wv'], solution['spectra']])
fh = StringIO()
tab.write(fh, format='ascii.no_header')
if write_table:
tab.write('generic.dat')
# Plot
fig = figure(title='Rescaled Generic Grid Transmission Spectra'.upper(),
plot_width=1100,
plot_height=400)
fig.x_range.start = 0.3
fig.x_range.end = 5
fig.line(solution['wv'], solution['spectra'], color='Black', line_width=1)
fig.xaxis.axis_label = 'Wavelength (um)'
fig.yaxis.axis_label = 'Transit Depth (Rp/R*)^2'
if write_plot:
output_file('generic.html')
save(fig)
return fig, fh, closest_match, error_message
def new_vs_active(start,
end,
x_range=None,
y_range=None,
variable='active',
country='world',
use_top_n=False,
log=False):
""" Show graph of new cases vs. active """
# Start and end indices
start_index, end_index = get_start_end(start, end)
# Choose title
title = "Casos nuevos vs. casos %s en " % ws.trans[variable]
if country == 'world':
addstr = 'el mundo'
elif country == 'Spain':
addstr = 'España'
elif country == 'Colombia':
addstr = 'Colombia'
title = title + addstr
data = get_new_7_days(start_index,
end_index,
variable,
country=country,
avg=True)
# Bokeh figure for the country
# Add hover tool
hover = HoverTool(tooltips=[(
'Fecha',
'@date'), ('Casos %s' % ws.trans[variable],
'@%s' % variable), ('Casos nuevos (últ. 7d)',
'@new_7_days')])
# Figure
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="datetime",
title=title,
# tools = [hover]
)
p.add_tools(hover)
# Add a circle renderer with a size, color and alpha
p.circle(variable,
'new_7_days',
source=data,
size=5,
color="black",
alpha=0.5)
p.line(variable,
'new_7_days',
source=data,
line_width=2,
color="black",
alpha=0.5)
# Arrange figure
p.xaxis.axis_label = 'Casos %s' % ws.trans[variable]
p.yaxis.axis_label = 'Casos nuevos en los últimos 7 días'
p.yaxis.axis_label = 'Casos nuevos (promedio de 7 días)'
p.xaxis.formatter = NumeralTickFormatter(format="0")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# Output to static HTML file
output_file(
os.path.join(
ws.folders["website/static/images"],
"new_vs_%s_last7days_%s.html" % (variable, country.lower())))
# show(p)
save(p)
######################################
# Bokeh figure for all the countries
if country == 'world':
# Copy previous hover tool
hover_copy = HoverTool(tooltips=[(
'Fecha',
'@date'), ('Casos %s' % ws.trans[variable], '@%s' %
variable), ('Casos nuevos (últ. 7d)', '@new_7_days')])
# Hover tool indicating the country
hover2 = HoverTool(tooltips=[('País', '@country'), (
'Fecha',
'@date'), ('Casos %s' % ws.trans[variable], '@%s' %
variable), ('Casos nuevos (últ. 7d)', '@new_7_days')])
# Figure
if log:
p = figure(
plot_width=800,
plot_height=400,
x_axis_type="log",
y_axis_type="log",
title="Casos nuevos vs. casos %s. Ejes en escala logarítmica" %
ws.trans[variable],
# tools = [hover2],
)
else:
p = figure(
plot_width=800,
plot_height=400,
title="Casos nuevos vs. casos %s" % ws.trans[variable],
# tools = [hover2],
)
p.add_tools(hover2)
if x_range is not None:
p.x_range = Range1d(*x_range)
if y_range is not None:
p.y_range = Range1d(*y_range)
df = ws.data_countries_only
countries = set(df.country_region)
if use_top_n:
countries = ws.top_ten
for country in countries:
data = get_new_7_days(start_index,
end_index,
variable,
country=country,
avg=True)
# Add a circle renderer with a size, color and alpha
p.circle(variable,
'new_7_days',
source=data,
size=5,
color="black",
alpha=0.2)
p.line(variable,
'new_7_days',
source=data,
line_width=2,
color="black",
alpha=0.2)
# Last circle indicating the country
data_last = {k: [v[-1]] for k, v in data.items()}
data_last['country'] = [country]
p.circle(variable,
'new_7_days',
source=data_last,
size=10,
color="magenta",
alpha=1.)
p.text(x=data_last[variable],
y=data_last['new_7_days'],
text=[country],
text_baseline="middle",
text_align="left")
# Arrange figure
p.xaxis.axis_label = 'Casos %s' % ws.trans[variable]
p.yaxis.axis_label = 'Casos nuevos en los últimos 7 días'
p.yaxis.axis_label = 'Casos nuevos (promedio de 7 días)'
p.xaxis.formatter = NumeralTickFormatter(format="0")
p.yaxis.formatter = NumeralTickFormatter(format="0")
p.toolbar.logo = None
# Output to static HTML file
use_log_str = 'logscale' if log else ''
if use_top_n:
output_file(
os.path.join(
ws.folders["website/static/images"],
"new_vs_%s_last7days_top_n%01i_%s.html" %
(variable, ws.ntop, use_log_str)))
else:
output_file(
os.path.join(
ws.folders["website/static/images"],
" new_vs_%s_last7days_whole_world_%s.html" %
(variable, use_log_str)))
# show(p)
save(p)
def save(self, filename):
self.output_file = bp.output_file(filename,
title="Progression plot")
bp.save(self.p, self.output_file)
def create_graph(results_df):
# Create Data Source
y_axis = ['Min', 'Avg', 'Max', '90th', '95th', '99th']
x_axis = [
round(results_df['Transfer_Time'].min(), 2),
round(results_df['Transfer_Time'].mean(), 2),
round(results_df['Transfer_Time'].max(), 2),
round(results_df['Transfer_Time'].quantile(0.90), 2),
round(results_df['Transfer_Time'].quantile(0.95), 2),
round(results_df['Transfer_Time'].quantile(0.99), 2)
]
# Create Source for 2nd Graph
graph2_source = ColumnDataSource(dict(y=y_axis, right=x_axis))
# Write results to csv
aggregate_results_df = pd.DataFrame.from_dict(
dict(Metrics=y_axis, Value=x_axis))
# Save Dataframe to csv
agg_report_name = "Perf_Summary_" + time.strftime("%H%M%S") + ".csv"
aggregate_results_df.to_csv(agg_report_name, sep=',', index=False)
print(
"Performance Metrics Summary report generated! Please check file: {}".
format(agg_report_name))
# Plot 1st Graph
x_axis_graph1 = results_df['File_Number'].tolist()
temp_list = results_df['Transfer_Time'].tolist()
y_axis_graph1 = [round(elem, 2) for elem in temp_list]
graph1_source = ColumnDataSource(dict(x=x_axis_graph1, y=y_axis_graph1))
transfer_rate_time_graph = figure(
x_range=x_axis_graph1,
plot_width=1900,
plot_height=400,
y_axis_label="Time Taken to Transfer File(secs)",
toolbar_location="below",
tools="save")
# Bar Graph
transfer_rate_time_graph.vbar(x='x',
top='y',
bottom=0,
source=graph1_source,
width=0.3,
color="#FFBF00")
# Line Graph
transfer_rate_time_graph.line(x='x',
y='y',
source=graph1_source,
line_width=2,
color="#FFFF00")
# Format axises
transfer_rate_time_graph_final = set_graph_and_legend_properties(
transfer_rate_time_graph, "Transfer Rate Time (sec)")
transfer_rate_time_graph_final.xaxis.major_label_orientation = pi / 4
# Add Tool - Hovertool
transfer_rate_time_graph_final.add_tools(set_hover_tool_tips_graph1())
# Format x-axis labels
# transfer_rate_time_graph_final.xaxis.formatter = PrintfTickFormatter(format="File_%1.0f")
# Plot 2nd Graph
# Create Fig
aggregates_graph = figure(y_range=y_axis,
x_range=ranges.Range1d(
start=0,
end=results_df['Transfer_Time'].max() + 2),
toolbar_location='below',
plot_width=900,
plot_height=500,
tools="save")
# Add Labels
labels = LabelSet(x='right',
y='y',
text='right',
source=graph2_source,
level='glyph',
x_offset=5,
y_offset=5,
render_mode='canvas',
text_color='white')
# Create hBar
aggregates_graph.hbar(y='y',
right='right',
source=graph2_source,
left=0,
height=0.5)
# Format axises and add labels
aggregates_graph_final = set_graph_and_legend_properties(
aggregates_graph, "Performance Metrics (secs)")
aggregates_graph_final.add_layout(labels)
# output to static HTML file
graph_file_name = "perf_test_report_" + time.strftime("%H%M%S") + ".html"
output_file(graph_file_name)
# Save Graph
save(column(transfer_rate_time_graph_final, aggregates_graph_final))
# Success Message
print("Graph generated successfully! Please check file: {}".format(
graph_file_name))
def gen_animated_plot(df, boatnum):
"""
Creates an interactive bokeh plot.
:param df dataframe of data
:param boatnum boatnumber associated with this plot
"""
lat = df['lat'].values
lon = df['lon'].values
data = {'x': [lon2x(i) for i in lon], 'y': [lat2y(i) for i in lat]}
empty_data = {'x': [], 'y': []}
source_visible = ColumnDataSource(data=empty_data)
source_available = ColumnDataSource(data=data)
# plot = figure(plot_width=400, plot_height=400)
plot = figure(x_range=(min(data['x']), max(data['x'])),
y_range=(min(data['y']), max(data['y'])),
x_axis_type="linear",
y_axis_type="linear",
plot_width=400,
plot_height=400)
plot.add_tile(get_provider(Vendors.CARTODBPOSITRON))
# plot = figure(y_range=(-10, 10), plot_width=400, plot_height=400)
plot.line('x', 'y', source=source_visible, line_width=3, line_alpha=0.6)
plot.xaxis.major_tick_line_color = None # turn off x-axis major ticks
plot.xaxis.minor_tick_line_color = None # turn off x-axis minor ticks
plot.yaxis.major_tick_line_color = None # turn off y-axis major ticks
plot.yaxis.minor_tick_line_color = None
plot.xaxis.major_label_text_font_size = '0pt' # turn off x-axis tick labels
plot.yaxis.major_label_text_font_size = '0pt'
callback = CustomJS(args=dict(source_visible=source_visible,
source_available=source_available),
code="""
var visible_data = source_visible.data;
var available_data = source_available.data;
var time = time.value;
var x_avail = available_data['x']
var y_avail = available_data['y']
visible_data['x'] = []
visible_data['y'] = []
for (var i = 0; i < time; i++) {
visible_data['x'].push(x_avail[i]);
visible_data['y'].push(y_avail[i]);
}
source_visible.change.emit();
""")
time_slider = Slider(start=1,
end=len(data['x']),
value=1,
step=1,
title=None,
tooltips=False,
callback=callback)
callback.args["time"] = time_slider
layout = row(
plot,
column(time_slider),
)
output_file(home + "animation_" + str(boatnum) + ".html",
title="Boat Viewer")
save(layout)
return "animation_" + str(boatnum) + ".html"
p3.yaxis.axis_label = 'Delta Np/Np'
p4 = figure(title='DSCOVR CELIAS SHOCKS', tools=tools)
p4.scatter('shock', 'predict_shock_718', color='black', source=source)
p4.select_one(HoverTool).tooltips = tool_tips
p4.xaxis.axis_label = 'DSCOVR DB SHOCK'
p4.yaxis.axis_label = 'My Shock'
p5 = figure(title='DSCOVR CELIAS SHOCKS', tools=tools)
p5.scatter('SPEED', 'Np', color='black', source=source)
p5.select_one(HoverTool).tooltips = tool_tips
p5.xaxis.axis_label = '|V| [km/s]'
p5.yaxis.axis_label = 'Np [cm^-3]'
p6 = figure(title='DSCOVR CELIAS SHOCKS', tools=tools)
p6.scatter('SPEED', 'Vth', color='black', source=source)
p6.select_one(HoverTool).tooltips = tool_tips
p6.xaxis.axis_label = '|V| [km/s]'
p6.yaxis.axis_label = 'Vth [km/s]'
p7 = figure(title='DSCOVR CELIAS SHOCKS', tools=tools)
p7.scatter('Np', 'Vth', color='black', source=source)
p7.select_one(HoverTool).tooltips = tool_tips
p7.xaxis.axis_label = 'Np [cm^-3]'
p7.yaxis.axis_label = 'Vth [km/s]'
save(gridplot([p1, p2], [p3, p4], [p5, p6], [
p7,
]),
filename='../plots/bokeh_training_plot_dscovr.html')
label_opts2 = dict(x=-68, y=47, x_units='screen', y_units='screen')
label_opts3 = dict(x=627,
y=64,
x_units='screen',
y_units='screen',
text_align='right',
text_font_size='9pt')
msg1 = 'By Exoplots'
# when did the data last get updated
modtimestr = get_update_time().strftime('%Y %b %d')
msg3 = 'Data: NASA Exoplanet Archive'
caption1 = Label(text=msg1, **label_opts1)
caption2 = Label(text=modtimestr, **label_opts2)
caption3 = Label(text=msg3, **label_opts3)
fig.add_layout(caption1, 'below')
fig.add_layout(caption2, 'below')
fig.add_layout(caption3, 'below')
plotting.save(fig)
# save the individual pieces so we can just embed the figure without the whole
# html page
script, div = components(fig, theme=theme)
with open(embedfile, 'w') as ff:
ff.write(script)
ff.write(div)
