def create_plot(df, title, carbon_unit, cost_unit, ylimit=None):
"""
:param df:
:param title: string, plot title
:param carbon_unit: string, the unit of carbon emissions used in the
database/model, e.g. "tCO2"
:param cost_unit: string, the unit of cost used in the database/model,
e.g. "USD"
:param ylimit: float/int, upper limit of y-axis; optional
:return:
"""
if df.empty:
return figure()
# Set up data source
source = ColumnDataSource(data=df)
# Determine column types for plotting, legend and colors
# Order of stacked_cols will define order of stacked areas in chart
x_col = "period"
line_col = "carbon_cap"
stacked_cols = ["in_zone_project_emissions", "import_emissions_degen"]
# Stacked Area Colors
colors = ["#666666", "#999999"]
# Set up the figure
plot = figure(
plot_width=800,
plot_height=500,
tools=["pan", "reset", "zoom_in", "zoom_out", "save", "help"],
title=title,
x_range=df[x_col]
# sizing_mode="scale_both"
)
# Add stacked bar chart to plot
bar_renderers = plot.vbar_stack(
stackers=stacked_cols,
x=x_col,
source=source,
color=colors,
width=0.5,
)
# Add Carbon Cap target line chart to plot
target_renderer = plot.circle(
x=x_col,
y=line_col,
source=source,
size=20,
color="black",
fill_alpha=0.2,
line_width=2,
)
# Create legend items
legend_items = [
("Project Emissions", [bar_renderers[0]]),
("Import Emissions", [bar_renderers[1]]),
("Carbon Target", [target_renderer]),
]
# Add Legend
legend = Legend(items=legend_items)
plot.add_layout(legend, "right")
plot.legend[0].items.reverse() # Reverse legend to match stacked order
plot.legend.click_policy = "hide" # Add interactivity to the legend
# Note: Doesn't rescale the graph down, simply hides the area
# Note2: There's currently no way to auto-size legend based on graph size(?)
# except for maybe changing font size automatically?
show_hide_legend(plot=plot) # Hide legend on double click
# Format Axes (labels, number formatting, range, etc.)
plot.xaxis.axis_label = "Period"
plot.yaxis.axis_label = "Emissions ({})".format(carbon_unit)
plot.yaxis.formatter = NumeralTickFormatter(format="0,0")
plot.y_range.end = ylimit # will be ignored if ylimit is None
# Add delivered RPS HoverTool
r_delivered = bar_renderers[0] # renderer for delivered RPS
hover = HoverTool(
tooltips=[
("Period", "@period"),
(
"Project Emissions",
"@%s{0,0} %s (@fraction_of_project_emissions{0%%})"
% (stacked_cols[0], carbon_unit),
),
],
renderers=[r_delivered],
toggleable=False,
)
plot.add_tools(hover)
# Add curtailed RPS HoverTool
r_curtailed = bar_renderers[1] # renderer for curtailed RPS
hover = HoverTool(
tooltips=[
("Period", "@period"),
(
"Import Emissions",
"@%s{0,0} %s (@fraction_of_import_emissions{0%%})"
% (stacked_cols[1], carbon_unit),
),
],
renderers=[r_curtailed],
toggleable=False,
)
plot.add_tools(hover)
# Add RPS Target HoverTool
hover = HoverTool(
tooltips=[
("Period", "@period"),
("Carbon Target", "@%s{0,0} %s" % (line_col, carbon_unit)),
(
"Marginal Cost",
"@carbon_cap_marginal_cost_per_emission{0,0} %s/%s"
% (cost_unit, carbon_unit),
),
],
renderers=[target_renderer],
toggleable=False,
)
plot.add_tools(hover)
return plot
def generateBucketChartForFile(figureName, dataframe, y_max, x_min, x_max):
global funcToColor
global plotWidth
global timeUnitString
colorAlreadyUsedInLegend = {}
MAX_ITEMS_PER_LEGEND = 10
numLegends = 0
legendItems = {}
pixelsPerStackLevel = 30
pixelsPerLegend = 60
pixelsForTitle = 30
cds = ColumnDataSource(dataframe)
hover = HoverTool(
tooltips=[("function", "@function"), (
"duration",
"@durations{0,0}"), ("log file begin timestamp",
"@origstart{0,0}")])
TOOLS = [hover]
p = figure(title=figureName,
plot_width=plotWidth,
x_range=(x_min, x_max),
y_range=(0, y_max + 1),
x_axis_label="Time (" + timeUnitString + ")",
y_axis_label="Stack depth",
tools=TOOLS,
toolbar_location="above")
# No minor ticks or labels on the y-axis
p.yaxis.major_tick_line_color = None
p.yaxis.minor_tick_line_color = None
p.yaxis.major_label_text_font_size = '0pt'
p.yaxis.ticker = FixedTicker(ticks=list(range(0, y_max + 1)))
p.ygrid.ticker = FixedTicker(ticks=list(range(0, y_max + 1)))
p.xaxis.formatter = NumeralTickFormatter(format="0,")
p.title.text_font_style = "bold"
p.quad(left='start',
right='end',
bottom='stackdepth',
top='stackdepthNext',
color='color',
line_color="lightgrey",
line_width=0.5,
source=cds)
for func, fColor in funcToColor.items():
# If this function is not present in this dataframe,
# we don't care about it.
#
boolVec = (dataframe['function'] == func)
fDF = dataframe[boolVec]
if (fDF.size == 0):
continue
# If we already added a color to any legend, we don't
# add it again to avoid redundancy in the charts and
# in order not to waste space.
#
if fColor in colorAlreadyUsedInLegend:
continue
else:
colorAlreadyUsedInLegend[fColor] = True
legendItems[func] = fColor
# Plot height is the function of the maximum call stack and the number of
# legends
p.plot_height = max(
(y_max + 1) * pixelsPerStackLevel, 100) + pixelsForTitle
return p, legendItems
"#80FFFF", "#8CFFFF", "#99FFFF", "#A6FFFF", "#B2FFFF", "#BFFFFF",
"#CCFFFF", "#D9FFFF", "#E6FFFF", "#F2FFFF", "#FFFFFF"
]
tempColor = [
'#000066', '#000080', '#000099', '#0019A3', '#0040B2', '#0066C2',
'#008CD1', '#00B2E0', '#00D9F0', '#00FFFF', '#73FFFF', '#B2FFFF',
'#FFF2F2', '#FFCCCC', '#FFB2B2', '#FF9999', '#FF7373', '#FF4D4D',
'#FF0D0D', '#CC0000', '#A00000', '#800000'
]
mapper = LinearColorMapper(palette=tempColor, low=-20, high=120)
color_bar = ColorBar(color_mapper=mapper,
ticker=BasicTicker(desired_num_ticks=5),
formatter=NumeralTickFormatter(format="0.0"),
label_standoff=15,
border_line_color=None,
location=(0, 0),
title="Temp.(°F)",
major_label_text_color="black",
title_text_color="black",
major_label_text_font_size="8pt",
title_text_font_size="13pt",
title_standoff=15)
#This is the same
color_bar2 = ColorBar(color_mapper=mapper,
ticker=BasicTicker(desired_num_ticks=5),
formatter=NumeralTickFormatter(format="0.0"),
label_standoff=15,
y = cleaned_purchases.accumulated_purchases
lr = LinearRegression()
lr.fit(x[:, np.newaxis], y)
y_lr = lr.predict(x[:, np.newaxis])
days_in_a_lifetime = [[600]]
lifetime_value = "{:,}".format(int(lr.predict(days_in_a_lifetime)[0]) * 10)
# create and render a scatter plot for accumulated purchases
plt = figure(width=280,
height=200,
x_axis_type="datetime",
title='Spend Rate',
toolbar_location=None)
plt.xaxis.formatter = DatetimeTickFormatter(months=["%b %Y"])
plt.yaxis.formatter = NumeralTickFormatter(format="`$0,0`")
plt.circle('date', 'accumulated_purchases', source=purchase_history, size=2)
# plt.line([purchases['Date'].iloc[0] + np.timedelta64(x.min(), 'D'),
# purchases['Date'].iloc[0] + np.timedelta64(x.max(), 'D')],
# [y_lr.min(), y_lr.max()], color='red', line_width=2)
plt.xaxis.major_label_orientation = pi / 4
x = [
purchases['Date'].iloc[0] + np.timedelta64(int(x.min() - 7), 'D'),
purchases['Date'].iloc[0] + np.timedelta64(int(x.max() + 7), 'D')
]
y = [y_lr.min(), y_lr.max()]
lines_source = ColumnDataSource(data=dict(x=x, y=y))
line = Line(x='x', y='y', line_color="red", line_width=2, line_alpha=.8)
plt.add_glyph(lines_source, line)
def plot_it(CsvFullName, CsvFileType, InterestingColumns, DateTimeIndexed,
IndexColumn, data):
'''
Generic plotter
'''
graph_style = args.style
TOOLS = "pan,box_zoom,reset,save" # Defaults for Bokeh
for ColumnName in InterestingColumns:
if graph_style == 'interactive':
BokehChart = figure(tools=TOOLS,
x_axis_type='datetime',
title=ColumnName,
width=1024,
height=768,
x_axis_label='time')
BokehChart.line(data.index,
data[ColumnName],
legend=ColumnName,
line_width=2)
BokehChart.yaxis[0].formatter = NumeralTickFormatter(format="0,0")
output_file(CsvFileType + '_' + ColumnName.replace('/', '_') +
'_interactive.html')
save(BokehChart)
else:
plt.figure(num=None,
figsize=(10, 6),
dpi=80,
facecolor='w',
edgecolor='dimgrey')
if DateTimeIndexed == 'NoIndex':
if graph_style == 'dot':
plt.plot(data[ColumnName],
".",
markersize=2,
color='dimgrey')
else:
plt.plot(data[ColumnName], color='dimgrey')
elif DateTimeIndexed == 'DateTimeIndexed' or DateTimeIndexed == 'WinDateTimeIndexed':
if graph_style == 'dot':
plt.plot(data.DateTime,
data[ColumnName],
".",
markersize=2,
color='dimgrey')
else:
plt.plot(data.DateTime, data[ColumnName], color='dimgrey')
elif DateTimeIndexed == 'TimeIndexed' or DateTimeIndexed == 'WinTimeIndexed':
if graph_style == 'dot':
plt.plot(data.Time,
data[ColumnName],
".",
markersize=2,
color='dimgrey')
else:
plt.plot(data.Time, data[ColumnName], color='dimgrey')
plt.grid()
plt.title(ColumnName, fontsize=10)
plt.xlabel("Time", fontsize=10)
plt.tick_params(labelsize=8)
#plt.title(ColumnName + " of %s" %(CsvFullName), fontsize=10)
#plt.ylabel(ColumnName, fontsize=10)
ax = plt.gca()
ax.set_ylim(ymin=0) # Always zero start
if '%' in ColumnName:
ax.set_ylim(ymax=100)
elif CsvFileType == 'vmstat' and ColumnName in 'us sy wa id':
ax.set_ylim(ymax=100)
ax.get_yaxis().set_major_formatter(
plt.FuncFormatter(lambda x, loc: "{:,}".format(int(x))))
plt.savefig(CsvFileType + '_' + ColumnName.replace('/', '_') +
'_' + graph_style + '.png')
plt.close('all')
def __init__(self, **kwargs):
self.source = ColumnDataSource(data={'time': [], 'cpu': [],
'memory_percent':[], 'network-send':[], 'network-recv':[]}
)
x_range = DataRange1d(follow='end', follow_interval=30000, range_padding=0)
resource_plot = Plot(
x_range=x_range, y_range=Range1d(start=0, end=1),
toolbar_location=None, min_border_bottom=10, **kwargs
)
line_opts = dict(line_width=2, line_alpha=0.8)
g1 = resource_plot.add_glyph(
self.source,
Line(x='time', y='memory_percent', line_color="#33a02c", **line_opts)
)
g2 = resource_plot.add_glyph(
self.source,
Line(x='time', y='cpu', line_color="#1f78b4", **line_opts)
)
resource_plot.add_layout(
LinearAxis(formatter=NumeralTickFormatter(format="0 %")),
'left'
)
legend_opts = dict(
location='top_left', orientation='horizontal', padding=5, margin=5,
label_height=5)
resource_plot.add_layout(
Legend(items=[('Memory', [g1]), ('CPU', [g2])], **legend_opts)
)
network_plot = Plot(
x_range=x_range, y_range=DataRange1d(start=0),
toolbar_location=None, **kwargs
)
g1 = network_plot.add_glyph(
self.source,
Line(x='time', y='network-send', line_color="#a6cee3", **line_opts)
)
g2 = network_plot.add_glyph(
self.source,
Line(x='time', y='network-recv', line_color="#b2df8a", **line_opts)
)
network_plot.add_layout(DatetimeAxis(axis_label="Time"), "below")
network_plot.add_layout(LinearAxis(axis_label="MB/s"), 'left')
network_plot.add_layout(
Legend(items=[('Network Send', [g1]), ('Network Recv', [g2])], **legend_opts)
)
tools = [
PanTool(dimensions='width'), WheelZoomTool(dimensions='width'),
BoxZoomTool(), ResetTool()
]
if 'sizing_mode' in kwargs:
sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
else:
sizing_mode = {}
combo_toolbar = ToolbarBox(
tools=tools, logo=None, toolbar_location='right', **sizing_mode
)
self.root = row(
column(resource_plot, network_plot, **sizing_mode),
column(combo_toolbar, **sizing_mode),
id='bk-resource-profiles-plot',
**sizing_mode
)
# Required for update callback
self.resource_index = [0]
def __init__(self, worker, height=150, **kwargs):
self.worker = worker
names = worker.monitor.quantities
self.last = 0
self.source = ColumnDataSource({name: [] for name in names})
update(self.source, self.get_data())
x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
tools = "reset,xpan,xwheel_zoom"
self.cpu = figure(
title="CPU",
x_axis_type="datetime",
height=height,
tools=tools,
x_range=x_range,
**kwargs
)
self.cpu.line(source=self.source, x="time", y="cpu")
self.cpu.yaxis.axis_label = "Percentage"
self.mem = figure(
title="Memory",
x_axis_type="datetime",
height=height,
tools=tools,
x_range=x_range,
**kwargs
)
self.mem.line(source=self.source, x="time", y="memory")
self.mem.yaxis.axis_label = "Bytes"
self.bandwidth = figure(
title="Bandwidth",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
**kwargs
)
self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
self.bandwidth.yaxis.axis_label = "Bytes / second"
# self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
plots = [self.cpu, self.mem, self.bandwidth]
if not WINDOWS:
self.num_fds = figure(
title="Number of File Descriptors",
x_axis_type="datetime",
height=height,
x_range=x_range,
tools=tools,
**kwargs
)
self.num_fds.line(source=self.source, x="time", y="num_fds")
plots.append(self.num_fds)
if "sizing_mode" in kwargs:
kw = {"sizing_mode": kwargs["sizing_mode"]}
else:
kw = {}
if not WINDOWS:
self.num_fds.y_range.start = 0
self.mem.y_range.start = 0
self.cpu.y_range.start = 0
self.bandwidth.y_range.start = 0
self.root = column(*plots, **kw)
self.worker.monitor.update()
def create_plots(data_rand, strat_columns, pure_randomization_boolean,
sample_p, session_update):
viz_list = []
sample_p = sample_p / 100
for i in range(len(strat_columns)):
# print(strat_columns[i])
# print(data_rand[strat_columns[i]])
# print(data_rand[strat_columns[i]].dtype)
try:
data_rand[strat_columns[i]] = pd.to_numeric(
data_rand[strat_columns[i]])
print("SOMETHING NUMERIC")
# fruits = ['Apples', 'Pears', 'Nectarines', 'Plums', 'Grapes', 'Strawberries']
# counts = [5, 3, 4, 2, 4, 6]
# p = figure(x_range=fruits,
# plot_height=250,
# title="Fruit Counts",
# toolbar_location=None,
# tools="")
# p.vbar(x=fruits, top=counts, width=0.9)number):
df = data_rand.groupby('group-rct').agg(['mean',
'std'])[strat_columns[i]]
# print(df)
#(100*(pd.crosstab(data_rand['group-rct'], data_rand[strat_columns[i]], normalize='columns')))
#df = df.stack().reset_index().rename(columns={0:'Percentage'})
colors = ["#c9d9d3", "#718dbf"]
p = figure(x_range=list(pd.unique(df.index)),
plot_height=250,
title=strat_columns[i],
toolbar_location=None,
tools="")
p.vbar(x=list(df.index.values),
top=list(df['mean'].values),
width=0.9,
color=["#c9d9d3", "#718dbf"])
except ValueError:
print("NOT NUMERIC")
#print(data_rand)
df = pd.crosstab(data_rand['group-rct'],
data_rand[strat_columns[i]],
normalize='columns') #.set_index([])
# df = df.stack().reset_index().rename(columns={0:'Percentage'})
# print(df)
palette = ["#c9d9d3", "#718dbf", "#e84d60"]
# p = figure(x_range=list(df.index.values),
# plot_height=250,
# title=strat_columns[i],
# toolbar_location=None,
# tools="")
# p.vbar(x=list(df.index.values),
# top=list(df['Percentage'].values),
# width=0.9)
x = [(col, group) for col in df.columns
for group in df.index.values]
print(x)
pcts_ = list(df.transpose().stack().values) # like an hstack
print(pcts_)
source = ColumnDataSource(data=dict(
x=x, counts=pcts_, sample_p=list(repeat(sample_p, len(x)))))
hover = HoverTool(tooltips=[('Percentage', '@counts{%0.2f}'),
('Goal for control group',
'@sample_p{%0.2f}')],
formatters={
'Percentage': 'numeral',
'Goal for control group': 'numeral'
})
p = figure(x_range=FactorRange(*x),
plot_height=350,
title=strat_columns[i],
toolbar_location=None,
tools=[hover])
p.yaxis.formatter = NumeralTickFormatter(format='0 %')
p.vbar(x='x',
top='counts',
width=0.9,
source=source,
line_color="white",
fill_color=factor_cmap('x',
palette=palette,
factors=list(df.index.values),
start=1,
end=2))
sample_p_ref = Span(location=sample_p,
dimension='width',
line_color='red',
line_dash='dashed',
line_width=3)
p.add_layout(sample_p_ref)
script, div = components(p)
viz_list.append((script, div))
return viz_list
x_range=(0, 100),
y_range=(0, 1),
plot_width=700,
plot_height=400,
title='Life Expectancy (based on United States Life Tables, 2017 CDC/NVSS)'
)
plot.line('x', 'y', source=source, line_width=3, line_alpha=0.6)
# plot.varea('x', 'y1','y2', source=source2,fill_color='#CFEBE6')
plot.varea('x', 'y1', 'y2', source=source2, fill_color='#F4D8D8')
plot.xaxis.ticker = SingleIntervalTicker(interval=10)
plot.xgrid.ticker = SingleIntervalTicker(interval=10)
plot.xaxis.axis_label = "Projected Age"
plot.yaxis.ticker = SingleIntervalTicker(interval=0.25)
plot.ygrid.ticker = SingleIntervalTicker(interval=0.25)
plot.yaxis.axis_label = "Expected Survival rate at projected age"
plot.yaxis.formatter = NumeralTickFormatter(format='0 %')
label50 = Label(x=5,
y=0.5,
text=('50% probability still alive at age = ' +
str(np.around(MedianAge, decimals=1))),
text_font_size='15px',
text_color='#0000E6')
label75 = Label(x=5,
y=0.75,
text=('75% probability still alive at age = ' +
str(np.around(Age75, decimals=1))),
text_font_size='15px',
text_color='#0000E6')
label25 = Label(x=5,
y=0.25,
text=('25% probability still alive at age = ' +
def grafico(lista_dati, df_eff, choice):
lista_dati['jd'] = pd.to_numeric(lista_dati['jd'], errors='coerce')
lista_dati['magnitudine'] = pd.to_numeric(lista_dati['magnitudine'],
errors='coerce')
p = figure(plot_width=1920,
plot_height=1080,
sizing_mode='scale_both',
tools=['box_zoom,save,hover,reset,wheel_zoom'],
output_backend="webgl")
elenco_filtro = lista_dati['filtro'].to_list()
set_filtro = set(elenco_filtro)
p.xaxis.formatter = NumeralTickFormatter(format="0.00")
i = 0
while i < len(set_filtro):
filtro_sel = list(set_filtro)[i]
dati = lista_dati[lista_dati['filtro'] == filtro_sel]
source = ColumnDataSource(dati)
p.asterisk(x='jd',
y='magnitudine',
size=4,
source=source,
color=COLORE_FILTRO.get(filtro_sel, 'black'),
legend_label=list(set_filtro)[i])
intest = list(dati.columns.values)
t = pd.to_numeric(lista_dati[intest[1]])
for item in range(len(df_eff)):
# se la stella ha effemeridi
if (df_eff.loc[item]['stella'] == choice):
epocaP = float(df_eff.loc[item]['epoca'])
periodo = float(df_eff.loc[item]['periodo'])
numoss = 0
# cerca il giorno in cui c'è l'osservazione
for i in range(len(t)):
while ((numoss * periodo) + epocaP) < (t.iloc[i]):
# nel giorno in cui c'è l'osservazione
if int(t.iloc[i]) == int(epocaP +
numoss * periodo): # or \
#int(t.iloc[i]) == (int(epocaP + numoss*periodo)+1):
# disegna la riga del minimo
eff_primario = (epocaP + (numoss * periodo))
# eff_secondario = 1+(epocaP + (numoss * periodo))
line = Span(location=eff_primario,
dimension='height',
line_dash='dashed',
line_color='red',
line_width=1)
# line2 = Span(location=eff_secondario,
# dimension='height',
# line_dash='dashed',
# line_color='green',
# line_width=0.8)
p.add_layout(line)
#p.add_layout(line2)
numoss = numoss + 1
i += 1
p.y_range.flipped = True
ColumnDataSource(lista_dati)
# genero i dati da visualizzare quando passo col mouse
hover = p.select(dict(type=HoverTool))
hover.tooltips = [
("Data [JD]", "@jd{0.00000}"),
("Magnitudine", "@magnitudine{0.0000}"),
("Osservatore", "@osservatore"),
]
hover.mode = 'mouse'
titolo = choice
p.title.text = titolo
p.title.align = "center"
p.title.text_font_size = "21px"
p.xaxis.axis_label = 'Data [JD]'
p.yaxis.axis_label = 'Magnitudine'
p.legend.location = "top_left"
p.legend.orientation = 'horizontal'
p.legend.click_policy = "hide"
script, div = components(p)
return script, div
def plotOverTime(df, aggregation='median', resample_freq='d'):
# Remove outliers and resample
df = removeOutliers(df,
remove_price_nulls=True,
remove_price_outliers=True)
df['date'] = pd.to_datetime(df['datetime'],
infer_datetime_format=True).dt.date
df = df[df['date'] > (df['date'].max() - timedelta(days=7))]
# Group
df_grouped = df.groupby(['date',
'borough'])[['price']].median().reset_index()
# Create the chart
manhattanSource = ColumnDataSource(
df_grouped[df_grouped['borough'] == 'Manhattan'])
brooklynSource = ColumnDataSource(
df_grouped[df_grouped['borough'] == 'Brooklyn'])
bronxSource = ColumnDataSource(
df_grouped[df_grouped['borough'] == 'Bronx'])
queensSource = ColumnDataSource(
df_grouped[df_grouped['borough'] == 'Queens'])
statenIslandSource = ColumnDataSource(
df_grouped[df_grouped['borough'] == 'Staten Island'])
# Create the plot
p = figure(x_axis_type='datetime', tools=[])
p.line(x='date',
y='price',
line_width=2,
color=colorA,
source=manhattanSource,
legend='Manhattan')
p.line(x='date',
y='price',
line_width=2,
color=colorB,
source=brooklynSource,
legend='Brooklyn')
p.line(x='date',
y='price',
line_width=2,
color=colorC,
source=bronxSource,
legend='Bronx')
p.line(x='date',
y='price',
line_width=2,
color=colorD,
source=queensSource,
legend='Queens')
p.line(x='date',
y='price',
line_width=2,
color=colorE,
source=statenIslandSource,
legend='Staten Island')
# Style chart
p.y_range = Range1d(0, df_grouped['price'].max() * 1.1)
p.yaxis.formatter = NumeralTickFormatter(format="0a")
# Move legend
p.legend.location = 'bottom_left'
p.legend.border_line_color = None
return returnFigure(p)
def get_expTimePerTile(exposures,
width=250,
height=250,
min_border_left=50,
min_border_right=50):
'''
Generates three overlaid histogram of the total exposure time per tile for the given
exposures table. Each of the histograms correspond to different
PROGRAM type: DARK, GREY, BRIGHT
Args:
exposures: Table of exposures with columns "PROGRAM", "EXPTIME"
Options:
width, height: plot width and height in pixels
min_border_left, min_border_right: set minimum width for external labels in pixels
Returns bokeh Figure object
'''
keep = exposures['PROGRAM'] != 'CALIB'
exposures_nocalib = exposures[keep]
exposures_nocalib = exposures_nocalib["PROGRAM", "TILEID", "EXPTIME"]
fig = bk.figure(plot_width=width,
plot_height=height,
title='title',
x_axis_label="Total Exposure Time (Minutes)",
min_border_left=min_border_left,
min_border_right=min_border_right)
fig.yaxis.major_label_text_font_size = '0pt'
fig.title.text_color = '#ffffff'
def total_exptime_dgb(program, color):
'''
Adds a histogram to fig that correspond to the program of the argument with the color provided.
The histogram will be Total Exposure Time per Tile.
Args:
program: String of the desired program name
color: Color of histogram
'''
thisprogram = (exposures_nocalib["PROGRAM"] == program)
if not any(thisprogram):
return
a = exposures_nocalib["TILEID", "EXPTIME"][thisprogram].group_by(
"TILEID").groups.aggregate(np.sum)
hist, edges = np.histogram(np.array(a["EXPTIME"]) / 60,
density=True,
bins=50)
fig.quad(top=hist,
bottom=0,
left=edges[:-1],
right=edges[1:],
fill_color=color,
alpha=0.5,
legend=program)
total_exptime_dgb("DARK", "red")
total_exptime_dgb("GRAY", "blue")
total_exptime_dgb("BRIGHT", "green")
fig.legend.click_policy = "hide"
fig.yaxis[0].formatter = NumeralTickFormatter(format="0.000")
fig.yaxis.major_label_orientation = np.pi / 4
fig.legend.label_text_font_size = '8.5pt'
fig.legend.glyph_height = 15
fig.legend.glyph_width = 15
return fig
def visualize(data, columnName='Mean'):
output_file("%s_trends.html" % columnName)
f = lambda x: str(x)[:10]
data["datetime_s"] = map(f, data.index.date)
cds = ColumnDataSource(data)
print(cds.data.keys())
TOOLS = "crosshair,pan,wheel_zoom,box_zoom,reset,previewsave"
p = figure(plot_width=800,
plot_height=400,
title="BTC %s vs BTC Trend" % columnName,
x_axis_type="datetime",
tools=TOOLS)
r = p.line('Date',
columnName,
line_width=2,
legend="BTC Prices" if columnName == 'Mean' else "BTC %s" %
columnName,
source=cds)
p.extra_y_ranges = {
'Trend': Range1d(data['Trend'].min(), data['Trend'].max())
}
p.line('Date',
'Trend',
color="red",
y_range_name="Trend",
line_width=2,
legend="BTC Trends",
source=cds)
yaxis2 = LinearAxis(y_range_name="Trend")
yaxis2.axis_label = "BTC Trend"
p.add_layout(yaxis2, 'right')
p.xaxis.axis_label = 'Date'
p.yaxis[0].axis_label = 'BTC %s' % columnName
if columnName == "Mean":
p.yaxis[0].formatter = NumeralTickFormatter(format="$0.00")
else:
p.yaxis[0].formatter = NumeralTickFormatter(format="0,0")
if columnName == "Mean":
tool_tip_col = 'Price'
tool_tip_val = '$@Mean{0,0}'
tool_tip_fmt = 'printf'
else:
tool_tip_col = columnName
tool_tip_val = '@%s{0,0}' % columnName
tool_tip_fmt = 'numeral'
p.add_tools(
HoverTool(tooltips=[
("Date", '@datetime_s'),
(tool_tip_col, tool_tip_val),
('Trend', '@Trend'),
],
formatters={
"Date": "datetime",
tool_tip_col: tool_tip_fmt,
"Trend": "numeral"
},
renderers=[r],
mode='vline'))
p.legend.location = "top_left"
show(p)
def aggregate_plot(tb):
"""
Function for creating a bokeh plot that shows aggregate tax liabilities for
each year the TaxBrain instance was run
Parameters
----------
tb: An instance of the TaxBrain object
Returns
-------
Bokeh figure
"""
# Pull aggregate data by year and transpose it for plotting
varlist = ["iitax", "payrolltax", "combined"]
base_data = tb.multi_var_table(varlist, "base").transpose()
base_data["calc"] = "Base"
reform_data = tb.multi_var_table(varlist, "reform").transpose()
reform_data["calc"] = "Reform"
base_cds = ColumnDataSource(base_data)
reform_cds = ColumnDataSource(reform_data)
num_ticks = len(base_data)
del base_data, reform_data
fig = figure(title="Aggregate Tax Liability by Year",
width=700,
height=500,
tools="save")
ii_base = fig.line(x="index",
y="iitax",
line_width=4,
line_color="#12719e",
legend="Income Tax - Base",
source=base_cds)
ii_reform = fig.line(x="index",
y="iitax",
line_width=4,
line_color="#73bfe2",
legend="Income Tax - Reform",
source=reform_cds)
proll_base = fig.line(x="index",
y="payrolltax",
line_width=4,
line_color="#408941",
legend="Payroll Tax - Base",
source=base_cds)
proll_reform = fig.line(x="index",
y="payrolltax",
line_width=4,
line_color="#98cf90",
legend="Payroll Tax - Reform",
source=reform_cds)
comb_base = fig.line(x="index",
y="combined",
line_width=4,
line_color="#a4201d",
legend="Combined - Base",
source=base_cds)
comb_reform = fig.line(x="index",
y="combined",
line_width=4,
line_color="#e9807d",
legend="Combined - Reform",
source=reform_cds)
# format figure
fig.legend.location = "top_left"
fig.yaxis.formatter = NumeralTickFormatter(format="$0.00a")
fig.yaxis.axis_label = "Aggregate Tax Liability"
fig.xaxis.minor_tick_line_color = None
fig.xaxis[0].ticker.desired_num_ticks = num_ticks
# Add hover tool
tool_str = """
<p><b>@calc - {}</b></p>
<p>${}</p>
"""
ii_hover = HoverTool(tooltips=tool_str.format("Individual Income Tax",
"@iitax{0,0}"),
renderers=[ii_base, ii_reform])
proll_hover = HoverTool(tooltips=tool_str.format("Payroll Tax",
"@payrolltax{0,0}"),
renderers=[proll_base, proll_reform])
combined_hover = HoverTool(tooltips=tool_str.format(
"Combined Tax", "@combined{0,0}"),
renderers=[comb_base, comb_reform])
fig.add_tools(ii_hover, proll_hover, combined_hover)
# toggle which lines are shown
plot_js = """
object1.visible = toggle.active
object2.visible = toggle.active
object3.visible = toggle.active
"""
base_callback = CustomJS.from_coffeescript(code=plot_js, args={})
base_toggle = Toggle(label="Base",
button_type="primary",
callback=base_callback,
active=True)
base_callback.args = {
"toggle": base_toggle,
"object1": ii_base,
"object2": proll_base,
"object3": comb_base
}
reform_callback = CustomJS.from_coffeescript(code=plot_js, args={})
reform_toggle = Toggle(label="Reform",
button_type="primary",
callback=reform_callback,
active=True)
reform_callback.args = {
"toggle": reform_toggle,
"object1": ii_reform,
"object2": proll_reform,
"object3": comb_reform
}
fig_layout = layout([fig], [base_toggle, reform_toggle])
# Components needed to embed the figure
js, div = components(fig_layout)
outputs = {
"media_type": "bokeh",
"title": "",
"data": {
"javascript": js,
"html": div
}
}
return outputs
circle = Circle(x="lon",
y="lat",
size=15,
fill_color="fill",
line_color="black")
plot.add_glyph(source, circle)
pan = PanTool()
wheel_zoom = WheelZoomTool()
box_select = BoxSelectTool()
plot.add_tools(pan, wheel_zoom, box_select)
xaxis = LinearAxis(axis_label="lat",
major_tick_in=0,
formatter=NumeralTickFormatter(format="0.000"))
plot.add_layout(xaxis, 'below')
yaxis = LinearAxis(axis_label="lon",
major_tick_in=0,
formatter=PrintfTickFormatter(format="%.3f"))
plot.add_layout(yaxis, 'left')
overlay = BoxSelectionOverlay(tool=box_select)
plot.add_layout(overlay)
doc = Document()
doc.add(plot)
if __name__ == "__main__":
filename = "maps.html"
def build_gaps_block(path, product_id, output_file=None, show=False):
# Open and read the file into lines
with open(path, 'r') as infile:
lines = [line.strip() for line in infile.readlines()]
# Get min and max from the range line
line = lines[1][len('Range: ('):-1]
split = line.split(', ')
min_trade_id = int(split[0])
max_trade_id = int(split[1])
# Get gaps
gaps = []
for line in lines[3:]:
gaps.append(_parse_gap(line))
# Get assignment for each trade id (missing or not)
trade_ids = []
indicators = []
missing = set(num for gap in gaps for num in range(gap[0], gap[1] + 1))
for i in range(min_trade_id, max_trade_id + 1):
trade_ids.append(i)
indicators.append(i not in missing)
# Group into missing and not missing
quads = []
cur_indicator = indicators[0]
quad_left = trade_ids[0]
quad_indicators = []
i = 0
while i < len(trade_ids):
if indicators[i] == cur_indicator:
quad_right = trade_ids[i]
if indicators[i] != cur_indicator or i == len(trade_ids) - 1:
quad_indicators.append(cur_indicator)
quads.append((quad_left, quad_right))
quad_left = trade_ids[i]
quad_right = trade_ids[i]
cur_indicator = indicators[i]
i += 1
# Tools
tools = 'save,xpan,box_zoom,xwheel_zoom,reset'
# Set window range
span = max_trade_id - min_trade_id
space = span * .05
left = min_trade_id - space
right = max_trade_id + space
top = 1.2
bottom = -.2
# Set window range (For some reason, y_range isn't working right)
x_range = Range1d(start=left, end=right, bounds=(left, right))
y_range = Range1d(start=bottom, end=top, bounds=(bottom, top))
p = bplot.figure(
title='Gaps in trade ids',
tools=tools,
plot_width=1000,
plot_height=300,
x_range=x_range,
# y_range=y_range,
active_scroll='xwheel_zoom',
toolbar_location='above',
background_fill_color=C_CHART_BG)
#p.add_tools(WheelZoomTool(dimensions='width'))
source = ColumnDataSource(
data={
'quad_left': [quad[0] for quad in quads],
'quad_right': [quad[1] + 1 for quad in quads],
'top': [1 for x in quads],
'bottom': [0 for x in quads],
'range_left': [quad[0] for quad in quads],
'range_right': [quad[1] for quad in quads],
'fill': [C_BAR if x else C_MISSING for x in quad_indicators],
'indicator': quad_indicators,
'size': [quad[1] - quad[0] + 1 for quad in quads],
})
p.quad(top='top',
bottom='bottom',
left='quad_left',
right='quad_right',
line_color=None,
fill_color='fill',
source=source)
p.xaxis.axis_label = 'trade_id'
p.yaxis.visible = False
p.add_tools(
HoverTool(tooltips=[
('trade_ids', '@range_left - @range_right'),
('size', '@size'),
]))
# Numbers formatted as ints on the x-axis
p.xaxis[0].formatter = NumeralTickFormatter(format='0' *
len(str(min_trade_id)))
# Apply desired styles
styles = {'ygrid.grid_line_color': None}
_apply_figure_styles(p, **styles)
if output_file:
bplot.output_file(output_file)
bplot.save(p)
if show:
bplot.show(p)
return p
def line(self,
query: "query to get the data for difference and confidence",
title: "String of plot title",
x: "string, colomn name for x values",
y: "string, column name for y values",
file_name: "string, output file name",
plot_width=800,
plot_height=600):
"""Create difference and confidence line plot"""
from bokeh.io import output_file, show, save
from bokeh.models import ColumnDataSource, LabelSet, NumeralTickFormatter
from bokeh.plotting import figure
from bokeh.layouts import gridplot
from bokeh.models.tools import HoverTool
from sqlalchemy import create_engine
import pandas as pd
import os
engine = create_engine(self.engineStr)
df_con_diff = pd.read_sql_query(query, con=engine)
df_con_diff['date'] = pd.to_datetime(df_con_diff['date'])
# df_con_diff = df_con_diff.iloc[1:, :]
df_con_diff["label"] = [
str(round(i * 100, 2)) + "%" for i in df_con_diff[y].tolist()
]
output_file(
f"{os.getcwd().replace('/private', '')}/static/plots/{file_name}.html"
)
conf_diff = figure(plot_width=plot_width,
plot_height=plot_height,
title=title,
x_axis_type="datetime",
tools="pan,box_zoom,wheel_zoom,undo,reset,save")
source = ColumnDataSource(df_con_diff)
conf_diff.circle(x=df_con_diff[x],
y=df_con_diff[y],
size=10,
alpha=0.8,
color=(lambda x: '#756bb1'
if y == 'diff' else '#31a354')(y))
conf_diff.line(x,
y,
source=source,
line_width=2,
line_alpha=0.3,
color=(lambda x: '#756bb1'
if y == 'diff' else '#31a354')(y))
conf_diff.add_tools(
HoverTool(tooltips=[("Date", "@date{%F}"),
(f"Inventory {file_name.capitalize()}",
"@label")],
formatters={'@date': 'datetime'}))
conf_diff.yaxis.axis_label = file_name.capitalize()
conf_diff.xaxis.axis_label = "Date"
conf_diff.yaxis.formatter = NumeralTickFormatter(format='%0.0f %%')
conf_diff.outline_line_color = None
conf_diff.title.text_font_size = '25px'
label = LabelSet(x=x,
y=y,
text='label',
x_offset=-10,
y_offset=5,
text_font_size='10px',
source=source,
render_mode='canvas')
conf_diff.add_layout(label)
grid = gridplot([[conf_diff]],
toolbar_location='right',
merge_tools=True,
toolbar_options=dict(logo=None),
sizing_mode='stretch_both')
save(grid)
return grid
def display_timeseries_anomolies(
data: pd.DataFrame,
y: str = "Total",
time_column: str = "TimeGenerated",
anomalies_column: str = "anomalies",
source_columns: list = None,
period: int = 30,
**kwargs,
) -> figure:
"""
Display time series anomalies visualization.
Parameters
----------
data : pd.DataFrame
DataFrame as a time series data set retreived from KQL time series
functions. Dataframe must have columns specified in `y`, `time_column`
and `anomalies_column` parameters
y : str, optional
Name of column holding numeric values to plot against time series to
determine anomolies
(the default is 'Total')
time_column : str, optional
Name of the timestamp column
(the default is 'TimeGenerated')
anomalies_column : str, optional
Name of the column holding binary status(1/0) for anomaly/benign
(the default is 'anomalies')
source_columns : list, optional
List of default source columns to use in tooltips
(the default is None)
period : int, optional
Period of the dataset for hourly-no of days, for daily-no of weeks.
This is used to correctly calculate the plot height.
(the default is 30)
Other Parameters
----------------
ref_time : datetime, optional
Input reference line to display (the default is None)
title : str, optional
Title to display (the default is None)
legend: str, optional
Where to position the legend
None, left, right or inline (default is None)
yaxis : bool, optional
Whether to show the yaxis and labels
range_tool : bool, optional
Show the the range slider tool (default is True)
height : int, optional
The height of the plot figure
(the default is auto-calculated height)
width : int, optional
The width of the plot figure (the default is 900)
xgrid : bool, optional
Whether to show the xaxis grid (default is True)
ygrid : bool, optional
Whether to show the yaxis grid (default is False)
color : list, optional
List of colors to use in 3 plots as specified in order
3 plots- line(observed), circle(baseline), circle_x/user specified(anomalies).
(the default is ["navy", "green", "firebrick"])
Returns
-------
figure
The bokeh plot figure.
"""
check_kwargs(kwargs, _DEFAULT_KWARGS + _TL_VALUE_KWARGS)
reset_output()
output_notebook()
height: int = kwargs.pop("height", None)
width: int = kwargs.pop("width", 1200)
title: str = kwargs.pop("title", None)
time_column = kwargs.get("x", time_column)
show_range: bool = kwargs.pop("range_tool", True)
color: list = kwargs.get("color", ["navy", "green", "firebrick"])
color = [
col1 or col2 for col1, col2 in zip_longest(
color[:3], ["navy", "green", "firebrick"])
]
legend_pos: str = kwargs.pop("legend", "top_left")
xgrid: bool = kwargs.pop("xgrid", False)
ygrid: bool = kwargs.pop("ygrid", False)
kind: str = kwargs.pop("kind", "circle_x")
ref_time, ref_label = _get_ref_event_time(**kwargs)
source = ColumnDataSource(data)
series_count = int(len(data) / period)
# Filtering anomalies to create new dataframe
source_columns = [
col for col in data.columns if col not in [anomalies_column]
]
data_anomaly = data[data[anomalies_column] ==
1][source_columns].reset_index()
hover = HoverTool(
tooltips=_create_tool_tips(data, source_columns),
formatters={
time_column: "datetime",
"Tooltip": "printf"
},
)
# Create the Plot figure
title = title if title else "Time Series Anomalies Visualization"
min_time = data[time_column].min()
max_time = data[time_column].max()
start_range = min_time - ((max_time - min_time) * 0.05)
end_range = max_time + ((max_time - min_time) * 0.05)
height = height if height else _calc_auto_plot_height(series_count)
plot = figure(
x_range=(start_range, end_range),
min_border_left=50,
plot_height=height,
plot_width=width,
x_axis_label=time_column,
x_axis_type="datetime",
y_axis_label=y,
x_minor_ticks=10,
tools=[hover, "xwheel_zoom", "box_zoom", "reset", "save", "xpan"],
toolbar_location="above",
title=title,
)
if xgrid:
plot.xgrid.minor_grid_line_color = "navy"
plot.xgrid.minor_grid_line_alpha = 0.1
plot.xgrid.grid_line_color = "navy"
plot.xgrid.grid_line_alpha = 0.3
else:
plot.xgrid.grid_line_color = None
if ygrid:
plot.ygrid.minor_grid_line_color = "navy"
plot.ygrid.minor_grid_line_alpha = 0.1
plot.ygrid.grid_line_color = "navy"
plot.ygrid.grid_line_alpha = 0.3
else:
plot.ygrid.grid_line_color = None
# set the tick formatter
plot.xaxis[0].formatter = _get_tick_formatter()
plot.yaxis.formatter = NumeralTickFormatter(format="00")
plot.circle(
time_column,
y,
line_color=color[0],
size=4,
source=source,
legend_label="observed",
)
plot.line(
time_column,
"baseline",
line_color=color[1],
source=source,
legend_label="baseline",
)
# create default plot args
arg_dict: Dict[str, Any] = dict(
x=time_column,
y=y,
size=12,
color=color[2],
fill_alpha=0.2,
legend_label="anomalies",
source=ColumnDataSource(data_anomaly),
)
# setting the visualization types for anomalies based on user input to kind
if kind == "diamond_cross":
plot.diamond_cross(**arg_dict)
elif kind == "cross":
plot.cross(**arg_dict)
elif kind == "diamond":
plot.diamond(**arg_dict)
else:
plot.circle_x(**arg_dict)
# interactive legend to hide single/multiple plots if selected
plot.legend.location = legend_pos
plot.legend.click_policy = "hide"
# Create plot for the score column to act as as range selector
rng_select = _create_range_tool(
data=data,
y="score",
min_time=min_time,
max_time=max_time,
plot_range=plot.x_range,
width=width,
height=height,
time_column=time_column,
)
# if we have a reference timestamp, plot the time as a line
if ref_time is not None:
_add_ref_line(plot, ref_time, ref_label, data[y].max())
if show_range:
show(column(plot, rng_select))
return column(plot, rng_select)
show(plot)
return plot
pctFig = figure(title='2PT FG % vs 3PT FG %, 2017-18 Regular Season',
plot_height=400,
plot_width=400,
tools=toolList,
x_axis_label='2PT FG%',
y_axis_label='3PT FG%')
# Draw with circle markers
pctFig.circle(x='team2P%',
y='team3P%',
source=gm_stats_cds,
size=12,
color='black')
# Format the y-axis tick labels as percenages
pctFig.xaxis[0].formatter = NumeralTickFormatter(format='00.0%')
pctFig.yaxis[0].formatter = NumeralTickFormatter(format='00.0%')
# Create a figure relating the totals
totFig = figure(title='Team Points vs Opponent Points, 2017-18 Regular Season',
plot_height=400,
plot_width=400,
tools=toolList,
x_axis_label='Team Points',
y_axis_label='Opponent Points')
# Draw with square markers
totFig.square(x='teamPTS',
y='opptPTS',
source=gm_stats_cds,
size=10,
def test_element_yformatter_instance(self):
formatter = NumeralTickFormatter()
curve = Curve(range(10)).options(yformatter=formatter)
plot = bokeh_renderer.get_plot(curve)
yaxis = plot.handles['yaxis']
self.assertIs(yaxis.formatter, formatter)
def gpu_resource_timeline(doc):
memory_list = [
pynvml.nvmlDeviceGetMemoryInfo(handle).total / (1024 * 1024)
for handle in gpu_handles
]
gpu_mem_max = max(memory_list) * (1024 * 1024)
gpu_mem_sum = sum(memory_list)
# Shared X Range for all plots
x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
tools = "reset,xpan,xwheel_zoom"
item_dict = {
"time": [],
"gpu-total": [],
"memory-total": [],
"rx-total": [],
"tx-total": [],
}
for i in range(ngpus):
item_dict["gpu-" + str(i)] = []
item_dict["memory-" + str(i)] = []
source = ColumnDataSource(item_dict)
def _get_color(ind):
color_list = [
"blue",
"red",
"green",
"black",
"brown",
"cyan",
"orange",
"pink",
"purple",
"gold",
]
return color_list[ind % len(color_list)]
memory_fig = figure(
title="Memory Utilization (per Device)",
sizing_mode="stretch_both",
x_axis_type="datetime",
y_range=[0, gpu_mem_max],
x_range=x_range,
tools=tools,
)
for i in range(ngpus):
memory_fig.line(source=source,
x="time",
y="memory-" + str(i),
color=_get_color(i))
memory_fig.yaxis.formatter = NumeralTickFormatter(format="0.0b")
gpu_fig = figure(
title="GPU Utilization (per Device) [%]",
sizing_mode="stretch_both",
x_axis_type="datetime",
y_range=[0, 100],
x_range=x_range,
tools=tools,
)
for i in range(ngpus):
gpu_fig.line(source=source,
x="time",
y="gpu-" + str(i),
color=_get_color(i))
tot_fig = figure(
title="Total Utilization [%]",
sizing_mode="stretch_both",
x_axis_type="datetime",
y_range=[0, 100],
x_range=x_range,
tools=tools,
)
tot_fig.line(source=source,
x="time",
y="gpu-total",
color="blue",
legend="Total-GPU")
tot_fig.line(source=source,
x="time",
y="memory-total",
color="red",
legend="Total-Memory")
tot_fig.legend.location = "top_left"
pci_fig = figure(
title="Total PCI Throughput [B/s]",
sizing_mode="stretch_both",
x_axis_type="datetime",
x_range=x_range,
tools=tools,
)
pci_fig.line(source=source,
x="time",
y="tx-total",
color="blue",
legend="TX")
pci_fig.line(source=source,
x="time",
y="rx-total",
color="red",
legend="RX")
pci_fig.yaxis.formatter = NumeralTickFormatter(format="0.0b")
pci_fig.legend.location = "top_left"
doc.title = "Resource Timeline"
doc.add_root(
column(gpu_fig,
memory_fig,
tot_fig,
pci_fig,
sizing_mode="stretch_both"))
last_time = time.time()
def cb():
nonlocal last_time
now = time.time()
src_dict = {"time": [now * 1000]}
gpu_tot = 0
mem_tot = 0
tx_tot = 0
rx_tot = 0
for i in range(ngpus):
gpu = pynvml.nvmlDeviceGetUtilizationRates(gpu_handles[i]).gpu
mem = pynvml.nvmlDeviceGetMemoryInfo(gpu_handles[i]).used
tx = (pynvml.nvmlDeviceGetPcieThroughput(
gpu_handles[i], pynvml.NVML_PCIE_UTIL_TX_BYTES) * 1024)
rx = (pynvml.nvmlDeviceGetPcieThroughput(
gpu_handles[i], pynvml.NVML_PCIE_UTIL_RX_BYTES) * 1024)
gpu_tot += gpu
mem_tot += mem / (1024 * 1024)
rx_tot += rx
tx_tot += tx
src_dict["gpu-" + str(i)] = [gpu]
src_dict["memory-" + str(i)] = [mem]
src_dict["gpu-total"] = [gpu_tot / ngpus]
src_dict["memory-total"] = [(mem_tot / gpu_mem_sum) * 100]
src_dict["tx-total"] = [tx_tot]
src_dict["rx-total"] = [rx_tot]
source.stream(src_dict, 1000)
last_time = now
doc.add_periodic_callback(cb, 200)
def plot_region(ds,
city,
plot_cols=cts.CATEGORIES[:3],
dt_col="date",
legend_map=cts.LEGEND_MAP,
log_y=False,
dt_fmt="%d-%m-%Y",
fmt="{0,0}",
set_yticks=False,
color_ramp=cts.COLOR_RAMP,
alpha=0.7,
line_width=2,
ms=8,
bar=False,
bar_width=cfg.DATE_WIDTH,
bar_bottom=100,
glyphs={},
alphas={},
height=cfg.MAX_MAIN_HEIGHT,
height_policy="fit",
width=cfg.MAX_MAIN_WIDTH,
width_policy="fixed",
additional_tools=[],
legend_loc="top_left",
xaxis_ticks=None,
yrange=None):
"""Make a single region plot."""
# Setting tools
active_zooms = [tool for tool in additional_tools if "zoom" in tool]
if len(active_zooms) != 0:
active_scroll = active_zooms[0]
else:
active_scroll = "auto"
tools = ",".join(["save"] + additional_tools)
# Preparing data
ds = ds.reset_index()
ds["date_str"] = ds.date.dt.strftime(dt_fmt)
# Fix any keys with spaces
ds = ds.rename(lambda cl: cl.replace(" ", "_"), axis=1)
plot_cols = [cl.replace(" ", "_") for cl in plot_cols]
glyphs = {cl.replace(" ", "_"): glyph for cl, glyph in glyphs.items()}
alphas = {cl.replace(" ", "_"): a for cl, a in alphas.items()}
# Creating figures
if log_y:
p = figure(x_axis_type="datetime",
y_axis_type="log",
tools=tools,
plot_height=height,
max_height=height,
min_height=height,
height_policy=height_policy,
plot_width=width,
max_width=width,
width_policy=width_policy,
active_drag=None,
active_scroll=active_scroll)
p.yaxis[0].formatter = PrintfTickFormatter(format="%7.0f")
else:
p = figure(x_axis_type="datetime",
tools=tools,
plot_height=height,
max_height=height,
min_height=height,
height_policy=height_policy,
plot_width=width,
max_width=width,
width_policy=width_policy,
active_drag=None,
active_scroll=active_scroll)
p.toolbar.logo = None
# Plot each column
for ci, cl in enumerate(plot_cols):
current_data = ds[ds[cl].notnull()].copy() # remove any NaNs
if bar or glyphs.get(cl, None) == "bar":
cr = p.vbar(x=dt_col,
top=cl,
width=bar_width,
bottom=bar_bottom,
color=color_ramp[ci % len(color_ramp)],
source=current_data,
alpha=alphas.get(cl, alpha),
legend_label=legend_map[ci],
line_width=0)
else:
p.line(x=dt_col,
y=cl,
color=color_ramp[ci % len(color_ramp)],
source=current_data,
alpha=alpha,
line_width=line_width)
cr = p.circle(x=dt_col,
y=cl,
color=color_ramp[ci % len(color_ramp)],
source=current_data,
alpha=alpha,
size=ms,
fill_color="white",
legend_label=legend_map[ci])
# Formatting tooltips
tooltip_header = cts.CASES_TOOLTIP.format(city=city)
tooltip_footer = cts.CASES_TOOLTIP_FOOTER.format(
value_type=data.capitalize(legend_map[ci]), col=cl, fmt=fmt)
# Adding series specific hover
p.add_tools(
HoverTool(renderers=[cr],
tooltips=tooltip_header + tooltip_footer,
formatters={dt_col: "datetime"},
toggleable=False))
# Setting ticks
p.xaxis[0].formatter = DatetimeTickFormatter(days=['%d %b'])
if set_yticks:
p.yaxis[0].formatter = NumeralTickFormatter(format="0 %")
# Setting legend
p.legend.location = legend_loc
p.legend.background_fill_alpha = 0.4
# Setting X-ticker
if xaxis_ticks is not None:
p.xaxis[0].ticker.desired_num_ticks = xaxis_ticks
# Setting Y-axis range
if yrange is not None:
p.y_range.start = yrange[0]
p.y_range.end = yrange[1]
# Applying theme
doc = curdoc()
doc.theme = cfg.THEME
return p
fig = figure(title='Order Map', x_axis_type='datetime', x_range=x_range, y_range=y_range)
best_bid_glyph = Step(x='datetime', y='order_bid_0_price', line_width=2, line_color='green', mode='after')
best_ask_glyph = Step(x='datetime', y='order_ask_0_price', line_width=2, line_color='red', mode='after')
fig.add_glyph(source, best_bid_glyph)
fig.add_glyph(source, best_ask_glyph)
fig.circle_x(source=source, x='datetime', y='trade_price', color='trade_aggressor', size='trade_size')
for i in range(0, num_levels):
fig.circle(source=source, x='datetime', y='order_bid_{0}_price'.format(i), fill_color='white', line_color='green',
size='order_bid_{0}_scaled_qty'.format(i))
fig.circle(source=source, x='datetime', y='order_ask_{0}_price'.format(i), fill_color='white', line_color='red',
size='order_ask_{0}_scaled_qty'.format(i))
fig.xaxis.formatter = DatetimeTickFormatter(seconds=["%M:%S.%3n"])
fig.xaxis.major_label_orientation = np.pi/4
fig.yaxis.formatter = NumeralTickFormatter(format="$0,0.00")
def test_update():
new = dict.fromkeys(
['datetime',
'trade_price',
'trade_size',
'trade_aggressor'] +
[part.format(i) for i in range(0, num_levels) for part in ['order_bid_{0}_price',
'order_ask_{0}_price',
'order_bid_{0}_qty',
'order_ask_{0}_qty',
'order_bid_{0}_scaled_qty',
'order_ask_{0}_scaled_qty']], [])
def plot_swabs(ds,
city,
start_date=cts.SWABS_START_DATE,
dt_col="date",
plot_cols=cts.SWABS_CATEGORIES,
legend_map=cts.SWABS_LEGEND_MAP,
x_col="date",
y_range=cts.SWABS_RANGE,
alpha_factor=cts.SWABS_ALPHA,
size_factor=cts.SWABS_SIZE,
delay_factor=cts.SWABS_DELAY,
log_y=False,
dt_fmt="%d-%m-%Y",
fmt="{0,0}",
set_yticks=False,
color_ramp=cts.COLOR_RAMP,
alpha=0.7,
line_width=2,
ms=8,
glyphs={},
alphas={},
height=cfg.MAX_MAIN_HEIGHT,
height_policy="fit",
width=cfg.MAX_MAIN_WIDTH,
width_policy="fixed",
legend_loc="top_left",
xaxis_ticks=None,
yrange=None):
"""Custom plot for swabs."""
tools = "save"
# Preparing data
ds = ds.reset_index()
ds["date_str"] = ds[dt_col].dt.strftime(dt_fmt)
# Creating figures
if log_y:
if x_col == dt_col:
p = figure(x_axis_type="datetime",
y_range=y_range,
y_axis_type="log",
tools=tools)
p.xaxis[0].formatter = DatetimeTickFormatter(days=['%d %b'])
else:
p = figure(y_range=y_range, y_axis_type="log", tools=tools)
p.xaxis[0].formatter = NumeralTickFormatter(format="0a")
else:
if x_col == dt_col:
p = figure(x_axis_type="datetime", y_range=y_range, tools=tools)
p.xaxis[0].formatter = DatetimeTickFormatter(days=['%d %b'])
else:
p = figure(y_range=y_range, tools=tools)
p.xaxis[0].formatter = NumeralTickFormatter(format="0a")
p.toolbar.logo = None
p.plot_height = height
p.max_height = height
p.min_height = height
p.height_policy = height_policy
p.plot_width = width
p.max_width = width
p.width_policy = width_policy
p.yaxis[0].formatter = NumeralTickFormatter(format="0a")
ds["alpha"] = (ds["total"].shift(delay_factor) /
ds["swabs_clean"]) * alpha_factor
ds["size"] = ds["swabs_derived_daily"] / size_factor
ds["positive"] = ds["total"].shift(delay_factor) / ds["swabs_clean"]
cr = p.circle(x=x_col,
y="swabs_clean",
size="size",
source=ds[ds["swabs_clean"].notnull()].loc[start_date:],
color="#990800",
alpha="alpha",
line_color="white")
p.yaxis[0].formatter = NumeralTickFormatter(format="0a")
p.add_tools(
HoverTool(renderers=[cr],
tooltips=cts.SWABS_TOOLTIP,
formatters={'date': 'datetime'},
toggleable=False))
p.toolbar.logo = None
# Setting axis labels
if x_col != dt_col:
x_label = Label(x=p.x_range.end,
y=p.y_range.start,
text="выявлено",
render_mode='css',
x_offset=350,
text_font_size="10pt")
y_label = Label(x=p.x_range.start,
y=p.y_range.end,
text="тестов всего",
render_mode='css',
text_baseline="top",
x_offset=-10,
text_font_size="10pt")
p.add_layout(x_label)
p.add_layout(y_label)
# Setting X-ticker
if xaxis_ticks is not None:
p.xaxis[0].ticker.desired_num_ticks = xaxis_ticks
# Applying theme
doc = curdoc()
doc.theme = cfg.THEME
return p
def plotOutlierHistogram(dataframe, maxOutliers, func,
statisticalOutlierThreshold, userLatencyThreshold,
averageDuration, maxDuration):
global pixelsForTitle
global pixelsPerHeightUnit
global plotWidth
global timeUnitString
cds = ColumnDataSource(dataframe)
figureTitle = "Occurrences of " + func + " that took longer than " \
+ statisticalOutlierThreshold + "."
hover = HoverTool(
tooltips=[("interval start",
"@lowerbound{0,0}"), ("interval end", "@upperbound{0,0}")])
TOOLS = [hover, "tap, reset"]
p = figure(title = figureTitle, plot_width = plotWidth,
plot_height = min(500, (max(5, (maxOutliers + 1)) \
* pixelsPerHeightUnit + \
pixelsForTitle)),
x_axis_label = "Execution timeline (" + timeUnitString + ")",
y_axis_label = "Number of outliers",
tools = TOOLS, toolbar_location="above")
y_ticker_max = p.plot_height // pixelsPerHeightUnit
y_ticker_step = max(1, (maxOutliers + 1) // y_ticker_max)
y_upper_bound = (maxOutliers // y_ticker_step + 2) * y_ticker_step
p.yaxis.ticker = FixedTicker(
ticks=list(range(0, y_upper_bound, y_ticker_step)))
p.ygrid.ticker = FixedTicker(
ticks=list(range(0, y_upper_bound, y_ticker_step)))
p.xaxis.formatter = NumeralTickFormatter(format="0,")
p.y_range = Range1d(0, y_upper_bound)
p.quad(left='lowerbound',
right='upperbound',
bottom='bottom',
top='height',
color=funcToColor[func],
source=cds,
nonselection_fill_color=funcToColor[func],
nonselection_fill_alpha=1.0,
line_color="lightgrey",
selection_fill_color=funcToColor[func],
selection_line_color="grey")
p.x(x='markerX',
y='markerY',
size='markersize',
color='navy',
line_width=1,
source=cds)
# Add an annotation to the chart
#
y_max = dataframe['height'].max()
text = "Average duration: " + '{0:,.0f}'.format(averageDuration) + " " + \
timeUnitString + \
". Maximum duration: " + '{0:,.0f}'.format(maxDuration) + " " + \
timeUnitString + ". "
if (userLatencyThreshold is not None):
text = text + \
"An \'x\' shows intervals with operations exceeding " + \
"a user-defined threshold of " + \
userLatencyThreshold + "."
mytext = Label(x=0,
y=y_upper_bound - y_ticker_step,
text=text,
text_color="grey",
text_font="helvetica",
text_font_size="10pt",
text_font_style="italic")
p.add_layout(mytext)
url = "@bucketfiles"
taptool = p.select(type=TapTool)
taptool.callback = OpenURL(url=url)
return p
def my_portfolio(request):
# GRAPHS
# Work-In-Progress: linking the 'Forecast' button to the variables in 'p'
selected_stocks = request.POST.getlist('idArr[]')
print(selected_stocks)
context = {'selected_stocks': selected_stocks}
if selected_stocks is None:
return render(request=request,
template_name='homepage.html',
context=context)
try:
print("Start")
stocks_db = Stocks.objects
user_stock_dic = {
} # to bring user_stock_dic to store in front end local storage for future usage
stock_name = []
forecast_return = []
for each in selected_stocks:
stock_data = stocks_db.filter(ticker=each).values()
print(stock_data)
stock_name.append(stock_data[0]['stock_name'])
forecast_return.append(stock_data[0]['forecast_return'])
user_stock_dic = {
'stock_name': stock_name,
'forecast_return': forecast_return
}
if user_stock_dic:
df = user_stock_dic
else:
# Dummy data
df = {'stock_name': ['AAPL', 'GOOGL'], 'forecast_return': [1, 2]}
print(df)
print("End")
except KeyError:
script = None
div = None
else:
p = figure(x_range=df['stock_name'],
plot_height=300,
plot_width=1000,
title="Predicted Returns",
toolbar_location=None,
tools="")
p.vbar(x=df['stock_name'],
top=df['forecast_return'],
width=0.9,
hover_color="pink")
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.background_fill_color = None
p.border_fill_color = None
p.title.text_font = "gill"
p.title.text_font_size = "24px"
p.title.text_color = "white"
p.yaxis.axis_label = "Predicted Returns"
p.yaxis.axis_label_text_font = "gill"
p.yaxis.axis_label_text_font_size = "16px"
p.yaxis.axis_label_text_color = "white"
p.yaxis[0].formatter = NumeralTickFormatter(format="0.000%")
p.xaxis.major_label_text_font = "gill"
p.xaxis.major_label_text_font_size = "20px"
p.xaxis.major_label_text_font_style = "bold"
p.xaxis.major_label_text_color = "white"
p.yaxis.major_label_text_font = "gill"
p.yaxis.major_label_text_font_size = "20px"
p.yaxis.major_label_text_color = "white"
p.add_tools(
HoverTool(tooltips=[(
"Stock",
"@stock_name"), ("Predicted Returns", "@forecast_return")]))
script, div = components(p)
''' Create Stocks Objects: Already done, do not uncomment to avoid duplicating stocks objects '''
# stocks_df = pd.read_csv('tickers_latest.csv')
# tup = stocks_df.values
# for each in tup:
# Stocks.objects.create(stock_id=each[0], stock_name=each[2], ticker=each[1], forecast_return=each[3], mse=each[4])
''' Link front end stocks selection inputs to back-end '''
# idArr contains a list of user-selected stock tickers
selected_stocks = request.POST.get("idArr")
# Retrieve Stock model object's ticker based on selected stocks, append the tickers into a dictionary (stock_dict) to return to user for future use
stock_dict = {}
if selected_stocks is not None:
for each in selected_stocks:
count = 1
# Assigns every user selected stock to a dictionary key. E.g. stock_dict = {'stock_1':{'ticker':'AAPL', 'forecasted_return': 0.01234}}
stock_dict["stock_{}".format(count)] = {
'ticker':
each,
'forecasted_return':
Stocks.objects.get(ticker=each).forecast_return
}
count += 1
''' ML Model: inputs- idArray'''
# Insert ML Model here
# store output to mlOutput to be passed to front end, which will then jump to optimize.html via jqeury
# Output: cumulative return, sharpe, weights
mlOutput = {}
context = {
'df': df,
'script': script,
'div': div,
'stock_dict': stock_dict,
'mlOutput': mlOutput
}
return render(request, "homepage.html", context=context)
def generateNavigatorFigure(dataframe, i, title):
global pixelsForTitle
global pixelsPerHeightUnit
global plotWidth
# Generate the colors, such that the current interval is shown in a
# different color than the rest.
#
numIntervals = dataframe['intervalnumber'].size
color = ["white" for x in range(numIntervals)]
color[i] = "salmon"
dataframe['color'] = color
cds = ColumnDataSource(dataframe)
title = title + " CLICK TO NAVIGATE"
hover = HoverTool(
tooltips=[("interval #",
"@intervalnumber"), ("interval start", "@intervalbegin{0,0}"
), ("interval end",
"@intervalend{0,0}")])
TOOLS = [hover, "tap"]
p = figure(title=title,
plot_width=plotWidth,
x_range=(0, numIntervals),
plot_height=2 * pixelsPerHeightUnit + pixelsForTitle,
x_axis_label="",
y_axis_label="",
tools=TOOLS,
toolbar_location="above")
# No minor ticks or labels on the y-axis
p.yaxis.major_tick_line_color = None
p.yaxis.minor_tick_line_color = None
p.yaxis.major_label_text_font_size = '0pt'
p.yaxis.ticker = FixedTicker(ticks=list(range(0, 1)))
p.ygrid.ticker = FixedTicker(ticks=list(range(0, 1)))
p.xaxis.formatter = NumeralTickFormatter(format="0,")
p.title.align = "center"
p.title.text_font_style = "normal"
p.quad(left='intervalnumber',
right='intervalnumbernext',
bottom=0,
top=2,
color='color',
source=cds,
nonselection_fill_color='color',
nonselection_fill_alpha=1.0,
line_color="aliceblue",
selection_fill_color="white",
selection_line_color="lightgrey")
url = "@bucketfiles"
taptool = p.select(type=TapTool)
taptool.callback = OpenURL(url=url)
return p
def compare(request):
#checkbox for previously saved portfolios (portfolio objects)
#button to run jquery to display charts and make YoY returns comparison for each portfolio
#Based on this, safe to say once a portfolio object is created, also need to save their charts and stats to load easily for comparison
if request.method == 'POST':
user_id = request.POST.get('user_id', None)
if user_id is not None:
for each in User_Portfolio.objects.get(
user_id=user_id
): # Get all portfolio objects associated with the current user
count = 1
user_port_dic = {}
user_port_dic['portfolio_{}'.format(count)] = {
'id': count,
'p_name': Portfolio.objects.get(id=each).p_name,
'sharpe': Portfolio.objects.get(id=each).sharpe,
'stocks': {
'ticker':
Stocks.objects.get(id=0).ticker,
'stock_weight':
Portfolio_Stocks.objects.get(id=each).stock_weight
}
}
count += 1
else:
user_portfolio = User_Portfolio.objects.all()
portfolio = Portfolio.objects.all()
p_name = []
for i in user_portfolio.values():
p_data = portfolio.filter(id=i['portfolio_id_id']).values()
p_name.append(p_data[0]['p_name'])
context = {'name': p_name}
return render(request, "compare.html", context=context)
# Saving output form the checkbox
try:
selected_portfolios = request.POST.getlist(
'checkbox1') # This will show [p_name, p_name, ...]
for each in selected_portfolios:
port_dic = {
} # to bring user_stock_dic to store in front end local storage for future usage
port_dic = pd.concat([
port_dic,
user_port_dic.loc[user_port_dic['p_name'] == each]
],
ignore_index=True)
df = port_dic #has p_name, sharpe, stocks[ticker, stock_weight]
# Dummy data; saving output from ML model (for loop?)
df = {'p_name': ['Port 1', 'Port 2'], 'sharpe': [1.234, 1.5637]}
context = {'selected_portfolios': selected_portfolios}
if selected_portfolios is None:
return render(request=request,
template_name='compare.html',
context=context)
else:
p = figure(x_range=df['p_name'],
plot_height=300,
plot_width=1000,
title="Portfolio Comparison",
toolbar_location=None,
tools="")
p.vbar(x=df['p_name'],
top=df['sharpe'],
width=0.9,
hover_color="pink")
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.background_fill_color = None
p.border_fill_color = None
p.title.text_font = "gill"
p.title.text_font_size = "24px"
p.title.text_color = "white"
p.yaxis.axis_label = "Sharpe Ratio"
p.yaxis.axis_label_text_font = "gill"
p.yaxis.axis_label_text_color = "white"
p.xaxis.major_label_text_font = "gill"
p.xaxis.major_label_text_font_size = "20px"
p.xaxis.major_label_text_font_style = "bold"
p.xaxis.major_label_text_color = "white"
p.yaxis.major_label_text_font = "gill"
p.yaxis.major_label_text_font_size = "20px"
p.yaxis.major_label_text_color = "white"
p.yaxis[0].formatter = NumeralTickFormatter(format="0.00")
p.add_tools(
HoverTool(tooltips=[("Portfolio",
"@p_name"), ("Sharpe Ratio",
"@sharpe")]))
except KeyError:
script = None
div = None
context = {}
# pie chart
'''
from selected_portfolios, find the id --> use it to reference to port_id --> get stock_id & stock_weight --> find stock_name & ticker
fill the below counter() with stock_name&ticker, stock_weight
df = #has id, p_name, sharpe, stocks[ticker, stock_weight]
'''
for each in df['id']:
div_dict = {}
div_list = []
x = Counter(df['stocks'])
data = pd.Series(x).reset_index(name='value').rename(
columns={'index': 'ticker'})
data['angle'] = data['value'] / sum(x.values()) * 2 * pi
data['color'] = Category20c[len(x)]
p_each = figure(plot_height=350,
plot_width=470,
title="Pie Chart",
toolbar_location=None,
tools="hover",
tooltips="@ticker: @value",
x_range=(-0.5, 1.0))
p_each.wedge(x=0,
y=1,
radius=0.4,
start_angle=cumsum('angle', include_zero=True),
end_angle=cumsum('angle'),
line_color="white",
fill_color='color',
legend='ticker',
source=data,
hover_color="pink")
p_each.title.text_font = "gill"
p_each.axis.axis_label = None
p_each.axis.visible = False
p_each.grid.grid_line_color = None
p_each.legend.label_text_font = "gill"
div_dict['p_each_div'] = 'p_each_div'
div_list.append('p_each_div')
context = {'script': script}
context['p_each_div'] = 'p_each_div'
components(div_dict)
df['div_list'] = div_list
return render(request, "compare.html", context=context)
else:
user_portfolio = User_Portfolio.objects.all()
portfolio = Portfolio.objects.all()
p_name = []
for i in user_portfolio.values():
p_data = portfolio.filter(id=i['portfolio_id_id']).values()
p_name.append(p_data[0]['p_name'])
context = {'name': p_name}
return render(request, "compare.html", context=context)
def __init__(self, scheduler, width=600, **kwargs):
with log_errors():
self.last = 0
self.scheduler = scheduler
self.source = ColumnDataSource({
"memory": [1, 2],
"memory-half": [0.5, 1],
"memory_text": ["1B", "2B"],
"utilization": [1, 2],
"utilization-half": [0.5, 1],
"worker": ["a", "b"],
"gpu-index": [0, 0],
"y": [1, 2],
"escaped_worker": ["a", "b"],
})
memory = figure(title="GPU Memory",
tools="",
id="bk-gpu-memory-worker-plot",
width=int(width / 2),
name="gpu_memory_histogram",
**kwargs)
rect = memory.rect(
source=self.source,
x="memory-half",
y="y",
width="memory",
height=1,
color="#76B900",
)
rect.nonselection_glyph = None
utilization = figure(title="GPU Utilization",
tools="",
id="bk-gpu-utilization-worker-plot",
width=int(width / 2),
name="gpu_utilization_histogram",
**kwargs)
rect = utilization.rect(
source=self.source,
x="utilization-half",
y="y",
width="utilization",
height=1,
color="#76B900",
)
rect.nonselection_glyph = None
memory.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512],
base=1024)
memory.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
memory.xaxis.major_label_orientation = -math.pi / 12
memory.x_range.start = 0
for fig in [memory, utilization]:
fig.xaxis.minor_tick_line_alpha = 0
fig.yaxis.visible = False
fig.ygrid.visible = False
tap = TapTool(callback=OpenURL(
url="./info/worker/@escaped_worker.html"))
fig.add_tools(tap)
fig.toolbar.logo = None
fig.toolbar_location = None
fig.yaxis.visible = False
hover = HoverTool()
hover.tooltips = "@worker : @utilization %"
hover.point_policy = "follow_mouse"
utilization.add_tools(hover)
hover = HoverTool()
hover.tooltips = "@worker : @memory_text"
hover.point_policy = "follow_mouse"
memory.add_tools(hover)
self.memory_figure = memory
self.utilization_figure = utilization
self.utilization_figure.y_range = memory.y_range
self.utilization_figure.x_range.start = 0
self.utilization_figure.x_range.end = 100
def resource_timeline(doc):
# Shared X Range for all plots
x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
tools = "reset,xpan,xwheel_zoom"
source = ColumnDataSource({
"time": [],
"memory": [],
"cpu": [],
"disk-read": [],
"disk-write": [],
"net-read": [],
"net-sent": [],
})
memory_fig = figure(
title="Memory",
sizing_mode="stretch_both",
x_axis_type="datetime",
y_range=[0, psutil.virtual_memory().total],
x_range=x_range,
tools=tools,
)
memory_fig.line(source=source, x="time", y="memory")
memory_fig.yaxis.formatter = NumeralTickFormatter(format="0.0b")
cpu_fig = figure(
title="CPU",
sizing_mode="stretch_both",
x_axis_type="datetime",
y_range=[0, 100],
x_range=x_range,
tools=tools,
)
cpu_fig.line(source=source, x="time", y="cpu")
disk_fig = figure(
title="Disk I/O Bandwidth",
sizing_mode="stretch_both",
x_axis_type="datetime",
x_range=x_range,
tools=tools,
)
disk_fig.line(source=source,
x="time",
y="disk-read",
color="blue",
legend="Read")
disk_fig.line(source=source,
x="time",
y="disk-write",
color="red",
legend="Write")
disk_fig.yaxis.formatter = NumeralTickFormatter(format="0.0b")
disk_fig.legend.location = "top_left"
net_fig = figure(
title="Network I/O Bandwidth",
sizing_mode="stretch_both",
x_axis_type="datetime",
x_range=x_range,
tools=tools,
)
net_fig.line(source=source,
x="time",
y="net-read",
color="blue",
legend="Recv")
net_fig.line(source=source,
x="time",
y="net-sent",
color="red",
legend="Send")
net_fig.yaxis.formatter = NumeralTickFormatter(format="0.0b")
net_fig.legend.location = "top_left"
doc.title = "Resource Timeline"
doc.add_root(
column(cpu_fig,
memory_fig,
disk_fig,
net_fig,
sizing_mode="stretch_both"))
last_disk_read = psutil.disk_io_counters().read_bytes
last_disk_write = psutil.disk_io_counters().write_bytes
last_net_recv = psutil.net_io_counters().bytes_recv
last_net_sent = psutil.net_io_counters().bytes_sent
last_time = time.time()
def cb():
nonlocal last_disk_read, last_disk_write, last_net_recv, last_net_sent, last_time
now = time.time()
cpu = psutil.cpu_percent()
mem = psutil.virtual_memory().used
disk = psutil.disk_io_counters()
disk_read = disk.read_bytes
disk_write = disk.write_bytes
net = psutil.net_io_counters()
net_read = net.bytes_recv
net_sent = net.bytes_sent
source.stream(
{
"time": [now * 1000], # bokeh measures in ms
"cpu": [cpu],
"memory": [mem],
"disk-read": [(disk_read - last_disk_read) /
(now - last_time)],
"disk-write": [(disk_write - last_disk_write) /
(now - last_time)],
"net-read": [(net_read - last_net_recv) / (now - last_time)],
"net-sent": [(net_sent - last_net_sent) / (now - last_time)],
},
1000,
)
last_disk_read = disk_read
last_disk_write = disk_write
last_net_recv = net_read
last_net_sent = net_sent
last_time = now
doc.add_periodic_callback(cb, 200)
