def _get_tick_formatter() -> DatetimeTickFormatter:
"""Return tick formatting for different zoom levels."""
# '%H:%M:%S.%3Nms
tick_format = DatetimeTickFormatter()
tick_format.days = ["%m-%d %H:%M"]
tick_format.hours = ["%H:%M:%S"]
tick_format.minutes = ["%H:%M:%S"]
tick_format.seconds = ["%H:%M:%S"]
tick_format.milliseconds = ["%H:%M:%S.%3N"]
return tick_format
def display_timeline(data,
alert=None,
overlay_data=None,
title: str = None,
time_column: str = 'TimeGenerated',
source_columns: list = None,
overlay_colums: list = None,
height: int = 300):
"""
Display a timeline of events.
Arguments:
data {pd.DataFrame} -- Input DataFrame
Keyword Arguments:
alert {SecurityAlert} -- Input alert (optional) (default: {None})
overlay_data {pd.DataFrame} -- Second event stream (DataFrame)
to display as overlay (default: {None})
title {str} -- [description] (default: {None})
time_column {str} -- The name of the time
property used in the Dataframe(s) (default: {'TimeGenerated'})
source_columns {list} -- List of source columns to use in
tooltips (default: {None})
overlay_colums {list} -- List of source columns to use in
overlay data tooltips (default: {None})
heigh {int} -- the height of the plot figure (under 300 limits access
to Bokeh tools)
"""
reset_output()
output_notebook()
# pylint: disable=C0103
WRAP = 50
WRAP_CMDL = 'WrapCmdl'
# pylint: enable=C0103
y_max = 1
if not source_columns:
source_columns = ['NewProcessName', 'EventID', 'CommandLine']
if time_column not in source_columns:
source_columns.append(time_column)
if 'CommandLine' in source_columns:
graph_df = data[source_columns].copy()
graph_df[WRAP_CMDL] = graph_df.apply(
lambda x: _wrap_text(x.CommandLine, WRAP), axis=1)
else:
graph_df = data[source_columns].copy()
# if we have an overlay - add this data and shift the y co-ordinates to
# show on two separate lines
if overlay_data is not None:
overlay_colums = (overlay_colums
if overlay_colums is not None else source_columns)
if time_column not in overlay_colums:
overlay_colums.append(time_column)
if 'CommandLine' in overlay_colums:
overlay_df = overlay_data[overlay_colums].copy()
overlay_df[WRAP_CMDL] = overlay_df.apply(
lambda x: _wrap_text(x.CommandLine, WRAP), axis=1)
else:
overlay_df = overlay_data[overlay_colums].copy()
graph_df['y_index'] = 2
overlay_df['y_index'] = 1
y_max = 2
else:
graph_df['y_index'] = 1
source = ColumnDataSource(graph_df)
# build the tool tips from columns (excluding these)
excl_cols = [time_column, 'CommandLine']
tool_tip_items = [(f'{col}', f'@{col}') for col in source_columns
if col not in excl_cols]
if WRAP_CMDL in graph_df:
tool_tip_items.append(('CommandLine', f'@{WRAP_CMDL}'))
hover = HoverTool(
tooltips=tool_tip_items,
formatters={'Tooltip': 'printf'}
# display a tooltip whenever the cursor is vertically in line with a glyph
# ,mode='vline'
)
if not title:
title = 'Event Timeline'
else:
title = 'Timeline {}'.format(title)
# tools = 'pan, box_zoom, wheel_zoom, reset, undo, redo, save, hover'
plot = figure(min_border_left=50,
plot_height=height,
plot_width=900,
x_axis_label='Event Time',
x_axis_type='datetime',
x_minor_ticks=10,
tools=[hover, 'pan', 'xwheel_zoom', 'box_zoom', 'reset'],
title=title)
plot.yaxis.visible = False
# Tick formatting for different zoom levels
# '%H:%M:%S.%3Nms
tick_format = DatetimeTickFormatter()
tick_format.days = ['%m-%d %H:%M']
tick_format.hours = ['%H:%M:%S']
tick_format.minutes = ['%H:%M:%S']
tick_format.seconds = ['%H:%M:%S']
tick_format.milliseconds = ['%H:%M:%S.%3N']
plot.xaxis[0].formatter = tick_format
plot.circle(x=time_column,
y='y_index',
color='navy',
alpha=0.5,
size=10,
source=source)
if overlay_data is not None:
overlay_source = ColumnDataSource(overlay_df)
plot.circle(x=time_column,
y='y_index',
color='green',
alpha=0.5,
size=10,
source=overlay_source)
# Adding data labels stops everything working!
# labels = LabelSet(x=time_column, y='y_index', y_offset=5,
# text='NewProcessName', source=source,
# angle='90deg', text_font_size='8pt')
# p.add_layout(labels)
# if we have an alert, plot the time as a line
if alert is not None:
x_alert_label = pd.Timestamp(alert['StartTimeUtc'])
plot.line(x=[x_alert_label, x_alert_label], y=[0, y_max + 1])
alert_label = Label(x=x_alert_label,
y=0,
y_offset=10,
x_units='data',
y_units='data',
text='< Alert time',
render_mode='css',
border_line_color='red',
border_line_alpha=1.0,
background_fill_color='white',
background_fill_alpha=1.0)
plot.add_layout(alert_label)
print('Alert start time = ', alert['StartTimeUtc'])
show(plot)
def main_charts(save=False):
### --------- PRICE CHART
data = {
'time':
[t_s(t) for t, p in get_gecko(models()['epic']).data['price_7d']][::2],
'price':
[p for t, p in get_gecko(models()['epic']).data['price_7d']][::2],
}
df = pd.DataFrame(data)
source = ColumnDataSource(df)
DTF = DatetimeTickFormatter()
DTF.hours = ["%H:%M"]
DTF.days = ["%d/%m"]
DTF.months = ["%d/%m/%Y"]
DTF.years = ["%d/%m/%Y"]
TOOLS = "pan, wheel_zoom, box_zoom, reset, save"
p = figure(x_axis_type="datetime",
tools=TOOLS,
plot_width=900,
plot_height=300,
sizing_mode='scale_width')
p.line(x='time', y='price', source=source, line_width=2, color='gold')
p.varea(x='time',
y1=0,
y2='price',
source=source,
alpha=0.2,
fill_color='gold')
hover = HoverTool(tooltips=[("Date: ", "@time{%y-%m-%d}"),
("Time: ", "@time{%H:%M}"),
("Price: ", "@price{$0.00f}")],
formatters={
"@time": "datetime",
'price': 'printf'
},
mode='vline')
p.add_tools(hover)
p.xaxis.major_label_orientation = 3.14 / 4
p.yaxis.visible = False
# x stuff
p.xaxis.visible = True
p.xgrid.visible = False
p.xaxis.major_label_text_color = "#cccac4"
p.xaxis[0].formatter = DTF
# Y - PRICE
p.y_range = Range1d(min(data['price']) * 0.9, max(data['price']) * 1.1)
# p.yaxis.axis_label = "Price in USD"
p.add_layout(LinearAxis(), 'right')
p.yaxis[1].formatter = NumeralTickFormatter(format="$0.000")
p.yaxis.major_label_text_color = "gold"
p.ygrid.visible = False
p.background_fill_color = None
p.border_fill_color = None
p.outline_line_color = None
p.toolbar.autohide = True
epic_7d_price, created = Chart.objects.get_or_create(name='epic_7d_price',
coin=models()['epic'])
epic_7d_price.script, epic_7d_price.div = components(p)
epic_7d_price.updated = timezone.now()
epic_7d_price.save()
### VOLUME CHART
colors = ["#f2a900", "#459e86"]
piles = ['Total', 'Citex', 'Vitex']
targets = ['Bitcoin', 'USD(T)']
vol_data = {
'piles':
piles,
'Bitcoin': [
volumes()['total']['btc'],
volumes()['citex']['btc'],
volumes()['vitex']['btc'],
],
'USD(T)': [volumes()['total']['usdt'],
volumes()['citex']['usdt'], 0]
}
p1 = figure(x_range=piles,
plot_height=50,
plot_width=150,
toolbar_location=None,
tools="hover",
tooltips="$name: @$name{0 a}",
sizing_mode='scale_width')
p1.vbar_stack(targets,
x='piles',
width=0.7,
color=colors,
source=vol_data,
legend_label=targets)
labels = LabelSet(x='piles',
y=0,
text='targets',
level='glyph',
x_offset=5,
y_offset=5,
source=source)
p1.yaxis.formatter = NumeralTickFormatter(format="0,0")
p1.ygrid.visible = False
p1.xgrid.visible = False
p1.legend.orientation = "horizontal"
p1.legend.background_fill_alpha = 0
p1.legend.border_line_color = None
p1.background_fill_color = None
p1.border_fill_color = None
p1.outline_line_color = None
p1.toolbar.autohide = True
vol_24h, created = Chart.objects.get_or_create(name='vol_24h',
coin=models()['epic'])
vol_24h.script, vol_24h.div = components(p1)
vol_24h.updated = timezone.now()
vol_24h.save()
return f'Main charts updated successfully!'
def ex_vol_chart():
for exchange in all_exchanges():
for target in pairs:
data = filters()['exchanges'][exchange.name.lower(
)]['tickers'][target].last().candles['c7x1440']
time = [time for time, value in data]
value = [value for time, value in data]
data = {'time': time, 'value': value}
df = pd.DataFrame(data)
df['time'] = pd.to_datetime(df['time'])
source = ColumnDataSource(df)
DTF = DatetimeTickFormatter()
DTF.hours = ["%H:%M"]
DTF.days = ["%d/%m/'%y"]
DTF.months = ["%d/%m/%Y"]
DTF.years = ["%d/%m/%Y"]
p = figure(x_axis_type="datetime",
plot_width=190,
plot_height=38,
toolbar_location=None)
p.line(x='time',
y='value',
line_width=1.5,
source=source,
color='blue')
p.varea(x='time',
y1=0,
y2='value',
alpha=0.1,
source=source,
fill_color='blue')
hover = HoverTool(tooltips=[("Date: ", "@time{%y-%m-%d}"),
("Time: ", "@time{%H:%M}"),
("Volume: ", "@value{0.00f}")],
formatters={
"@time": "datetime",
'price': 'printf'
},
mode='vline')
p.add_tools(hover)
p.xaxis.major_label_orientation = 3.14 / 4
p.yaxis.visible = False
p.xgrid.visible = False
p.xaxis.major_label_text_color = "grey"
p.xaxis[0].formatter = DTF
# Y - PRICE
p.y_range = Range1d(
min(data['value']) * 0.8,
max(data['value']) * 1.1)
# # p.yaxis.axis_label = "Price in USD"
# p.add_layout(LinearAxis(), 'right')
# p.yaxis[1].formatter = NumeralTickFormatter(format="$0.000")
# p.yaxis.major_label_text_color = "gold"
p.ygrid.visible = False
p.background_fill_color = None
p.border_fill_color = None
p.outline_line_color = None
p.toolbar.autohide = True
mw_chart, created = Chart.objects.get_or_create(
name=f'{exchange.name} vol_7d', coin=models()['epic'])
mw_chart.script, mw_chart.div = components(p)
mw_chart.updated = timezone.now()
mw_chart.save()
return f'MW charts updated successfully!'
def mw_charts(save=False):
for i, coin in enumerate(all_coins()):
if coin.mw_coin:
data = {
'time':
[t_s(t) for t, p in get_gecko(coin).data['price_7d']][::5],
'price': [p for t, p in get_gecko(coin).data['price_7d']][::5],
}
df = pd.DataFrame(data)
df['time'] = pd.to_datetime(df['time'])
source = ColumnDataSource(df)
DTF = DatetimeTickFormatter()
DTF.hours = ["%H:%M"]
DTF.days = ["%d/%m/'%y"]
DTF.months = ["%d/%m/%Y"]
DTF.years = ["%d/%m/%Y"]
p = figure(x_axis_type="datetime",
plot_width=190,
plot_height=45,
toolbar_location=None)
p.line(x='time',
y='price',
line_width=1.5,
source=source,
color=colors[coin.name])
p.varea(x='time',
y1=0,
y2='price',
alpha=0.1,
source=source,
fill_color=colors[coin.name])
hover = HoverTool(tooltips=[("Date: ", "@time{%y-%m-%d}"),
("Time: ", "@time{%H:%M}"),
("Price: ", "@price{$0.00f}")],
formatters={
"@time": "datetime",
'price': 'printf'
},
mode='vline')
p.add_tools(hover)
p.xaxis.major_label_orientation = 3.14 / 4
p.yaxis.visible = False
# x stuff
p.xaxis.visible = False
p.xgrid.visible = False
p.xaxis.major_label_text_color = "grey"
p.xaxis[0].formatter = DTF
# Y - PRICE
p.y_range = Range1d(min(data['price']) * 0.7, max(data['price']) * 1.1)
# # p.yaxis.axis_label = "Price in USD"
# p.add_layout(LinearAxis(), 'right')
# p.yaxis[1].formatter = NumeralTickFormatter(format="$0.000")
# p.yaxis.major_label_text_color = "gold"
p.ygrid.visible = False
p.background_fill_color = None
p.border_fill_color = None
p.outline_line_color = None
p.toolbar.autohide = True
mw_chart, created = Chart.objects.get_or_create(name='mw_chart',
coin=coin)
mw_chart.script, mw_chart.div = components(p)
mw_chart.updated = timezone.now()
mw_chart.save()
return f'MW charts updated successfully!'
def display_timeline(
data: pd.DataFrame,
alert: SecurityAlert = None,
overlay_data: pd.DataFrame = None,
title: str = None,
time_column: str = "TimeGenerated",
source_columns: list = None,
overlay_colums: list = None,
height: int = 300,
):
"""
Display a timeline of events.
Parameters
----------
data : pd.DataFrame
Input DataFrame
alert : SecurityAlert, optional
Input alert (the default is None)
overlay_data : pd.DataFrame, optional
Second event stream to display as overlay
(the default is None)
title : str, optional
Title to display (the default is None)
time_column : str, optional
Name of the timestamp column
(the default is 'TimeGenerated')
source_columns : list, optional
List of source columns to use in tooltips
(the default is None)
overlay_colums : list, optional
List of source columns to use in overlay data tooltips.
(the default is None)
height : int, optional
the height of the plot figure (under 300 limits access
to Bokeh tools)(the default is 300)
"""
reset_output()
output_notebook()
y_max = 1
if not source_columns:
source_columns = ["NewProcessName", "EventID", "CommandLine"]
if time_column not in source_columns:
source_columns.append(time_column)
if "CommandLine" in source_columns:
graph_df = data[source_columns].copy()
graph_df[_WRAP_CMDL] = graph_df.apply(
lambda x: _wrap_text(x.CommandLine, _WRAP), axis=1)
else:
graph_df = data[source_columns].copy()
# if we have an overlay - add this data and shift the y co-ordinates to
# show on two separate lines
if overlay_data is not None:
overlay_colums = (overlay_colums
if overlay_colums is not None else source_columns)
if time_column not in overlay_colums:
overlay_colums.append(time_column)
if "CommandLine" in overlay_colums:
overlay_df = overlay_data[overlay_colums].copy()
overlay_df[_WRAP_CMDL] = overlay_df.apply(
lambda x: _wrap_text(x.CommandLine, _WRAP), axis=1)
else:
overlay_df = overlay_data[overlay_colums].copy()
graph_df["y_index"] = 2
overlay_df["y_index"] = 1
y_max = 2
else:
graph_df["y_index"] = 1
source = ColumnDataSource(graph_df)
# build the tool tips from columns (excluding these)
excl_cols = [time_column, "CommandLine"]
tool_tip_items = [(f"{col}", f"@{col}") for col in source_columns
if col not in excl_cols]
if _WRAP_CMDL in graph_df:
tool_tip_items.append(("CommandLine", f"@{_WRAP_CMDL}"))
hover = HoverTool(
tooltips=tool_tip_items,
formatters={"Tooltip": "printf"}
# display a tooltip whenever the cursor is vertically in line with a glyph
# ,mode='vline'
)
if not title:
title = "Event Timeline"
else:
title = "Timeline {}".format(title)
# tools = 'pan, box_zoom, wheel_zoom, reset, undo, redo, save, hover'
plot = figure(
min_border_left=50,
plot_height=height,
plot_width=900,
x_axis_label="Event Time",
x_axis_type="datetime",
x_minor_ticks=10,
tools=[hover, "pan", "xwheel_zoom", "box_zoom", "reset"],
title=title,
)
plot.yaxis.visible = False
# Tick formatting for different zoom levels
# '%H:%M:%S.%3Nms
tick_format = DatetimeTickFormatter()
tick_format.days = ["%m-%d %H:%M"]
tick_format.hours = ["%H:%M:%S"]
tick_format.minutes = ["%H:%M:%S"]
tick_format.seconds = ["%H:%M:%S"]
tick_format.milliseconds = ["%H:%M:%S.%3N"]
plot.xaxis[0].formatter = tick_format
plot.circle(x=time_column,
y="y_index",
color="navy",
alpha=0.5,
size=10,
source=source)
if overlay_data is not None:
overlay_source = ColumnDataSource(overlay_df)
plot.circle(
x=time_column,
y="y_index",
color="green",
alpha=0.5,
size=10,
source=overlay_source,
)
# Adding data labels stops everything working!
# labels = LabelSet(x=time_column, y='y_index', y_offset=5,
# text='NewProcessName', source=source,
# angle='90deg', text_font_size='8pt')
# p.add_layout(labels)
# if we have an alert, plot the time as a line
if alert is not None:
x_alert_label = pd.Timestamp(alert["StartTimeUtc"])
plot.line(x=[x_alert_label, x_alert_label], y=[0, y_max + 1])
alert_label = Label(
x=x_alert_label,
y=0,
y_offset=10,
x_units="data",
y_units="data",
text="< Alert time",
render_mode="css",
border_line_color="red",
border_line_alpha=1.0,
background_fill_color="white",
background_fill_alpha=1.0,
)
plot.add_layout(alert_label)
print("Alert start time = ", alert["StartTimeUtc"])
show(plot)
