def _run_resampler(
data_timeframe,
data_compression,
resample_timeframe,
resample_compression,
runtime_seconds=27,
starting_value=200,
tick_interval=datetime.timedelta(seconds=11),
num_gen_bars=None,
start_delays=None,
num_data=1,
) -> bt.Strategy:
_logger.info("Constructing Cerebro")
cerebro = bt.Cerebro()
cerebro.addstrategy(LiveDemoStrategy)
cerebro.addlistener(bt.listeners.RecorderListener)
cerebro.addlistener(PlotListener,
volume=False,
scheme=Blackly(hovertool_timeformat='%F %R:%S'),
lookback=120)
cerebro.addanalyzer(bt.analyzers.TradeAnalyzer)
for i in range(0, num_data):
start_delay = 0
if start_delays is not None and i <= len(
start_delays) and start_delays[i] is not None:
start_delay = start_delays[i]
num_gen_bar = 0
if num_gen_bars is not None and i <= len(
num_gen_bars) and num_gen_bars[i] is not None:
num_gen_bar = num_gen_bars[i]
data = bt.feeds.FakeFeed(
timeframe=data_timeframe,
compression=data_compression,
run_duration=datetime.timedelta(seconds=runtime_seconds),
starting_value=starting_value,
tick_interval=tick_interval,
live=True,
num_gen_bars=num_gen_bar,
start_delay=start_delay,
name=f'data{i}',
)
cerebro.resampledata(data,
timeframe=resample_timeframe,
compression=resample_compression)
# return the recorded bars attribute from the first strategy
res = cerebro.run()
return cerebro, res[0]
cerebro.broker.setcommission(commission=0.0025)
data = bt.feeds.GenericCSVData(
dataname=f'./data/{target_stock}.csv',
nullvalue=0.0,
dtformat=('%Y-%m-%d'),
datetime=0,
open=1,
high=2,
low=3,
close=4,
volume=6,
)
cerebro.adddata(data)
cerebro.addstrategy(Buy_and_Hold)
cerebro.addanalyzer(bt.analyzers.TradeAnalyzer,
_name='TradeAnalyzer') # 交易分析 (策略勝率)
cerebro.addanalyzer(bt.analyzers.PeriodStats, _name='PeriodStats') # 交易基本統計分析
cerebro.addanalyzer(bt.analyzers.DrawDown, _name='DrawDown') # 回落統計
cerebro.addanalyzer(bt.analyzers.SQN,
_name='SQN') # 期望獲利/標準差 System Quality Number
cerebro.addanalyzer(bt.analyzers.SharpeRatio, _name='SharpeRatio') # 夏普指數
print('投資 > 起始資產 %.2f 💲' % cerebro.broker.getvalue())
cerebro.run()
print('投資 > 結束資產 %.2f 💲' % cerebro.broker.getvalue())
investment_plot = Bokeh(style='bar', plot_mode='single', scheme=Blackly())
cerebro.plot(investment_plot)
class Bokeh(metaclass=bt.MetaParams):
params = (('scheme', Blackly()), ('filename', None), ('plotconfig', None),
('output_mode', 'show'), ('show', True))
def __init__(self, **kwargs):
for pname, pvalue in kwargs.items():
setattr(self.p.scheme, pname, pvalue)
self._iplot: Optional[bool] = None
self._tablegen = TableGenerator(self.p.scheme)
if not isinstance(self.p.scheme, Scheme):
raise Exception(
"Provided scheme has to be a subclass of backtrader_plotting.schemes.scheme.Scheme"
)
self._initialized: bool = False
self._is_optreturn: bool = False # when optreturn is active during optimization then we get a thinned out result only
self._current_fig_idx: Optional[int] = None
self.figurepages: List[FigurePage] = []
def _configure_plotting(self, strategy: bt.Strategy):
datas, inds, obs = strategy.datas, strategy.getindicators(
), strategy.getobservers()
for objs in [datas, inds, obs]:
for idx, obj in enumerate(objs):
self._configure_plotobject(obj, idx, strategy)
def _configure_plotobject(self, obj, idx, strategy):
if self.p.plotconfig is None:
return
def apply_config(obj, config):
for k, v in config.items():
if k == 'plotmaster':
# this needs special treatment since a string is passed but we need to set the actual obj
v = find_by_plotid(strategy, v)
setattr(obj.plotinfo, k, v)
for k, config in self.p.plotconfig.items():
ctype, target = k.split(':')
if ctype == 'r': # regex
label = label_resolver.plotobj2label(obj)
m = re.match(target, label)
if m:
apply_config(obj, config)
elif ctype[0] == '#': # index
target_type, target_idx = target.split('-')
target_types = {
'i': bt.Indicator,
'o': bt.Observer,
'd': bt.AbstractDataBase,
}
# check if instance type matches
if not isinstance(obj, target_types[target_type]):
continue
if int(target_idx) != idx:
continue
apply_config(obj, config)
elif ctype == 'id': # plotid
plotid = getattr(obj.plotinfo, 'plotid', None)
if plotid is None or plotid != target:
continue
apply_config(obj, config)
else:
raise RuntimeError(
f'Unknown config type in plotting config: {k}')
def list_tradingdomains(self, strategy: bt.Strategy):
data_graph, volume_graph = self._build_graph(strategy.datas,
strategy.getindicators(),
strategy.getobservers())
lgs = list()
for master in itertools.chain(data_graph.keys(), volume_graph):
lg = FigureEnvelope._resolve_tradingdomain(master)
if isinstance(lg, str) and lg not in lgs:
lgs.append(lg)
return lgs
def _build_graph(self,
datas,
inds,
obs,
tradingdomain=None) -> Tuple[Dict, List]:
data_graph = {}
volume_graph = []
for d in datas:
if not d.plotinfo.plot or not FigureEnvelope.should_filter_by_tradingdomain(
d, tradingdomain):
continue
pmaster = Bokeh._resolve_plotmaster(d.plotinfo.plotmaster)
if pmaster is None:
data_graph[d] = []
else:
if pmaster not in data_graph:
data_graph[pmaster] = []
data_graph[pmaster].append(d)
if self.p.scheme.volume and self.p.scheme.voloverlay is False:
volume_graph.append(d)
for obj in itertools.chain(inds, obs):
if not hasattr(obj, 'plotinfo'):
# no plotting support - so far LineSingle derived classes
continue
# should this indicator be plotted?
if not obj.plotinfo.plot or obj.plotinfo.plotskip or not FigureEnvelope.should_filter_by_tradingdomain(
obj, tradingdomain):
continue
# subplot = create a new figure for this indicator
subplot: bool = obj.plotinfo.subplot
plotmaster: str = obj.plotinfo.plotmaster
if subplot and plotmaster is None:
data_graph[obj] = []
else:
plotmaster = plotmaster if plotmaster is not None else get_indicator_data(
obj)
if plotmaster not in data_graph:
data_graph[plotmaster] = []
data_graph[plotmaster].append(obj)
return data_graph, volume_graph
@property
def _cur_figurepage(self) -> FigurePage:
return self.figurepages[self._current_fig_idx]
@staticmethod
def _resolve_plotmaster(obj):
if obj is None:
return None
while True:
pm = obj.plotinfo.plotmaster
if pm is None:
break
else:
obj = pm
return obj
@staticmethod
def _get_start_end(strategy, start, end):
st_dtime = strategy.lines.datetime.array
if start is None:
start = 0
if end is None:
end = len(st_dtime)
if isinstance(start, datetime.date):
start = bisect.bisect_left(st_dtime, bt.date2num(start))
if isinstance(end, datetime.date):
end = bisect.bisect_right(st_dtime, bt.date2num(end))
if end < 0:
end = len(st_dtime) + 1 + end
return start, end
def _blueprint_strategy(self,
strategy: bt.Strategy,
start=None,
end=None,
tradingdomain=None,
**kwargs) -> None:
if not strategy.datas:
return
self._cur_figurepage.analyzers += [
a for _, a in strategy.analyzers.getitems()
]
data_graph, volume_graph = self._build_graph(strategy.datas,
strategy.getindicators(),
strategy.getobservers(),
tradingdomain)
start, end = Bokeh._get_start_end(strategy, start, end)
# reset hover container to not mix hovers with other strategies
hoverc = HoverContainer(
hover_tooltip_config=self.p.scheme.hover_tooltip_config,
is_multidata=len(strategy.datas) > 1)
strat_figures = []
for master, slaves in data_graph.items():
plotorder = getattr(master.plotinfo, 'plotorder', 0)
figure = FigureEnvelope(strategy, self._cur_figurepage.cds, hoverc,
start, end, self.p.scheme, master,
plotorder,
len(strategy.datas) > 1)
figure.plot(master, None)
for s in slaves:
figure.plot(s, master)
strat_figures.append(figure)
for f in strat_figures:
f.figure.legend.click_policy = self.p.scheme.legend_click
f.figure.legend.location = self.p.scheme.legend_location
f.figure.legend.background_fill_color = self.p.scheme.legend_background_color
f.figure.legend.label_text_color = self.p.scheme.legend_text_color
f.figure.legend.orientation = self.p.scheme.legend_orientation
# link axis
for i in range(1, len(strat_figures)):
strat_figures[i].figure.x_range = strat_figures[0].figure.x_range
# configure xaxis visibility
if self.p.scheme.xaxis_pos == "bottom":
for i, f in enumerate(strat_figures):
f.figure.xaxis.visible = False if i <= len(
strat_figures) else True
hoverc.apply_hovertips(strat_figures)
self._cur_figurepage.figure_envs += strat_figures
# volume graphs
for v in volume_graph:
plotorder = getattr(v.plotinfo, 'plotorder', 0)
figure = FigureEnvelope(strategy,
self._cur_figurepage.cds,
hoverc,
start,
end,
self.p.scheme,
v,
plotorder,
is_multidata=len(strategy.datas) > 1)
figure.plot_volume(v)
self._cur_figurepage.figure_envs.append(figure)
def plot_and_generate_optmodel(self, obj: Union[bt.Strategy,
bt.OptReturn]):
self._reset()
self.plot(obj)
# we support only one strategy at a time so pass fixed zero index
# if we ran optresults=False then we have a full strategy object -> pass it to get full plot
return self.generate_model(0)
@staticmethod
def _sort_plotobjects(objs: List[FigureEnvelope]) -> None:
objs.sort(key=lambda x: x.plotorder)
def get_figurepage(self, idx: int = 0):
return self.figurepages[idx]
# region Generator Methods
def generate_model(self, figurepage_idx: int = 0) -> Model:
"""Returns a model generated from internal blueprints"""
if figurepage_idx >= len(self.figurepages):
raise RuntimeError(
f'Cannot generate model for FigurePage with index {figurepage_idx} as there are only {len(self.figurepages)}.'
)
figurepage = self.figurepages[figurepage_idx]
if not self._is_optreturn:
tabs = self.generate_model_tabs(figurepage)
else:
tabs = []
# now append analyzer tab(s)
analyzers = figurepage.analyzers
panel_analyzer = self.get_analyzer_panel(analyzers)
if panel_analyzer is not None:
tabs.append(panel_analyzer)
# append meta tab
if not self._is_optreturn:
assert figurepage.strategy is not None
meta = Div(text=metadata.get_metadata_div(figurepage.strategy))
metapanel = Panel(child=meta, title="Meta")
tabs.append(metapanel)
model = Tabs(tabs=tabs)
# attach the model to the underlying figure for later reference (e.g. unit test)
figurepage.model = model
return model
def _get_nodata_panel(self):
chart_grid = gridplot([],
toolbar_location=self.p.scheme.toolbar_location,
toolbar_options={'logo': None})
return Panel(child=chart_grid, title="No Data")
@property
def is_tabs_single(self) -> bool:
if self.p.scheme.tabs == 'single':
return True
elif self.p.scheme.tabs == 'multi':
return False
else:
raise RuntimeError(
f'Invalid tabs parameter "{self.p.scheme.tabs}"')
def generate_model_tabs(self,
fp: FigurePage,
tradingdomain=None) -> List[Panel]:
observers = [
x for x in fp.figure_envs if isinstance(x.master, bt.Observer)
]
datas = [
x for x in fp.figure_envs if isinstance(x.master, bt.DataBase)
]
inds = [
x for x in fp.figure_envs if isinstance(x.master, bt.Indicator)
]
# now assign figures to tabs
# 1. assign default tabs if no manual tab is assigned
for figure in [x for x in datas if x.plottab is None]:
figure.plottab = 'Plots' if self.is_tabs_single else 'Datas'
for figure in [x for x in inds if x.plottab is None]:
figure.plottab = 'Plots' if self.is_tabs_single else 'Indicators'
for figure in [x for x in observers if x.plottab is None]:
figure.plottab = 'Plots' if self.is_tabs_single else 'Observers'
# 2. group panels by desired tabs
# groupby expects the groups to be sorted or else will produce duplicated groups
sorted_figs = list(itertools.chain(datas, inds, observers))
# 3. filter tradingdomains
if tradingdomain is not None:
filtered = []
for f in sorted_figs:
lgs = f.get_tradingdomains()
for lg in lgs:
if lg is True or lg == tradingdomain:
filtered.append(f)
sorted_figs = filtered
sorted_figs.sort(key=lambda x: x.plottab)
tabgroups = itertools.groupby(sorted_figs, lambda x: x.plottab)
panels = []
def build_panel(objects, panel_title):
if len(objects) == 0:
return
Bokeh._sort_plotobjects(objects)
g = gridplot(
[[x.figure] for x in objects],
toolbar_options={'logo': None},
toolbar_location=self.p.scheme.toolbar_location,
sizing_mode=self.p.scheme.plot_sizing_mode,
)
panels.append(Panel(title=panel_title, child=g))
for tabname, figures in tabgroups:
build_panel(list(figures), tabname)
return panels
# endregion
def get_analyzer_panel(self,
analyzers: List[bt.Analyzer]) -> Optional[Panel]:
if len(analyzers) == 0:
return None
table_width = int(self.p.scheme.analyzer_tab_width /
self.p.scheme.analyzer_tab_num_cols)
acolumns = []
for analyzer in analyzers:
table_header, elements = self._tablegen.get_analyzers_tables(
analyzer, table_width)
acolumns.append(column([table_header] + elements))
childs = gridplot(acolumns,
ncols=self.p.scheme.analyzer_tab_num_cols,
toolbar_options={'logo': None})
return Panel(child=childs, title='Analyzers')
def _output_stylesheet(self, template="basic.css.j2"):
return generate_stylesheet(self.p.scheme, template)
def _output_plot_file(self,
model,
idx,
filename=None,
template="basic.html.j2"):
if filename is None:
tmpdir = tempfile.gettempdir()
filename = os.path.join(tmpdir, f"bt_bokeh_plot_{idx}.html")
env = Environment(
loader=PackageLoader('backtrader_plotting.bokeh', 'templates'))
templ = env.get_template(template)
templ.globals['now'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
html = file_html(model,
template=templ,
resources=CDN,
template_variables=dict(
stylesheet=self._output_stylesheet(),
show_headline=self.p.scheme.show_headline,
))
with open(filename, 'w') as f:
f.write(html)
return filename
def savefig(self, fig, filename, width, height, dpi, tight):
self._generate_output(fig, filename)
def build_strategy_data(self,
strategy: bt.Strategy,
start: Optional[datetime.datetime] = None,
end: Optional[datetime.datetime] = None,
num_back: Optional[int] = None,
startidx: int = 0):
"""startidx: index number to write into the dataframe for the index column"""
strategydf = pd.DataFrame()
start, end = Bokeh._get_start_end(strategy, start, end)
strat_clk: array[float] = strategy.lines.datetime.plotrange(start, end)
# if patches occured then we see duplicate entries in the strategie clock -> clean them
strat_clk = np.unique(strat_clk)
if num_back is None:
num_back = len(strat_clk)
strat_clk = strat_clk[-num_back:]
# we use timezone of first data. we might see duplicated timestamps here
dtline = [bt.num2date(x, strategy.datas[0]._tz) for x in strat_clk]
# add an index line to use as x-axis (instead of datetime axis) to avoid datetime gaps (e.g. weekends)
indices = list(range(startidx, startidx + len(dtline)))
strategydf['index'] = indices
strategydf['datetime'] = dtline
for data in strategy.datas:
source_id = FigureEnvelope._source_id(data)
df_data = convert_to_pandas(strat_clk, data, start, end, source_id)
strategydf = strategydf.join(df_data)
df_colors = FigureEnvelope.build_color_lines(
df_data,
self.p.scheme,
col_open=source_id + 'open',
col_close=source_id + 'close',
col_prefix=source_id)
strategydf = strategydf.join(df_colors)
for obj in itertools.chain(strategy.getindicators(),
strategy.getobservers()):
for lineidx in range(obj.size()):
line = obj.lines[lineidx]
source_id = FigureEnvelope._source_id(line)
dataline = line.plotrange(start, end)
line_clk = get_clock_line(obj).plotrange(start, end)
dataline = convert_by_line_clock(dataline, line_clk, strat_clk)
strategydf[source_id] = dataline
# apply a proper index (should be identical to 'index' column)
if strategydf.shape[0] > 0:
strategydf.index = indices
return strategydf
# region interface for backtrader
def plot(self,
obj: Union[bt.Strategy, bt.OptReturn],
figid=0,
numfigs=1,
iplot=True,
start=None,
end=None,
use=None,
fill_data=True,
tradingdomain=None,
**kwargs):
"""Called by backtrader to plot either a strategy or an optimization result."""
# prepare new FigurePage
fp = FigurePage(obj)
self.figurepages.append(fp)
self._current_fig_idx = len(self.figurepages) - 1
self._is_optreturn = isinstance(obj, bt.OptReturn)
if isinstance(obj, bt.Strategy):
# only configure plotting for regular backtesting (not for optimization)
self._configure_plotting(obj)
if numfigs > 1:
raise Exception("numfigs must be 1")
if use is not None:
raise Exception("Different backends by 'use' not supported")
self._iplot = iplot and 'ipykernel' in sys.modules
if isinstance(obj, bt.Strategy):
self._blueprint_strategy(obj, start, end, tradingdomain, **kwargs)
if fill_data:
df: pd.DataFrame = self.build_strategy_data(obj, start, end)
new_cds = ColumnDataSource.from_df(df)
append_cds(fp.cds, new_cds)
elif isinstance(obj, bt.OptReturn):
# for optresults we only plot analyzers!
self._cur_figurepage.analyzers += [
a for _, a in obj.analyzers.getitems()
]
else:
raise Exception(
f'Unsupported plot source object: {str(type(obj))}')
return [self._cur_figurepage]
def show(self):
"""Display a figure (called by backtrader)."""
# as the plot() function only created the figures and the columndatasources with no data -> now we fill it
for idx in range(len(self.figurepages)):
model = self.generate_model(idx)
if self.p.output_mode in ['show', 'save']:
if self._iplot:
css = self._output_stylesheet()
display(HTML(css))
show(model)
else:
filename = self._output_plot_file(model, idx,
self.p.filename)
if self.p.output_mode == 'show':
view(filename)
elif self.p.output_mode == 'memory':
pass
else:
raise RuntimeError(
f'Invalid parameter "output_mode" with value: {self.p.output_mode}'
)
self._reset()
def _reset(self):
self.figurepages = []
self._is_optreturn = False
def run_cerebro_live(
strategycls,
data_timeframes,
data_compressions,
resample_timeframes,
resample_compressions,
runtime_secondss=27,
starting_values=200,
tick_intervals=datetime.timedelta(seconds=11),
num_gen_barss=None,
start_delays=None,
num_data=1,
fnc_name='resampledata',
):
def _listify(val):
return [val] * num_data if not isinstance(val, list) else val
data_timeframes = _listify(data_timeframes)
data_compressions = _listify(data_compressions)
resample_timeframes = _listify(resample_timeframes)
resample_compressions = _listify(resample_compressions)
runtimes_secondss = _listify(runtime_secondss)
starting_values = _listify(starting_values)
tick_intervals = _listify(tick_intervals)
num_gen_barss = _listify(num_gen_barss)
start_delays = _listify(start_delays)
_logger.info("Constructing Cerebro")
cerebro = bt.Cerebro()
cerebro.addstrategy(strategycls)
cerebro.addlistener(bt.listeners.RecorderListener)
cerebro.addanalyzer(bt.analyzers.TradeAnalyzer)
cerebro.addlistener(PlotListener,
volume=False,
scheme=Blackly(hovertool_timeformat='%F %R:%S',
tabs='multi'),
lookback=120)
for i in range(0, num_data):
num_gen_bars = 0 if num_gen_barss[i] is None else num_gen_barss[i]
data_timeframe = 0 if data_timeframes[i] is None else data_timeframes[i]
data_compression = 0 if data_compressions[
i] is None else data_compressions[i]
runtime_seconds = 0 if runtimes_secondss[
i] is None else runtimes_secondss[i]
starting_value = 0 if starting_values[i] is None else starting_values[i]
tick_interval = 0 if tick_intervals[i] is None else tick_intervals[i]
start_delay = 0 if start_delays[i] is None else start_delays[i]
resample_timeframe = 0 if resample_timeframes[
i] is None else resample_timeframes[i]
resample_compression = 0 if resample_compressions[
i] is None else resample_compressions[i]
data = bt.feeds.FakeFeed(
timeframe=data_timeframe,
compression=data_compression,
run_duration=datetime.timedelta(seconds=runtime_seconds),
starting_value=starting_value,
tick_interval=tick_interval,
live=True,
num_gen_bars=num_gen_bars,
start_delay=start_delay,
name=f'data{i}',
)
fnc = getattr(cerebro, fnc_name, None)
fnc(data,
timeframe=resample_timeframe,
compression=resample_compression)
# return the recorded bars attribute from the first strategy
res = cerebro.run()
return cerebro, res[0]
class PlotListener(bt.ListenerBase):
params = (
('scheme', Blackly()),
('style', 'bar'),
('lookback', 23),
('strategyidx', 0),
('http_port', 80),
('title', 'Live'),
)
class UpdateType(Enum):
APPEND = 1,
UPDATE_LAST = 2,
FILL_OR_PREPEND = 3,
def __init__(self, **kwargs):
self._cerebro: Optional[bt.Cerebro] = None
self._webapp = BokehWebapp(
self.p.title,
'basic.html.j2',
self.p.scheme,
self._bokeh_cb_build_root_model,
on_session_destroyed=self._on_session_destroyed,
port=self.p.http_port)
self._lock = Lock()
self._datastore = None
self._clients: Dict[str, LiveClient] = {}
self._bokeh_kwargs = kwargs
self._bokeh = self._create_bokeh()
self._pkgs_insert = defaultdict(lambda: [])
self._prev_strategy_len = 0
self._reset_patch_pkgs()
def _reset_patch_pkgs(self):
self._patch_pkgs: Dict[str, Dict[str, Any]] = defaultdict(lambda: {})
def _create_bokeh(self):
return Bokeh(
style=self.p.style, scheme=self.p.scheme, **self._bokeh_kwargs
) # get a copy of the scheme so we can modify it per client
def _on_session_destroyed(self, session_context):
with self._lock:
del self._clients[session_context.id]
def _bokeh_cb_build_root_model(self, doc: Document):
client = LiveClient(doc,
self._create_bokeh,
self._bokeh_cb_push_adds,
self._cerebro.runningstrats[self.p.strategyidx],
lookback=self.p.lookback)
with self._lock:
self._clients[doc.session_context.id] = client
self._bokeh_cb_push_adds(doc)
return client.model
def start(self, cerebro):
_logger.info("Starting PlotListener...")
self._cerebro = cerebro
self._datastore = self._bokeh.build_strategy_data(
self._cerebro.runningstrats[self.p.strategyidx])
t = threading.Thread(target=self._t_bokeh_server)
t.daemon = True
t.start()
def stop(self):
pass
def _t_bokeh_server(self):
asyncio.set_event_loop(asyncio.new_event_loop())
loop = tornado.ioloop.IOLoop.current()
self._webapp.start(loop)
def _bokeh_cb_push_adds(self, bootstrap_document=None):
if bootstrap_document is None:
document = curdoc()
else:
document = bootstrap_document
with self._lock:
client: LiveClient = self._clients[document.session_context.id]
updatepkg_df: pandas.DataFrame = self._datastore[
self._datastore['index'] > client.last_index]
# skip if we don't have new data
if updatepkg_df.shape[0] == 0:
return
updatepkg = ColumnDataSource.from_df(updatepkg_df)
client.push_adds(updatepkg)
def _bokeh_full_refresh(self):
document = curdoc()
session_id = document.session_context.id
with self._lock:
client: LiveClient = self._clients[session_id]
client.push_full_refresh(self._datastore)
# remove any pending patch packages as we just have issued a full update
self._reset_patch_pkgs()
def _bokeh_cb_push_patches(self):
document = curdoc()
session_id = document.session_context.id
with self._lock:
client: LiveClient = self._clients[session_id]
patch_pkgs = self._patch_pkgs[session_id]
self._patch_pkgs[session_id] = {} # reset
_logger.info("Patch pkg: " + str(patch_pkgs))
client.push_patches(patch_pkgs)
def _queue_patch_pkg(self, current_frame):
last_index = self._datastore.index[-1]
for column_name in current_frame.columns:
d = current_frame[column_name].iloc[0]
if isinstance(d, float) and np.isnan(d):
continue
self._datastore.at[last_index,
column_name] = d # update data in datastore
for sess_id in self._clients.keys():
self._patch_pkgs[sess_id][column_name] = d
# WIP: make curernt bar outline red
# if column_name.endswith('outline'):
# self._patch_pkgs[sess_id].append((column_name, None, '#ff0000'))
def _detect_update_type(self, strategy):
# treat as update of old data if strategy datetime is duplicated and we have already data stored
# in this case data in an older slot was added
if len(strategy) == self._prev_strategy_len:
return self.UpdateType.UPDATE_LAST
else:
assert len(strategy) > self._prev_strategy_len
if len(strategy) == 1 or self._datastore['datetime'].iloc[
-1] != bt.num2date(strategy.datetime[0]):
return self.UpdateType.APPEND
elif self._datastore['datetime'].iloc[-1] == bt.num2date(
strategy.datetime[0]):
# either data was added to the front or data in between was filled
return self.UpdateType.FILL_OR_PREPEND
else:
raise RuntimeError('Update type detection failed')
def next(self):
with self._lock:
strategy = self._cerebro.runningstrats[self.p.strategyidx]
update_type = self._detect_update_type(strategy)
self._prev_strategy_len = len(strategy)
_logger.info(f"next: update type: {update_type}")
if update_type == self.UpdateType.UPDATE_LAST:
startidx = int(self._datastore['index'].iloc[-1])
current_frame = self._bokeh.build_strategy_data(
strategy, num_back=1, startidx=startidx)
self._queue_patch_pkg(current_frame)
for client in self._clients.values():
client.document.add_next_tick_callback(
self._bokeh_cb_push_patches)
elif update_type == self.UpdateType.FILL_OR_PREPEND:
self._datastore = self._bokeh.build_strategy_data(strategy)
for client in self._clients.values():
_logger.info('Adding full refersh callback')
client.add_fullrefresh_callback(self._bokeh_full_refresh,
500)
elif update_type == self.UpdateType.APPEND:
nextidx = 0 if self._datastore.shape[0] == 0 else int(
self._datastore['index'].iloc[-1]) + 1
num_back = 1 if self._datastore.shape[
0] > 0 else None # if we have NO data yet then fetch all (first call)
new_frame = self._bokeh.build_strategy_data(strategy,
num_back=num_back,
startidx=nextidx)
# append data and remove old data
self._datastore = self._datastore.append(new_frame)
self._datastore = self._datastore.tail(self.p.lookback)
for client in self._clients.values():
doc = client.document
try:
doc.remove_next_tick_callback(self._bokeh_cb_push_adds)
except ValueError:
# there was no callback to remove
pass
doc.add_next_tick_callback(self._bokeh_cb_push_adds)
else:
raise RuntimeError(f'Unexepected update_type: {update_type}')
class PlotListener(bt.ListenerBase):
params = (
('scheme', Blackly()),
('style', 'bar'),
('lookback', 23),
('strategyidx', 0),
('http_port', 80),
('title', 'Live'),
)
def __init__(self, **kwargs):
self._cerebro: Optional[bt.Cerebro] = None
self._webapp = BokehWebapp(self.p.title,
'basic.html.j2',
self.p.scheme,
self._bokeh_cb_build_root_model,
on_session_destroyed=self._on_session_destroyed,
port=self.p.http_port)
self._lock = Lock()
self._datastore = None
self._clients: Dict[str, LiveClient] = {}
self._bokeh_kwargs = kwargs
self._bokeh = self._create_bokeh()
self._patch_pkgs = defaultdict(lambda: [])
def _create_bokeh(self):
return Bokeh(style=self.p.style, scheme=self.p.scheme, **self._bokeh_kwargs) # get a copy of the scheme so we can modify it per client
def _on_session_destroyed(self, session_context):
with self._lock:
del self._clients[session_context.id]
def _bokeh_cb_build_root_model(self, doc: Document):
client = LiveClient(doc,
self._bokeh_cb_push_adds,
self._create_bokeh,
self._bokeh_cb_push_adds,
self._cerebro.runningstrats[self.p.strategyidx],
lookback=self.p.lookback)
with self._lock:
self._clients[doc.session_context.id] = client
self._bokeh_cb_push_adds(doc)
return client.model
def start(self, cerebro):
_logger.info("Starting PlotListener...")
self._cerebro = cerebro
self._datastore = self._bokeh.build_strategy_data(self._cerebro.runningstrats[self.p.strategyidx])
t = threading.Thread(target=self._t_bokeh_server)
t.daemon = True
t.start()
def stop(self):
pass
def _t_bokeh_server(self):
asyncio.set_event_loop(asyncio.new_event_loop())
loop = tornado.ioloop.IOLoop.current()
self._webapp.start(loop)
def _bokeh_cb_push_adds(self, bootstrap_document=None):
if bootstrap_document is None:
document = curdoc()
else:
document = bootstrap_document
with self._lock:
client = self._clients[document.session_context.id]
updatepkg_df: pandas.DataFrame = self._datastore[self._datastore['index'] > client.last_data_index]
# skip if we don't have new data
if updatepkg_df.shape[0] == 0:
return
updatepkg = ColumnDataSource.from_df(updatepkg_df)
client.push_adds(updatepkg, new_last_index=updatepkg_df['index'].iloc[-1])
def _bokeh_cb_push_patches(self):
document = curdoc()
session_id = document.session_context.id
with self._lock:
client: LiveClient = self._clients[session_id]
patch_pkgs = self._patch_pkgs[session_id]
self._patch_pkgs[session_id] = []
client.push_patches(patch_pkgs)
def next(self):
strategy = self._cerebro.runningstrats[self.p.strategyidx]
# treat as update of old data if strategy datetime is duplicated and we have already data stored
is_update = len(strategy) > 1 and strategy.datetime[0] == strategy.datetime[-1] and self._datastore.shape[0] > 0
if is_update:
with self._lock:
fulldata = self._bokeh.build_strategy_data(strategy)
# generate series with number of missing values per column
new_count = fulldata.isnull().sum()
cur_count = self._datastore.isnull().sum()
# boolean series that indicates which column is missing data
patched_cols = new_count != cur_count
# get dataframe with only those columns that added data
patchcols = fulldata[fulldata.columns[patched_cols]]
for column_name in patchcols.columns:
for index, row in self._datastore.iterrows():
# compare all values in this column
od = row[column_name]
odt = row['datetime']
d = fulldata[column_name][index]
dt = fulldata['datetime'][index]
assert odt == dt
# if value is different then put to patch package
# either it WAS NaN and it's not anymore
# or both not NaN but different now
# and don't count it as True when both are NaN
if not (pandas.isna(d) and pandas.isna(od)) and ((pandas.isna(od) and not pandas.isna(d)) or d != od):
self._datastore.at[index, column_name] = d # update data in datastore
for sess_id in self._clients.keys():
self._patch_pkgs[sess_id].append((column_name, odt, d))
for client in self._clients.values():
client.document.add_next_tick_callback(self._bokeh_cb_push_patches)
else:
with self._lock:
nextidx = 0 if self._datastore.shape[0] == 0 else int(self._datastore['index'].iloc[-1]) + 1
num_back = 1 if self._datastore.shape[0] > 0 else None # fetch all on first call
new_frame = self._bokeh.build_strategy_data(strategy, num_back=num_back, startidx=nextidx)
# i have seen an empty line in the past. let's catch it here
assert new_frame['datetime'].iloc[0] != np.datetime64('NaT')
# append data and remove old data
self._datastore = self._datastore.append(new_frame)
self._datastore = self._datastore.tail(self.p.lookback)
for client in self._clients.values():
doc = client.document
try:
doc.remove_next_tick_callback(self._bokeh_cb_push_adds)
except ValueError:
# there was no callback to remove
pass
doc.add_next_tick_callback(self._bokeh_cb_push_adds)