def render_elements(self, elements_to_render_dict, title=None, output_filename=None, output_format='html',
pdf_converter='pdfkit', extra_head_code=None):
# Only take charts and tables, and ignore text
if 'charts' in elements_to_render_dict.keys() and 'tables' in elements_to_render_dict.keys():
elements_to_render_dict = {**elements_to_render_dict['charts'], **elements_to_render_dict['tables']}
elif 'charts' in elements_to_render_dict.keys():
elements_to_render_dict = elements_to_render_dict['charts']
elif 'tables' in elements_to_render_dict.keys():
elements_to_render_dict = elements_to_render_dict['tables']
elements_to_render = self._util_func.flatten_list_of_lists(list(elements_to_render_dict.values()))
canvas = Canvas(elements_to_render=elements_to_render)
# Generate HTML string with chartpy's Canvas object
html, _ = canvas.generate_canvas(output_filename=output_filename, silent_display=True,
canvas_plotter=self._canvas_plotter, page_title=title,
render_pdf=False,
return_html_binary=True, extra_head_code=extra_head_code)
if output_format not in ['html', 'pdf']:
raise Exception("Invalid output format selected")
# Taking HTML as input write to disk as HTML file (or PDF), and return the binary representation (which can
# be more easily handled by eg. a web server)
return self.write_html_pdf(html, output_filename=output_filename, output_format=output_format, pdf_converter=pdf_converter)
def run_strategy_returns_stats(self,
trading_model,
index=None,
engine='finmarketpy'):
"""Plots useful statistics for the trading strategy using various backends
Parameters
----------
trading_model : TradingModel
defining trading strategy
engine : str
'pyfolio' - use PyFolio as a backend
'finmarketpy' - use finmarketpy as a backend
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localize_index_as_UTC(pnl)
except:
pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
import matplotlib
import matplotlib.pyplot as plt
matplotlib.rcdefaults()
plt.style.use(ChartConstants().
chartfactory_style_sheet['chartpy-pyfolio'])
except:
pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig(trading_model.DUMP_PATH + "stats.png")
except:
pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
# temporarily make scale factor smaller so fits the window
old_scale_factor = trading_model.SCALE_FACTOR
trading_model.SCALE_FACTOR = 0.75
pnl = trading_model.plot_strategy_pnl(
silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(
silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(
silent_plot=True
) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(
silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(
silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(
silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual], [pnl_comp, ir_comp],
[leverage, ind_lev]])
canvas.generate_canvas(
page_title=trading_model.FINAL_STRATEGY + ' Return Statistics',
silent_display=False,
canvas_plotter='plain',
output_filename=trading_model.FINAL_STRATEGY + ".html",
render_pdf=False)
trading_model.SCALE_FACTOR = old_scale_factor
# choose run_example = 0 for everything
# run_example = 1 - create a plain and Keen.io based template for a chart webpage
run_example = 0
if run_example == 1 or run_example == 0:
df = Quandl.get(["FRED/A191RL1Q225SBEA"], authtoken=quandl_api_key)
df.columns = ["Real QoQ"]
# Chart object is initialised with the dataframe and our chart style
chart_bokeh = Chart(df=df, chart_type='line', engine='bokeh',
style=Style(title="US GDP", source="Quandl/Fred", scale_factor=-2, width=500, height=300, silent_display=True))
chart_plotly = Chart(df=df, chart_type='line', engine='plotly',
style=Style(title="US GDP", source="Quandl/Fred", scale_factor=-2, width=500, height=300, silent_display=True))
chart_matplotlib = Chart(df=df, chart_type='line', engine='matplotlib',
style=Style(title="US GDP", source="Quandl/Fred", scale_factor=-2, width=500, height=300, silent_display=True))
text = "A demo of chartpy canvas!!"
# using plain template
canvas = Canvas([[text, chart_bokeh], [chart_plotly, df.tail(n=5)]])
canvas.generate_canvas(silent_display=False, canvas_plotter='plain')
# using the Keen template (needs static folder in the same place as final HTML file)
canvas = Canvas([[chart_bokeh, chart_plotly], [chart_plotly, chart_matplotlib]])
canvas.generate_canvas(silent_display=False, canvas_plotter='keen')
def dataframe_compliance_tca_example():
"""Get a DataFrame of trades and apply compliance based TCA to it
"""
tca_engine = TCAEngineImpl(version=tca_version)
spread_to_mid_bp = 0.1
trade_order_list = ['trade_df']
# Read in CSV file as a DataFrame
trade_df = DatabaseSourceCSV(
trade_data_database_csv=csv_trade_order_mapping['trade_df']
).fetch_trade_order_data()
data_frame_trade_order_mapping = OrderedDict([('trade_df', trade_df)])
ticker_list = FXConv().correct_unique_notation_list(
trade_df['ticker'].unique().tolist())
start_date = trade_df.index[0]
finish_date = trade_df.index[-1]
# Specify the TCA request
tca_request = TCARequest(
start_date=start_date,
finish_date=finish_date,
ticker=ticker_list,
tca_type='aggregated',
dummy_market=True,
trade_data_store='dataframe',
market_data_store=market_data_store,
metric_calcs=[
MetricSlippage(trade_order_list=trade_order_list),
MetricTransientMarketImpact(
transient_market_impact_gap={'ms': 100},
trade_order_list=trade_order_list),
MetricPermanentMarketImpact(permanent_market_impact_gap={'h': 1},
trade_order_list=trade_order_list)
],
benchmark_calcs=[ # add spread to mid fields for every market data spot
BenchmarkSpreadToMid(bid_mid_bp=spread_to_mid_bp,
ask_mid_bp=spread_to_mid_bp),
],
results_form=[
# Display a table of all the anomalous trades by slippage (ie. outside bid/ask)
TableResultsForm(
trade_order_list=['trade_df'],
metric_name='slippage',
filter_by='worst_all', # Order by the worst slippage
tag_value_combinations={'slippage_anomalous': 1.0},
# Only flag trades outside bid/ask
keep_fields=[
'executed_notional_in_reporting_currency', 'side'
],
# Display only side and executed notionals
round_figures_by=None),
# Get the total notional executed by broker (in reporting currency)
BarResultsForm(
trade_order_list=['trade_df'], # trade
aggregate_by_field='broker_id', # aggregate by broker name
# keep_fields=['executed_notional_in_reporting_currency', 'executed_notional', 'side'],
metric_name='executed_notional_in_reporting_currency',
# analyse notional
aggregation_metric='sum', # sum the notional
scalar=1, # no need for a multipler
round_figures_by=0), # round to nearest unit
# Get average slippage per broker (weighted by notional)
BarResultsForm(
trade_order_list=['trade_df'],
aggregate_by_field='broker_id',
metric_name='slippage',
aggregation_metric='mean',
# keep_fields=['executed_notional_in_reporting_currency', 'executed_notional',
# 'side'],
weighting_field='executed_notional_in_reporting_currency',
# weight results by notional
scalar=10000.0,
round_figures_by=2)
],
# Aggregate the results (total notional and slippage) by broker
# into a single table for easy display to the user
join_tables=[
JoinTables(
tables_dict={
'table_name':
'jointables_broker_id',
# fetch the following calculated tables
'table_list': [
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id',
'bar_trade_df_slippage_by_broker_id'
],
# append to the columns of each table
'column_list': ['notional (rep cur)', 'slippage (bp)']
})
],
trade_order_mapping=data_frame_trade_order_mapping,
use_multithreading=False)
# Dictionary of dataframes as output from TCA calculation
dict_of_df = tca_engine.calculate_tca(tca_request)
# print all the output tables
print(dict_of_df.keys())
print('All trades')
print(dict_of_df['trade_df'])
print('Notional by broker ID')
print(dict_of_df[
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id'])
print('Notional by broker ID and weighted slippage')
print(dict_of_df['jointables_broker_id'])
print('Trades by worst slippage')
print(dict_of_df['table_trade_df_slippage_by_worst_all'])
from chartpy import Canvas, Chart
broker_notional_chart = Chart(
engine='plotly',
df=dict_of_df[
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id'],
chart_type='bar',
style=Style(title='Notional in USD per broker'))
broker_slippage_chart = Chart(
engine='plotly',
df=dict_of_df['bar_trade_df_slippage_by_broker_id'],
chart_type='bar',
style=Style(title='Slippage by broker (bp)'))
# Using plain template
canvas = Canvas([[broker_notional_chart, broker_slippage_chart]])
canvas.generate_canvas(silent_display=False, canvas_plotter='plain')
def run_strategy_returns_stats(self, trading_model, index = None, engine = 'pyfolio'):
"""Plots useful statistics for the trading strategy (using PyFolio)
Parameters
----------
trading_model : TradingModel
defining trading strategy
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.get_strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localise_index_as_UTC(pnl)
except: pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
matplotlib.rcdefaults()
plt.style.use(ChartConstants().chartfactory_style_sheet['chartpy-pyfolio'])
except: pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig (trading_model.DUMP_PATH + "stats.png")
except: pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
pnl = trading_model.plot_strategy_pnl(silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(silent_plot=True) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual],
[pnl_comp, ir_comp],
[leverage, ind_lev]]
)
canvas.generate_canvas(silent_display=False, canvas_plotter='plain')
def create_report(self,
output_filename=None,
output_format='html',
offline_js=False):
"""Creates an HTML/PDF report from a ComputationResult object, which can (optionally) be written to disk, alternatively
returns a binary representation of the HTML or PDF.
Parameters
----------
output_filename : str (optional)
File output, if this is not specified a binary object is returned
output_format : str
'html' (default) - output an HTML page
offline_js : bool
False (default) - download's Plotly.js in webpage to be rendered
True - includes Plotly.js in web page to be rendered
Returns
-------
pdf or HTML binary
"""
# extra code to put in the <head> part of HTML
extra_head_code = ''
if output_format == 'html':
# embed plotly.js in HTML (makes it bigger, but then doesn't require web connection)
if offline_js:
embed_chart = 'offline_embed_js_div'
else:
# otherwise put web link to plotly.js (but this means we need to download every time)
embed_chart = 'offline_div'
extra_head_code = '<head><script src="https://cdn.plot.ly/plotly-latest.min.js"></script></head>'
elif output_format == 'pdf':
# for PDFs we need to create static PNGs of plotly charts
embed_chart = 'offline_image_png_in_html'
# get a list of the HTML to render
elements_to_render = self._layout_computation_results_to_html(
embed_chart)
canvas = Canvas(elements_to_render=elements_to_render)
# should we return a binary string containing the HTML/PDF (this can be displayed by a web server for example)
# or later be written to disk
return_binary = False
# return a binary string, if we haven't specified a filename output
if output_filename is None:
return_binary = True
# generate the HTML or PDF with chartpy's Canvas object
if output_format == 'html':
html, _ = canvas.generate_canvas(
output_filename=output_filename,
silent_display=True,
canvas_plotter=self._canvas_plotter,
page_title=self._title,
render_pdf=False,
return_html_binary=return_binary,
extra_head_code=extra_head_code)
return html
elif output_format == 'pdf':
_, pdf = canvas.generate_canvas(
output_filename=output_filename,
silent_display=True,
canvas_plotter=self._canvas_plotter,
page_title=self._title,
render_pdf=True,
return_pdf_binary=return_binary,
extra_head_code=extra_head_code)
return pdf
else:
raise Exception("Invalid output format selected")
def run_strategy_returns_stats(self,
trading_model,
index=None,
engine='pyfolio'):
"""
run_strategy_returns_stats - Plots useful statistics for the trading strategy (using PyFolio)
Parameters
----------
trading_model : TradingModel
defining trading strategy
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.get_strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localise_index_as_UTC(pnl)
except:
pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
matplotlib.rcdefaults()
plt.style.use(ChartConstants().
chartfactory_style_sheet['chartpy-pyfolio'])
except:
pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig(trading_model.DUMP_PATH + "stats.png")
except:
pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
pnl = trading_model.plot_strategy_pnl(
silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(
silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(
silent_plot=True
) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(
silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(
silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(
silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual], [pnl_comp, ir_comp],
[leverage, ind_lev]])
canvas.generate_canvas(silent_display=False,
canvas_plotter='plain')
def single_ticker_tca_example():
"""Example for doing detailed TCA analysis on the trades of a single ticker, calculating metrics for slippage,
transient market impact & permanent market impact. It also calculates benchmarks for arrival price of each trade and
spread to mid).
Creates a TCAReport which generates standalone HTML and PDF files
Also on a lower level it collects results for slippage into a daily timeline and also average by venue (by default
weights by reporting currency)
"""
# Note: running Orca might not work in WSL
PLOT = False
# clear entire cache
# Mediator.get_volatile_cache(version='pro').clear_cache()
tca_engine = TCAEngineImpl(version=tca_version)
trade_order_type = 'trade_df'
trade_order_list = ['trade_df']
# Ensure orca is started, if want to convert to PDF (sometimes you may need to specify the path)
# Can be slow to start
if PLOT:
from chartpy.engine import EnginePlotly
EnginePlotly().start_orca() # constants.orca_server_path)
# specify the TCA request
tca_request = TCARequest(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
tca_type='aggregated',
dummy_market=False,
trade_data_store=trade_data_store,
market_data_store=market_data_store,
metric_calcs=[
MetricSlippage(trade_order_list=trade_order_list),
MetricTransientMarketImpact(
transient_market_impact_gap={'ms': 100},
trade_order_list=trade_order_list),
MetricPermanentMarketImpact(permanent_market_impact_gap={'h': 1},
trade_order_list=trade_order_list)
],
results_form=[
TimelineResultsForm(metric_name='slippage',
by_date='datehour',
scalar=10000.0),
TimelineResultsForm(
metric_name='executed_notional_in_reporting_currency',
by_date='datehour',
aggregation_metric='sum'),
BarResultsForm(metric_name='slippage',
aggregate_by_field='venue',
scalar=10000.0),
DistResultsForm(metric_name='slippage',
aggregate_by_field='side',
scalar=10000.0)
],
benchmark_calcs=[BenchmarkArrival(),
BenchmarkSpreadToMid()],
trade_order_mapping=trade_order_list,
use_multithreading=False)
# Dictionary of dataframes as output from TCA calculation
dict_of_df = tca_engine.calculate_tca(tca_request)
print(dict_of_df['trade_df'])
print(dict_of_df.keys())
timeline_slippage_df = dict_of_df['timeline_' + trade_order_type +
'_slippage_by_all']
timeline_executed_notional_df = dict_of_df[
'timeline_' + trade_order_type +
'_executed_notional_in_reporting_currency_by_all'] # average slippage per day
metric_df = dict_of_df[trade_order_type][
'permanent_market_impact'] # permanent market impact for every trade
print(metric_df.head(500))
if PLOT:
### Generate TCA report using high level object
# Use higher level TCAResults object to encapsulate results (easier to deal with than a dictionary of DataFrames)
tca_results = TCAResults(dict_of_df, tca_request)
tca_results.render_computation_charts()
tca_report = TCAReport(tca_results)
tca_report.create_report(output_filename='test_tca_report.htm',
output_format='html')
# Note needs plotly orca + wkhtmltopdf installed to render PDFs
try:
tca_report.create_report(output_filename='test_tca_report.pdf',
output_format='pdf')
except:
pass
### Lower level creation of TCA report
from chartpy import Chart, Style, Canvas
# Generate HTML file directly
Chart(engine='plotly').plot(
tca_results.sparse_market_charts['GBPUSD_trade_df'],
style=Style(plotly_plot_mode='offline_html'))
# Get an HTML string which can be used elsewhere (eg. could use these in other webpages!)
html_string = Chart(engine='plotly').plot(
tca_results.sparse_market_charts['GBPUSD_trade_df'],
style=Style(plotly_plot_mode='offline_embed_js_div'))
img_png_string = Chart(engine='plotly').plot(
tca_results.sparse_market_charts['GBPUSD_trade_df'],
style=Style(plotly_plot_mode='offline_image_png_in_html'))
# Using plain template
canvas = Canvas([[img_png_string]])
canvas.generate_canvas(silent_display=True,
canvas_plotter='plain',
page_title='Cuemacro TCA',
render_pdf=False)
with open('test_tca.html', "w") as text_file:
text_file.write(html_string)
### Plot charts individually
# Plot slippage by timeline
Chart(engine='plotly').plot(timeline_slippage_df)
# Plot total executed notional by timeline
Chart(engine='plotly').plot(timeline_executed_notional_df)
# Plot market impact (per trade)
Chart(engine='plotly').plot(metric_df.head(500))
subcounter += 1
counter += 1
return data
print(get_all_rates()[:-1])
maybe_regression = linear_model.LinearRegression()
maybe_regression.fit(concoct_variables(get_all_rates()), get_all_rates()[-1])
#LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,normalize=False)
print(maybe_regression.coef_)
graphdf = pd.DataFrame(data=sortedepr,
index=sortedeprsd,
columns=["Expected Returns"])
stockchart = Chart(df=graphdf,
engine='bokeh',
chart_type='scatter',
style=Style(title="Testing Chartpy",
source="Yahoo Finance/Findatapy"))
canvas = Canvas(stockchart)
canvas.generate_canvas(silent_display=False, canvas_plotter="plain")
end = time.time()
print(end - start)