def test_write_chunked_market_data_arctic():
"""For very large CSV files we might need to read them in chunks. tcapy supports this and also supports CSVs
which are sorted in reverse (ie. descending). We need to enable chunking and reverse reading with flags.
This tests whether chunked data is written correctly to Arctic, comparing it with that read from CSV directly
"""
if not (run_arctic_tests): return
market_loader = Mediator.get_tca_market_trade_loader(version=tcapy_version)
arctic_start_date = '01 Jan 2016'; arctic_finish_date = pd.Timestamp(datetime.datetime.utcnow())
# load data from CSVs directly (for comparison later)
market_df_csv_desc = DatabaseSourceCSV(market_data_database_csv=csv_reverse_market_data_store).fetch_market_data(
start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker)
market_df_csv_asc = DatabaseSourceCSV(market_data_database_csv=csv_market_data_store).fetch_market_data(
start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker)
for a in arctic_lib_type:
database_source = DatabaseSourceArctic(postfix='testharness', arctic_lib_type=a)
### write CSV data to Arctic which is sorted ascending (default!)
database_source.convert_csv_to_table(csv_market_data_store, ticker,
test_harness_arctic_market_data_table,
if_exists_table='replace',
if_exists_ticker='replace', market_trade_data='market',
csv_read_chunksize=100000, remove_duplicates=False)
market_request = MarketRequest(start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker,
data_store=test_harness_arctic_market_data_store,
market_data_database_table=test_harness_arctic_market_data_table)
market_df_load = market_loader.get_market_data(market_request=market_request)
# compare reading directly from the CSV vs. reading back from arctic
assert all(market_df_csv_asc['mid'] - market_df_load['mid'] < eps)
### write CSV data to Arctic which is sorted descending
database_source.convert_csv_to_table(csv_reverse_market_data_store, ticker,
test_harness_arctic_market_data_table,
if_exists_table='append',
if_exists_ticker='replace', market_trade_data='market',
csv_read_chunksize=100000, read_in_reverse=True, remove_duplicates=False)
market_request = MarketRequest(start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker,
data_store=test_harness_arctic_market_data_store,
market_data_database_table=test_harness_arctic_market_data_table)
market_df_load = market_loader.get_market_data(market_request=market_request)
# compare reading directly from the CSV vs. reading back from arctic
assert all(market_df_csv_desc['mid'] - market_df_load['mid'] < eps)
def test_write_trade_data_ms_sql_server():
"""Tests that trade data can be read from CSV and dumped to MS SQL server
"""
if not (run_ms_sql_server_tests): return
database_source = DatabaseSourceMSSQLServer()
for t in trade_order_list:
# dump trade_df to SQL test harness database and overwrite
database_source.convert_csv_to_table(csv_trade_order_mapping[t], None, sql_trade_order_mapping[t],
database_name=test_harness_sql_server_trade_data_database_name,
if_exists_table='replace', market_trade_data='trade')
trade_order_df_sql = database_source.fetch_trade_order_data(
start_date=start_date, finish_date=finish_date, ticker=ticker, table_name=sql_trade_order_mapping[t],
database_name=test_harness_sql_server_trade_data_database_name)
database_source_csv = DatabaseSourceCSV()
trade_order_df_csv = database_source_csv.fetch_trade_order_data(
start_date=start_date, finish_date=finish_date, ticker=ticker, table_name=csv_trade_order_mapping[t])
comp_fields = ['executed_price', 'notional', 'side']
# check that the data read back from MS SQL Server matches that from the original CSV
for c in comp_fields:
if c in trade_order_df_sql.columns and c in trade_order_df_csv.columns:
exec_sql = trade_order_df_sql[c]#.dropna()
exec_csv = trade_order_df_csv[c]#.dropna()
exec_diff = exec_sql - exec_csv
assert all(exec_diff < eps)
def dataframe_tca_example():
"""Example for doing detailed TCA analysis on all the trades in a CSV, calculating metrics for slippage,
transient market impact & permanent market impact. It also calculates benchmarks for arrival price of each trade and
spread to mid).
Collects results for slippage into a daily timeline and also average by venue (by default weights by reporting
currency)
"""
PLOT = False
# clear entire cache
# Mediator.get_volatile_cache(version='pro').clear_cache()
tca_engine = TCAEngineImpl(version=tca_version)
trade_order_type = 'trade_df'
trade_order_list = ['trade_df']
trade_df = DatabaseSourceCSV(trade_data_database_csv=csv_trade_order_mapping['trade_df']).fetch_trade_order_data()
data_frame_trade_order_mapping = OrderedDict([('trade_df', trade_df)])
start_date = trade_df.index[0]; finish_date = trade_df.index[-1]
ticker_list = FXConv().correct_unique_notation_list(trade_df['ticker'].unique().tolist())
# Specify the TCA request
tca_request = TCARequest(start_date=start_date, finish_date=finish_date, ticker=ticker_list,
tca_type='aggregated', dummy_market=True,
trade_data_store='dataframe', market_data_store=market_data_store,
metric_calcs=[MetricSlippage(trade_order_list=trade_order_list),
MetricTransientMarketImpact(transient_market_impact_gap={'ms': 100},
trade_order_list=trade_order_list),
MetricPermanentMarketImpact(permanent_market_impact_gap={'h': 1},
trade_order_list=trade_order_list)],
results_form=[TimelineResultsForm(metric_name='slippage', by_date='date'),
BarResultsForm(metric_name='slippage', aggregate_by_field='venue')],
benchmark_calcs=[BenchmarkArrival(), BenchmarkSpreadToMid()],
trade_order_mapping=data_frame_trade_order_mapping, use_multithreading=False)
# Dictionary of dataframes as output from TCA calculation
dict_of_df = tca_engine.calculate_tca(tca_request)
print(dict_of_df.keys())
timeline_df = dict_of_df['timeline_' + trade_order_type + '_slippage_by_all'] # average slippage per day
metric_df = dict_of_df[trade_order_type]['permanent_market_impact'] # permanent market impact for every trade
print(metric_df.head(500))
if PLOT:
from chartpy import Chart, Style
# plot slippage by timeline
Chart(engine='plotly').plot(timeline_df)
# plot market impact (per trade)
Chart(engine='plotly').plot(metric_df.head(500))
def test_write_market_data_arctic():
"""Tests we can write market data to Arctic
"""
if not (run_arctic_tests): return
market_loader = Mediator.get_tca_market_trade_loader(version=tcapy_version)
### Test we can read data from CSV and dump to Arctic (and when read back it matches CSV)
db_start_date = '01 Jan 2016'
db_finish_date = pd.Timestamp(datetime.datetime.utcnow())
replace_append = ['replace', 'append']
# Check first when replacing full table and then appending
for a in arctic_lib_type:
for i in replace_append:
database_source = DatabaseSourceArctic(postfix='testharness',
arctic_lib_type=a)
# Write CSV to Arctic
database_source.convert_csv_to_table(
csv_market_data_store,
ticker,
test_harness_arctic_market_data_table,
if_exists_table=i,
if_exists_ticker='replace',
market_trade_data='market',
remove_duplicates=False)
# Fetch data directly from CSV
database_source_csv = DatabaseSourceCSV(
market_data_database_csv=csv_market_data_store)
market_df_csv = database_source_csv.fetch_market_data(
start_date=db_start_date,
finish_date=db_finish_date,
ticker=ticker)
# Read back data from Arctic and compare with CSV
market_request = MarketRequest(
start_date=db_start_date,
finish_date=db_finish_date,
ticker=ticker,
data_store=
database_source, # test_harness_arctic_market_data_store,
market_data_database_table=test_harness_arctic_market_data_table
)
market_df_load = market_loader.get_market_data(
market_request=market_request)
diff_df = market_df_csv['mid'] - market_df_load['mid']
diff_df.to_csv('test' + i + '.csv')
assert all(diff_df < eps)
def test_write_market_data_db():
"""Tests we can write market data to KDB/Influxdb/PyStore
"""
database_source_list, test_harness_market_data_table_list, test_harness_data_store_list = _get_db_market_database_source(
)
market_loader = Mediator.get_tca_market_trade_loader(version=tcapy_version)
for i in range(0, len(database_source_list)):
database_source = database_source_list[i]
test_harness_market_data_table = test_harness_market_data_table_list[i]
test_harness_data_store = test_harness_data_store_list[i]
### Test we can read data from CSV and dump to InfluxDB/KDB/PyStore (and when read back it matches CSV)
db_start_date = '01 Jan 2016'
db_finish_date = pd.Timestamp(datetime.datetime.utcnow())
replace_append = ['replace', 'append']
database_source_csv = DatabaseSourceCSV(
market_data_database_csv=csv_market_data_store)
market_df_csv = database_source_csv.fetch_market_data(
start_date=db_start_date,
finish_date=db_finish_date,
ticker=ticker)
# Check first when replacing full table and then appending (will still replace ticker though)
for i in replace_append:
database_source.convert_csv_to_table(
csv_market_data_store,
ticker,
test_harness_market_data_table,
if_exists_table=i,
if_exists_ticker='replace',
market_trade_data='market',
remove_duplicates=False)
market_request = MarketRequest(
start_date=db_start_date,
finish_date=db_finish_date,
ticker=ticker,
data_store=test_harness_data_store,
market_data_database_table=test_harness_market_data_table)
market_df_load = market_loader.get_market_data(
market_request=market_request)
diff_df = market_df_csv['mid'] - market_df_load['mid']
assert all(diff_df < eps)
def test_append_market_data_arctic():
"""Tests we can append market data to arctic (we will have already written data to the test harness database)
"""
if not (run_arctic_tests): return
market_loader = Mediator.get_tca_market_trade_loader(version=tcapy_version)
### Test we can append (non-overlapping) data to Arctic
arctic_start_date = '01 Jan 2016'; arctic_finish_date = pd.Timestamp(datetime.datetime.utcnow())
# use this market request later when reading back from Arctic
market_request = MarketRequest(start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker,
data_store=test_harness_arctic_market_data_store,
market_data_database_table=test_harness_arctic_market_data_table)
# load data from CSV for comparison later
database_source_csv = DatabaseSourceCSV(market_data_database_csv=csv_market_data_store)
market_df_csv = database_source_csv.fetch_market_data(
start_date=arctic_start_date, finish_date=arctic_finish_date, ticker=ticker)
market_df_list = TimeSeriesOps().split_array_chunks(market_df_csv, chunks=2)
for a in arctic_lib_type:
database_source = DatabaseSourceArctic(postfix='testharness', arctic_lib_type=a)
market_df_lower = market_df_list[0];
market_df_higher = market_df_list[1]
database_source.append_market_data(market_df_lower, ticker, table_name=test_harness_arctic_market_data_table,
if_exists_table='replace', if_exists_ticker='replace', remove_duplicates=False)
overlap_error = False
## Try to append overlapping data (this will fail!)
try:
database_source.append_market_data(market_df_lower, ticker,
table_name=test_harness_arctic_market_data_table,
if_exists_table='append', if_exists_ticker='append', remove_duplicates=False)
except ErrorWritingOverlapDataException as e:
overlap_error = True
assert overlap_error
# Append non-overlapping data which follows (writing overlapping data into Arctic will mess up the datastore!)
database_source.append_market_data(market_df_higher, ticker, table_name=test_harness_arctic_market_data_table,
if_exists_table='append', if_exists_ticker='append', remove_duplicates=False)
market_df_all_read_back = market_loader.get_market_data(market_request=market_request)
assert all(market_df_all_read_back['mid'] - market_df_csv['mid'] < eps)
def test_write_trade_data_sql():
"""Tests that trade data can be read from CSV and dumped to various SQL dialect
"""
database_source_list, test_harness_trade_database_list, test_harness_data_store_list = _get_db_trade_database_source(
)
for i in range(0, len(database_source_list)):
database_source = database_source_list[i]
test_harness_trade_database = test_harness_trade_database_list[i]
test_harness_data_store = test_harness_data_store_list[i]
for t in trade_order_list:
# Dump trade_df to SQL test harness database and overwrite
database_source.convert_csv_to_table(
csv_trade_order_mapping[t],
None, (sql_trade_order_mapping[test_harness_data_store])[t],
test_harness_trade_database,
if_exists_table='replace',
market_trade_data='trade')
trade_order_df_sql = database_source.fetch_trade_order_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
table_name=sql_trade_order_mapping[test_harness_data_store][t],
database_name=test_harness_trade_database)
database_source_csv = DatabaseSourceCSV()
trade_order_df_csv = database_source_csv.fetch_trade_order_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
table_name=csv_trade_order_mapping[t])
comp_fields = ['executed_price', 'notional', 'side']
# Check that the data read back from SQL database matches that from the original CSV
for c in comp_fields:
if c in trade_order_df_sql.columns and c in trade_order_df_csv.columns:
exec_sql = trade_order_df_sql[c] #.dropna()
exec_csv = trade_order_df_csv[c] #.dropna()
exec_diff = exec_sql - exec_csv
assert all(exec_diff < eps)
def test_randomized_trade_data_generation():
"""Tests randomized trade generation data (and writing to database)
"""
data_test_creator = DataTestCreator(write_to_db=False)
# use database source as Arctic for market data (assume we are using market data as a source)
if use_test_csv:
data_test_creator._database_source_market = DatabaseSourceCSV(
market_data_database_csv=market_data_store)
else:
data_test_creator._database_source_market = DatabaseSourceArctic(
postfix=postfix)
# create randomised trade/order data
trade_order = data_test_creator.create_test_trade_order(
ticker, start_date=start_date, finish_date=finish_date)
# trade_order has dictionary of trade_df and order_df
# make sure the number of trades > number of orders
assert (len(trade_order['trade_df'].index) > len(
trade_order['order_df'].index))
def test_randomized_trade_data_generation():
"""Tests randomized trade generation data (and writing to database)
"""
from tcapy.data.datatestcreator import DataTestCreator
data_test_creator = DataTestCreator(market_data_postfix=postfix, write_to_db=False,
market_data_database_table=test_harness_arctic_market_data_table,
trade_data_database_name=test_harness_mysql_trade_data_database)
# Use database source as Arctic for market data (assume we are using market data as a source)
if use_market_data_test_csv:
data_test_creator._database_source_market = DatabaseSourceCSV(market_data_database_csv=market_data_store)
data_test_creator._market_data_source = market_data_store
else:
data_test_creator._database_source_market = DatabaseSourceArctic(postfix=postfix)
# Create randomised trade/order data
trade_order = data_test_creator.create_test_trade_order(ticker, start_date=start_date, finish_date=finish_date)
# Trade_order has dictionary of trade_df and order_df
# Make sure the number of trades > number of orders
assert (len(trade_order['trade_df'].index) > len(trade_order['order_df'].index))
def test_write_multiple_wildcard_market_data_csvs_arctic():
"""Tests we can write sequential market data CSVs (or HDF5) whose path has been specified by a wildcard (eg. EURUSD*.csv).
It is assumed that the CSVs are in chronological orders, from their filenames.
"""
if not (run_arctic_tests): return
market_loader = Mediator.get_tca_market_trade_loader(version=tcapy_version)
arctic_start_date = '01 Jan 2016'
arctic_finish_date = pd.Timestamp(datetime.datetime.utcnow())
for a in arctic_lib_type:
database_source = DatabaseSourceArctic(postfix='testharness',
arctic_lib_type=a)
### Read CSV data which is sorted ascending (default!)
database_source.convert_csv_to_table(
csv_market_data_store,
ticker,
test_harness_arctic_market_data_table,
if_exists_table='replace',
if_exists_ticker='replace',
market_trade_data='market',
csv_read_chunksize=10**6,
remove_duplicates=False)
database_source_csv = DatabaseSourceCSV(
market_data_database_csv=csv_market_data_store)
market_df_csv = database_source_csv.fetch_market_data(
start_date=arctic_start_date,
finish_date=arctic_finish_date,
ticker=ticker)
# Prepare the CSV folder first
csv_folder = os.path.join(constants.test_data_harness_folder,
'csv_arctic_mult')
# Empty the CSV test harness folder, where we shall dump the mini CSVs
UtilFunc().forcibly_create_empty_folder(csv_folder)
# Split the CSV file into several mini CSV files (and also HDF5 files)
market_df_list = TimeSeriesOps().split_array_chunks(market_df_csv,
chunks=3)
chunk_no = 0
for m in market_df_list:
m.to_csv(
os.path.join(csv_folder, "EURUSD" + str(chunk_no) + '.csv'))
UtilFunc().write_dataframe_to_binary(
m,
os.path.join(csv_folder,
"EURUSD" + str(chunk_no) + '.parquet'),
format='parquet')
chunk_no = chunk_no + 1
file_ext = ['csv', 'parquet']
for f in file_ext:
### Read CSV data from the mini CSVs (using wildcard char) and dump to Arctic
database_source.convert_csv_to_table(
os.path.join(csv_folder, "EURUSD*." + f),
ticker,
test_harness_arctic_market_data_table,
if_exists_table='append',
if_exists_ticker='replace',
market_trade_data='market',
csv_read_chunksize=10**6,
remove_duplicates=False)
market_request = MarketRequest(
start_date=arctic_start_date,
finish_date=arctic_finish_date,
ticker=ticker,
data_store=database_source,
market_data_database_table=test_harness_arctic_market_data_table
)
# Read back from Arctic
market_df_load = market_loader.get_market_data(
market_request=market_request)
# Compare reading directly from the original large CSV vs. reading back from arctic (which was dumped from split CSVs)
diff_df = abs(market_df_load['mid'] - market_df_csv['mid'])
outside_bounds = diff_df[diff_df >= eps]
assert len(outside_bounds) == 0
file_extension = 'parquet' # 'parquet' (recommended) or 'csv' or 'h5' on disk
# Files dumped by DatabasePopulator look like this
## 'AUDUSD_dukascopy_2016-01-03_22_00_01.868000+00_002016-01-31_23_59_57.193000+00_00.parquet'
csv_file = [x + '_' + data_vendor + '_20*.' + file_extension for x in
ticker_mkt] # assume that ALL TIME IN UTC!
date_format = None
read_in_reverse = False
remove_duplicates = False
####################################################################################################################
# Load market data
data_source_csv = DatabaseSourceCSV()
# Create market data store for database and associated data vendor
if market_data_store == 'arctic':
database_source = DatabaseSourceArctic(postfix=data_vendor)
market_data_database_table = constants.arctic_market_data_database_table
if market_data_store == 'pystore':
database_source = DatabaseSourcePyStore(postfix=data_vendor)
market_data_database_table = constants.pystore_market_data_database_table
if market_data_store == 'influxdb':
database_source = DatabaseSourceInfluxDB(postfix=data_vendor)
market_data_database_table = constants.influxdb_market_data_database_table
if market_data_store == 'kdb':
def test_fetch_market_trade_data_csv():
"""Tests downloading of market and trade/order data from CSV files
"""
### Get market data
market_loader = Mediator.get_tca_market_trade_loader()
market_request = MarketRequest(start_date=start_date,
finish_date=finish_date,
ticker=ticker,
data_store=csv_market_data_store)
market_df = market_loader.get_market_data(market_request)
assert not(market_df.empty) \
and market_df.index[0] >= pd.Timestamp(start_date).tz_localize('utc') \
and market_df.index[-1] <= pd.Timestamp(finish_date).tz_localize('utc')
# For a high level trade data request, we need to use TCA request, because it usually involves some
# market data download (we are assuming that the market data is being downloaded from our arctic database)
# eg. for converting notionals to reporting currency
tca_request = TCARequest(start_date=start_date,
finish_date=finish_date,
ticker=ticker,
trade_data_store='csv',
market_data_store=arctic_market_data_store,
trade_order_mapping=csv_trade_order_mapping)
for t in trade_order_list:
trade_order_df = market_loader.get_trade_order_data(tca_request, t)
try:
trade_order_df = Mediator.get_volatile_cache(
).get_dataframe_handle(trade_order_df)
except:
pass
assert not trade_order_df.empty \
and trade_order_df.index[0] >= pd.Timestamp(start_date).tz_localize('utc') \
and trade_order_df.index[-1] <= pd.Timestamp(finish_date).tz_localize('utc')
### Test using DataFactory and DatabaseSource
from tcapy.data.datafactory import DataFactory
data_factory = DataFactory()
for t in trade_order_list:
### Test using DataFactory
trade_request = TradeRequest(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
data_store='csv',
trade_order_mapping=csv_trade_order_mapping,
trade_order_type=t)
trade_order_df = data_factory.fetch_table(trade_request)
assert not trade_order_df.empty \
and trade_order_df.index[0] >= pd.Timestamp(start_date).tz_localize('utc') \
and trade_order_df.index[-1] <= pd.Timestamp(finish_date).tz_localize('utc')
### Test using DatabaseSourceCSV
from tcapy.data.databasesource import DatabaseSourceCSV
database_source = DatabaseSourceCSV()
trade_order_df = database_source.fetch_trade_order_data(
start_date,
finish_date,
ticker,
table_name=csv_trade_order_mapping[t])
assert not trade_order_df.empty \
and trade_order_df.index[0] >= pd.Timestamp(start_date).tz_localize('utc') \
and trade_order_df.index[-1] <= pd.Timestamp(finish_date).tz_localize('utc')
def dataframe_compliance_tca_example():
"""Get a DataFrame of trades and apply compliance based TCA to it
"""
tca_engine = TCAEngineImpl(version=tca_version)
spread_to_mid_bp = 0.1
trade_order_list = ['trade_df']
# Read in CSV file as a DataFrame
trade_df = DatabaseSourceCSV(
trade_data_database_csv=csv_trade_order_mapping['trade_df']
).fetch_trade_order_data()
data_frame_trade_order_mapping = OrderedDict([('trade_df', trade_df)])
ticker_list = FXConv().correct_unique_notation_list(
trade_df['ticker'].unique().tolist())
start_date = trade_df.index[0]
finish_date = trade_df.index[-1]
# Specify the TCA request
tca_request = TCARequest(
start_date=start_date,
finish_date=finish_date,
ticker=ticker_list,
tca_type='aggregated',
dummy_market=True,
trade_data_store='dataframe',
market_data_store=market_data_store,
metric_calcs=[
MetricSlippage(trade_order_list=trade_order_list),
MetricTransientMarketImpact(
transient_market_impact_gap={'ms': 100},
trade_order_list=trade_order_list),
MetricPermanentMarketImpact(permanent_market_impact_gap={'h': 1},
trade_order_list=trade_order_list)
],
benchmark_calcs=[ # add spread to mid fields for every market data spot
BenchmarkSpreadToMid(bid_mid_bp=spread_to_mid_bp,
ask_mid_bp=spread_to_mid_bp),
],
results_form=[
# Display a table of all the anomalous trades by slippage (ie. outside bid/ask)
TableResultsForm(
trade_order_list=['trade_df'],
metric_name='slippage',
filter_by='worst_all', # Order by the worst slippage
tag_value_combinations={'slippage_anomalous': 1.0},
# Only flag trades outside bid/ask
keep_fields=[
'executed_notional_in_reporting_currency', 'side'
],
# Display only side and executed notionals
round_figures_by=None),
# Get the total notional executed by broker (in reporting currency)
BarResultsForm(
trade_order_list=['trade_df'], # trade
aggregate_by_field='broker_id', # aggregate by broker name
# keep_fields=['executed_notional_in_reporting_currency', 'executed_notional', 'side'],
metric_name='executed_notional_in_reporting_currency',
# analyse notional
aggregation_metric='sum', # sum the notional
scalar=1, # no need for a multipler
round_figures_by=0), # round to nearest unit
# Get average slippage per broker (weighted by notional)
BarResultsForm(
trade_order_list=['trade_df'],
aggregate_by_field='broker_id',
metric_name='slippage',
aggregation_metric='mean',
# keep_fields=['executed_notional_in_reporting_currency', 'executed_notional',
# 'side'],
weighting_field='executed_notional_in_reporting_currency',
# weight results by notional
scalar=10000.0,
round_figures_by=2)
],
# Aggregate the results (total notional and slippage) by broker
# into a single table for easy display to the user
join_tables=[
JoinTables(
tables_dict={
'table_name':
'jointables_broker_id',
# fetch the following calculated tables
'table_list': [
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id',
'bar_trade_df_slippage_by_broker_id'
],
# append to the columns of each table
'column_list': ['notional (rep cur)', 'slippage (bp)']
})
],
trade_order_mapping=data_frame_trade_order_mapping,
use_multithreading=False)
# Dictionary of dataframes as output from TCA calculation
dict_of_df = tca_engine.calculate_tca(tca_request)
# print all the output tables
print(dict_of_df.keys())
print('All trades')
print(dict_of_df['trade_df'])
print('Notional by broker ID')
print(dict_of_df[
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id'])
print('Notional by broker ID and weighted slippage')
print(dict_of_df['jointables_broker_id'])
print('Trades by worst slippage')
print(dict_of_df['table_trade_df_slippage_by_worst_all'])
from chartpy import Canvas, Chart
broker_notional_chart = Chart(
engine='plotly',
df=dict_of_df[
'bar_trade_df_executed_notional_in_reporting_currency_by_broker_id'],
chart_type='bar',
style=Style(title='Notional in USD per broker'))
broker_slippage_chart = Chart(
engine='plotly',
df=dict_of_df['bar_trade_df_slippage_by_broker_id'],
chart_type='bar',
style=Style(title='Slippage by broker (bp)'))
# Using plain template
canvas = Canvas([[broker_notional_chart, broker_slippage_chart]])
canvas.generate_canvas(silent_display=False, canvas_plotter='plain')
from tcapy.conf.constants import Constants
import os
constants = Constants()
if __name__ == '__main__':
folder = constants.test_data_harness_folder
csv_market_data_files = ['small_test_market_df.csv.gz', 'small_test_market_df_reverse.csv.gz']
# Can either dump to Parquet (default) or HDF (optional)
# format = 'hdf5'; file_ext = 'h5'
format = 'parquet'; file_ext = 'parquet'
for csv_market_data in csv_market_data_files:
csv_market_data = os.path.join(folder, csv_market_data)
REVERSE_SORT = False
from tcapy.data.databasesource import DatabaseSourceCSV
# Read CSV and parse the main field
df = DatabaseSourceCSV()._fetch_table(csv_market_data)
if REVERSE_SORT:
df = df.sort_index(ascending=False)
h5_market_data = csv_market_data.replace('.csv.gz', '.' + file_ext)
UtilFunc().write_dataframe_to_binary(df, h5_market_data, format=format)
