def __init__(self,
temp_data_folder=constants.temp_data_folder,
temp_large_data_folder=constants.temp_large_data_folder,
tickers=None,
data_store=None,
access_control=AccessControl()):
self._temp_data_folder = temp_data_folder
self._temp_large_data_folder = temp_large_data_folder
self._tickers = None
self._util_func = UtilFunc()
self._time_series_ops = TimeSeriesOps()
self._data_store = data_store
self._access_control = access_control
logger = LoggerManager().getLogger(__name__)
if not (os.path.isdir(self._temp_data_folder)):
logger.warning("Temp data folder " + self._temp_data_folder +
" does not exist")
if not (os.path.isdir(self._temp_large_data_folder)):
logger.warning("Temp large data folder " + self._temp_data_folder +
" does not exist")
if tickers is not None:
self._tickers = tickers
def get_trade_order_data(self,
tca_request,
trade_order_type,
start_date=None,
finish_date=None):
"""Gets trade data for specified parameters (eg. start/finish dates _tickers). Will also try to find trades
when they have booked in the inverted market convention, and change the fields appropriately. For example, if
we ask for GBPUSD trade data, it will also search for USDGBP and convert those trades in the correct convention.
Parameters
----------
tca_request : TCARequest
What type of trade data do we want
trade_order_type : str
Do we want trade or order data?
Returns
-------
DataFrame
"""
logger = LoggerManager().getLogger(__name__)
# by default, assume we want trade data (rather than order data)
if trade_order_type is None:
trade_order_type = 'trade_df'
if start_date is None and finish_date is None:
start_date = tca_request.start_date
finish_date = tca_request.finish_date
# Create request for actual executed trades
trade_request = TradeRequest(trade_request=tca_request)
trade_request.start_date = start_date
trade_request.finish_date = finish_date
trade_request.trade_order_type = trade_order_type
# Fetch all the trades done in that ticker (will be sparse-like randomly spaced tick data)
# assumed to be the correct convention (eg. GBPUSD)
trade_df = self._data_factory.fetch_table(data_request=trade_request)
# if fx see if inverted or not
if tca_request.asset_class == 'fx' and tca_request.instrument == 'spot':
# Also fetch data in the inverted cross (eg. USDGBP) as some trades may be recorded this way
inv_trade_request = TradeRequest(trade_request=tca_request)
inv_trade_request.start_date = start_date
inv_trade_request.finish_date = finish_date
inv_trade_request.trade_order_type = trade_order_type
inv_trade_request.ticker = self._fx_conv.reverse_notation(
trade_request.ticker)
trade_inverted_df = self._data_factory.fetch_table(
data_request=inv_trade_request)
# Only add inverted trades if they exist!
if trade_inverted_df is not None:
if not (trade_inverted_df.empty):
invert_price_columns = [
'executed_price', 'price_limit', 'market_bid',
'market_mid', 'market_ask', 'arrival_price'
]
invert_price_columns = [
x for x in invert_price_columns
if x in trade_inverted_df.columns
]
# For trades (but not orders), there is an executed price field, which needs to be inverted
if invert_price_columns != []:
trade_inverted_df[
invert_price_columns] = 1.0 / trade_inverted_df[
invert_price_columns].values
trade_inverted_df['side'] = -trade_inverted_df[
'side'] # buys become sells, and vice versa!
trade_inverted_df['ticker'] = trade_request.ticker
if trade_df is not None:
trade_df = trade_df.append(trade_inverted_df)
trade_df = trade_df.sort_index()
else:
trade_df = trade_inverted_df
# Check if trade data is not empty? if it is return None
if self._check_is_empty_trade_order(trade_df, tca_request, start_date,
finish_date, trade_order_type):
return None
if tca_request.asset_class == 'fx' and tca_request.instrument == 'spot':
# Check if any notionals of any trade/order are quoted in the TERMS currency?
terms_notionals = trade_df[
'notional_currency'] == tca_request.ticker[3:6]
# If any notional are quoted as terms, we should invert these so we quote notionals with base currency
# for consistency
if terms_notionals.any():
inversion_ticker = tca_request.ticker[
3:6] + tca_request.ticker[0:3]
inversion_spot, trade_df = self._fill_reporting_spot(
inversion_ticker, trade_df, start_date, finish_date,
tca_request)
notional_fields = [
'notional', 'order_notional', 'executed_notional'
]
# Need to check terms notionals again, as trade data could have shrunk (because can only get trades, where we have market data)
terms_notionals = trade_df['notional_currency'] == str(
tca_request.ticker[3:6])
# Only get the inversion spot if any terms notionals are quoted wrong way around
if terms_notionals.any():
if inversion_spot is not None:
for n in notional_fields:
if n in trade_inverted_df.columns:
# trade_df[n][terms_notionals] = trade_df[n][terms_notionals].values * inversion_spot[terms_notionals].values
trade_df[n][terms_notionals] = pd.Series(
index=trade_df.index[
terms_notionals.values],
data=trade_df[n][terms_notionals].values *
inversion_spot[terms_notionals].values)
else:
logger.warning(
"Couldn't get spot data for " + inversion_ticker +
" to invert notionals. Hence not returning trading data."
)
if terms_notionals.any():
trade_df['notional_currency'][
terms_notionals] = trade_request.ticker[0:3]
# Also represent notional is reporting currency notional amount (eg. if we are USD based investors, convert
# notional to USDs)
# Using a reporting currency can be particularly useful if we are trying to aggregate metrics from many different
# currency pairs (and wish to weight by a commonly measured reporting notional)
# Eg. if we don't have USDUSD, then we need to convert
if trade_request.ticker[0:3] != tca_request.reporting_currency:
# So if we have EURJPY, we want to download EURUSD data
reporting_ticker = trade_request.ticker[
0:3] + tca_request.reporting_currency
reporting_spot, trade_df = self._fill_reporting_spot(
reporting_ticker, trade_df, start_date, finish_date,
tca_request)
if reporting_spot is not None:
trade_df[
'notional_reporting_currency_mid'] = reporting_spot.values
# trade_df['notional_reporting_currency_mid'] = \
# self._time_series_ops.vlookup_style_data_frame(trade_df.index, market_conversion_df, 'mid')[0].values
trade_df[
'reporting_currency'] = tca_request.reporting_currency
columns_to_report = [
'executed_notional', 'notional', 'order_notional'
]
for c in columns_to_report:
if c in trade_df.columns:
trade_df[c + '_in_reporting_currency'] = \
trade_df['notional_reporting_currency_mid'].values * trade_df[c]
else:
logger.warning(
"Couldn't get spot data to convert notionals into reporting currency. Hence not returning trading data."
)
return None
else:
# ie. USDUSD, so spot is 1
trade_df['notional_reporting_currency_mid'] = 1.0
# Reporting currency is the same as the notional of the trade, so no need to convert, just
# replicate columns
trade_df['reporting_currency'] = tca_request.reporting_currency
columns_to_report = [
'executed_notional', 'notional', 'order_notional'
]
for c in columns_to_report:
if c in trade_df.columns:
trade_df[c + '_in_reporting_currency'] = trade_df[c]
return trade_df
def fetch_table(self, data_request):
"""Fetches table from underlying DatabaseSource
Parameters
----------
data_request : DataRequest
Request for data with start/finish date etc.
Returns
-------
DataFrame
"""
# Fetch table from the underlying database (CSV, SQL or RESTful etc.)
logger = LoggerManager().getLogger(__name__)
data_norm = data_request.data_norm
if data_norm is None:
data_norm = Mediator.get_data_norm(version=self._version)
# Where do we get data from?
database_source = Mediator.get_database_source_picker(
).get_database_source(data_request)
if database_source is None:
Exception("User asked for an unsupported database source")
# Extract the start/finish dates and ticker we wish to download data for
start_date = data_request.start_date
finish_date = data_request.finish_date
ticker = data_request.ticker
# Are we requesting market data or trade/order data of our own executions?
if isinstance(data_request, MarketRequest):
df = database_source.fetch_market_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
table_name=data_request.market_data_database_table)
df = data_norm.normalize_market_data(df, None, data_request)
elif isinstance(data_request, TradeRequest):
trade_order_type = data_request.trade_order_type
trade_order_mapping = data_request.trade_order_mapping
trade_data_database_name = data_request.trade_data_database_name
if data_request.data_store == 'csv' and trade_order_type != None and trade_order_mapping != None:
df = database_source.fetch_trade_order_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
table_name=trade_order_mapping[trade_order_type])
elif trade_order_mapping is not None:
df = database_source.fetch_trade_order_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker,
table_name=trade_order_mapping[trade_order_type],
database_name=trade_data_database_name)
else:
# Otherwise we have a CSV file without any sort of mapping, which we assume only contains trade_df data
df = database_source.fetch_trade_order_data(
start_date=start_date,
finish_date=finish_date,
ticker=ticker)
df = data_norm.normalize_trade_data(df, None, data_request)
if df is None:
pass
if df is not None and df.empty:
logger.warning('Dataframe empty for ticker ' + ticker)
return df