def translate_from_vendor_ticker(self, vendor_tickers_list,
market_data_request):
"""Converts all the fields from vendor tickers to findatapy tickers
Parameters
----------
market_data_request : MarketDataRequest
contains all the various parameters detailing time series start and finish, tickers etc
Returns
-------
List of Strings
"""
if market_data_request.vendor_tickers is not None:
dictionary = dict(
zip(market_data_request.vendor_tickers,
market_data_request.tickers))
tickers_stuff = []
for vendor_ticker in vendor_tickers_list:
tickers_stuff.append(dictionary[vendor_ticker])
return tickers_stuff # [item for sublist in tickers_stuff for item in sublist]
data_source = market_data_request.data_source
# tickers_list = market_data_request.tickers
if isinstance(vendor_tickers_list, str):
vendor_tickers_list = [vendor_tickers_list]
if self.config is None: return vendor_tickers_list
tickers_converted = []
for vendor_ticker in vendor_tickers_list:
try:
v = self.config.convert_vendor_to_library_ticker(
data_source, vendor_ticker)
except:
logger = LoggerManager().getLogger(__name__)
logger.error(
"Couldn't find ticker conversion, did you type it correctly: "
+ vendor_ticker)
return
tickers_converted.append(v)
return tickers_converted
def fetch_group_time_series(self, market_data_request_list):
logger = LoggerManager().getLogger(__name__)
df_agg = None
thread_no = constants.market_thread_no["other"]
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(
thread_technique=constants.market_thread_technique,
thread_no=thread_no)
# Open the market data downloads in their own threads and return
# the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
df_group = result.get()
pool.close()
pool.join()
else:
df_group = []
for md_request in market_data_request_list:
df_group.append(
self.fetch_single_time_series(md_request))
# Collect together all the time series
if df_group is not None:
df_group = [i for i in df_group if i is not None]
if df_group is not None:
try:
df_agg = self._calculations.join(df_group,
how="outer")
# Force ordering to be the same!
# df_agg = df_agg[columns]
except Exception as e:
logger.warning(
"Possible overlap of columns? Have you specifed same "
"ticker several times: " + str(e))
return df_agg
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self._all_econ_tickers = pandas.read_csv(
DataConstants().all_econ_tickers)
self._econ_country_codes = pandas.read_csv(
DataConstants().econ_country_codes)
self._econ_country_groups = pandas.read_csv(
DataConstants().econ_country_groups)
if market_data_generator is None:
self.market_data_generator = MarketDataGenerator()
else:
self.market_data_generator = market_data_generator
def fetch_group_time_series(self, market_data_request_list):
logger = LoggerManager().getLogger(__name__)
data_frame_agg = None
thread_no = constants.market_thread_no['other']
if market_data_request_list[0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(thread_technique = constants.market_thread_technique, thread_no=thread_no)
# Open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series, market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(self.fetch_single_time_series(md_request))
# Collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
# import itertools
# columns = list(itertools.chain.from_iterable([i.columns for i in data_frame_group if i is not None]))
# For debugging!
# import pickle
# import datetime
# pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb"))
if data_frame_group is not None:
try:
data_frame_agg = self.calculations.join(data_frame_group, how='outer')
# Force ordering to be the same!
# data_frame_agg = data_frame_agg[columns]
except Exception as e:
logger.warning('Possible overlap of columns? Have you specifed same ticker several times: ' + str(e))
return data_frame_agg
def translate_to_vendor_ticker(self, md_request):
"""Converts all the tickers from findatapy tickers to vendor tickers
Parameters
----------
md_request : MarketDataRequest
contains all the various parameters detailing time series start
and finish, tickers etc
Returns
-------
List of Strings
"""
if md_request.vendor_tickers is not None:
return md_request.vendor_tickers
category = md_request.category
source = md_request.data_source
freq = md_request.freq
cut = md_request.cut
tickers_list = md_request.tickers
if isinstance(tickers_list, str):
tickers_list = [tickers_list]
if self.config is None: return tickers_list
tickers_list_converted = []
for ticker in tickers_list:
try:
t = self.config.convert_library_to_vendor_ticker(category,
source, freq,
cut, ticker)
except:
logger = LoggerManager().getLogger(__name__)
logger.error(
"Couldn't find ticker conversion, did you type "
"it correctly: " + ticker)
return
tickers_list_converted.append(t)
return tickers_list_converted
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self.cache = {}
self.calculations = Calculations()
self.market_data_generator = market_data_generator
return
def convert_to_numeric_dataframe(self, data_frame):
logger = LoggerManager().getLogger(__name__)
failed_conversion_cols = []
for c in data_frame.columns:
is_date = False
# If it's a date column don't append to convert to a float
for d in constants.always_date_columns:
if d in c or 'release-dt' in c:
is_date = True
break
if is_date:
try:
data_frame[c] = pd.to_datetime(data_frame[c],
errors='coerce')
except:
pass
else:
try:
data_frame[c] = data_frame[c].astype('float32')
except:
if '.' in c:
if c.split('.')[1] in constants.always_numeric_column:
data_frame[c] = data_frame[c].astype(
'float32', errors='coerce')
else:
failed_conversion_cols.append(c)
else:
failed_conversion_cols.append(c)
try:
data_frame[c] = data_frame[c].fillna(value=np.nan)
except:
pass
if failed_conversion_cols != []:
logger.warning('Could not convert to float for ' +
str(failed_conversion_cols))
return data_frame
def __init__(self):
super(EventStudy, self).__init__()
self.config = ConfigManager()
self.logger = LoggerManager().getLogger(__name__)
self.filter = Filter()
self.io_engine = IOEngine()
if (EventsFactory._econ_data_frame is None):
self.load_economic_events()
return
def __init__(self, market_data_generator = None):
self.logger = LoggerManager().getLogger(__name__)
self._all_econ_tickers = pandas.read_csv(DataConstants().all_econ_tickers)
self._econ_country_codes = pandas.read_csv(DataConstants().econ_country_codes)
self._econ_country_groups = pandas.read_csv(DataConstants().econ_country_groups)
if market_data_generator is None:
self.market_data_generator = MarketDataGenerator()
else:
self.market_data_generator = market_data_generator
def translate_to_vendor_field(self, md_request):
"""Converts all the fields from findatapy fields to vendor fields
Parameters
----------
md_request : MarketDataRequest
contains all the various parameters detailing time series start
and finish, tickers etc
Returns
-------
List of Strings
"""
if md_request.vendor_fields is not None:
return md_request.vendor_fields
source = md_request.data_source
fields_list = md_request.fields
if isinstance(fields_list, str):
fields_list = [fields_list]
if self.config is None: return fields_list
fields_converted = []
for field in fields_list:
try:
f = self.config.convert_library_to_vendor_field(source, field)
except:
logger = LoggerManager().getLogger(__name__)
logger.warn(
"Couldn't find field conversion, "
"did you type it correctly: " + field)
return
fields_converted.append(f)
return fields_converted
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self.market_data_generator = market_data_generator
self.calculations = Calculations()
self.filter = Filter()
self.timezone = Timezone()
self.rates = RatesFactory()
return
def __init__(self, df=None):
super(EventStudy, self).__init__()
self.config = ConfigManager()
self.logger = LoggerManager().getLogger(__name__)
self.filter = Filter()
self.io_engine = IOEngine()
if df is not None:
self._econ_data_frame = df
else:
self.load_economic_events()
return
def load_tickers():
logger = LoggerManager.getLogger(__name__)
market = Market(market_data_generator=MarketDataGenerator())
DataConstants.market_thread_technique = 'thread'
# load S&P 500 ticker via wikipedia
snp = pd.read_html(
'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
tickers = snp[0]['Symbol'].to_list()
# download equities data from Yahoo
md_request = MarketDataRequest(
start_date=START_DATE,
data_source='yahoo', # use Bloomberg as data source
tickers=tickers, # ticker (findatapy)
fields=['close', 'open', 'high', 'low',
'volume'], # which fields to download
vendor_tickers=tickers, # ticker (Yahoo)
vendor_fields=['Close', 'Open', 'High', 'Low',
'Volume']) # which Bloomberg fields to download)
logger.info("Loading data with threading")
df = market.fetch_market(md_request)
logger.info("Loading data with multiprocessing")
DataConstants.market_thread_technique = 'multiprocessing'
df = market.fetch_market(md_request)
logger.info("Loaded data with multiprocessing")
df.to_csv("temp_downloads/snp.csv")
class FXCrossFactory(object):
"""Generates FX spot time series and FX total return time series (assuming we already have
total return indices available from xxxUSD form) from underlying series. Can also produce cross rates from the USD
crosses.
"""
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self.fxconv = FXConv()
self.cache = {}
self.calculations = Calculations()
self.market_data_generator = market_data_generator
return
def get_fx_cross_tick(self,
start,
end,
cross,
cut="NYC",
data_source="dukascopy",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['bid', 'ask']):
if isinstance(cross, str):
cross = [cross]
market_data_request = MarketDataRequest(
gran_freq="tick",
freq_mult=1,
freq='tick',
cut=cut,
fields=['bid', 'ask', 'bidv', 'askv'],
cache_algo=cache_algo,
environment=environment,
start_date=start,
finish_date=end,
data_source=data_source,
category='fx')
market_data_generator = self.market_data_generator
data_frame_agg = None
for cr in cross:
if (type == 'spot'):
market_data_request.tickers = cr
cross_vals = market_data_generator.fetch_market_data(
market_data_request)
if cross_vals is not None:
# if user only wants 'close' calculate that from the bid/ask fields
if fields == ['close']:
cross_vals = cross_vals[[cr + '.bid',
cr + '.ask']].mean(axis=1)
cross_vals.columns = [cr + '.close']
else:
filter = Filter()
filter_columns = [cr + '.' + f for f in fields]
cross_vals = filter.filter_time_series_by_columns(
filter_columns, cross_vals)
if data_frame_agg is None:
data_frame_agg = cross_vals
else:
data_frame_agg = data_frame_agg.join(cross_vals, how='outer')
if data_frame_agg is not None:
# strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def get_fx_cross(self,
start,
end,
cross,
cut="NYC",
data_source="bloomberg",
freq="intraday",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['close']):
if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick':
return self.get_fx_cross_tick(start,
end,
cross,
cut=cut,
data_source=data_source,
cache_algo=cache_algo,
type='spot',
fields=fields)
if isinstance(cross, str):
cross = [cross]
market_data_request_list = []
freq_list = []
type_list = []
for cr in cross:
market_data_request = MarketDataRequest(freq_mult=1,
cut=cut,
fields=['close'],
freq=freq,
cache_algo=cache_algo,
start_date=start,
finish_date=end,
data_source=data_source,
environment=environment)
market_data_request.type = type
market_data_request.cross = cr
if freq == 'intraday':
market_data_request.gran_freq = "minute" # intraday
elif freq == 'daily':
market_data_request.gran_freq = "daily" # daily
market_data_request_list.append(market_data_request)
data_frame_agg = []
# depends on the nature of operation as to whether we should use threading or multiprocessing library
if constantsmarket_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocessing_on_dill library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocessing_on_dill import Pool
thread_no = constants.market_thread_no['other']
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
# fudge, issue with multithreading and accessing HDF5 files
# if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
# thread_no = 0
thread_no = 0
if (thread_no > 0):
pool = Pool(thread_no)
# open the market data downloads in their own threads and return the results
df_list = pool.map_async(self._get_individual_fx_cross,
market_data_request_list).get()
data_frame_agg = self.calculations.iterative_outer_join(df_list)
# data_frame_agg = self.calculations.pandas_outer_join(result.get())
try:
pool.close()
pool.join()
except:
pass
else:
for md_request in market_data_request_list:
data_frame_agg.append(
self._get_individual_fx_cross(md_request))
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_agg)
# strip the nan elements
data_frame_agg = data_frame_agg.dropna(how='all')
# self.speed_cache.put_dataframe(key, data_frame_agg)
return data_frame_agg
def _get_individual_fx_cross(self, market_data_request):
cr = market_data_request.cross
type = market_data_request.type
freq = market_data_request.freq
base = cr[0:3]
terms = cr[3:6]
if (type == 'spot'):
# non-USD crosses
if base != 'USD' and terms != 'USD':
base_USD = self.fxconv.correct_notation('USD' + base)
terms_USD = self.fxconv.correct_notation('USD' + terms)
# TODO check if the cross exists in the database
# download base USD cross
market_data_request.tickers = base_USD
market_data_request.category = 'fx'
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# download terms USD cross
market_data_request.tickers = terms_USD
market_data_request.category = 'fx'
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# if quoted USD/base flip to get USD terms
if (base_USD[0:3] == 'USD'):
base_vals = 1 / base_vals
# if quoted USD/terms flip to get USD terms
if (terms_USD[0:3] == 'USD'):
terms_vals = 1 / terms_vals
base_vals.columns = ['temp']
terms_vals.columns = ['temp']
cross_vals = base_vals.div(terms_vals, axis='index')
cross_vals.columns = [cr + '.close']
base_vals.columns = [base_USD + '.close']
terms_vals.columns = [terms_USD + '.close']
else:
# if base == 'USD': non_USD = terms
# if terms == 'USD': non_USD = base
correct_cr = self.fxconv.correct_notation(cr)
market_data_request.tickers = correct_cr
market_data_request.category = 'fx'
cross_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# special case for USDUSD!
if base + terms == 'USDUSD':
if freq == 'daily':
cross_vals = pandas.DataFrame(
1,
index=cross_vals.index,
columns=cross_vals.columns)
filter = Filter()
cross_vals = filter.filter_time_series_by_holidays(
cross_vals, cal='WEEKDAY')
else:
# flip if not convention (eg. JPYUSD)
if (correct_cr != cr):
cross_vals = 1 / cross_vals
# cross_vals = self.market_data_generator.harvest_time_series(market_data_request)
cross_vals.columns = [cr + '.close']
elif type[0:3] == "tot":
if freq == 'daily':
# download base USD cross
market_data_request.tickers = base + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
x = 0
# download terms USD cross
market_data_request.tickers = terms + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
pass
# base_rets = self.calculations.calculate_returns(base_vals)
# terms_rets = self.calculations.calculate_returns(terms_vals)
# special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == 'USDUSD':
base_rets = self.calculations.calculate_returns(base_vals)
cross_rets = pandas.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + 'USD' == 'USDUSD':
cross_rets = -self.calculations.calculate_returns(
terms_vals)
elif terms + 'USD' == 'USDUSD':
cross_rets = self.calculations.calculate_returns(base_vals)
else:
base_rets = self.calculations.calculate_returns(base_vals)
terms_rets = self.calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# first returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self.calculations.create_mult_index(cross_rets)
cross_vals.columns = [cr + '-tot.close']
elif freq == 'intraday':
self.logger.info(
'Total calculated returns for intraday not implemented yet'
)
return None
return cross_vals
class MarketDataGenerator(object):
"""Returns market data time series by directly calling market data sources.
At present it supports Bloomberg (bloomberg), Yahoo (yahoo), Quandl (quandl), FRED (fred) etc. which are implemented
in subclasses of DataVendor class. This provides a common wrapper for all these data sources.
"""
def __init__(self):
self.config = ConfigManager().get_instance()
self.logger = LoggerManager().getLogger(__name__)
self.filter = Filter()
self.calculations = Calculations()
self.io_engine = IOEngine()
self._intraday_code = -1
self.days_expired_intraday_contract_download = -1
return
def set_intraday_code(self, code):
self._intraday_code = code
def get_data_vendor(self, source):
"""Loads appropriate data service class
Parameters
----------
source : str
the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc.
we can also have forms like "bloomberg-boe" separated by hyphens
Returns
-------
DataVendor
"""
data_vendor = None
try:
source = source.split("-")[0]
except:
self.logger.error("Was data source specified?")
return None
if source == 'bloomberg':
try:
from findatapy.market.datavendorbbg import DataVendorBBGOpen
data_vendor = DataVendorBBGOpen()
except:
self.logger.warn("Bloomberg needs to be installed")
elif source == 'quandl':
from findatapy.market.datavendorweb import DataVendorQuandl
data_vendor = DataVendorQuandl()
elif source == 'ons':
from findatapy.market.datavendorweb import DataVendorONS
data_vendor = DataVendorONS()
elif source == 'boe':
from findatapy.market.datavendorweb import DataVendorBOE
data_vendor = DataVendorBOE()
elif source == 'dukascopy':
from findatapy.market.datavendorweb import DataVendorDukasCopy
data_vendor = DataVendorDukasCopy()
elif source == 'fxcm':
from findatapy.market.datavendorweb import DataVendorFXCM
data_vendor = DataVendorFXCM()
elif source == 'alfred':
from findatapy.market.datavendorweb import DataVendorALFRED
data_vendor = DataVendorALFRED()
elif source == 'yahoo':
from findatapy.market.datavendorweb import DataVendorYahoo
data_vendor = DataVendorYahoo()
elif source in ['google', 'fred', 'oecd', 'eurostat', 'edgar-index']:
from findatapy.market.datavendorweb import DataVendorPandasWeb
data_vendor = DataVendorPandasWeb()
elif source == 'bitcoincharts':
from findatapy.market.datavendorweb import DataVendorBitcoincharts
data_vendor = DataVendorBitcoincharts()
elif source == 'poloniex':
from findatapy.market.datavendorweb import DataVendorPoloniex
data_vendor = DataVendorPoloniex()
elif source == 'binance':
from findatapy.market.datavendorweb import DataVendorBinance
data_vendor = DataVendorBinance()
elif source == 'bitfinex':
from findatapy.market.datavendorweb import DataVendorBitfinex
data_vendor = DataVendorBitfinex()
elif source == 'gdax':
from findatapy.market.datavendorweb import DataVendorGdax
data_vendor = DataVendorGdax()
elif source == 'kraken':
from findatapy.market.datavendorweb import DataVendorKraken
data_vendor = DataVendorKraken()
elif source == 'bitmex':
from findatapy.market.datavendorweb import DataVendorBitmex
data_vendor = DataVendorBitmex()
elif '.csv' in source or '.h5' in source:
from findatapy.market.datavendorweb import DataVendorFlatFile
data_vendor = DataVendorFlatFile()
elif source == 'alphavantage':
from findatapy.market.datavendorweb import DataVendorAlphaVantage
data_vendor = DataVendorAlphaVantage()
elif source == 'huobi':
from findatapy.market.datavendorweb import DataVendorHuobi
data_vendor = DataVendorHuobi()
# TODO add support for other data sources (like Reuters)
return data_vendor
def fetch_market_data(self, market_data_request, kill_session=True):
"""Loads time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
# data_vendor = self.get_data_vendor(market_data_request.data_source)
# check if tickers have been specified (if not load all of them for a category)
# also handle single tickers/list tickers
create_tickers = False
if market_data_request.vendor_tickers is not None and market_data_request.tickers is None:
market_data_request.tickers = market_data_request.vendor_tickers
tickers = market_data_request.tickers
if tickers is None:
create_tickers = True
elif isinstance(tickers, str):
if tickers == '': create_tickers = True
elif isinstance(tickers, list):
if tickers == []: create_tickers = True
if create_tickers:
market_data_request.tickers = ConfigManager().get_instance(
).get_tickers_list_for_category(market_data_request.category,
market_data_request.data_source,
market_data_request.freq,
market_data_request.cut)
# intraday or tick: only one ticker per cache file
if (market_data_request.freq
in ['intraday', 'tick', 'second', 'hour', 'minute']):
data_frame_agg = self.download_intraday_tick(market_data_request)
# return data_frame_agg
# daily: multiple tickers per cache file - assume we make one API call to vendor library
else:
data_frame_agg = self.download_daily(market_data_request)
if ('internet_load' in market_data_request.cache_algo):
self.logger.debug("Internet loading.. ")
# signal to data_vendor template to exit session
# if data_vendor is not None and kill_session == True: data_vendor.kill_session()
if (market_data_request.cache_algo == 'cache_algo'):
self.logger.debug(
"Only caching data in memory, do not return any time series.")
return
# only return time series if specified in the algo
if 'return' in market_data_request.cache_algo:
# special case for events/events-dt which is not indexed like other tables (also same for downloading futures
# contracts dates)
if market_data_request.category is not None:
if 'events' in market_data_request.category:
return data_frame_agg
# pad columns a second time (is this necessary to do here again?)
# TODO only do this for not daily data?
try:
if data_frame_agg is not None:
data_frame_agg = self.filter.filter_time_series(market_data_request, data_frame_agg, pad_columns=True)\
.dropna(how = 'all')
# resample data using pandas if specified in the MarketDataRequest
if market_data_request.resample is not None:
if 'last' in market_data_request.resample_how:
data_frame_agg = data_frame_agg.resample(
market_data_request.resample).last()
elif 'first' in market_data_request.resample_how:
data_frame_agg = data_frame_agg.resample(
market_data_request.resample).first()
if 'dropna' in market_data_request.resample_how:
data_frame_agg = data_frame_agg.dropna(how='all')
else:
self.logger.warn("No data returned for " +
str(market_data_request.tickers))
return data_frame_agg
except Exception as e:
print(str(e))
if data_frame_agg is not None:
return data_frame_agg
import traceback
self.logger.warn("No data returned for " +
str(market_data_request.tickers))
return None
def create_time_series_hash_key(self, market_data_request, ticker=None):
"""Creates a hash key for retrieving the time series
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
str
"""
if (isinstance(ticker, list)):
ticker = ticker[0]
return self.create_cache_file_name(
MarketDataRequest().create_category_key(market_data_request,
ticker))
def download_intraday_tick(self, market_data_request):
"""Loads intraday time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
data_frame_agg = None
calcuations = Calculations()
ticker_cycle = 0
data_frame_group = []
# single threaded version
# handle intraday ticker calls separately one by one
if len(market_data_request.tickers) == 1 or DataConstants(
).market_thread_no['other'] == 1:
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [
market_data_request.vendor_tickers[ticker_cycle]
]
ticker_cycle = ticker_cycle + 1
# we downscale into float32, to avoid memory problems in Python (32 bit)
# data is stored on disk as float32 anyway
# old_finish_date = market_data_request_single.finish_date
#
# market_data_request_single.finish_date = self.refine_expiry_date(market_data_request)
#
# if market_data_request_single.finish_date >= market_data_request_single.start_date:
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
# else:
# data_frame_single = None
#
# market_data_request_single.finish_date = old_finish_date
#
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
data_frame_single = self.fetch_single_time_series(
market_data_request)
# if the vendor doesn't provide any data, don't attempt to append
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
data_frame_single = data_frame_single.astype('float32')
data_frame_group.append(data_frame_single)
# # if you call for returning multiple tickers, be careful with memory considerations!
# if data_frame_agg is not None:
# data_frame_agg = data_frame_agg.join(data_frame_single, how='outer')
# else:
# data_frame_agg = data_frame_single
# key = self.create_category_key(market_data_request, ticker)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday)
# if you call for returning multiple tickers, be careful with memory considerations!
if data_frame_group is not None:
data_frame_agg = calcuations.pandas_outer_join(
data_frame_group)
return data_frame_agg
else:
market_data_request_list = []
# create a list of MarketDataRequests
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [
market_data_request.vendor_tickers[ticker_cycle]
]
ticker_cycle = ticker_cycle + 1
market_data_request_list.append(market_data_request_single)
return self.fetch_group_time_series(market_data_request_list)
def fetch_single_time_series(self, market_data_request):
market_data_request = MarketDataRequest(md_request=market_data_request)
# only includes those tickers have not expired yet!
start_date = pandas.Timestamp(market_data_request.start_date).date()
import datetime
current_date = datetime.datetime.utcnow().date()
from datetime import timedelta
tickers = market_data_request.tickers
vendor_tickers = market_data_request.vendor_tickers
expiry_date = market_data_request.expiry_date
config = ConfigManager().get_instance()
# in many cases no expiry is defined so skip them
for i in range(0, len(tickers)):
try:
expiry_date = config.get_expiry_for_ticker(
market_data_request.data_source, tickers[i])
except:
pass
if expiry_date is not None:
expiry_date = pandas.Timestamp(expiry_date).date()
# use pandas Timestamp, a bit more robust with weird dates (can fail if comparing date vs datetime)
# if the expiry is before the start date of our download don't bother downloading this ticker
if expiry_date < start_date:
tickers[i] = None
# special case for futures-contracts which are intraday
# avoid downloading if the expiry date is very far in the past
# (we need this before there might be odd situations where we run on an expiry date, but still want to get
# data right till expiry time)
if market_data_request.category == 'futures-contracts' and market_data_request.freq == 'intraday' \
and self.days_expired_intraday_contract_download > 0:
if expiry_date + timedelta(
days=self.days_expired_intraday_contract_download
) < current_date:
tickers[i] = None
if vendor_tickers is not None and tickers[i] is None:
vendor_tickers[i] = None
market_data_request.tickers = [e for e in tickers if e != None]
if vendor_tickers is not None:
market_data_request.vendor_tickers = [
e for e in vendor_tickers if e != None
]
data_frame_single = None
if len(market_data_request.tickers) > 0:
data_frame_single = self.get_data_vendor(
market_data_request.data_source).load_ticker(
market_data_request)
#print(data_frame_single.head(n=10))
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
# will fail for dataframes which includes dates/strings (eg. futures contract names)
try:
data_frame_single = data_frame_single.astype('float32')
except:
self.logger.warning('Could not convert to float')
if market_data_request.freq == "second":
data_frame_single = data_frame_single.resample("1s")
return data_frame_single
def fetch_group_time_series(self, market_data_request_list):
data_frame_agg = None
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants(
).market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(
thread_technique=DataConstants().market_thread_technique,
thread_no=thread_no)
# open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(
self.fetch_single_time_series(md_request))
# collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
# for debugging!
# import pickle
# import datetime
# pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb"))
if data_frame_group is not None:
try:
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_group)
except Exception as e:
self.logger.warning(
'Possible overlap of columns? Have you specifed same ticker several times: '
+ str(e))
return data_frame_agg
def download_daily(self, market_data_request):
"""Loads daily time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
key = MarketDataRequest().create_category_key(market_data_request)
is_key_overriden = False
for k in DataConstants().override_multi_threading_for_categories:
if k in key:
is_key_overriden = True
break
# by default use other
thread_no = DataConstants().market_thread_no['other']
if market_data_request.data_source in DataConstants().market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request.data_source]
# daily data does not include ticker in the key, as multiple tickers in the same file
if thread_no == 1:
# data_frame_agg = data_vendor.load_ticker(market_data_request)
data_frame_agg = self.fetch_single_time_series(market_data_request)
else:
market_data_request_list = []
# when trying your example 'equitiesdata_example' I had a -1 result so it went out of the comming loop and I had errors in execution
group_size = max(
int(len(market_data_request.tickers) / thread_no - 1), 0)
if group_size == 0: group_size = 1
# split up tickers into groups related to number of threads to call
for i in range(0, len(market_data_request.tickers), group_size):
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = market_data_request.tickers[
i:i + group_size]
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = \
market_data_request.vendor_tickers[i:i + group_size]
market_data_request_list.append(market_data_request_single)
# special case where we make smaller calls one after the other
if is_key_overriden:
data_frame_list = []
for md in market_data_request_list:
data_frame_list.append(self.fetch_single_time_series(md))
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_list)
else:
data_frame_agg = self.fetch_group_time_series(
market_data_request_list)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (ok for daily data)
return data_frame_agg
def refine_expiry_date(self, market_data_request):
# expiry date
if market_data_request.expiry_date is None:
ConfigManager().get_instance().get_expiry_for_ticker(
market_data_request.data_source, market_data_request.ticker)
return market_data_request
def create_cache_file_name(self, filename):
return DataConstants().folder_time_series_data + "/" + filename
class HistEconDataFactory(object):
def __init__(self, market_data_generator = None):
self.logger = LoggerManager().getLogger(__name__)
self._all_econ_tickers = pandas.read_csv(DataConstants().all_econ_tickers)
self._econ_country_codes = pandas.read_csv(DataConstants().econ_country_codes)
self._econ_country_groups = pandas.read_csv(DataConstants().econ_country_groups)
if market_data_generator is None:
self.market_data_generator = MarketDataGenerator()
else:
self.market_data_generator = market_data_generator
def get_economic_data_history(self, start_date, finish_date, country_group, data_type,
source = 'fred', cache_algo = "internet_load_return"):
#vendor_country_codes = self.fred_country_codes[country_group]
#vendor_pretty_country = self.fred_nice_country_codes[country_group]
if isinstance(country_group, list):
pretty_country_names = country_group
else:
# get all the country names in the country_group
pretty_country_names = list(self._econ_country_groups[
self._econ_country_groups["Country Group"] == country_group]['Country'])
# construct the pretty tickers
pretty_tickers = [x + '-' + data_type for x in pretty_country_names]
# get vendor tickers
vendor_tickers = []
for pretty_ticker in pretty_tickers:
vendor_ticker = list(self._all_econ_tickers[
self._all_econ_tickers["Full Code"] == pretty_ticker][source].values)
if vendor_ticker == []:
vendor_ticker = None
self.logger.error('Could not find match for ' + pretty_ticker)
else:
vendor_ticker = vendor_ticker[0]
vendor_tickers.append(vendor_ticker)
vendor_fields = ['close']
if source == 'bloomberg': vendor_fields = ['PX_LAST']
md_request = MarketDataRequest(
start_date = start_date, # start date
finish_date = finish_date, # finish date
category = 'economic',
freq = 'daily', # intraday data
data_source = source, # use Bloomberg as data source
cut = 'LOC',
tickers = pretty_tickers,
fields = ['close'], # which fields to download
vendor_tickers = vendor_tickers,
vendor_fields = vendor_fields, # which Bloomberg fields to download
cache_algo = cache_algo) # how to return data
return self.market_data_generator.fetch_market_data(md_request)
def grasp_coded_entry(self, df, index):
df = df.ix[index:].stack()
df = df.reset_index()
df.columns = ['Date', 'Name', 'Val']
countries = df['Name']
countries = [x.split('-', 1)[0] for x in countries]
df['Code'] = sum(
[list(self._econ_country_codes[self._econ_country_codes["Country"] == x]['Code']) for x in countries],
[])
return df
def __init__(self):
self.logger = LoggerManager().getLogger(__name__)
self._pnl = None
self._portfolio = None
return
def __init__(self):
super(DataVendorALFRED, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
class DataVendorDukasCopy(DataVendor):
tick_name = "{symbol}/{year}/{month}/{day}/{hour}h_ticks.bi5"
def __init__(self):
super(DataVendor, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
import logging
logging.getLogger("requests").setLevel(logging.WARNING)
self.config = ConfigManager()
# implement method in abstract superclass
def load_ticker(self, market_data_request):
"""
load_ticker - Retrieves market data from external data source (in this case Bloomberg)
Parameters
----------
market_data_request : TimeSeriesRequest
contains all the various parameters detailing time series start and finish, tickers etc
Returns
-------
DataFrame
"""
market_data_request_vendor = self.construct_vendor_market_data_request(market_data_request)
data_frame = None
self.logger.info("Request Dukascopy data")
# doesn't support non-tick data
if (market_data_request.freq in ['daily', 'weekly', 'monthly', 'quarterly', 'yearly', 'intraday', 'minute', 'hourly']):
self.logger.warning("Dukascopy loader is for tick data only")
return None
# assume one ticker only (MarketDataGenerator only calls one ticker at a time)
if (market_data_request.freq in ['tick']):
# market_data_request_vendor.tickers = market_data_request_vendor.tickers[0]
data_frame = self.get_tick(market_data_request, market_data_request_vendor)
if data_frame is not None: data_frame.tz_localize('UTC')
self.logger.info("Completed request from Dukascopy")
return data_frame
def kill_session(self):
return
def get_tick(self, market_data_request, market_data_request_vendor):
data_frame = self.download_tick(market_data_request_vendor)
# convert from vendor to findatapy tickers/fields
if data_frame is not None:
returned_fields = data_frame.columns
returned_tickers = [market_data_request_vendor.tickers[0]] * (len(returned_fields))
if data_frame is not None:
fields = self.translate_from_vendor_field(returned_fields, market_data_request)
tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)
ticker_combined = []
for i in range(0, len(fields)):
ticker_combined.append(tickers[i] + "." + fields[i])
data_frame.columns = ticker_combined
data_frame.index.name = 'Date'
return data_frame
def download_tick(self, market_data_request):
symbol = market_data_request.tickers[0]
df_list = []
self.logger.info("About to download from Dukascopy... for " + symbol)
# single threaded
df_list = [self.fetch_file(time, symbol) for time in
self.hour_range(market_data_request.start_date, market_data_request.finish_date)]
# TODO parallel (has pickle issues)
# time_list = self.hour_range(market_data_request.start_date, market_data_request.finish_date)
# import multiprocessing_on_dill as multiprocessing
#
# pool = multiprocessing.Pool(processes=4)
# results = [pool.apply_async(self.fetch_file, args=(time, symbol)) for time in time_list]
# df_list = [p.get() for p in results]
try:
return pandas.concat(df_list)
except:
return None
def fetch_file(self, time, symbol):
if time.hour % 24 == 0: self.logger.info("Downloading... " + str(time))
tick_path = self.tick_name.format(
symbol = symbol,
year = str(time.year).rjust(4, '0'),
month = str(time.month).rjust(2, '0'),
day = str(time.day).rjust(2, '0'),
hour = str(time.hour).rjust(2, '0')
)
tick = self.fetch_tick(DataConstants().dukascopy_base_url + tick_path)
if DataConstants().dukascopy_write_temp_tick_disk:
out_path = DataConstants().temp_folder + "/dkticks/" + tick_path
if not os.path.exists(out_path):
if not os.path.exists(os.path.dirname(out_path)):
os.makedirs(os.path.dirname(out_path))
self.write_tick(tick, out_path)
try:
return self.retrieve_df(lzma.decompress(tick), symbol, time)
except:
return None
def fetch_tick(self, tick_url):
i = 0
tick_request = None
# try up to 5 times to download
while i < 5:
try:
tick_request = requests.get(tick_url)
i = 5
except:
i = i + 1
if (tick_request is None):
self.logger("Failed to download from " + tick_url)
return None
return tick_request.content
def write_tick(self, content, out_path):
data_file = open(out_path, "wb+")
data_file.write(content)
data_file.close()
def chunks(self, list, n):
if n < 1:
n = 1
return [list[i:i + n] for i in range(0, len(list), n)]
def retrieve_df(self, data, symbol, epoch):
date, tuple = self.parse_tick_data(data, epoch)
df = pandas.DataFrame(data = tuple, columns=['temp', 'ask', 'bid', 'askv', 'bidv'], index = date)
df.drop('temp', axis = 1)
df.index.name = 'Date'
divisor = 100000
# where JPY is the terms currency we have different divisor
if symbol[3:6] == 'JPY':
divisor = 1000
# prices are returned without decimal point
df['bid'] = df['bid'] / divisor
df['ask'] = df['ask'] / divisor
return df
def hour_range(self, start_date, end_date):
delta_t = end_date - start_date
delta_hours = (delta_t.days * 24.0) + (delta_t.seconds / 3600.0)
for n in range(int (delta_hours)):
yield start_date + timedelta(0, 0, 0, 0, 0, n) # Hours
def parse_tick_data(self, data, epoch):
import struct
# tick = namedtuple('Tick', 'Date ask bid askv bidv')
chunks_list = self.chunks(data, 20)
parsed_list = []
date = []
# note: Numba can speed up for loops
for row in chunks_list:
d = struct.unpack(">LLLff", row)
date.append((epoch + timedelta(0,0,0, d[0])))
# SLOW: no point using named tuples!
# row_data = tick._asdict(tick._make(d))
# row_data['Date'] = (epoch + timedelta(0,0,0,row_data['Date']))
parsed_list.append(d)
return date, parsed_list
def chunks(self, list, n):
if n < 1: n = 1
return [list[i:i + n] for i in range(0, len(list), n)]
def get_daily_data(self):
pass
class DataVendorQuandl(DataVendor):
def __init__(self):
super(DataVendorQuandl, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
# implement method in abstract superclass
def load_ticker(self, market_data_request):
market_data_request_vendor = self.construct_vendor_market_data_request(market_data_request)
self.logger.info("Request Quandl data")
data_frame = self.download_daily(market_data_request_vendor)
if data_frame is None or data_frame.index is []: return None
# convert from vendor to findatapy tickers/fields
if data_frame is not None:
returned_tickers = data_frame.columns
if data_frame is not None:
# tidy up tickers into a format that is more easily translatable
# we can often get multiple fields returned (even if we don't ask for them!)
# convert to lower case
returned_fields = [(x.split(' - ')[1]).lower().replace(' ', '-').replace('.', '-').replace('--', '-') for x in returned_tickers]
returned_fields = [x.replace('value', 'close') for x in returned_fields] # special case for close
# replace time fields (can cause problems later for times to start with 0)
for i in range(0, 10):
returned_fields = [x.replace('0'+ str(i) + ':00', str(i) + ':00') for x in returned_fields]
returned_tickers = [x.replace('.', '/') for x in returned_tickers]
returned_tickers = [x.split(' - ')[0] for x in returned_tickers]
try:
fields = self.translate_from_vendor_field(returned_fields, market_data_request)
tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)
except:
print('error')
ticker_combined = []
for i in range(0, len(fields)):
ticker_combined.append(tickers[i] + "." + fields[i])
data_frame.columns = ticker_combined
data_frame.index.name = 'Date'
self.logger.info("Completed request from Quandl for " + str(ticker_combined))
return data_frame
def download_daily(self, market_data_request):
trials = 0
data_frame = None
while(trials < 5):
try:
data_frame = Quandl.get(market_data_request.tickers, authtoken=DataConstants().quandl_api_key, trim_start=market_data_request.start_date,
trim_end=market_data_request.finish_date)
break
except:
trials = trials + 1
self.logger.info("Attempting... " + str(trials) + " request to download from Quandl")
if trials == 5:
self.logger.error("Couldn't download from Quandl after several attempts!")
return data_frame
class MarketDataGenerator(object):
_time_series_cache = {} # shared across all instances of object!
def __init__(self):
self.config = ConfigManager()
self.logger = LoggerManager().getLogger(__name__)
self.filter = Filter()
self.calculations = Calculations()
self.io_engine = IOEngine()
self._intraday_code = -1
return
def flush_cache(self):
"""
flush_cache - Flushs internal cache of time series
"""
self._time_series_cache = {}
def set_intraday_code(self, code):
self._intraday_code = code
def get_data_vendor(self, source):
"""
get_loader - Loads appropriate data service class
Parameters
----------
source : str
the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc.
we can also have forms like "bloomberg-boe" separated by hyphens
Returns
-------
DataVendor
"""
data_vendor = None
source = source.split("-")[0]
if source == 'bloomberg':
from findatapy.market.datavendorbbg import DataVendorBBGOpen
data_vendor = DataVendorBBGOpen()
elif source == 'quandl':
from findatapy.market.datavendorweb import DataVendorQuandl
data_vendor = DataVendorQuandl()
elif source == 'ons':
from findatapy.market.datavendorweb import DataVendorONS
data_vendor = DataVendorONS()
elif source == 'boe':
from findatapy.market.datavendorweb import DataVendorBOE
data_vendor = DataVendorBOE()
elif source == 'dukascopy':
from findatapy.market.datavendorweb import DataVendorDukasCopy
data_vendor = DataVendorDukasCopy()
elif source in [
'yahoo', 'google', 'fred', 'oecd', 'eurostat', 'edgar-index'
]:
from findatapy.market.datavendorweb import DataVendorPandasWeb
data_vendor = DataVendorPandasWeb()
# TODO add support for other data sources (like Reuters)
return data_vendor
def fetch_market_data(self, market_data_request, kill_session=True):
"""
fetch_market_data - Loads time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
tickers = market_data_request.tickers
data_vendor = self.get_data_vendor(market_data_request.data_source)
# check if tickers have been specified (if not load all of them for a category)
# also handle single tickers/list tickers
create_tickers = False
if tickers is None:
create_tickers = True
elif isinstance(tickers, str):
if tickers == '': create_tickers = True
elif isinstance(tickers, list):
if tickers == []: create_tickers = True
if create_tickers:
market_data_request.tickers = self.config.get_tickers_list_for_category(
market_data_request.category, market_data_request.data_source,
market_data_request.freq, market_data_request.cut)
# intraday or tick: only one ticker per cache file
if (market_data_request.freq
in ['intraday', 'tick', 'second', 'hour', 'minute']):
data_frame_agg = self.download_intraday_tick(
market_data_request, data_vendor)
# daily: multiple tickers per cache file - assume we make one API call to vendor library
else:
data_frame_agg = self.download_daily(market_data_request,
data_vendor)
if ('internet_load' in market_data_request.cache_algo):
self.logger.debug("Internet loading.. ")
# signal to data_vendor template to exit session
# if data_vendor is not None and kill_session == True: data_vendor.kill_session()
if (market_data_request.cache_algo == 'cache_algo'):
self.logger.debug(
"Only caching data in memory, do not return any time series.")
return
# only return time series if specified in the algo
if 'return' in market_data_request.cache_algo:
# special case for events/events-dt which is not indexed like other tables
if market_data_request.category is not None:
if 'events' in market_data_request.category:
return data_frame_agg
try:
return self.filter.filter_time_series(market_data_request,
data_frame_agg,
pad_columns=True)
except:
import traceback
self.logger.error(traceback.format_exc())
return None
def get_market_data_cached(self, market_data_request):
"""
get_time_series_cached - Loads time series from cache (if it exists)
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
if (market_data_request.freq == "intraday"):
ticker = market_data_request.tickers
else:
ticker = None
fname = self.create_time_series_hash_key(market_data_request, ticker)
if (fname in self._time_series_cache):
data_frame = self._time_series_cache[fname]
return self.filter.filter_time_series(market_data_request,
data_frame)
return None
def create_time_series_hash_key(self, market_data_request, ticker=None):
"""
create_time_series_hash_key - Creates a hash key for retrieving the time series
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
str
"""
if (isinstance(ticker, list)):
ticker = ticker[0]
return self.create_cache_file_name(
self.create_category_key(market_data_request, ticker))
def download_intraday_tick(self, market_data_request, data_vendor):
"""
download_intraday_tick - Loads intraday time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
data_frame_agg = None
calcuations = Calculations()
ticker_cycle = 0
data_frame_group = []
# single threaded version
# handle intraday ticker calls separately one by one
if len(market_data_request.tickers) == 1 or DataConstants(
).market_thread_no['other'] == 1:
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [
market_data_request.vendor_tickers[ticker_cycle]
]
ticker_cycle = ticker_cycle + 1
# we downscale into float32, to avoid memory problems in Python (32 bit)
# data is stored on disk as float32 anyway
data_frame_single = data_vendor.load_ticker(
market_data_request_single)
# if the vendor doesn't provide any data, don't attempt to append
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
data_frame_single = data_frame_single.astype('float32')
data_frame_group.append(data_frame_single)
# # if you call for returning multiple tickers, be careful with memory considerations!
# if data_frame_agg is not None:
# data_frame_agg = data_frame_agg.join(data_frame_single, how='outer')
# else:
# data_frame_agg = data_frame_single
# key = self.create_category_key(market_data_request, ticker)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday)
# if you call for returning multiple tickers, be careful with memory considerations!
if data_frame_group is not None:
data_frame_agg = calcuations.pandas_outer_join(
data_frame_group)
return data_frame_agg
else:
market_data_request_list = []
# create a list of MarketDataRequests
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if hasattr(market_data_request, 'vendor_tickers'):
market_data_request_single.vendor_tickers = [
market_data_request.vendor_tickers[ticker_cycle]
]
ticker_cycle = ticker_cycle + 1
market_data_request_list.append(market_data_request_single)
return self.fetch_group_time_series(market_data_request_list)
def fetch_single_time_series(self, market_data_request):
data_frame_single = self.get_data_vendor(
market_data_request.data_source).load_ticker(market_data_request)
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
# will fail for dataframes which includes dates
try:
data_frame_single = data_frame_single.astype('float32')
except:
pass
if market_data_request.freq == "second":
data_frame_single = data_frame_single.resample("1s")
return data_frame_single
def fetch_group_time_series(self, market_data_request_list):
data_frame_agg = None
# depends on the nature of operation as to whether we should use threading or multiprocessing library
if DataConstants().market_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocessing_on_dill library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocessing_on_dill import Pool
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants(
).market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = Pool(thread_no)
# open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(
self.fetch_single_time_series(md_request))
# collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
if data_frame_group is not None:
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_group)
return data_frame_agg
def download_daily(self, market_data_request, data_vendor):
"""
download_daily - Loads daily time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
# daily data does not include ticker in the key, as multiple tickers in the same file
if DataConstants().market_thread_no['other'] == 1:
data_frame_agg = data_vendor.load_ticker(market_data_request)
else:
market_data_request_list = []
group_size = int(
len(market_data_request.tickers) /
DataConstants().market_thread_no['other'] - 1)
if group_size == 0: group_size = 1
# split up tickers into groups related to number of threads to call
for i in range(0, len(market_data_request.tickers), group_size):
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = market_data_request.tickers[
i:i + group_size]
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = \
market_data_request.vendor_tickers[i:i + group_size]
market_data_request_list.append(market_data_request_single)
data_frame_agg = self.fetch_group_time_series(
market_data_request_list)
key = self.create_category_key(market_data_request)
fname = self.create_cache_file_name(key)
self._time_series_cache[
fname] = data_frame_agg # cache in memory (ok for daily data)
return data_frame_agg
def create_category_key(self, market_data_request, ticker=None):
"""
create_category_key - Returns a category key for the associated MarketDataRequest
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
str
"""
category = 'default-cat'
cut = 'default-cut'
if market_data_request.category is not None:
category = market_data_request.category
environment = market_data_request.environment
source = market_data_request.data_source
freq = market_data_request.freq
if market_data_request.cut is not None: cut = market_data_request.cut
if (ticker is not None):
key = environment + "." + category + '.' + source + '.' + freq + '.' + cut + '.' + ticker
else:
key = environment + "." + category + '.' + source + '.' + freq + '.' + cut
return key
def create_cache_file_name(self, filename):
return DataConstants().folder_time_series_data + "/" + filename
def price_instrument(self,
cross,
horizon_date,
strike,
expiry_date=None,
vol=None,
notional=1000000,
contract_type='european-call',
tenor=None,
fx_vol_surface=None,
premium_output=None,
delta_output=None,
depo_tenor=None,
return_as_df=True):
"""Prices FX options for horizon dates/expiry dates given by the user from FX spot rates, FX volatility surface
and deposit rates.
Parameters
----------
cross : str
Currency pair
horizon_date : DateTimeIndex
Horizon dates for options
expiry_date : DateTimeIndex
expiry dates for options
market_df : DataFrame
Contains FX spot, FX vol surface quotes, FX forwards and base depos
Returns
-------
DataFrame
"""
# if market_df is None: market_df = self._market_df
if fx_vol_surface is None: fx_vol_surface = self._fx_vol_surface
if premium_output is None: premium_output = self._premium_output
if delta_output is None: delta_output = self._delta_output
logger = LoggerManager().getLogger(__name__)
field = fx_vol_surface._field
# Make horizon date and expiry date pandas DatetimeIndex
if isinstance(horizon_date, pd.Timestamp):
horizon_date = pd.DatetimeIndex([horizon_date])
else:
horizon_date = pd.DatetimeIndex(horizon_date)
if expiry_date is not None:
if isinstance(expiry_date, pd.Timestamp):
expiry_date = pd.DatetimeIndex([expiry_date])
else:
expiry_date = pd.DatetimeIndex(expiry_date)
else:
expiry_date = self._calendar.get_expiry_date_from_horizon_date(
horizon_date, tenor, cal=cross)
# If the strike hasn't been supplied need to work this out
if not (isinstance(strike, np.ndarray)):
old_strike = strike
if isinstance(strike, str):
strike = np.empty(len(horizon_date), dtype=object)
else:
strike = np.empty(len(horizon_date))
strike.fill(old_strike)
# If the vol hasn't been supplied need to work this out
if not (isinstance(vol, np.ndarray)):
if vol is None:
vol = np.nan
old_vol = vol
vol = np.empty(len(horizon_date))
vol.fill(old_vol)
option_values = np.zeros(len(horizon_date))
spot = np.zeros(len(horizon_date))
delta = np.zeros(len(horizon_date))
intrinsic_values = np.zeros(len(horizon_date))
def _price_option(contract_type_, contract_type_fin_):
for i in range(len(expiry_date)):
built_vol_surface = False
# If we have a "key strike" need to fit the vol surface
if isinstance(strike[i], str):
if not (built_vol_surface):
fx_vol_surface.build_vol_surface(horizon_date[i])
fx_vol_surface.extract_vol_surface(
num_strike_intervals=None)
built_vol_surface = True
# Delta neutral strike/or whatever strike is quoted as ATM
# usually this is ATM delta neutral strike, but can sometimes be ATMF for some Latam
# Take the vol directly quoted, rather than getting it from building vol surface
if strike[i] == 'atm':
strike[i] = fx_vol_surface.get_atm_strike(tenor)
vol[i] = fx_vol_surface.get_atm_quoted_vol(
tenor) / 100.0
# vol[i] = fx_vol_surface.get_atm_vol(tenor) / 100.0 # interpolated
elif strike[i] == 'atms':
strike[i] = fx_vol_surface.get_spot(
) # Interpolate vol later
elif strike[i] == 'atmf':
# Quoted tenor, no need to interpolate
strike[i] = float(fx_vol_surface.get_all_market_data()[cross + ".close"][horizon_date[i]]) \
+ (float(fx_vol_surface.get_all_market_data()[cross + tenor + ".close"][horizon_date[i]]) \
/ self._fx_forwards_pricer.get_forwards_divisor(cross[3:6]))
# Interpolate vol later
elif strike[i] == '25d-otm':
if 'call' in contract_type_:
strike[i] = fx_vol_surface.get_25d_call_strike(
tenor)
vol[i] = fx_vol_surface.get_25d_call_vol(
tenor) / 100.0
elif 'put' in contract_type_:
strike[i] = fx_vol_surface.get_25d_put_strike(
tenor)
vol[i] = fx_vol_surface.get_25d_put_vol(
tenor) / 100.0
elif strike[i] == '10d-otm':
if 'call' in contract_type_:
strike[i] = fx_vol_surface.get_10d_call_strike(
tenor)
vol[i] = fx_vol_surface.get_10d_call_vol(
tenor) / 100.0
elif 'put' in contract_type_:
strike[i] = fx_vol_surface.get_10d_put_strike(
tenor)
vol[i] = fx_vol_surface.get_10d_put_vol(
tenor) / 100.0
if not (built_vol_surface):
try:
fx_vol_surface.build_vol_surface(horizon_date[i])
except:
logger.warn("Failed to build vol surface for " +
str(horizon_date) +
", won't be able to interpolate vol")
# fx_vol_surface.extract_vol_surface(num_strike_intervals=None)
# If an implied vol hasn't been provided, interpolate that one, fit the vol surface (if hasn't already been
# done)
if np.isnan(vol[i]):
if tenor is None:
vol[i] = fx_vol_surface.calculate_vol_for_strike_expiry(
strike[i], expiry_date=expiry_date[i], tenor=None)
else:
vol[i] = fx_vol_surface.calculate_vol_for_strike_expiry(
strike[i], expiry_date=None, tenor=tenor)
model = FinModelBlackScholes(float(vol[i]))
logger.info("Pricing " + contract_type_ +
" option, horizon date = " + str(horizon_date[i]) +
", expiry date = " + str(expiry_date[i]))
option = FinFXVanillaOption(self._findate(expiry_date[i]),
strike[i], cross,
contract_type_fin_, notional,
cross[0:3])
spot[i] = fx_vol_surface.get_spot()
""" FinancePy will return the value in the following dictionary for values
{'v': vdf,
"cash_dom": cash_dom,
"cash_for": cash_for,
"pips_dom": pips_dom,
"pips_for": pips_for,
"pct_dom": pct_dom,
"pct_for": pct_for,
"not_dom": notional_dom,
"not_for": notional_for,
"ccy_dom": self._domName,
"ccy_for": self._forName}
"""
option_values[i] = option_values[i] + option.value(
self._findate(horizon_date[i]), spot[i],
fx_vol_surface.get_dom_discount_curve(),
fx_vol_surface.get_for_discount_curve(),
model)[premium_output.replace('-', '_')]
intrinsic_values[i] = intrinsic_values[i] + option.value(
self._findate(expiry_date[i]), spot[i],
fx_vol_surface.get_dom_discount_curve(),
fx_vol_surface.get_for_discount_curve(),
model)[premium_output.replace('-', '_')]
"""FinancePy returns this dictionary for deltas
{"pips_spot_delta": pips_spot_delta,
"pips_fwd_delta": pips_fwd_delta,
"pct_spot_delta_prem_adj": pct_spot_delta_prem_adj,
"pct_fwd_delta_prem_adj": pct_fwd_delta_prem_adj}
"""
delta[i] = delta[i] + option.delta(
self._findate(horizon_date[i]), spot[i],
fx_vol_surface.get_dom_discount_curve(),
fx_vol_surface.get_for_discount_curve(),
model)[delta_output.replace('-', '_')]
if contract_type == 'european-call':
contract_type_fin = FinOptionTypes.EUROPEAN_CALL
_price_option(contract_type, contract_type_fin)
elif contract_type == 'european-put':
contract_type_fin = FinOptionTypes.EUROPEAN_PUT
_price_option(contract_type, contract_type_fin)
elif contract_type == 'european-straddle' or contract_type == 'european-strangle':
contract_type = 'european-call'
contract_type_fin = FinOptionTypes.EUROPEAN_CALL
_price_option(contract_type, contract_type_fin)
contract_type = 'european-put'
contract_type_fin = FinOptionTypes.EUROPEAN_PUT
_price_option(contract_type, contract_type_fin)
if return_as_df:
option_prices_df = pd.DataFrame(index=horizon_date)
option_prices_df[cross + '-option-price.' + field] = option_values
option_prices_df[cross + '.' + field] = spot
option_prices_df[cross + '-strike.' + field] = strike
option_prices_df[cross + '-vol.' + field] = vol
option_prices_df[cross + '-delta.' + field] = delta
option_prices_df[cross + '.expiry-date'] = expiry_date
option_prices_df[cross + '-intrinsic-value.' +
field] = intrinsic_values
return option_prices_df
return option_values, spot, strike, vol, delta, expiry_date, intrinsic_values
class TradingModel(object):
#### Default parameters for outputting of results from trading model
SAVE_FIGURES = True
DEFAULT_PLOT_ENGINE = ChartConstants().chartfactory_default_engine
SCALE_FACTOR = ChartConstants().chartfactory_scale_factor
CHART_SOURCE = 'Web'
DUMP_CSV = ''
DUMP_PATH = datetime.date.today().strftime("%Y%m%d") + ' '
chart = Chart(engine=DEFAULT_PLOT_ENGINE)
logger = LoggerManager().getLogger(__name__)
def __init__(self):
pass
# to be implemented by every trading strategy
@abc.abstractmethod
def load_parameters(self):
"""
load_parameters - Fills parameters for the backtest, such as start-end dates, transaction costs etc. To
be implemented by subclass.
"""
return
@abc.abstractmethod
def load_assets(self):
"""
load_assets - Loads time series for the assets to be traded and also for data for generating signals.
"""
return
@abc.abstractmethod
def construct_signal(self, spot_df, spot_df2, tech_params):
"""
construct_signal - Constructs signal from pre-loaded time series
Parameters
----------
spot_df : pandas.DataFrame
Market time series for generating signals
spot_df2 : pandas.DataFrame
Market time series for generated signals (can be of different frequency)
tech_params : TechParams
Parameters for generating signals
"""
return
####### Generic functions for every backtest
def construct_strategy(self, br=None):
"""
construct_strategy - Constructs the returns for all the strategies which have been specified.
- gets parameters form fill_backtest_request
- market data from fill_assets
"""
calculations = Calculations()
# get the parameters for backtesting
if hasattr(self, 'br'):
br = self.br
elif br is None:
br = self.load_parameters()
# get market data for backtest
asset_df, spot_df, spot_df2, basket_dict = self.load_assets()
if hasattr(br, 'tech_params'):
tech_params = br.tech_params
else:
tech_params = TechParams()
cumresults = pandas.DataFrame(index=asset_df.index)
portleverage = pandas.DataFrame(index=asset_df.index)
from collections import OrderedDict
ret_statsresults = OrderedDict()
# each portfolio key calculate returns - can put parts of the portfolio in the key
for key in basket_dict.keys():
asset_cut_df = asset_df[[x + '.close' for x in basket_dict[key]]]
spot_cut_df = spot_df[[x + '.close' for x in basket_dict[key]]]
self.logger.info("Calculating " + key)
results, backtest = self.construct_individual_strategy(
br, spot_cut_df, spot_df2, asset_cut_df, tech_params, key)
cumresults[results.columns[0]] = results
portleverage[results.columns[0]] = backtest.get_porfolio_leverage()
ret_statsresults[key] = backtest.get_portfolio_pnl_ret_stats()
# for a key, designated as the final strategy save that as the "strategy"
if key == self.FINAL_STRATEGY:
self._strategy_pnl = results
self._strategy_pnl_ret_stats = backtest.get_portfolio_pnl_ret_stats(
)
self._strategy_leverage = backtest.get_porfolio_leverage()
self._strategy_signal = backtest.get_porfolio_signal()
self._strategy_pnl_trades = backtest.get_pnl_trades()
# get benchmark for comparison
benchmark = self.construct_strategy_benchmark()
cumresults_benchmark = self.compare_strategy_vs_benchmark(
br, cumresults, benchmark)
self._strategy_group_benchmark_ret_stats = ret_statsresults
if hasattr(self, '_benchmark_ret_stats'):
ret_statslist = ret_statsresults
ret_statslist['Benchmark'] = (self._benchmark_ret_stats)
self._strategy_group_benchmark_ret_stats = ret_statslist
# calculate annualised returns
years = calculations.average_by_annualised_year(
calculations.calculate_returns(cumresults_benchmark))
self._strategy_group_pnl = cumresults
self._strategy_group_pnl_ret_stats = ret_statsresults
self._strategy_group_benchmark_pnl = cumresults_benchmark
self._strategy_group_leverage = portleverage
self._strategy_group_benchmark_annualised_pnl = years
def construct_individual_strategy(self, br, spot_df, spot_df2, asset_df,
tech_params, key):
"""
construct_individual_strategy - Combines the signal with asset returns to find the returns of an individual
strategy
Parameters
----------
br : BacktestRequest
Parameters for backtest such as start and finish dates
spot_df : pandas.DataFrame
Market time series for generating signals
spot_df2 : pandas.DataFrame
Secondary Market time series for generated signals (can be of different frequency)
tech_params : TechParams
Parameters for generating signals
Returns
-------
cumportfolio : pandas.DataFrame
backtest : Backtest
"""
backtest = Backtest()
signal_df = self.construct_signal(spot_df, spot_df2, tech_params,
br) # get trading signal
backtest.calculate_trading_PnL(br, asset_df,
signal_df) # calculate P&L
cumpnl = backtest.get_cumpnl()
if br.write_csv: cumpnl.to_csv(self.DUMP_CSV + key + ".csv")
cumportfolio = backtest.get_cumportfolio()
if br.calc_stats:
cumportfolio.columns = [
key + ' ' + str(backtest.get_portfolio_pnl_desc()[0])
]
else:
cumportfolio.columns = [key]
return cumportfolio, backtest
def compare_strategy_vs_benchmark(self, br, strategy_df, benchmark_df):
"""
compare_strategy_vs_benchmark - Compares the trading strategy we are backtesting against a benchmark
Parameters
----------
br : BacktestRequest
Parameters for backtest such as start and finish dates
strategy_df : pandas.DataFrame
Strategy time series
benchmark_df : pandas.DataFrame
Benchmark time series
"""
include_benchmark = False
calc_stats = False
if hasattr(br, 'include_benchmark'):
include_benchmark = br.include_benchmark
if hasattr(br, 'calc_stats'): calc_stats = br.calc_stats
if include_benchmark:
ret_stats = RetStats()
risk_engine = RiskEngine()
filter = Filter()
calculations = Calculations()
# align strategy time series with that of benchmark
strategy_df, benchmark_df = strategy_df.align(benchmark_df,
join='left',
axis=0)
# if necessary apply vol target to benchmark (to make it comparable with strategy)
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
benchmark_df = risk_engine.calculate_vol_adjusted_index_from_prices(
benchmark_df, br=br)
# only calculate return statistics if this has been specified (note when different frequencies of data
# might underrepresent vol
if calc_stats:
benchmark_df = benchmark_df.fillna(method='ffill')
ret_stats.calculate_ret_stats_from_prices(
benchmark_df, br.ann_factor)
benchmark_df.columns = ret_stats.summary()
# realign strategy & benchmark
strategy_benchmark_df = strategy_df.join(benchmark_df, how='inner')
strategy_benchmark_df = strategy_benchmark_df.fillna(
method='ffill')
strategy_benchmark_df = filter.filter_time_series_by_date(
br.plot_start, br.finish_date, strategy_benchmark_df)
strategy_benchmark_df = calculations.create_mult_index_from_prices(
strategy_benchmark_df)
self._benchmark_pnl = benchmark_df
self._benchmark_ret_stats = ret_stats
return strategy_benchmark_df
return strategy_df
def get_strategy_name(self):
return self.FINAL_STRATEGY
def get_individual_leverage(self):
return self._individual_leverage
def get_strategy_group_pnl_trades(self):
return self._strategy_pnl_trades
def get_strategy_pnl(self):
return self._strategy_pnl
def get_strategy_pnl_ret_stats(self):
return self._strategy_pnl_ret_stats
def get_strategy_leverage(self):
return self._strategy_leverage
def get_strategy_group_benchmark_pnl(self):
return self._strategy_group_benchmark_pnl
def get_strategy_group_benchmark_ret_stats(self):
return self._strategy_group_benchmark_ret_stats
def get_strategy_leverage(self):
return self._strategy_group_leverage
def get_strategy_signal(self):
return self._strategy_signal
def get_benchmark(self):
return self._benchmark_pnl
def get_benchmark_ret_stats(self):
return self._benchmark_ret_stats
def get_strategy_group_benchmark_annualised_pnl(self):
return self._strategy_group_benchmark_annualised_pnl
#### Plotting
def reduce_plot(self, data_frame):
"""
reduce_plot - Reduces the frequency of a time series to every business day so it can be plotted more easily
Parameters
----------
data_frame: pandas.DataFrame
Strategy time series
Returns
-------
pandas.DataFrame
"""
try:
# make plots on every business day (will downsample intraday data)
data_frame = data_frame.resample('B')
data_frame = data_frame.fillna(method='pad')
return data_frame
except:
return data_frame
##### Quick helper functions to plot aspects of the strategy such as P&L, leverage etc.
def plot_individual_leverage(self):
style = self.create_style("Leverage", "Individual Leverage")
try:
self.chart.plot(self.reduce_plot(self._individual_leverage),
chart_type='line',
style=style)
except:
pass
def plot_strategy_group_pnl_trades(self):
style = self.create_style("(bp)", "Individual Trade PnL")
# zero when there isn't a trade exit
# strategy_pnl_trades = self._strategy_pnl_trades * 100 * 100
# strategy_pnl_trades = strategy_pnl_trades.dropna()
# note only works with single large basket trade
try:
strategy_pnl_trades = self._strategy_pnl_trades.fillna(
0) * 100 * 100
self.chart.plot(self.reduce_plot(strategy_pnl_trades),
chart_type='line',
style=style)
except:
pass
def plot_strategy_pnl(self):
style = self.create_style("", "Strategy PnL")
try:
self.chart.plot(self.reduce_plot(self._strategy_pnl),
chart_type='line',
style=style)
except:
pass
def plot_strategy_signal_proportion(self, strip=None):
signal = self._strategy_signal
# count number of long, short and flat periods in our sample
long = signal[signal > 0].count()
short = signal[signal < 0].count()
flat = signal[signal == 0].count()
keys = long.index
# how many trades have there been (ignore size of the trades)
trades = abs(signal - signal.shift(-1))
trades = trades[trades > 0].count()
df_trades = pandas.DataFrame(index=keys,
columns=['Trades'],
data=trades)
df = pandas.DataFrame(index=keys, columns=['Long', 'Short', 'Flat'])
df['Long'] = long
df['Short'] = short
df['Flat'] = flat
if strip is not None: keys = [k.replace(strip, '') for k in keys]
df.index = keys
df_trades.index = keys
# df = df.sort_index()
style = self.create_style("", "")
try:
style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Strategy signal proportion).png'
style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Strategy signal proportion).html'
self.chart.plot(self.reduce_plot(df),
chart_type='bar',
style=style)
style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Strategy trade no).png'
style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Strategy trade no).html'
self.chart.plot(self.reduce_plot(df_trades),
chart_type='bar',
style=style)
except:
pass
def plot_strategy_leverage(self):
style = self.create_style("Leverage", "Strategy Leverage")
try:
self.chart.plot(self.reduce_plot(self._strategy_leverage),
chart_type='line',
style=style)
except:
pass
def plot_strategy_group_benchmark_pnl(self, strip=None):
style = self.create_style("", "Group Benchmark PnL - cumulative")
strat_list = self._strategy_group_benchmark_pnl.columns #.sort_values()
for line in strat_list:
self.logger.info(line)
# plot cumulative line of returns
self.chart.plot(self.reduce_plot(self._strategy_group_benchmark_pnl),
style=style)
# needs write stats flag turned on
try:
keys = self._strategy_group_benchmark_ret_stats.keys()
ir = []
for key in keys:
ir.append(
self._strategy_group_benchmark_ret_stats[key].inforatio()
[0])
if strip is not None: keys = [k.replace(strip, '') for k in keys]
ret_stats = pandas.DataFrame(index=keys, data=ir, columns=['IR'])
# ret_stats = ret_stats.sort_index()
style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - IR) ' + style.SCALE_FACTOR + '.png'
style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - IR) ' + style.SCALE_FACTOR + '.html'
style.display_brand_label = False
self.chart.plot(ret_stats, chart_type='bar', style=style)
except:
pass
def plot_strategy_group_benchmark_annualised_pnl(self, cols=None):
# TODO - unfinished, needs checking!
if cols is None:
cols = self._strategy_group_benchmark_annualised_pnl.columns
style = self.create_style("", "Group Benchmark Annualised PnL")
style.color = [
'red', 'blue', 'purple', 'gray', 'yellow', 'green', 'pink'
]
self.chart.plot(self.reduce_plot(
self._strategy_group_benchmark_annualised_pnl[cols]),
chart_type='line',
style=style)
def plot_strategy_group_leverage(self):
style = self.create_style("Leverage", "Group Leverage")
self.chart.plot(self.reduce_plot(self._strategy_group_leverage),
chart_type='line',
style=style)
def plot_strategy_signals(self, date=None, strip=None):
######## plot signals
strategy_signal = self._strategy_signal
strategy_signal = 100 * (strategy_signal)
if date is None:
last_day = strategy_signal.ix[-1].transpose().to_frame()
else:
if not (isinstance(date, list)):
date = [date]
last_day = []
for d in date:
last_day.append(strategy_signal.ix[d].transpose().to_frame())
last_day = pandas.concat(last_day, axis=1)
last_day = last_day.sort_index(axis=1)
if strip is not None:
last_day.index = [x.replace(strip, '') for x in last_day.index]
style = self.create_style("positions (% portfolio notional)",
"Positions")
self.chart.plot(last_day, chart_type='bar', style=style)
def create_style(self, title, file_add):
style = Style()
style.title = self.FINAL_STRATEGY + " " + title
style.display_legend = True
style.scale_factor = self.SCALE_FACTOR
style.source = self.CHART_SOURCE
if self.DEFAULT_PLOT_ENGINE not in ['plotly', 'cufflinks'
] and self.SAVE_FIGURES:
style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (' + file_add + ') ' + str(
style.scale_factor) + '.png'
style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (' + file_add + ') ' + str(
style.scale_factor) + '.html'
try:
style.silent_display = self.SILENT_DISPLAY
except:
pass
return style
class HistEconDataFactory(object):
def __init__(self, market_data_generator = None):
self.logger = LoggerManager().getLogger(__name__)
self._all_econ_tickers = pandas.read_csv(DataConstants().all_econ_tickers)
self._econ_country_codes = pandas.read_csv(DataConstants().econ_country_codes)
self._econ_country_groups = pandas.read_csv(DataConstants().econ_country_groups)
if market_data_generator is None:
self.market_data_generator = MarketDataGenerator()
else:
self.market_data_generator = market_data_generator
def get_economic_data_history(self, start_date, finish_date, country_group, data_type,
source = 'fred', cache_algo = "internet_load_return"):
if isinstance(country_group, list):
pretty_country_names = country_group
else:
# get all the country names in the country_group
pretty_country_names = list(self._econ_country_groups[
self._econ_country_groups["Country Group"] == country_group]['Country'])
# construct the pretty tickers
pretty_tickers = [x + '-' + data_type for x in pretty_country_names]
# get vendor tickers
vendor_tickers = []
for pretty_ticker in pretty_tickers:
vendor_ticker = list(self._all_econ_tickers[
self._all_econ_tickers["Full Code"] == pretty_ticker][source].values)
if vendor_ticker == []:
vendor_ticker = None
self.logger.error('Could not find match for ' + pretty_ticker)
else:
vendor_ticker = vendor_ticker[0]
vendor_tickers.append(vendor_ticker)
vendor_fields = ['close']
if source == 'bloomberg': vendor_fields = ['PX_LAST']
md_request = MarketDataRequest(
start_date = start_date, # start date
finish_date = finish_date, # finish date
category = 'economic',
freq = 'daily', # intraday data
data_source = source, # use Bloomberg as data source
cut = 'LOC',
tickers = pretty_tickers,
fields = ['close'], # which fields to download
vendor_tickers = vendor_tickers,
vendor_fields = vendor_fields, # which Bloomberg fields to download
cache_algo = cache_algo) # how to return data
return self.market_data_generator.fetch_market_data(md_request)
def grasp_coded_entry(self, df, index):
df = df.ix[index:].stack()
df = df.reset_index()
df.columns = ['Date', 'Name', 'Val']
countries = df['Name']
countries = [x.split('-', 1)[0] for x in countries]
df['Code'] = sum(
[list(self._econ_country_codes[self._econ_country_codes["Country"] == x]['Code']) for x in countries],
[])
return df
class DataVendorALFRED(DataVendor):
def __init__(self):
super(DataVendorALFRED, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
# implement method in abstract superclass
def load_ticker(self, market_data_request):
market_data_request_vendor = self.construct_vendor_market_data_request(market_data_request)
self.logger.info("Request ALFRED/FRED data")
data_frame = self.download_daily(market_data_request_vendor)
if data_frame is None or data_frame.index is []: return None
# convert from vendor to findatapy tickers/fields
if data_frame is not None:
returned_tickers = data_frame.columns
if data_frame is not None:
# tidy up tickers into a format that is more easily translatable
# we can often get multiple fields returned (even if we don't ask for them!)
# convert to lower case
returned_fields = [(x.split('.')[1]) for x in returned_tickers]
returned_tickers = [(x.split('.')[0]) for x in returned_tickers]
try:
fields = self.translate_from_vendor_field(returned_fields, market_data_request)
tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)
except:
print('error')
ticker_combined = []
for i in range(0, len(fields)):
ticker_combined.append(tickers[i] + "." + fields[i])
data_frame.columns = ticker_combined
data_frame.index.name = 'Date'
self.logger.info("Completed request from ALFRED/FRED for " + str(ticker_combined))
return data_frame
def download_daily(self, market_data_request):
trials = 0
data_frame_list = []
data_frame_release = []
# TODO refactor this code, a bit messy at the moment!
for i in range(0, len(market_data_request.tickers)):
while (trials < 5):
try:
fred = Fred(api_key=DataConstants().fred_api_key)
# acceptable fields: close, actual-release, release-date-time-full
if 'close' in market_data_request.fields and 'release-date-time-full' in market_data_request.fields:
data_frame = fred.get_series_all_releases(market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame.columns = ['Date', market_data_request.tickers[i] + '.release-date-time-full',
market_data_request.tickers[i] + '.close']
data_frame = data_frame.sort_values(by=['Date', market_data_request.tickers[i] + '.release-date-time-full'])
data_frame = data_frame.drop_duplicates(subset=['Date'], keep='last')
data_frame = data_frame.set_index(['Date'])
filter = Filter()
data_frame = filter.filter_time_series_by_date(market_data_request.start_date,
market_data_request.finish_date, data_frame)
data_frame_list.append(data_frame)
elif 'close' in market_data_request.fields:
data_frame = fred.get_series(series_id=market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame = pandas.DataFrame(data_frame)
data_frame.columns = [market_data_request.tickers[i] + '.close']
data_frame_list.append(data_frame)
if 'first-revision' in market_data_request.fields:
data_frame = fred.get_series_first_revision(market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame = pandas.DataFrame(data_frame)
data_frame.columns = [market_data_request.tickers[i] + '.first-revision']
filter = Filter()
data_frame = filter.filter_time_series_by_date(market_data_request.start_date,
market_data_request.finish_date, data_frame)
data_frame_list.append(data_frame)
if 'actual-release' in market_data_request.fields and 'release-date-time-full' in market_data_request.fields:
data_frame = fred.get_series_all_releases(market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame.columns = ['Date', market_data_request.tickers[i] + '.release-date-time-full',
market_data_request.tickers[i] + '.actual-release']
data_frame = data_frame.sort_values(by=['Date', market_data_request.tickers[i] + '.release-date-time-full'])
data_frame = data_frame.drop_duplicates(subset=['Date'], keep='first')
data_frame = data_frame.set_index(['Date'])
filter = Filter()
data_frame = filter.filter_time_series_by_date(market_data_request.start_date,
market_data_request.finish_date, data_frame)
data_frame_list.append(data_frame)
elif 'actual-release' in market_data_request.fields:
data_frame = fred.get_series_first_release(market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame = pandas.DataFrame(data_frame)
data_frame.columns = [market_data_request.tickers[i] + '.actual-release']
filter = Filter()
data_frame = filter.filter_time_series_by_date(market_data_request.start_date,
market_data_request.finish_date, data_frame)
data_frame_list.append(data_frame)
elif 'release-date-time-full' in market_data_request.fields:
data_frame = fred.get_series_all_releases(market_data_request.tickers[i],
observation_start=market_data_request.start_date,
observation_end=market_data_request.finish_date)
data_frame = data_frame['realtime_start']
data_frame = pandas.DataFrame(data_frame)
data_frame.columns = [market_data_request.tickers[i] + '.release-date-time-full']
data_frame.index = data_frame[market_data_request.tickers[i] + '.release-date-time-full']
data_frame = data_frame.sort()
data_frame = data_frame.drop_duplicates()
filter = Filter()
data_frame_release.append(filter.filter_time_series_by_date(market_data_request.start_date,
market_data_request.finish_date, data_frame))
break
except:
trials = trials + 1
self.logger.info("Attempting... " + str(trials) + " request to download from ALFRED/FRED")
if trials == 5:
self.logger.error("Couldn't download from ALFRED/FRED after several attempts!")
calc = Calculations()
data_frame1 = calc.pandas_outer_join(data_frame_list)
data_frame2 = calc.pandas_outer_join(data_frame_release)
data_frame = pandas.concat([data_frame1, data_frame2], axis=1)
return data_frame
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.timeseries import Calculations
from findatapy.util import LoggerManager
from finmarketpy.economics import EventStudy
# choose run_example = 0 for everything
# run_example = 1 - download recent NFP times and do event study for USD/JPY
run_example = 0
###### download recent NFP times and do event study for USD/JPY (using Bloomberg data)
if run_example == 1 or run_example == 0:
logger = LoggerManager().getLogger(__name__)
import datetime
from datetime import timedelta
###### Get intraday data for USD/JPY from the past few months from Bloomberg, NFP date/times from Bloomberg
###### then plot intraday price action around NFP for EUR/USD
start_date = datetime.date.today() - timedelta(days=180)
finish_date = datetime.datetime.utcnow()
market = Market(market_data_generator=MarketDataGenerator())
# Fetch NFP times from Bloomberg
md_request = MarketDataRequest(
class DataVendorBOE(DataVendor):
def __init__(self):
super(DataVendorBOE, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
# implement method in abstract superclass
def load_ticker(self, market_data_request):
market_data_request_vendor = self.construct_vendor_market_data_request(market_data_request)
self.logger.info("Request BOE data")
data_frame = self.download_daily(market_data_request_vendor)
if data_frame is None or data_frame.index is []: return None
# convert from vendor to findatapy tickers/fields
if data_frame is not None:
returned_tickers = data_frame.columns
if data_frame is not None:
# tidy up tickers into a format that is more easily translatable
# we can often get multiple fields returned (even if we don't ask for them!)
# convert to lower case
returned_fields = [(x.split(' - ')[1]).lower().replace(' ', '-') for x in returned_tickers]
returned_fields = [x.replace('value', 'close') for x in returned_fields] # special case for close
returned_tickers = [x.replace('.', '/') for x in returned_tickers]
returned_tickers = [x.split(' - ')[0] for x in returned_tickers]
fields = self.translate_from_vendor_field(returned_fields, market_data_request)
tickers = self.translate_from_vendor_ticker(returned_tickers, market_data_request)
ticker_combined = []
for i in range(0, len(fields)):
ticker_combined.append(tickers[i] + "." + fields[i])
data_frame.columns = ticker_combined
data_frame.index.name = 'Date'
self.logger.info("Completed request from BOE.")
return data_frame
def download_daily(self, market_data_request):
trials = 0
data_frame = None
while (trials < 5):
try:
# TODO
break
except:
trials = trials + 1
self.logger.info("Attempting... " + str(trials) + " request to download from BOE")
if trials == 5:
self.logger.error("Couldn't download from ONS after several attempts!")
return data_frame
def get_data_vendor(self, source):
"""Loads appropriate data service class
Parameters
----------
source : str
the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc.
we can also have forms like "bloomberg-boe" separated by hyphens
Returns
-------
DataVendor
"""
logger = LoggerManager().getLogger(__name__)
data_vendor = None
try:
source = source.split("-")[0]
except:
logger.error("Was data source specified?")
return None
if source == 'bloomberg':
try:
from findatapy.market.datavendorbbg import DataVendorBBGOpen
data_vendor = DataVendorBBGOpen()
except:
logger.warn("Bloomberg needs to be installed")
elif source == 'quandl':
from findatapy.market.datavendorweb import DataVendorQuandl
data_vendor = DataVendorQuandl()
elif source == 'eikon':
from findatapy.market.datavendorweb import DataVendorEikon
data_vendor = DataVendorEikon()
elif source == 'ons':
from findatapy.market.datavendorweb import DataVendorONS
data_vendor = DataVendorONS()
elif source == 'boe':
from findatapy.market.datavendorweb import DataVendorBOE
data_vendor = DataVendorBOE()
elif source == 'dukascopy':
from findatapy.market.datavendorweb import DataVendorDukasCopy
data_vendor = DataVendorDukasCopy()
elif source == 'fxcm':
from findatapy.market.datavendorweb import DataVendorFXCM
data_vendor = DataVendorFXCM()
elif source == 'alfred':
from findatapy.market.datavendorweb import DataVendorALFRED
data_vendor = DataVendorALFRED()
elif source == 'yahoo':
from findatapy.market.datavendorweb import DataVendorYahoo
data_vendor = DataVendorYahoo()
elif source in ['google', 'fred', 'oecd', 'eurostat', 'edgar-index']:
from findatapy.market.datavendorweb import DataVendorPandasWeb
data_vendor = DataVendorPandasWeb()
elif source == 'bitcoincharts':
from findatapy.market.datavendorweb import DataVendorBitcoincharts
data_vendor = DataVendorBitcoincharts()
elif source == 'poloniex':
from findatapy.market.datavendorweb import DataVendorPoloniex
data_vendor = DataVendorPoloniex()
elif source == 'binance':
from findatapy.market.datavendorweb import DataVendorBinance
data_vendor = DataVendorBinance()
elif source == 'bitfinex':
from findatapy.market.datavendorweb import DataVendorBitfinex
data_vendor = DataVendorBitfinex()
elif source == 'gdax':
from findatapy.market.datavendorweb import DataVendorGdax
data_vendor = DataVendorGdax()
elif source == 'kraken':
from findatapy.market.datavendorweb import DataVendorKraken
data_vendor = DataVendorKraken()
elif source == 'bitmex':
from findatapy.market.datavendorweb import DataVendorBitmex
data_vendor = DataVendorBitmex()
elif '.csv' in source or '.h5' in source or '.parquet' in source:
from findatapy.market.datavendorweb import DataVendorFlatFile
data_vendor = DataVendorFlatFile()
elif source == 'alphavantage':
from findatapy.market.datavendorweb import DataVendorAlphaVantage
data_vendor = DataVendorAlphaVantage()
elif source == 'huobi':
from findatapy.market.datavendorweb import DataVendorHuobi
data_vendor = DataVendorHuobi()
# TODO add support for other data sources (like Reuters)
return data_vendor
def __init__(self):
super(DataVendorPandasWeb, self).__init__()
self.logger = LoggerManager().getLogger(__name__)
def __init__(self, *args, **kwargs):
self.logger = LoggerManager().getLogger(__name__)
def _get_individual_fx_cross(self, market_data_request):
cr = market_data_request.cross
type = market_data_request.type
freq = market_data_request.freq
base = cr[0:3]
terms = cr[3:6]
if (type == 'spot'):
# Non-USD crosses
if base != 'USD' and terms != 'USD':
base_USD = self.fxconv.correct_notation('USD' + base)
terms_USD = self.fxconv.correct_notation('USD' + terms)
# TODO check if the cross exists in the database
# Download base USD cross
market_data_request.tickers = base_USD
market_data_request.category = 'fx'
base_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Download terms USD cross
market_data_request.tickers = terms_USD
market_data_request.category = 'fx'
terms_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# If quoted USD/base flip to get USD terms
if (base_USD[0:3] == 'USD'):
base_vals = 1 / base_vals
# If quoted USD/terms flip to get USD terms
if (terms_USD[0:3] == 'USD'):
terms_vals = 1 / terms_vals
base_vals.columns = ['temp']
terms_vals.columns = ['temp']
cross_vals = base_vals.div(terms_vals, axis='index')
cross_vals.columns = [cr + '.close']
base_vals.columns = [base_USD + '.close']
terms_vals.columns = [terms_USD + '.close']
else:
# if base == 'USD': non_USD = terms
# if terms == 'USD': non_USD = base
correct_cr = self.fxconv.correct_notation(cr)
market_data_request.tickers = correct_cr
market_data_request.category = 'fx'
cross_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Special case for USDUSD!
if base + terms == 'USDUSD':
if freq == 'daily':
cross_vals = pd.DataFrame(1,
index=cross_vals.index,
columns=cross_vals.columns)
filter = Filter()
cross_vals = filter.filter_time_series_by_holidays(
cross_vals, cal='WEEKDAY')
else:
# Flip if not convention (eg. JPYUSD)
if (correct_cr != cr):
cross_vals = 1 / cross_vals
# cross_vals = self._market_data_generator.harvest_time_series(market_data_request)
cross_vals.columns = [cr + '.close']
elif type[0:3] == "tot":
if freq == 'daily':
# Download base USD cross
market_data_request.tickers = base + 'USD'
market_data_request.category = 'fx-' + type
if type[0:3] == "tot":
base_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Download terms USD cross
market_data_request.tickers = terms + 'USD'
market_data_request.category = 'fx-' + type
if type[0:3] == "tot":
terms_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# base_rets = self._calculations.calculate_returns(base_vals)
# terms_rets = self._calculations.calculate_returns(terms_vals)
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == 'USDUSD':
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + 'USD' == 'USDUSD':
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + 'USD' == 'USDUSD':
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [cr + '-' + type + '.close']
elif freq == 'intraday':
LoggerManager().getLogger(__name__).info(
'Total calculated returns for intraday not implemented yet'
)
return None
return cross_vals
def __init__(self):
self.logger = LoggerManager().getLogger(__name__)
return
