class FXCrossFactory(object):
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self.fxconv = FXConv()
self.cache = {}
self.calculations = Calculations()
self.market_data_generator = market_data_generator
return
def flush_cache(self):
self.cache = {}
def get_fx_cross_tick(self,
start,
end,
cross,
cut="NYC",
source="dukascopy",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['bid', 'ask']):
if isinstance(cross, str):
cross = [cross]
market_data_request = MarketDataRequest(
gran_freq="tick",
freq_mult=1,
freq='tick',
cut=cut,
fields=['bid', 'ask', 'bidv', 'askv'],
cache_algo=cache_algo,
environment=environment,
start_date=start,
finish_date=end,
data_source=source,
category='fx')
market_data_generator = self.market_data_generator
data_frame_agg = None
for cr in cross:
if (type == 'spot'):
market_data_request.tickers = cr
cross_vals = market_data_generator.fetch_market_data(
market_data_request)
# if user only wants 'close' calculate that from the bid/ask fields
if fields == ['close']:
cross_vals = cross_vals[[cr + '.bid',
cr + '.ask']].mean(axis=1)
cross_vals.columns = [cr + '.close']
if data_frame_agg is None:
data_frame_agg = cross_vals
else:
data_frame_agg = data_frame_agg.join(cross_vals, how='outer')
# strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def get_fx_cross(self,
start,
end,
cross,
cut="NYC",
source="bloomberg",
freq="intraday",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['close']):
if source == "gain" or source == 'dukascopy' or freq == 'tick':
return self.get_fx_cross_tick(start,
end,
cross,
cut=cut,
source=source,
cache_algo=cache_algo,
type='spot',
fields=fields)
if isinstance(cross, str):
cross = [cross]
market_data_request_list = []
freq_list = []
type_list = []
for cr in cross:
market_data_request = MarketDataRequest(freq_mult=1,
cut=cut,
fields=['close'],
freq=freq,
cache_algo=cache_algo,
start_date=start,
finish_date=end,
data_source=source,
environment=environment)
market_data_request.type = type
market_data_request.cross = cr
if freq == 'intraday':
market_data_request.gran_freq = "minute" # intraday
elif freq == 'daily':
market_data_request.gran_freq = "daily" # daily
market_data_request_list.append(market_data_request)
data_frame_agg = []
# depends on the nature of operation as to whether we should use threading or multiprocessing library
if DataConstants().market_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocessing_on_dill library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocessing_on_dill import Pool
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants(
).market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request_list[0].data_source]
# fudge, issue with multithreading and accessing HDF5 files
# if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
# thread_no = 0
if (thread_no > 0):
pool = Pool(thread_no)
# open the market data downloads in their own threads and return the results
result = pool.map_async(self._get_individual_fx_cross,
market_data_request_list)
data_frame_agg = self.calculations.iterative_outer_join(
result.get())
# data_frame_agg = self.calculations.pandas_outer_join(result.get())
# pool would have already been closed earlier
# try:
# pool.close()
# pool.join()
# except: pass
else:
for md_request in market_data_request_list:
data_frame_agg.append(
self._get_individual_fx_cross(md_request))
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_agg)
# strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def _get_individual_fx_cross(self, market_data_request):
cr = market_data_request.cross
type = market_data_request.type
freq = market_data_request.freq
base = cr[0:3]
terms = cr[3:6]
if (type == 'spot'):
# non-USD crosses
if base != 'USD' and terms != 'USD':
base_USD = self.fxconv.correct_notation('USD' + base)
terms_USD = self.fxconv.correct_notation('USD' + terms)
# TODO check if the cross exists in the database
# download base USD cross
market_data_request.tickers = base_USD
market_data_request.category = 'fx'
if base_USD + '.close' in self.cache:
base_vals = self.cache[base_USD + '.close']
else:
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
self.cache[base_USD + '.close'] = base_vals
# download terms USD cross
market_data_request.tickers = terms_USD
market_data_request.category = 'fx'
if terms_USD + '.close' in self.cache:
terms_vals = self.cache[terms_USD + '.close']
else:
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
self.cache[terms_USD + '.close'] = terms_vals
# if quoted USD/base flip to get USD terms
if (base_USD[0:3] == 'USD'):
if 'USD' + base in '.close' in self.cache:
base_vals = self.cache['USD' + base + '.close']
else:
base_vals = 1 / base_vals
self.cache['USD' + base + '.close'] = base_vals
# if quoted USD/terms flip to get USD terms
if (terms_USD[0:3] == 'USD'):
if 'USD' + terms in '.close' in self.cache:
terms_vals = self.cache['USD' + terms + '.close']
else:
terms_vals = 1 / terms_vals
self.cache['USD' + terms + '.close'] = base_vals
base_vals.columns = ['temp']
terms_vals.columns = ['temp']
cross_vals = base_vals.div(terms_vals, axis='index')
cross_vals.columns = [cr + '.close']
base_vals.columns = [base_USD + '.close']
terms_vals.columns = [terms_USD + '.close']
else:
# if base == 'USD': non_USD = terms
# if terms == 'USD': non_USD = base
correct_cr = self.fxconv.correct_notation(cr)
market_data_request.tickers = correct_cr
market_data_request.category = 'fx'
if correct_cr + '.close' in self.cache:
cross_vals = self.cache[correct_cr + '.close']
else:
cross_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# flip if not convention
if (correct_cr != cr):
if cr + '.close' in self.cache:
cross_vals = self.cache[cr + '.close']
else:
cross_vals = 1 / cross_vals
self.cache[cr + '.close'] = cross_vals
self.cache[correct_cr + '.close'] = cross_vals
# cross_vals = self.market_data_generator.harvest_time_series(market_data_request)
cross_vals.columns.names = [cr + '.close']
elif type[0:3] == "tot":
if freq == 'daily':
# download base USD cross
market_data_request.tickers = base + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
x = 0
# download terms USD cross
market_data_request.tickers = terms + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
pass
base_rets = self.calculations.calculate_returns(base_vals)
terms_rets = self.calculations.calculate_returns(terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# first returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self.calculations.create_mult_index(cross_rets)
cross_vals.columns = [cr + '-tot.close']
elif freq == 'intraday':
self.logger.info(
'Total calculated returns for intraday not implemented yet'
)
return None
return cross_vals
class FXCrossFactory(object):
"""Generates FX spot time series and FX total return time series (assuming we already have
total return indices available from xxxUSD form) from underlying series. Can also produce cross rates from the USD
crosses.
"""
def __init__(self, market_data_generator=None):
self.fxconv = FXConv()
self.cache = {}
self._calculations = Calculations()
self._market_data_generator = market_data_generator
return
def get_fx_cross_tick(self,
start,
end,
cross,
cut="NYC",
data_source="dukascopy",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['bid', 'ask']):
if isinstance(cross, str):
cross = [cross]
market_data_request = MarketDataRequest(
gran_freq="tick",
freq_mult=1,
freq='tick',
cut=cut,
fields=['bid', 'ask', 'bidv', 'askv'],
cache_algo=cache_algo,
environment=environment,
start_date=start,
finish_date=end,
data_source=data_source,
category='fx')
market_data_generator = self._market_data_generator
data_frame_agg = None
for cr in cross:
if (type == 'spot'):
market_data_request.tickers = cr
cross_vals = market_data_generator.fetch_market_data(
market_data_request)
if cross_vals is not None:
# If user only wants 'close' calculate that from the bid/ask fields
if fields == ['close']:
cross_vals = cross_vals[[cr + '.bid',
cr + '.ask']].mean(axis=1)
cross_vals.columns = [cr + '.close']
else:
filter = Filter()
filter_columns = [cr + '.' + f for f in fields]
cross_vals = filter.filter_time_series_by_columns(
filter_columns, cross_vals)
if data_frame_agg is None:
data_frame_agg = cross_vals
else:
data_frame_agg = data_frame_agg.join(cross_vals, how='outer')
if data_frame_agg is not None:
# Strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def get_fx_cross(self,
start,
end,
cross,
cut="NYC",
data_source="bloomberg",
freq="intraday",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['close']):
if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick':
return self.get_fx_cross_tick(start,
end,
cross,
cut=cut,
data_source=data_source,
cache_algo=cache_algo,
type='spot',
fields=fields)
if isinstance(cross, str):
cross = [cross]
market_data_request_list = []
freq_list = []
type_list = []
for cr in cross:
market_data_request = MarketDataRequest(freq_mult=1,
cut=cut,
fields=['close'],
freq=freq,
cache_algo=cache_algo,
start_date=start,
finish_date=end,
data_source=data_source,
environment=environment)
market_data_request.type = type
market_data_request.cross = cr
if freq == 'intraday':
market_data_request.gran_freq = "minute" # intraday
elif freq == 'daily':
market_data_request.gran_freq = "daily" # daily
market_data_request_list.append(market_data_request)
data_frame_agg = []
# Depends on the nature of operation as to whether we should use threading or multiprocessing library
if constants.market_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# Most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocess library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocess import Pool
thread_no = constants.market_thread_no['other']
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
# Fudge, issue with multithreading and accessing HDF5 files
# if self._market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
# thread_no = 0
thread_no = 0
if (thread_no > 0):
pool = Pool(thread_no)
# Open the market data downloads in their own threads and return the results
df_list = pool.map_async(self._get_individual_fx_cross,
market_data_request_list).get()
data_frame_agg = self._calculations.iterative_outer_join(df_list)
# data_frame_agg = self._calculations.pandas_outer_join(result.get())
try:
pool.close()
pool.join()
except:
pass
else:
for md_request in market_data_request_list:
data_frame_agg.append(
self._get_individual_fx_cross(md_request))
data_frame_agg = self._calculations.pandas_outer_join(
data_frame_agg)
# Strip the nan elements
data_frame_agg = data_frame_agg.dropna(how='all')
# self.speed_cache.put_dataframe(key, data_frame_agg)
return data_frame_agg
def _get_individual_fx_cross(self, market_data_request):
cr = market_data_request.cross
type = market_data_request.type
freq = market_data_request.freq
base = cr[0:3]
terms = cr[3:6]
if (type == 'spot'):
# Non-USD crosses
if base != 'USD' and terms != 'USD':
base_USD = self.fxconv.correct_notation('USD' + base)
terms_USD = self.fxconv.correct_notation('USD' + terms)
# TODO check if the cross exists in the database
# Download base USD cross
market_data_request.tickers = base_USD
market_data_request.category = 'fx'
base_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Download terms USD cross
market_data_request.tickers = terms_USD
market_data_request.category = 'fx'
terms_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# If quoted USD/base flip to get USD terms
if (base_USD[0:3] == 'USD'):
base_vals = 1 / base_vals
# If quoted USD/terms flip to get USD terms
if (terms_USD[0:3] == 'USD'):
terms_vals = 1 / terms_vals
base_vals.columns = ['temp']
terms_vals.columns = ['temp']
cross_vals = base_vals.div(terms_vals, axis='index')
cross_vals.columns = [cr + '.close']
base_vals.columns = [base_USD + '.close']
terms_vals.columns = [terms_USD + '.close']
else:
# if base == 'USD': non_USD = terms
# if terms == 'USD': non_USD = base
correct_cr = self.fxconv.correct_notation(cr)
market_data_request.tickers = correct_cr
market_data_request.category = 'fx'
cross_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Special case for USDUSD!
if base + terms == 'USDUSD':
if freq == 'daily':
cross_vals = pd.DataFrame(1,
index=cross_vals.index,
columns=cross_vals.columns)
filter = Filter()
cross_vals = filter.filter_time_series_by_holidays(
cross_vals, cal='WEEKDAY')
else:
# Flip if not convention (eg. JPYUSD)
if (correct_cr != cr):
cross_vals = 1 / cross_vals
# cross_vals = self._market_data_generator.harvest_time_series(market_data_request)
cross_vals.columns = [cr + '.close']
elif type[0:3] == "tot":
if freq == 'daily':
# Download base USD cross
market_data_request.tickers = base + 'USD'
market_data_request.category = 'fx-' + type
if type[0:3] == "tot":
base_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# Download terms USD cross
market_data_request.tickers = terms + 'USD'
market_data_request.category = 'fx-' + type
if type[0:3] == "tot":
terms_vals = self._market_data_generator.fetch_market_data(
market_data_request)
# base_rets = self._calculations.calculate_returns(base_vals)
# terms_rets = self._calculations.calculate_returns(terms_vals)
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == 'USDUSD':
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + 'USD' == 'USDUSD':
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + 'USD' == 'USDUSD':
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [cr + '-' + type + '.close']
elif freq == 'intraday':
LoggerManager().getLogger(__name__).info(
'Total calculated returns for intraday not implemented yet'
)
return None
return cross_vals
def fetch_continuous_time_series(self,
md_request,
market_data_generator,
fx_vol_surface=None,
enter_trading_dates=None,
fx_options_trading_tenor=None,
roll_days_before=None,
roll_event=None,
construct_via_currency=None,
fx_options_tenor_for_interpolation=None,
base_depos_tenor=None,
roll_months=None,
cum_index=None,
strike=None,
contract_type=None,
premium_output=None,
position_multiplier=None,
depo_tenor_for_option=None,
freeze_implied_vol=None,
tot_label=None,
cal=None,
output_calculation_fields=None):
if fx_vol_surface is None: fx_vol_surface = self._fx_vol_surface
if enter_trading_dates is None:
enter_trading_dates = self._enter_trading_dates
if market_data_generator is None:
market_data_generator = self._market_data_generator
if fx_options_trading_tenor is None:
fx_options_trading_tenor = self._fx_options_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if construct_via_currency is None:
construct_via_currency = self._construct_via_currency
if fx_options_tenor_for_interpolation is None:
fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation
if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor
if roll_months is None: roll_months = self._roll_months
if strike is None: strike = self._strike
if contract_type is None: contract_type = self._contact_type
if premium_output is None: premium_output = self._premium_output
if position_multiplier is None:
position_multiplier = self._position_multiplier
if depo_tenor_for_option is None:
depo_tenor_for_option = self._depo_tenor_for_option
if freeze_implied_vol is None:
freeze_implied_vol = self._freeze_implied_vol
if tot_label is None: tot_label = self._tot_label
if cal is None: cal = self._cal
if output_calculation_fields is None:
output_calculation_fields = self._output_calculation_fields
# Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient options/forward data for this)
if construct_via_currency == 'no':
if fx_vol_surface is None:
# Download FX spot, FX forwards points and base depos etc.
market = Market(market_data_generator=market_data_generator)
md_request_download = MarketDataRequest(md_request=md_request)
fx_conv = FXConv()
# CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data
# should be fetched in correct convention
md_request_download.tickers = [
fx_conv.correct_notation(x) for x in md_request.tickers
]
md_request_download.category = 'fx-vol-market'
md_request_download.fields = 'close'
md_request_download.abstract_curve = None
md_request_download.fx_options_tenor = fx_options_tenor_for_interpolation
md_request_download.base_depos_tenor = base_depos_tenor
# md_request_download.base_depos_currencies = []
forwards_market_df = market.fetch_market(md_request_download)
else:
forwards_market_df = None
# Now use the original tickers
return self.construct_total_return_index(
md_request.tickers,
forwards_market_df,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
freeze_implied_vol=freeze_implied_vol,
depo_tenor_for_option=depo_tenor_for_option,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields)
else:
# eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency
# Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns)
total_return_indices = []
for tick in md_request.tickers:
base = tick[0:3]
terms = tick[3:6]
md_request_base = MarketDataRequest(md_request=md_request)
md_request_base.tickers = base + construct_via_currency
md_request_terms = MarketDataRequest(md_request=md_request)
md_request_terms.tickers = terms + construct_via_currency
# Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD)
base_vals = self.fetch_continuous_time_series(
md_request_base,
market_data_generator,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
terms_vals = self.fetch_continuous_time_series(
md_request_terms,
market_data_generator,
fx_vol_surface=fx_vol_surface,
enter_trading_dates=enter_trading_dates,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == construct_via_currency + construct_via_currency:
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [tick + '-option-tot.close']
total_return_indices.append(cross_vals)
return self._calculations.join(total_return_indices, how='outer')
