def plot_strategy_group_benchmark_pnl_yoy(self, strip = None, silent_plot = False): style = self.create_style("", "Group Benchmark PnL YoY - cumulative") keys = self._strategy_group_benchmark_ret_stats.keys() yoy = [] for key in keys: col = self._strategy_group_benchmark_ret_stats[key].yoy_rets() col.columns = [key] yoy.append(col) calculations = Calculations() ret_stats = calculations.pandas_outer_join(yoy) ret_stats.index = ret_stats.index.year if strip is not None: ret_stats.columns = [k.replace(strip, '') for k in ret_stats.columns] # ret_stats = ret_stats.sort_index() style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - YoY) ' + str(style.scale_factor) + '.png' style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - YoY) ' + str(style.scale_factor) + '.html' style.display_brand_label = False style.date_formatter = "%Y" chart = Chart(ret_stats * 100, engine=self.DEFAULT_PLOT_ENGINE, chart_type='bar', style=style) if not (silent_plot): chart.plot() return chart
def calculate_diagnostic_trading_PnL(self, asset_a_df, signal_df, further_df = [], further_df_labels = []): """Calculates P&L table which can be used for debugging purposes, The table is populated with asset, signal and further dataframes provided by the user, can be used to check signalling methodology. It does not apply parameters such as transaction costs, vol adjusment and so on. Parameters ---------- asset_a_df : DataFrame Asset prices signal_df : DataFrame Trade signals (typically +1, -1, 0 etc) further_df : DataFrame Further dataframes user wishes to output in the diagnostic output (typically inputs for the signals) further_df_labels Labels to append to the further dataframes Returns ------- DataFrame with asset, trading signals and returns of the trading strategy for diagnostic purposes """ calculations = Calculations() asset_rets_df = calculations.calculate_returns(asset_a_df) strategy_rets = calculations.calculate_signal_returns(signal_df, asset_rets_df) reset_points = ((signal_df - signal_df.shift(1)).abs()) asset_a_df_entry = asset_a_df.copy(deep=True) asset_a_df_entry[reset_points == 0] = numpy.nan asset_a_df_entry = asset_a_df_entry.ffill() asset_a_df_entry.columns = [x + '_entry' for x in asset_a_df_entry.columns] asset_rets_df.columns = [x + '_asset_rets' for x in asset_rets_df.columns] strategy_rets.columns = [x + '_strat_rets' for x in strategy_rets.columns] signal_df.columns = [x + '_final_signal' for x in signal_df.columns] for i in range(0, len(further_df)): further_df[i].columns = [x + '_' + further_df_labels[i] for x in further_df[i].columns] flatten_df =[asset_a_df, asset_a_df_entry, asset_rets_df, strategy_rets, signal_df] for f in further_df: flatten_df.append(f) return calculations.pandas_outer_join(flatten_df)
def create_tech_ind( self, data_frame_non_nan, name, tech_params, data_frame_non_nan_early=None): self._signal = None self._techind = None if tech_params.fillna: data_frame = data_frame_non_nan.fillna(method="ffill") else: data_frame = data_frame_non_nan if data_frame_non_nan_early is not None: data_frame_early = data_frame_non_nan_early.fillna(method="ffill") if name == "SMA": if (data_frame_non_nan_early is not None): # calculate the lagged sum of the n-1 point if pd.__version__ < '0.17': rolling_sum = pd.rolling_sum( data_frame.shift(1).rolling, window=tech_params.sma_period - 1) else: rolling_sum = data_frame.shift(1).rolling( center=False, window=tech_params.sma_period - 1).sum() # add non-nan one for today rolling_sum = rolling_sum + data_frame_early # calculate average = sum / n self._techind = rolling_sum / tech_params.sma_period narray = np.where(data_frame_early > self._techind, 1, -1) else: if pd.__version__ < '0.17': self._techind = pd.rolling_sum( data_frame, window=tech_params.sma_period) else: self._techind = data_frame.rolling( window=tech_params.sma_period, center=False).mean() narray = np.where(data_frame > self._techind, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.sma_period] = np.nan self._signal.columns = [ x + " SMA Signal" for x in data_frame.columns.values] self._techind.columns = [ x + " SMA" for x in data_frame.columns.values] elif name == "EMA": # self._techind = pd.ewma(data_frame, span = tech_params.ema_period) self._techind = data_frame.ewm( ignore_na=False, span=tech_params.ema_period, min_periods=0, adjust=True).mean() narray = np.where(data_frame > self._techind, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.ema_period] = np.nan self._signal.columns = [ x + " EMA Signal" for x in data_frame.columns.values] self._techind.columns = [ x + " EMA" for x in data_frame.columns.values] elif name == "ROC": if (data_frame_non_nan_early is not None): self._techind = data_frame_early / \ data_frame.shift(tech_params.roc_period) - 1 else: self._techind = data_frame / \ data_frame.shift(tech_params.roc_period) - 1 narray = np.where(self._techind > 0, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.roc_period] = np.nan self._signal.columns = [ x + " ROC Signal" for x in data_frame.columns.values] self._techind.columns = [ x + " ROC" for x in data_frame.columns.values] elif name == "polarity": self._techind = data_frame narray = np.where(self._techind > 0, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " Polarity Signal" for x in data_frame.columns.values] self._techind.columns = [ x + " Polarity" for x in data_frame.columns.values] elif name == "SMA2": sma = data_frame.rolling( window=tech_params.sma_period, center=False).mean() sma2 = data_frame.rolling( window=tech_params.sma2_period, center=False).mean() narray = np.where(sma > sma2, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " SMA2 Signal" for x in data_frame.columns.values] sma.columns = [x + " SMA" for x in data_frame.columns.values] sma2.columns = [x + " SMA2" for x in data_frame.columns.values] most = max(tech_params.sma_period, tech_params.sma2_period) self._signal.loc[0:most] = np.nan self._techind = pd.concat([sma, sma2], axis=1) elif name in ['RSI']: # delta = data_frame.diff() # # dUp, dDown = delta.copy(), delta.copy() # dUp[dUp < 0] = 0 # dDown[dDown > 0] = 0 # # rolUp = pd.rolling_mean(dUp, tech_params.rsi_period) # rolDown = pd.rolling_mean(dDown, tech_params.rsi_period).abs() # # rsi = rolUp / rolDown # Get the difference in price from previous step delta = data_frame.diff() # Get rid of the first row, which is NaN since it did not have a previous # row to calculate the differences delta = delta[1:] # Make the positive gains (up) and negative gains (down) Series up, down = delta.copy(), delta.copy() up[up < 0] = 0 down[down > 0] = 0 # Calculate the EWMA roll_up1 = pd.stats.moments.ewma(up, tech_params.rsi_period) roll_down1 = pd.stats.moments.ewma( down.abs(), tech_params.rsi_period) # Calculate the RSI based on EWMA RS1 = roll_up1 / roll_down1 RSI1 = 100.0 - (100.0 / (1.0 + RS1)) # Calculate the SMA roll_up2 = up.rolling( window=tech_params.rsi_period, center=False).mean() roll_down2 = down.abs().rolling( window=tech_params.rsi_period, center=False).mean() # Calculate the RSI based on SMA RS2 = roll_up2 / roll_down2 RSI2 = 100.0 - (100.0 / (1.0 + RS2)) self._techind = RSI2 self._techind.columns = [ x + " RSI" for x in data_frame.columns.values] signal = data_frame.copy() sells = (signal.shift(-1) < tech_params.rsi_lower) & (signal > tech_params.rsi_lower) buys = (signal.shift(-1) > tech_params.rsi_upper) & (signal < tech_params.rsi_upper) # print (buys[buys == True]) # buys signal[buys] = 1 signal[sells] = -1 signal[~(buys | sells)] = np.nan signal = signal.fillna(method='ffill') self._signal = signal self._signal.loc[0:tech_params.rsi_period] = np.nan self._signal.columns = [ x + " RSI Signal" for x in data_frame.columns.values] elif name in ["BB"]: # calcuate Bollinger bands mid = data_frame.rolling( center=False, window=tech_params.bb_period).mean() mid.columns = [x + " BB Mid" for x in data_frame.columns.values] std_dev = data_frame.rolling( center=False, window=tech_params.bb_period).std() BB_std = tech_params.bb_mult * std_dev lower = pd.DataFrame( data=mid.values - BB_std.values, index=mid.index, columns=data_frame.columns) upper = pd.DataFrame( data=mid.values + BB_std.values, index=mid.index, columns=data_frame.columns) # calculate signals signal = data_frame.copy() buys = signal > upper sells = signal < lower signal[buys] = 1 signal[sells] = -1 signal[~(buys | sells)] = np.nan signal = signal.fillna(method='ffill') self._signal = signal self._signal.loc[0:tech_params.bb_period] = np.nan self._signal.columns = [ x + " " + name + " Signal" for x in data_frame.columns.values] lower.columns = [ x + " BB Lower" for x in data_frame.columns.values] upper.columns = [x + " BB Mid" for x in data_frame.columns.values] upper.columns = [ x + " BB Lower" for x in data_frame.columns.values] self._techind = pd.concat([lower, mid, upper], axis=1) elif name == "long-only": # have +1 signals only self._techind = data_frame # the technical indicator is just "prices" narray = np.ones((len(data_frame.index), len(data_frame.columns))) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " Long Only Signal" for x in data_frame.columns.values] self._techind.columns = [ x + " Long Only" for x in data_frame.columns.values] elif name == "ATR": # get all the asset names (assume we have names 'close', 'low', 'high' in the Data) # keep ordering of assets asset_name = list(OrderedDict.fromkeys( [x.split('.')[0] for x in data_frame.columns])) df = [] # can improve the performance of this if vectorise more! for a in asset_name: close = [a + '.close'] low = [a + '.low'] high = [a + '.high'] # if we don't fill NaNs, we need to remove those rows and then # calculate the ATR if not(tech_params.fillna): data_frame_short = data_frame[[close[0], low[0], high[0]]] data_frame_short = data_frame_short.dropna() else: data_frame_short = data_frame prev_close = data_frame_short[close].shift(1) c1 = data_frame_short[high].values - \ data_frame_short[low].values c2 = np.abs(data_frame_short[high].values - prev_close.values) c3 = np.abs(data_frame_short[low].values - prev_close.values) true_range = np.max((c1, c2, c3), axis=0) true_range = pd.DataFrame( index=data_frame_short.index, data=true_range, columns=[ close[0] + ' True Range']) # put back NaNs into ATR if necessary if (not(tech_params.fillna)): true_range = true_range.reindex( data_frame.index, fill_value=np.nan) df.append(true_range) calc = Calculations() true_range = calc.pandas_outer_join(df) self._techind = true_range.rolling( window=tech_params.atr_period, center=False).mean() # self._techind = true_range.ewm(ignore_na=False, span=tech_params.atr_period, min_periods=0, adjust=True).mean() self._techind.columns = [x + ".close ATR" for x in asset_name] elif name in ["VWAP"]: asset_name = list(OrderedDict.fromkeys( [x.split('.')[0] for x in data_frame.columns])) df = [] for a in asset_name: high = [a + '.high'] low = [a + '.low'] close = [a + '.close'] volume = [a + '.volume'] if not tech_params.fillna: df_mod = data_frame[[high[0], low[0], close[0], volume[0]]] df_mod.dropna(inplace=True) else: df_mod = data_frame l = df_mod[low].values h = df_mod[high].values c = df_mod[close].values v = df_mod[volume].values vwap = np.cumsum(((h + l + c) / 3) * v) / np.cumsum(v) vwap = pd.DataFrame(index=df_mod.index, data=vwap, columns=[close[0] + ' VWAP']) print(vwap.columns) if not tech_params.fillna: vwap = vwap.reindex(data_frame.index, fill_value=np.nan) df.append(vwap) calc = Calculations() vwap = calc.pandas_outer_join(df) self._techind = vwap self._techind.columns = [x + ".close VWAP" for x in asset_name] self.create_custom_tech_ind( data_frame_non_nan, name, tech_params, data_frame_non_nan_early) # TODO create other indicators if hasattr(tech_params, 'only_allow_longs'): self._signal[self._signal < 0] = 0 # TODO create other indicators if hasattr(tech_params, 'only_allow_shorts'): self._signal[self._signal > 0] = 0 # apply signal multiplier (typically to flip signals) if hasattr(tech_params, 'signal_mult'): self._signal = self._signal * tech_params.signal_mult if hasattr(tech_params, 'strip_signal_name'): if tech_params.strip_signal_name: self._signal.columns = data_frame.columns return self._techind, self._signal
class MarketDataGenerator(object): _time_series_cache = {} # shared across all instances of object! def __init__(self): self.config = ConfigManager() self.logger = LoggerManager().getLogger(__name__) self.filter = Filter() self.calculations = Calculations() self.io_engine = IOEngine() self._intraday_code = -1 return def flush_cache(self): """ flush_cache - Flushs internal cache of time series """ self._time_series_cache = {} def set_intraday_code(self, code): self._intraday_code = code def get_data_vendor(self, source): """ get_loader - Loads appropriate data service class Parameters ---------- source : str the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc. we can also have forms like "bloomberg-boe" separated by hyphens Returns ------- DataVendor """ data_vendor = None source = source.split("-")[0] if source == 'bloomberg': from findatapy.market.datavendorbbg import DataVendorBBGOpen data_vendor = DataVendorBBGOpen() elif source == 'quandl': from findatapy.market.datavendorweb import DataVendorQuandl data_vendor = DataVendorQuandl() elif source == 'ons': from findatapy.market.datavendorweb import DataVendorONS data_vendor = DataVendorONS() elif source == 'boe': from findatapy.market.datavendorweb import DataVendorBOE data_vendor = DataVendorBOE() elif source == 'dukascopy': from findatapy.market.datavendorweb import DataVendorDukasCopy data_vendor = DataVendorDukasCopy() elif source in [ 'yahoo', 'google', 'fred', 'oecd', 'eurostat', 'edgar-index' ]: from findatapy.market.datavendorweb import DataVendorPandasWeb data_vendor = DataVendorPandasWeb() # TODO add support for other data sources (like Reuters) return data_vendor def fetch_market_data(self, market_data_request, kill_session=True): """ fetch_market_data - Loads time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ tickers = market_data_request.tickers data_vendor = self.get_data_vendor(market_data_request.data_source) # check if tickers have been specified (if not load all of them for a category) # also handle single tickers/list tickers create_tickers = False if tickers is None: create_tickers = True elif isinstance(tickers, str): if tickers == '': create_tickers = True elif isinstance(tickers, list): if tickers == []: create_tickers = True if create_tickers: market_data_request.tickers = self.config.get_tickers_list_for_category( market_data_request.category, market_data_request.data_source, market_data_request.freq, market_data_request.cut) # intraday or tick: only one ticker per cache file if (market_data_request.freq in ['intraday', 'tick', 'second', 'hour', 'minute']): data_frame_agg = self.download_intraday_tick( market_data_request, data_vendor) # daily: multiple tickers per cache file - assume we make one API call to vendor library else: data_frame_agg = self.download_daily(market_data_request, data_vendor) if ('internet_load' in market_data_request.cache_algo): self.logger.debug("Internet loading.. ") # signal to data_vendor template to exit session # if data_vendor is not None and kill_session == True: data_vendor.kill_session() if (market_data_request.cache_algo == 'cache_algo'): self.logger.debug( "Only caching data in memory, do not return any time series.") return # only return time series if specified in the algo if 'return' in market_data_request.cache_algo: # special case for events/events-dt which is not indexed like other tables if market_data_request.category is not None: if 'events' in market_data_request.category: return data_frame_agg try: return self.filter.filter_time_series(market_data_request, data_frame_agg, pad_columns=True) except: import traceback self.logger.error(traceback.format_exc()) return None def get_market_data_cached(self, market_data_request): """ get_time_series_cached - Loads time series from cache (if it exists) Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ if (market_data_request.freq == "intraday"): ticker = market_data_request.tickers else: ticker = None fname = self.create_time_series_hash_key(market_data_request, ticker) if (fname in self._time_series_cache): data_frame = self._time_series_cache[fname] return self.filter.filter_time_series(market_data_request, data_frame) return None def create_time_series_hash_key(self, market_data_request, ticker=None): """ create_time_series_hash_key - Creates a hash key for retrieving the time series Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- str """ if (isinstance(ticker, list)): ticker = ticker[0] return self.create_cache_file_name( self.create_category_key(market_data_request, ticker)) def download_intraday_tick(self, market_data_request, data_vendor): """ download_intraday_tick - Loads intraday time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ data_frame_agg = None calcuations = Calculations() ticker_cycle = 0 data_frame_group = [] # single threaded version # handle intraday ticker calls separately one by one if len(market_data_request.tickers) == 1 or DataConstants( ).market_thread_no['other'] == 1: for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 # we downscale into float32, to avoid memory problems in Python (32 bit) # data is stored on disk as float32 anyway data_frame_single = data_vendor.load_ticker( market_data_request_single) # if the vendor doesn't provide any data, don't attempt to append if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' data_frame_single = data_frame_single.astype('float32') data_frame_group.append(data_frame_single) # # if you call for returning multiple tickers, be careful with memory considerations! # if data_frame_agg is not None: # data_frame_agg = data_frame_agg.join(data_frame_single, how='outer') # else: # data_frame_agg = data_frame_single # key = self.create_category_key(market_data_request, ticker) # fname = self.create_cache_file_name(key) # self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday) # if you call for returning multiple tickers, be careful with memory considerations! if data_frame_group is not None: data_frame_agg = calcuations.pandas_outer_join( data_frame_group) return data_frame_agg else: market_data_request_list = [] # create a list of MarketDataRequests for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if hasattr(market_data_request, 'vendor_tickers'): market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 market_data_request_list.append(market_data_request_single) return self.fetch_group_time_series(market_data_request_list) def fetch_single_time_series(self, market_data_request): data_frame_single = self.get_data_vendor( market_data_request.data_source).load_ticker(market_data_request) if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' # will fail for dataframes which includes dates try: data_frame_single = data_frame_single.astype('float32') except: pass if market_data_request.freq == "second": data_frame_single = data_frame_single.resample("1s") return data_frame_single def fetch_group_time_series(self, market_data_request_list): data_frame_agg = None # depends on the nature of operation as to whether we should use threading or multiprocessing library if DataConstants().market_thread_technique is "thread": from multiprocessing.dummy import Pool else: # most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing # must use the multiprocessing_on_dill library otherwise can't pickle objects correctly # note: currently not very stable from multiprocessing_on_dill import Pool thread_no = DataConstants().market_thread_no['other'] if market_data_request_list[0].data_source in DataConstants( ).market_thread_no: thread_no = DataConstants().market_thread_no[ market_data_request_list[0].data_source] if thread_no > 0: pool = Pool(thread_no) # open the market data downloads in their own threads and return the results result = pool.map_async(self.fetch_single_time_series, market_data_request_list) data_frame_group = result.get() pool.close() pool.join() else: data_frame_group = [] for md_request in market_data_request_list: data_frame_group.append( self.fetch_single_time_series(md_request)) # collect together all the time series if data_frame_group is not None: data_frame_group = [i for i in data_frame_group if i is not None] if data_frame_group is not None: data_frame_agg = self.calculations.pandas_outer_join( data_frame_group) return data_frame_agg def download_daily(self, market_data_request, data_vendor): """ download_daily - Loads daily time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ # daily data does not include ticker in the key, as multiple tickers in the same file if DataConstants().market_thread_no['other'] == 1: data_frame_agg = data_vendor.load_ticker(market_data_request) else: market_data_request_list = [] group_size = int( len(market_data_request.tickers) / DataConstants().market_thread_no['other'] - 1) if group_size == 0: group_size = 1 # split up tickers into groups related to number of threads to call for i in range(0, len(market_data_request.tickers), group_size): market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = market_data_request.tickers[ i:i + group_size] if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = \ market_data_request.vendor_tickers[i:i + group_size] market_data_request_list.append(market_data_request_single) data_frame_agg = self.fetch_group_time_series( market_data_request_list) key = self.create_category_key(market_data_request) fname = self.create_cache_file_name(key) self._time_series_cache[ fname] = data_frame_agg # cache in memory (ok for daily data) return data_frame_agg def create_category_key(self, market_data_request, ticker=None): """ create_category_key - Returns a category key for the associated MarketDataRequest Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- str """ category = 'default-cat' cut = 'default-cut' if market_data_request.category is not None: category = market_data_request.category environment = market_data_request.environment source = market_data_request.data_source freq = market_data_request.freq if market_data_request.cut is not None: cut = market_data_request.cut if (ticker is not None): key = environment + "." + category + '.' + source + '.' + freq + '.' + cut + '.' + ticker else: key = environment + "." + category + '.' + source + '.' + freq + '.' + cut return key def create_cache_file_name(self, filename): return DataConstants().folder_time_series_data + "/" + filename
minute_frequencies = [1, 2, 5, 10, 15, 30, 60] realized_vol = [] for min in minute_frequencies: min_df = pd.DataFrame(intraday_spot_df.resample(str(min) + 'min').last().dropna()) rv = vol_stats.calculate_realized_vol('GBPUSD', spot_df=min_df, tenor_label="ON", freq='intraday', freq_min_mult=min, hour_of_day=10, minute_of_day=0, field='close', timezone_hour_minute='America/New_York') * 100 rv.columns=[str(min) + 'min'] realized_vol.append(rv) realized_vol = calc.pandas_outer_join(realized_vol) style = Style() style.title = 'GBPUSD ON realized volatility over Brexit with different minute sampling frequencies' style.scale_factor = 3 style.source = 'Bloomberg' style.color = 'Blues' # Plot the volatilities with different sampling frequencies chart.plot(realized_vol, style=style) ###### Look at the addon in the ON GBPUSD implied vol around Brexit, note the first month will be empty given the nature # of the model if run_example == 3 or run_example == 0: # Download the whole all market data for GBPUSD for pricing options (vol surface) # Note: 10AM prints for vol no longer published by Bloomberg, so later values are a weighted average of TOK and LDN
class FXForwardsCurve(object): """Constructs continuous forwards time series total return indices from underlying forwards contracts. """ def __init__(self, market_data_generator=None, fx_forwards_trading_tenor=market_constants.fx_forwards_trading_tenor, roll_days_before=market_constants.fx_forwards_roll_days_before, roll_event=market_constants.fx_forwards_roll_event, construct_via_currency='no', fx_forwards_tenor_for_interpolation=market_constants.fx_forwards_tenor_for_interpolation, base_depos_tenor=data_constants.base_depos_tenor, roll_months=market_constants.fx_forwards_roll_months, cum_index=market_constants.fx_forwards_cum_index, output_calculation_fields=market_constants.output_calculation_fields): """Initializes FXForwardsCurve Parameters ---------- market_data_generator : MarketDataGenerator Used for downloading market data fx_forwards_trading_tenor : str What is primary forward contract being used to trade (default - '1M') roll_days_before : int Number of days before roll event to enter into a new forwards contract roll_event : str What constitutes a roll event? ('month-end', 'quarter-end', 'year-end', 'expiry') construct_via_currency : str What currency should we construct the forward via? Eg. if we asked for AUDJPY we can construct it via AUDUSD & JPYUSD forwards, as opposed to AUDJPY forwards (default - 'no') fx_forwards_tenor_for_interpolation : str(list) Which forwards should we use for interpolation base_depos_tenor : str(list) Which base deposits tenors do we need (this is only necessary if we want to start inferring depos) roll_months : int After how many months should we initiate a roll. Typically for trading 1M this should 1, 3M this should be 3 etc. cum_index : str In total return index, do we compute in additive or multiplicative way ('add' or 'mult') output_calculation_fields : bool Also output additional data should forward expiries etc. alongside total returns indices """ self._market_data_generator = market_data_generator self._calculations = Calculations() self._calendar = Calendar() self._filter = Filter() self._fx_forwards_trading_tenor = fx_forwards_trading_tenor self._roll_days_before = roll_days_before self._roll_event = roll_event self._construct_via_currency = construct_via_currency self._fx_forwards_tenor_for_interpolation = fx_forwards_tenor_for_interpolation self._base_depos_tenor = base_depos_tenor self._roll_months = roll_months self._cum_index = cum_index self._output_calcultion_fields = output_calculation_fields def generate_key(self): from findatapy.market.ioengine import SpeedCache # Don't include any "large" objects in the key return SpeedCache().generate_key(self, ['_market_data_generator', '_calculations', '_calendar', '_filter']) def fetch_continuous_time_series(self, md_request, market_data_generator, fx_forwards_trading_tenor=None, roll_days_before=None, roll_event=None, construct_via_currency=None, fx_forwards_tenor_for_interpolation=None, base_depos_tenor=None, roll_months=None, cum_index=None, output_calculation_fields=False): if market_data_generator is None: market_data_generator = self._market_data_generator if fx_forwards_trading_tenor is None: fx_forwards_trading_tenor = self._fx_forwards_trading_tenor if roll_days_before is None: roll_days_before = self._roll_days_before if roll_event is None: roll_event = self._roll_event if construct_via_currency is None: construct_via_currency = self._construct_via_currency if fx_forwards_tenor_for_interpolation is None: fx_forwards_tenor_for_interpolation = self._fx_forwards_tenor_for_interpolation if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor if roll_months is None: roll_months = self._roll_months if cum_index is None: cum_index = self._cum_index if output_calculation_fields is None: output_calculation_fields # Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient forward data for this) if construct_via_currency == 'no': # Download FX spot, FX forwards points and base depos etc. market = Market(market_data_generator=market_data_generator) md_request_download = MarketDataRequest(md_request=md_request) fx_conv = FXConv() # CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data # should be fetched in correct convention md_request_download.tickers = [fx_conv.correct_notation(x) for x in md_request.tickers] md_request_download.category = 'fx-forwards-market' md_request_download.fields = 'close' md_request_download.abstract_curve = None md_request_download.fx_forwards_tenor = fx_forwards_tenor_for_interpolation md_request_download.base_depos_tenor = base_depos_tenor forwards_market_df = market.fetch_market(md_request_download) # Now use the original tickers return self.construct_total_return_index(md_request.tickers, forwards_market_df, fx_forwards_trading_tenor=fx_forwards_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation, roll_months=roll_months, cum_index=cum_index, output_calculation_fields=output_calculation_fields) else: # eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency # Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns) total_return_indices = [] for tick in md_request.tickers: base = tick[0:3] terms = tick[3:6] md_request_base = MarketDataRequest(md_request=md_request) md_request_base.tickers = base + construct_via_currency md_request_terms = MarketDataRequest(md_request=md_request) md_request_terms.tickers = terms + construct_via_currency # Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD) base_vals = self.fetch_continuous_time_series(md_request_base, market_data_generator, fx_forwards_trading_tenor=fx_forwards_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation, base_depos_tenor=base_depos_tenor, roll_months=roll_months, output_calculation_fields=False, cum_index=cum_index, construct_via_currency='no') terms_vals = self.fetch_continuous_time_series(md_request_terms, market_data_generator, fx_forwards_trading_tenor=fx_forwards_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation, base_depos_tenor=base_depos_tenor, roll_months=roll_months, cum_index=cum_index, output_calculation_fields=False, construct_via_currency='no') # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == construct_via_currency + construct_via_currency: base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = -self._calculations.calculate_returns(terms_vals) elif terms + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = self._calculations.calculate_returns(base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns(terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [tick + '-forward-tot.close'] total_return_indices.append(cross_vals) return self._calculations.pandas_outer_join(total_return_indices) def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None): pass def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None, total_return_indices_df=None): pass def get_day_count_conv(self, currency): if currency in market_constants.currencies_with_365_basis: return 365.0 return 360.0 def construct_total_return_index(self, cross_fx, forwards_market_df, fx_forwards_trading_tenor=None, roll_days_before=None, roll_event=None, roll_months=None, fx_forwards_tenor_for_interpolation=None, cum_index=None, output_calculation_fields=False): if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] if fx_forwards_trading_tenor is None: fx_forwards_trading_tenor = self._fx_forwards_trading_tenor if roll_days_before is None: roll_days_before = self._roll_days_before if roll_event is None: roll_event = self._roll_event if roll_months is None: roll_months = self._roll_months if fx_forwards_tenor_for_interpolation is None: fx_forwards_tenor_for_interpolation = self._fx_forwards_tenor_for_interpolation if cum_index is None: cum_index = self._cum_index total_return_index_df_agg = [] # Remove columns where there is no data (because these points typically aren't quoted) forwards_market_df = forwards_market_df.dropna(how='all', axis=1) fx_forwards_pricer = FXForwardsPricer() def get_roll_date(horizon_d, delivery_d, asset_hols, month_adj=1): if roll_event == 'month-end': roll_d = horizon_d + CustomBusinessMonthEnd(roll_months + month_adj, holidays=asset_hols) elif roll_event == 'delivery-date': roll_d = delivery_d return (roll_d - CustomBusinessDay(n=roll_days_before, holidays=asset_hols)) for cross in cross_fx: # Eg. if we specify USDUSD if cross[0:3] == cross[3:6]: total_return_index_df_agg.append( pd.DataFrame(100, index=forwards_market_df.index, columns=[cross + "-forward-tot.close"])) else: # Is the FX cross in the correct convention old_cross = cross cross = FXConv().correct_notation(cross) horizon_date = forwards_market_df.index delivery_date = [] roll_date = [] new_trade = np.full(len(horizon_date), False, dtype=bool) asset_holidays = self._calendar.get_holidays(cal=cross) # Get first delivery date delivery_date.append( self._calendar.get_delivery_date_from_horizon_date(horizon_date[0], fx_forwards_trading_tenor, cal=cross, asset_class='fx')[0]) # For first month want it to expire within that month (for consistency), hence month_adj=0 ONLY here roll_date.append(get_roll_date(horizon_date[0], delivery_date[0], asset_holidays, month_adj=0)) # New trade => entry at beginning AND on every roll new_trade[0] = True # Get all the delivery dates and roll dates # At each "roll/trade" day we need to reset them for the new contract for i in range(1, len(horizon_date)): # If the horizon date has reached the roll date (from yesterday), we're done, and we have a # new roll/trade if (horizon_date[i] - roll_date[i-1]).days == 0: new_trade[i] = True # else: # new_trade[i] = False # If we're entering a new trade/contract, we need to get new delivery and roll dates if new_trade[i]: delivery_date.append(self._calendar.get_delivery_date_from_horizon_date(horizon_date[i], fx_forwards_trading_tenor, cal=cross, asset_class='fx')[0]) roll_date.append(get_roll_date(horizon_date[i], delivery_date[i], asset_holidays)) else: # Otherwise use previous delivery and roll dates, because we're still holding same contract delivery_date.append(delivery_date[i-1]) roll_date.append(roll_date[i-1]) interpolated_forward = fx_forwards_pricer.price_instrument(cross, horizon_date, delivery_date, market_df=forwards_market_df, fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation)[cross + '-interpolated-outright-forward.close'].values # To record MTM prices mtm = np.copy(interpolated_forward) # Note: may need to add discount factor when marking to market forwards? # Special case: for very first trading day # mtm[0] = interpolated_forward[0] # On rolling dates, MTM will be the previous forward contract (interpolated) # otherwise it will be the current forward contract for i in range(1, len(horizon_date)): if new_trade[i]: mtm[i] = fx_forwards_pricer.price_instrument(cross, horizon_date[i], delivery_date[i-1], market_df=forwards_market_df, fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation) \ [cross + '-interpolated-outright-forward.close'].values # else: # mtm[i] = interpolated_forward[i] # Eg. if we asked for USDEUR, we first constructed spot/forwards for EURUSD # and then need to invert it if old_cross != cross: mtm = 1.0 / mtm interpolated_forward = 1.0 / interpolated_forward forward_rets = mtm / np.roll(interpolated_forward, 1) - 1.0 forward_rets[0] = 0 if cum_index == 'mult': cum_rets = 100 * np.cumprod(1.0 + forward_rets) elif cum_index == 'add': cum_rets = 100 + 100 * np.cumsum(forward_rets) total_return_index_df = pd.DataFrame(index=horizon_date, columns=[cross + "-forward-tot.close"]) total_return_index_df[cross + "-forward-tot.close"] = cum_rets if output_calculation_fields: total_return_index_df[cross + '-interpolated-outright-forward.close'] = interpolated_forward total_return_index_df[cross + '-mtm.close'] = mtm total_return_index_df[cross + '-roll.close'] = new_trade total_return_index_df[cross + '.roll-date'] = roll_date total_return_index_df[cross + '.delivery-date'] = delivery_date total_return_index_df[cross + '-forward-return.close'] = forward_rets total_return_index_df_agg.append(total_return_index_df) return self._calculations.pandas_outer_join(total_return_index_df_agg)
# Get Bloomberg calculated total return indices (for spot) md_request.category = 'fx-tot' md_request.cut = 'NYC' df_bbg_tot = market.fetch_market(md_request) df_bbg_tot.columns = [x + '-bbg' for x in df_bbg_tot.columns] # Calculate a hedged portfolio of spot + 2*options (can we reduce drawdowns?) calculations = Calculations() ret_stats = RetStats() df_hedged = calculations.pandas_outer_join([ df_bbg_tot[cross + '-tot.close-bbg'].to_frame(), df_cuemacro_option_put_tc[cross + '-option-tot-with-tc.close'].to_frame() ]) df_hedged = df_hedged.fillna(method='ffill') df_hedged = df_hedged.pct_change() df_hedged['Spot + 2*option hedge'] = df_hedged[ cross + '-tot.close-bbg'] + df_hedged[cross + '-option-tot-with-tc.close'] df_hedged.columns = RetStats(returns_df=df_hedged, ann_factor=252).summary() # Plot everything chart.plot( calculations.create_mult_index_from_prices(
def download_daily(self, market_data_request): trials = 0 data_frame_list = [] data_frame_release = [] # TODO refactor this code, a bit messy at the moment! for i in range(0, len(market_data_request.tickers)): while (trials < 5): try: fred = Fred(api_key=DataConstants().fred_api_key) # acceptable fields: close, actual-release, release-date-time-full if 'close' in market_data_request.fields and 'release-date-time-full' in market_data_request.fields: data_frame = fred.get_series_all_releases(market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame.columns = ['Date', market_data_request.tickers[i] + '.release-date-time-full', market_data_request.tickers[i] + '.close'] data_frame = data_frame.sort_values(by=['Date', market_data_request.tickers[i] + '.release-date-time-full']) data_frame = data_frame.drop_duplicates(subset=['Date'], keep='last') data_frame = data_frame.set_index(['Date']) filter = Filter() data_frame = filter.filter_time_series_by_date(market_data_request.start_date, market_data_request.finish_date, data_frame) data_frame_list.append(data_frame) elif 'close' in market_data_request.fields: data_frame = fred.get_series(series_id=market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame = pandas.DataFrame(data_frame) data_frame.columns = [market_data_request.tickers[i] + '.close'] data_frame_list.append(data_frame) if 'first-revision' in market_data_request.fields: data_frame = fred.get_series_first_revision(market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame = pandas.DataFrame(data_frame) data_frame.columns = [market_data_request.tickers[i] + '.first-revision'] filter = Filter() data_frame = filter.filter_time_series_by_date(market_data_request.start_date, market_data_request.finish_date, data_frame) data_frame_list.append(data_frame) if 'actual-release' in market_data_request.fields and 'release-date-time-full' in market_data_request.fields: data_frame = fred.get_series_all_releases(market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame.columns = ['Date', market_data_request.tickers[i] + '.release-date-time-full', market_data_request.tickers[i] + '.actual-release'] data_frame = data_frame.sort_values(by=['Date', market_data_request.tickers[i] + '.release-date-time-full']) data_frame = data_frame.drop_duplicates(subset=['Date'], keep='first') data_frame = data_frame.set_index(['Date']) filter = Filter() data_frame = filter.filter_time_series_by_date(market_data_request.start_date, market_data_request.finish_date, data_frame) data_frame_list.append(data_frame) elif 'actual-release' in market_data_request.fields: data_frame = fred.get_series_first_release(market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame = pandas.DataFrame(data_frame) data_frame.columns = [market_data_request.tickers[i] + '.actual-release'] filter = Filter() data_frame = filter.filter_time_series_by_date(market_data_request.start_date, market_data_request.finish_date, data_frame) data_frame_list.append(data_frame) elif 'release-date-time-full' in market_data_request.fields: data_frame = fred.get_series_all_releases(market_data_request.tickers[i], observation_start=market_data_request.start_date, observation_end=market_data_request.finish_date) data_frame = data_frame['realtime_start'] data_frame = pandas.DataFrame(data_frame) data_frame.columns = [market_data_request.tickers[i] + '.release-date-time-full'] data_frame.index = data_frame[market_data_request.tickers[i] + '.release-date-time-full'] data_frame = data_frame.sort() data_frame = data_frame.drop_duplicates() filter = Filter() data_frame_release.append(filter.filter_time_series_by_date(market_data_request.start_date, market_data_request.finish_date, data_frame)) break except: trials = trials + 1 self.logger.info("Attempting... " + str(trials) + " request to download from ALFRED/FRED") if trials == 5: self.logger.error("Couldn't download from ALFRED/FRED after several attempts!") calc = Calculations() data_frame1 = calc.pandas_outer_join(data_frame_list) data_frame2 = calc.pandas_outer_join(data_frame_release) data_frame = pandas.concat([data_frame1, data_frame2], axis=1) return data_frame
class FXCrossFactory(object): def __init__(self, market_data_generator=None): self.logger = LoggerManager().getLogger(__name__) self.fxconv = FXConv() self.cache = {} self.calculations = Calculations() self.market_data_generator = market_data_generator return def flush_cache(self): self.cache = {} def get_fx_cross_tick(self, start, end, cross, cut="NYC", source="dukascopy", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['bid', 'ask']): if isinstance(cross, str): cross = [cross] market_data_request = MarketDataRequest( gran_freq="tick", freq_mult=1, freq='tick', cut=cut, fields=['bid', 'ask', 'bidv', 'askv'], cache_algo=cache_algo, environment=environment, start_date=start, finish_date=end, data_source=source, category='fx') market_data_generator = self.market_data_generator data_frame_agg = None for cr in cross: if (type == 'spot'): market_data_request.tickers = cr cross_vals = market_data_generator.fetch_market_data( market_data_request) # if user only wants 'close' calculate that from the bid/ask fields if fields == ['close']: cross_vals = cross_vals[[cr + '.bid', cr + '.ask']].mean(axis=1) cross_vals.columns = [cr + '.close'] if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def get_fx_cross(self, start, end, cross, cut="NYC", source="bloomberg", freq="intraday", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['close']): if source == "gain" or source == 'dukascopy' or freq == 'tick': return self.get_fx_cross_tick(start, end, cross, cut=cut, source=source, cache_algo=cache_algo, type='spot', fields=fields) if isinstance(cross, str): cross = [cross] market_data_request_list = [] freq_list = [] type_list = [] for cr in cross: market_data_request = MarketDataRequest(freq_mult=1, cut=cut, fields=['close'], freq=freq, cache_algo=cache_algo, start_date=start, finish_date=end, data_source=source, environment=environment) market_data_request.type = type market_data_request.cross = cr if freq == 'intraday': market_data_request.gran_freq = "minute" # intraday elif freq == 'daily': market_data_request.gran_freq = "daily" # daily market_data_request_list.append(market_data_request) data_frame_agg = [] # depends on the nature of operation as to whether we should use threading or multiprocessing library if DataConstants().market_thread_technique is "thread": from multiprocessing.dummy import Pool else: # most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing # must use the multiprocessing_on_dill library otherwise can't pickle objects correctly # note: currently not very stable from multiprocessing_on_dill import Pool thread_no = DataConstants().market_thread_no['other'] if market_data_request_list[0].data_source in DataConstants( ).market_thread_no: thread_no = DataConstants().market_thread_no[ market_data_request_list[0].data_source] # fudge, issue with multithreading and accessing HDF5 files # if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator': # thread_no = 0 if (thread_no > 0): pool = Pool(thread_no) # open the market data downloads in their own threads and return the results result = pool.map_async(self._get_individual_fx_cross, market_data_request_list) data_frame_agg = self.calculations.iterative_outer_join( result.get()) # data_frame_agg = self.calculations.pandas_outer_join(result.get()) # pool would have already been closed earlier # try: # pool.close() # pool.join() # except: pass else: for md_request in market_data_request_list: data_frame_agg.append( self._get_individual_fx_cross(md_request)) data_frame_agg = self.calculations.pandas_outer_join( data_frame_agg) # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def _get_individual_fx_cross(self, market_data_request): cr = market_data_request.cross type = market_data_request.type freq = market_data_request.freq base = cr[0:3] terms = cr[3:6] if (type == 'spot'): # non-USD crosses if base != 'USD' and terms != 'USD': base_USD = self.fxconv.correct_notation('USD' + base) terms_USD = self.fxconv.correct_notation('USD' + terms) # TODO check if the cross exists in the database # download base USD cross market_data_request.tickers = base_USD market_data_request.category = 'fx' if base_USD + '.close' in self.cache: base_vals = self.cache[base_USD + '.close'] else: base_vals = self.market_data_generator.fetch_market_data( market_data_request) self.cache[base_USD + '.close'] = base_vals # download terms USD cross market_data_request.tickers = terms_USD market_data_request.category = 'fx' if terms_USD + '.close' in self.cache: terms_vals = self.cache[terms_USD + '.close'] else: terms_vals = self.market_data_generator.fetch_market_data( market_data_request) self.cache[terms_USD + '.close'] = terms_vals # if quoted USD/base flip to get USD terms if (base_USD[0:3] == 'USD'): if 'USD' + base in '.close' in self.cache: base_vals = self.cache['USD' + base + '.close'] else: base_vals = 1 / base_vals self.cache['USD' + base + '.close'] = base_vals # if quoted USD/terms flip to get USD terms if (terms_USD[0:3] == 'USD'): if 'USD' + terms in '.close' in self.cache: terms_vals = self.cache['USD' + terms + '.close'] else: terms_vals = 1 / terms_vals self.cache['USD' + terms + '.close'] = base_vals base_vals.columns = ['temp'] terms_vals.columns = ['temp'] cross_vals = base_vals.div(terms_vals, axis='index') cross_vals.columns = [cr + '.close'] base_vals.columns = [base_USD + '.close'] terms_vals.columns = [terms_USD + '.close'] else: # if base == 'USD': non_USD = terms # if terms == 'USD': non_USD = base correct_cr = self.fxconv.correct_notation(cr) market_data_request.tickers = correct_cr market_data_request.category = 'fx' if correct_cr + '.close' in self.cache: cross_vals = self.cache[correct_cr + '.close'] else: cross_vals = self.market_data_generator.fetch_market_data( market_data_request) # flip if not convention if (correct_cr != cr): if cr + '.close' in self.cache: cross_vals = self.cache[cr + '.close'] else: cross_vals = 1 / cross_vals self.cache[cr + '.close'] = cross_vals self.cache[correct_cr + '.close'] = cross_vals # cross_vals = self.market_data_generator.harvest_time_series(market_data_request) cross_vals.columns.names = [cr + '.close'] elif type[0:3] == "tot": if freq == 'daily': # download base USD cross market_data_request.tickers = base + 'USD' market_data_request.category = 'fx-tot' if type == "tot": base_vals = self.market_data_generator.fetch_market_data( market_data_request) else: x = 0 # download terms USD cross market_data_request.tickers = terms + 'USD' market_data_request.category = 'fx-tot' if type == "tot": terms_vals = self.market_data_generator.fetch_market_data( market_data_request) else: pass base_rets = self.calculations.calculate_returns(base_vals) terms_rets = self.calculations.calculate_returns(terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # first returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self.calculations.create_mult_index(cross_rets) cross_vals.columns = [cr + '-tot.close'] elif freq == 'intraday': self.logger.info( 'Total calculated returns for intraday not implemented yet' ) return None return cross_vals
cache_algo='cache_algo_return', abstract_curve=FXSpotCurve(construct_via_currency='USD', depo_tenor='ON')) df_tot = market.fetch_market(md_request=md_request) df_tot.columns = [x + '-tot-cuemacro' for x in df_tot.columns] # Get spot data md_request.abstract_curve = None df_spot = market.fetch_market(md_request=md_request) df_spot.columns = [x + '-spot' for x in df_spot.columns] # Get Bloomberg calculated total return indices (for spot) md_request.category = 'fx-tot' df_bbg_tot = market.fetch_market(md_request) df_bbg_tot.columns = [x + '-bbg' for x in df_bbg_tot.columns] # Get Bloomberg calculated total return indices (for 1M forwards rolled) md_request.category = 'fx-tot-forwards' df_bbg_tot_forwards = market.fetch_market(md_request) df_bbg_tot_forwards.columns = [x + '-bbg' for x in df_bbg_tot_forwards.columns] # Combine into a single data frame and plot, we note that the Cuemacro constructed indices track the Bloomberg # indices relatively well (both from spot and 1M forwards). Also note the large difference with spot indices # CAREFUL to fill down, before reindexing because 1M forwards indices are likely to have different publishing dates df = calculations.pandas_outer_join([df_tot, df_bbg_tot, df_spot, df_bbg_tot_forwards]).fillna(method='ffill') df = calculations.create_mult_index_from_prices(df) chart.plot(df)
class FXSpotCurve(object): """Construct total return (spot) indices for FX. In future will also convert assets from local currency to foreign currency denomination and construct indices from forwards series. """ def __init__(self, market_data_generator=None, depo_tenor=market_constants.spot_depo_tenor, construct_via_currency='no', output_calculation_fields=market_constants. output_calculation_fields): self._market_data_generator = market_data_generator self._calculations = Calculations() self._depo_tenor = depo_tenor self._construct_via_currency = construct_via_currency self._output_calculation_fields = output_calculation_fields def generate_key(self): from findatapy.market.ioengine import SpeedCache # Don't include any "large" objects in the key return SpeedCache().generate_key( self, ['_market_data_generator', '_calculations']) def fetch_continuous_time_series(self, md_request, market_data_generator, depo_tenor=None, construct_via_currency=None, output_calculation_fields=None): if market_data_generator is None: market_data_generator = self._market_data_generator if depo_tenor is None: depo_tenor = self._depo_tenor if construct_via_currency is None: construct_via_currency = self._construct_via_currency if output_calculation_fields is None: output_calculation_fields = self._output_calculation_fields # Eg. we construct AUDJPY via AUDJPY directly if construct_via_currency == 'no': base_depo_tickers = [ x[0:3] + self._depo_tenor for x in md_request.tickers ] terms_depo_tickers = [ x[3:6] + self._depo_tenor for x in md_request.tickers ] depo_tickers = list(set(base_depo_tickers + terms_depo_tickers)) market = Market(market_data_generator=market_data_generator) # Deposit data for base and terms currency md_request_download = MarketDataRequest(md_request=md_request) md_request_download.tickers = depo_tickers md_request_download.category = 'base-depos' md_request_download.fields = 'close' md_request_download.abstract_curve = None depo_df = market.fetch_market(md_request_download) # Spot data md_request_download.tickers = md_request.tickers md_request_download.category = 'fx' spot_df = market.fetch_market(md_request_download) return self.construct_total_return_index( md_request.tickers, self._calculations.pandas_outer_join([spot_df, depo_df]), depo_tenor=depo_tenor, output_calculation_fields=output_calculation_fields) else: # eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency # Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns) total_return_indices = [] for tick in md_request.tickers: base = tick[0:3] terms = tick[3:6] md_request_base = MarketDataRequest(md_request=md_request) md_request_base.tickers = base + construct_via_currency md_request_terms = MarketDataRequest(md_request=md_request) md_request_terms.tickers = terms + construct_via_currency base_vals = self.fetch_continuous_time_series( md_request_base, market_data_generator, construct_via_currency='no') terms_vals = self.fetch_continuous_time_series( md_request_terms, market_data_generator, construct_via_currency='no') # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == construct_via_currency + construct_via_currency: base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [tick + '-tot.close'] total_return_indices.append(cross_vals) return self._calculations.pandas_outer_join(total_return_indices) def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None): pass def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None, total_return_indices_df=None): pass def get_day_count_conv(self, currency): if currency in market_constants.currencies_with_365_basis: return 365.0 return 360.0 def construct_total_return_index(self, cross_fx, market_df, depo_tenor=None, output_calculation_fields=False): """Creates total return index for selected FX crosses from spot and deposit data Parameters ---------- cross_fx : String Crosses to construct total return indices (can be a list) tenor : String Tenor of deposit rates to use to compute carry (typically ON for spot) spot_df : pd.DataFrame Spot data (must include crosses we select) deposit_df : pd.DataFrame Deposit data Returns ------- pd.DataFrame """ if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] if depo_tenor is None: depo_tenor = self._depo_tenor total_return_index_df_agg = [] for cross in cross_fx: # Get the spot series, base deposit base_deposit = market_df[cross[0:3] + depo_tenor + ".close"].to_frame() terms_deposit = market_df[cross[3:6] + depo_tenor + ".close"].to_frame() # Eg. if we specify USDUSD if cross[0:3] == cross[3:6]: total_return_index_df_agg.append( pd.DataFrame(100, index=base_deposit.index, columns=[cross + "-tot.close"])) else: carry = base_deposit.join(terms_deposit, how='inner') spot = market_df[cross + ".close"].to_frame() base_daycount = self.get_day_count_conv(cross[0:3]) terms_daycount = self.get_day_count_conv(cross[4:6]) # Align the base & terms deposits series to spot (this should already be done by construction) # spot, carry = spot.align(carry, join='left', axis=0) # Sometimes depo data can be patchy, ok to fill down, given not very volatile (don't do this with spot!) carry = carry.fillna(method='ffill') / 100.0 # In case there are values missing at start of list (fudge for old data!) carry = carry.fillna(method='bfill') spot = spot[cross + ".close"].to_frame() spot_vals = spot[cross + ".close"].values base_deposit_vals = carry[cross[0:3] + depo_tenor + ".close"].values terms_deposit_vals = carry[cross[3:6] + depo_tenor + ".close"].values # Calculate the time difference between each data point (flooring it to whole days, because carry # is accured when there's a new day) spot['index_col'] = spot.index.floor('D') time = spot['index_col'].diff() spot = spot.drop('index_col', 1) time_diff = time.values.astype( float) / 86400000000000.0 # get time difference in days time_diff[0] = 0.0 # Use Numba to do total return index calculation given has many loops total_return_index_df = pd.DataFrame( index=spot.index, columns=[cross + "-tot.close"], data=_spot_index_numba(spot_vals, time_diff, base_deposit_vals, terms_deposit_vals, base_daycount, terms_daycount)) if output_calculation_fields: total_return_index_df[cross + '-carry.close'] = carry total_return_index_df[ cross + '-tot-return.close'] = total_return_index_df / total_return_index_df.shift( 1) - 1.0 total_return_index_df[ cross + '-spot-return.close'] = spot / spot.shift(1) - 1.0 total_return_index_df_agg.append(total_return_index_df) return self._calculations.pandas_outer_join(total_return_index_df_agg)
# Get Bloomberg calculated total return indices (for 1M forwards rolled) md_request.category = 'fx-tot-forwards' df_bbg_tot_forwards = market.fetch_market(md_request) df_bbg_tot_forwards.columns = [ x + '-bbg' for x in df_bbg_tot_forwards.columns ] # Combine into a single data frame and plot, we note that the Cuemacro constructed indices track the Bloomberg # indices relatively well (both from spot and forwards). Also note the large difference with spot indices # CAREFUL to fill down, before reindexing because forwards indices are likely to have different publishing dates df = calculations.pandas_outer_join([ pd.DataFrame(df_cuemacro_tot_1M[cross + '-forward-tot-1M-cuemacro.close']), pd.DataFrame(df_cuemacro_tot_3M[cross + '-forward-tot-3M-cuemacro.close']), df_bbg_tot, df_spot, df_bbg_tot_forwards ]).fillna(method='ffill') df = calculations.create_mult_index_from_prices(df) chart.plot(df) ###### Create total return indices plot for AUDJPY using the underlying USD legs (ie. AUDUSD & JPYUSD) if run_example == 2 or run_example == 0: cross = 'AUDJPY' # Download more tenors fx_forwards_tenors = ['1W', '1M', '2M', '3M']
class FXOptionsCurve(object): """Constructs continuous forwards time series total return indices from underlying forwards contracts. """ def __init__( self, market_data_generator=None, fx_options_trading_tenor=market_constants.fx_options_trading_tenor, roll_days_before=market_constants.fx_options_roll_days_before, roll_event=market_constants.fx_options_roll_event, construct_via_currency='no', fx_options_tenor_for_interpolation=market_constants. fx_options_tenor_for_interpolation, base_depos_tenor=data_constants.base_depos_tenor, roll_months=market_constants.fx_options_roll_months, cum_index=market_constants.fx_options_cum_index, strike=market_constants.fx_options_index_strike, contract_type=market_constants.fx_options_index_contract_type, premium_output=market_constants.fx_options_index_premium_output, position_multiplier=1, depo_tenor_for_option=market_constants.fx_options_depo_tenor, freeze_implied_vol=market_constants.fx_options_freeze_implied_vol, tot_label='', cal=None, output_calculation_fields=market_constants. output_calculation_fields): """Initializes FXForwardsCurve Parameters ---------- market_data_generator : MarketDataGenerator Used for downloading market data fx_options_trading_tenor : str What is primary forward contract being used to trade (default - '1M') roll_days_before : int Number of days before roll event to enter into a new forwards contract roll_event : str What constitutes a roll event? ('month-end', 'quarter-end', 'year-end', 'expiry') cum_index : str In total return index, do we compute in additive or multiplicative way ('add' or 'mult') construct_via_currency : str What currency should we construct the forward via? Eg. if we asked for AUDJPY we can construct it via AUDUSD & JPYUSD forwards, as opposed to AUDJPY forwards (default - 'no') fx_options_tenor_for_interpolation : str(list) Which forwards should we use for interpolation base_depos_tenor : str(list) Which base deposits tenors do we need (this is only necessary if we want to start inferring depos) roll_months : int After how many months should we initiate a roll. Typically for trading 1M this should 1, 3M this should be 3 etc. tot_label : str Postfix for the total returns field cal : str Calendar to use for expiry (if None, uses that of FX pair) output_calculation_fields : bool Also output additional data should forward expiries etc. alongside total returns indices """ self._market_data_generator = market_data_generator self._calculations = Calculations() self._calendar = Calendar() self._filter = Filter() self._fx_options_trading_tenor = fx_options_trading_tenor self._roll_days_before = roll_days_before self._roll_event = roll_event self._construct_via_currency = construct_via_currency self._fx_options_tenor_for_interpolation = fx_options_tenor_for_interpolation self._base_depos_tenor = base_depos_tenor self._roll_months = roll_months self._cum_index = cum_index self._contact_type = contract_type self._strike = strike self._premium_output = premium_output self._position_multiplier = position_multiplier self._depo_tenor_for_option = depo_tenor_for_option self._freeze_implied_vol = freeze_implied_vol self._tot_label = tot_label self._cal = cal self._output_calculation_fields = output_calculation_fields def generate_key(self): from findatapy.market.ioengine import SpeedCache # Don't include any "large" objects in the key return SpeedCache().generate_key(self, [ '_market_data_generator', '_calculations', '_calendar', '_filter' ]) def fetch_continuous_time_series(self, md_request, market_data_generator, fx_options_trading_tenor=None, roll_days_before=None, roll_event=None, construct_via_currency=None, fx_options_tenor_for_interpolation=None, base_depos_tenor=None, roll_months=None, cum_index=None, strike=None, contract_type=None, premium_output=None, position_multiplier=None, depo_tenor_for_option=None, freeze_implied_vol=None, tot_label=None, cal=None, output_calculation_fields=None): if market_data_generator is None: market_data_generator = self._market_data_generator if fx_options_trading_tenor is None: fx_options_trading_tenor = self._fx_options_trading_tenor if roll_days_before is None: roll_days_before = self._roll_days_before if roll_event is None: roll_event = self._roll_event if construct_via_currency is None: construct_via_currency = self._construct_via_currency if fx_options_tenor_for_interpolation is None: fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor if roll_months is None: roll_months = self._roll_months if strike is None: strike = self._strike if contract_type is None: contract_type = self._contact_type if premium_output is None: premium_output = self._premium_output if position_multiplier is None: position_multiplier = self._position_multiplier if depo_tenor_for_option is None: depo_tenor_for_option = self._depo_tenor_for_option if freeze_implied_vol is None: freeze_implied_vol = self._freeze_implied_vol if tot_label is None: tot_label = self._tot_label if cal is None: cal = self._cal if output_calculation_fields is None: output_calculation_fields = self._output_calculation_fields # Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient options/forward data for this) if construct_via_currency == 'no': # Download FX spot, FX forwards points and base depos etc. market = Market(market_data_generator=market_data_generator) md_request_download = MarketDataRequest(md_request=md_request) fx_conv = FXConv() # CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data # should be fetched in correct convention md_request_download.tickers = [ fx_conv.correct_notation(x) for x in md_request.tickers ] md_request_download.category = 'fx-vol-market' md_request_download.fields = 'close' md_request_download.abstract_curve = None md_request_download.fx_options_tenor = fx_options_tenor_for_interpolation md_request_download.base_depos_tenor = base_depos_tenor # md_request_download.base_depos_currencies = [] forwards_market_df = market.fetch_market(md_request_download) # Now use the original tickers return self.construct_total_return_index( md_request.tickers, forwards_market_df, fx_options_trading_tenor=fx_options_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_options_tenor_for_interpolation= fx_options_tenor_for_interpolation, roll_months=roll_months, cum_index=cum_index, strike=strike, contract_type=contract_type, premium_output=premium_output, position_multiplier=position_multiplier, freeze_implied_vol=freeze_implied_vol, depo_tenor_for_option=depo_tenor_for_option, tot_label=tot_label, cal=cal, output_calculation_fields=output_calculation_fields) else: # eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency # Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns) total_return_indices = [] for tick in md_request.tickers: base = tick[0:3] terms = tick[3:6] md_request_base = MarketDataRequest(md_request=md_request) md_request_base.tickers = base + construct_via_currency md_request_terms = MarketDataRequest(md_request=md_request) md_request_terms.tickers = terms + construct_via_currency # Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD) base_vals = self.fetch_continuous_time_series( md_request_base, market_data_generator, fx_options_trading_tenor=fx_options_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_options_tenor_for_interpolation= fx_options_tenor_for_interpolation, base_depos_tenor=base_depos_tenor, roll_months=roll_months, cum_index=cum_index, strike=strike, contract_type=contract_type, premium_output=premium_output, position_multiplier=position_multiplier, depo_tenor_for_option=depo_tenor_for_option, freeze_implied_vol=freeze_implied_vol, tot_label=tot_label, cal=cal, output_calculation_fields=output_calculation_fields, construct_via_currency='no') terms_vals = self.fetch_continuous_time_series( md_request_terms, market_data_generator, fx_options_trading_tenor=fx_options_trading_tenor, roll_days_before=roll_days_before, roll_event=roll_event, fx_options_tenor_for_interpolation= fx_options_tenor_for_interpolation, base_depos_tenor=base_depos_tenor, roll_months=roll_months, cum_index=cum_index, strike=strike, contract_type=contract_type, position_multiplier=position_multiplier, depo_tenor_for_option=depo_tenor_for_option, freeze_implied_vol=freeze_implied_vol, tot_label=tot_label, cal=cal, output_calculation_fields=output_calculation_fields, construct_via_currency='no') # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == construct_via_currency + construct_via_currency: base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + construct_via_currency == construct_via_currency + construct_via_currency: cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [tick + '-option-tot.close'] total_return_indices.append(cross_vals) return self._calculations.pandas_outer_join(total_return_indices) def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None): pass def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None, total_return_indices_df=None): pass def get_day_count_conv(self, currency): if currency in market_constants.currencies_with_365_basis: return 365.0 return 360.0 def construct_total_return_index(self, cross_fx, market_df, fx_options_trading_tenor=None, roll_days_before=None, roll_event=None, roll_months=None, cum_index=None, strike=None, contract_type=None, premium_output=None, position_multiplier=None, fx_options_tenor_for_interpolation=None, freeze_implied_vol=None, depo_tenor_for_option=None, tot_label=None, cal=None, output_calculation_fields=None): if fx_options_trading_tenor is None: fx_options_trading_tenor = self._fx_options_trading_tenor if roll_days_before is None: roll_days_before = self._roll_days_before if roll_event is None: roll_event = self._roll_event if roll_months is None: roll_months = self._roll_months if cum_index is None: cum_index = self._cum_index if strike is None: strike = self._strike if contract_type is None: contract_type = self._contact_type if premium_output is None: premium_output = self._premium_output if position_multiplier is None: position_multiplier = self._position_multiplier if fx_options_tenor_for_interpolation is None: fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation if freeze_implied_vol is None: freeze_implied_vol = self._freeze_implied_vol if depo_tenor_for_option is None: depo_tenor_for_option = self._depo_tenor_for_option if tot_label is None: tot_label = self._tot_label if cal is None: cal = self._cal if output_calculation_fields is None: output_calculation_fields = self._output_calculation_fields if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] total_return_index_df_agg = [] # Remove columns where there is no data (because these vols typically aren't quoted) market_df = market_df.dropna(how='all', axis=1) fx_options_pricer = FXOptionsPricer(premium_output=premium_output) def get_roll_date(horizon_d, expiry_d, asset_hols, month_adj=0): if roll_event == 'month-end': roll_d = horizon_d + CustomBusinessMonthEnd( roll_months + month_adj, holidays=asset_hols) # Special case so always rolls on month end date, if specify 0 days if roll_days_before > 0: return (roll_d - CustomBusinessDay(n=roll_days_before, holidays=asset_hols)) elif roll_event == 'expiry-date': roll_d = expiry_d # Special case so always rolls on expiry date, if specify 0 days if roll_days_before > 0: return (roll_d - CustomBusinessDay(n=roll_days_before, holidays=asset_hols)) return roll_d for cross in cross_fx: if cal is None: cal = cross # Eg. if we specify USDUSD if cross[0:3] == cross[3:6]: total_return_index_df_agg.append( pd.DataFrame(100, index=market_df.index, columns=[cross + "-option-tot.close"])) else: # Is the FX cross in the correct convention old_cross = cross cross = FXConv().correct_notation(cross) # TODO also specification of non-standard crosses like USDGBP if old_cross != cross: pass fx_vol_surface = FXVolSurface( market_df=market_df, asset=cross, tenors=fx_options_tenor_for_interpolation, depo_tenor=depo_tenor_for_option) horizon_date = market_df.index expiry_date = [] roll_date = [] new_trade = np.full(len(horizon_date), False, dtype=bool) asset_holidays = self._calendar.get_holidays(cal=cross) # Get first expiry date expiry_date.append( self._calendar.get_expiry_date_from_horizon_date( pd.DatetimeIndex([horizon_date[0]]), fx_options_trading_tenor, cal=cal, asset_class='fx-vol')[0]) # For first month want it to expire within that month (for consistency), hence month_adj=0 ONLY here roll_date.append( get_roll_date(horizon_date[0], expiry_date[0], asset_holidays, month_adj=0)) # New trade => entry at beginning AND on every roll new_trade[0] = True # Get all the expiry dates and roll dates # At each "roll/trade" day we need to reset them for the new contract for i in range(1, len(horizon_date)): # If the horizon date has reached the roll date (from yesterday), we're done, and we have a # new roll/trade if (horizon_date[i] - roll_date[i - 1]).days >= 0: new_trade[i] = True else: new_trade[i] = False # If we're entering a new trade/contract, we need to get new expiry and roll dates if new_trade[i]: exp = self._calendar.get_expiry_date_from_horizon_date( pd.DatetimeIndex([horizon_date[i]]), fx_options_trading_tenor, cal=cal, asset_class='fx-vol')[0] # Make sure we don't expire on a date in the history where there isn't market data # It is ok for future values to expire after market data (just not in the backtest!) if exp not in market_df.index: exp_index = market_df.index.searchsorted(exp) if exp_index < len(market_df.index): exp_index = min(exp_index, len(market_df.index)) exp = market_df.index[exp_index] expiry_date.append(exp) roll_date.append( get_roll_date(horizon_date[i], expiry_date[i], asset_holidays)) else: # Otherwise use previous expiry and roll dates, because we're still holding same contract expiry_date.append(expiry_date[i - 1]) roll_date.append(roll_date[i - 1]) # Note: may need to add discount factor when marking to market option mtm = np.zeros(len(horizon_date)) calculated_strike = np.zeros(len(horizon_date)) interpolated_option = np.zeros(len(horizon_date)) implied_vol = np.zeros(len(horizon_date)) delta = np.zeros(len(horizon_date)) # For debugging df_temp = pd.DataFrame() df_temp['expiry-date'] = expiry_date df_temp['horizon-date'] = horizon_date df_temp['roll-date'] = roll_date # Special case: for first day of history (given have no previous positions) option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \ fx_options_pricer.price_instrument(cross, horizon_date[0], strike, expiry_date[0], contract_type=contract_type, tenor=fx_options_trading_tenor, fx_vol_surface=fx_vol_surface, return_as_df=False) interpolated_option[0] = option_values_ calculated_strike[0] = strike_ implied_vol[0] = vol_ mtm[0] = 0 # Now price options for rest of history # On rolling dates: MTM will be the previous option contract (interpolated) # On non-rolling dates: it will be the current option contract for i in range(1, len(horizon_date)): if new_trade[i]: # Price option trade being exited option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \ fx_options_pricer.price_instrument(cross, horizon_date[i], calculated_strike[i-1], expiry_date[i-1], contract_type=contract_type, tenor=fx_options_trading_tenor, fx_vol_surface=fx_vol_surface, return_as_df=False) # Store as MTM mtm[i] = option_values_ # option_output[cross + '-option-price.close'].values # Price new option trade being entered option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \ fx_options_pricer.price_instrument(cross, horizon_date[i], strike, expiry_date[i], contract_type=contract_type, tenor=fx_options_trading_tenor, fx_vol_surface=fx_vol_surface, return_as_df=False) calculated_strike[ i] = strike_ # option_output[cross + '-strike.close'].values implied_vol[i] = vol_ interpolated_option[ i] = option_values_ # option_output[cross + '-option-price.close'].values else: # Price current option trade # - strike/expiry the same as yesterday # - other market inputs taken live, closer to expiry calculated_strike[i] = calculated_strike[i - 1] if freeze_implied_vol: frozen_vol = implied_vol[i - 1] else: frozen_vol = None option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \ fx_options_pricer.price_instrument(cross, horizon_date[i], calculated_strike[i], expiry_date[i], vol=frozen_vol, contract_type=contract_type, tenor=fx_options_trading_tenor, fx_vol_surface=fx_vol_surface, return_as_df=False) interpolated_option[ i] = option_values_ # option_output[cross + '-option-price.close'].values implied_vol[i] = vol_ mtm[i] = interpolated_option[i] delta[ i] = delta_ # option_output[cross + '-delta.close'].values # Calculate delta hedging P&L spot_rets = (market_df[cross + ".close"] / market_df[cross + ".close"].shift(1) - 1).values if tot_label == '': tot_rets = spot_rets else: tot_rets = ( market_df[cross + "-" + tot_label + ".close"] / market_df[cross + "-" + tot_label + ".close"].shift(1) - 1).values # Remember to take the inverted sign, eg. if call is +20%, we need to -20% of spot to flatten delta # Also invest for whether we are long or short the option delta_hedging_pnl = -np.roll( delta, 1) * tot_rets * position_multiplier delta_hedging_pnl[0] = 0 # Calculate options P&L (given option premium is already percentage, only need to subtract) # Again need to invert if we are short option option_rets = (mtm - np.roll(interpolated_option, 1)) * position_multiplier option_rets[0] = 0 # Calculate option + delta hedging P&L option_delta_rets = delta_hedging_pnl + option_rets if cum_index == 'mult': cum_rets = 100 * np.cumprod(1.0 + option_rets) cum_delta_rets = 100 * np.cumprod(1.0 + delta_hedging_pnl) cum_option_delta_rets = 100 * np.cumprod(1.0 + option_delta_rets) elif cum_index == 'add': cum_rets = 100 + 100 * np.cumsum(option_rets) cum_delta_rets = 100 + 100 * np.cumsum(delta_hedging_pnl) cum_option_delta_rets = 100 + 100 * np.cumsum( option_delta_rets) total_return_index_df = pd.DataFrame( index=horizon_date, columns=[cross + "-option-tot.close"]) total_return_index_df[cross + "-option-tot.close"] = cum_rets if output_calculation_fields: total_return_index_df[ cross + '-interpolated-option.close'] = interpolated_option total_return_index_df[cross + '-mtm.close'] = mtm total_return_index_df[cross + '-implied-vol.close'] = implied_vol total_return_index_df[cross + '-roll.close'] = new_trade total_return_index_df[cross + '.roll-date'] = roll_date total_return_index_df[cross + '.expiry-date'] = expiry_date total_return_index_df[ cross + '-calculated-strike.close'] = calculated_strike total_return_index_df[cross + '-option-return.close'] = option_rets total_return_index_df[cross + '-spot-return.close'] = spot_rets total_return_index_df[cross + '-tot-return.close'] = tot_rets total_return_index_df[cross + '-delta.close'] = delta total_return_index_df[ cross + '-delta-pnl-return.close'] = delta_hedging_pnl total_return_index_df[ cross + '-delta-pnl-index.close'] = cum_delta_rets total_return_index_df[ cross + '-option-delta-return.close'] = option_delta_rets total_return_index_df[ cross + '-option-delta-tot.close'] = cum_option_delta_rets total_return_index_df_agg.append(total_return_index_df) return self._calculations.pandas_outer_join(total_return_index_df_agg) def apply_tc_to_total_return_index(self, cross_fx, total_return_index_orig_df, option_tc_bp, spot_tc_bp, cum_index=None): if cum_index is None: cum_index = self._cum_index total_return_index_df_agg = [] if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] option_tc = option_tc_bp / (2 * 100 * 100) spot_tc = spot_tc_bp / (2 * 100 * 100) total_return_index_df = total_return_index_orig_df.copy() for cross in cross_fx: p = abs(total_return_index_df[cross + '-roll.close'].shift(1)) * option_tc q = abs(total_return_index_df[cross + '-delta.close'] - total_return_index_df[cross + '-delta.close'].shift(1)) * spot_tc # Additional columns to include P&L with transaction costs total_return_index_df[cross + '-option-return-with-tc.close'] = \ total_return_index_df[cross + '-option-return.close'] - abs(total_return_index_df[cross + '-roll.close'].shift(1)) * option_tc total_return_index_df[cross + '-delta-pnl-return-with-tc.close'] = \ total_return_index_df[cross + '-delta-pnl-return.close'] \ - abs(total_return_index_df[cross + '-delta.close'] - total_return_index_df[cross + '-delta.close'].shift(1)) * spot_tc total_return_index_df[cross + '-option-return-with-tc.close'][0] = 0 total_return_index_df[cross + '-delta-pnl-return-with-tc.close'][0] = 0 total_return_index_df[cross + '-option-delta-return-with-tc.close'] = \ total_return_index_df[cross + '-option-return-with-tc.close'] + total_return_index_df[cross + '-delta-pnl-return-with-tc.close'] if cum_index == 'mult': cum_rets = 100 * np.cumprod(1.0 + total_return_index_df[ cross + '-option-return-with-tc.close'].values) cum_delta_rets = 100 * np.cumprod(1.0 + total_return_index_df[ cross + '-delta-pnl-return-with-tc.close'].values) cum_option_delta_rets = 100 * np.cumprod( 1.0 + total_return_index_df[ cross + '-option-delta-return-with-tc.close'].values) elif cum_index == 'add': cum_rets = 100 + 100 * np.cumsum(total_return_index_df[ cross + '-option-return-with-tc.close'].values) cum_delta_rets = 100 + 100 * np.cumsum(total_return_index_df[ cross + '-delta-pnl-return-with-tc.close'].values) cum_option_delta_rets = 100 + 100 * np.cumsum( total_return_index_df[ cross + '-option-delta-return-with-tc.close'].values) total_return_index_df[cross + "-option-tot-with-tc.close"] = cum_rets total_return_index_df[ cross + '-delta-pnl-index-with-tc.close'] = cum_delta_rets total_return_index_df[ cross + '-option-delta-tot-with-tc.close'] = cum_option_delta_rets total_return_index_df_agg.append(total_return_index_df) return self._calculations.pandas_outer_join(total_return_index_df_agg)
class MarketDataGenerator(object): """Returns market data time series by directly calling market data sources. At present it supports Bloomberg (bloomberg), Yahoo (yahoo), Quandl (quandl), FRED (fred) etc. which are implemented in subclasses of DataVendor class. This provides a common wrapper for all these data sources. """ def __init__(self): self.config = ConfigManager().get_instance() self.logger = LoggerManager().getLogger(__name__) self.filter = Filter() self.calculations = Calculations() self.io_engine = IOEngine() self._intraday_code = -1 self.days_expired_intraday_contract_download = -1 return def set_intraday_code(self, code): self._intraday_code = code def get_data_vendor(self, source): """Loads appropriate data service class Parameters ---------- source : str the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc. we can also have forms like "bloomberg-boe" separated by hyphens Returns ------- DataVendor """ data_vendor = None try: source = source.split("-")[0] except: self.logger.error("Was data source specified?") return None if source == 'bloomberg': try: from findatapy.market.datavendorbbg import DataVendorBBGOpen data_vendor = DataVendorBBGOpen() except: self.logger.warn("Bloomberg needs to be installed") elif source == 'quandl': from findatapy.market.datavendorweb import DataVendorQuandl data_vendor = DataVendorQuandl() elif source == 'ons': from findatapy.market.datavendorweb import DataVendorONS data_vendor = DataVendorONS() elif source == 'boe': from findatapy.market.datavendorweb import DataVendorBOE data_vendor = DataVendorBOE() elif source == 'dukascopy': from findatapy.market.datavendorweb import DataVendorDukasCopy data_vendor = DataVendorDukasCopy() elif source == 'fxcm': from findatapy.market.datavendorweb import DataVendorFXCM data_vendor = DataVendorFXCM() elif source == 'alfred': from findatapy.market.datavendorweb import DataVendorALFRED data_vendor = DataVendorALFRED() elif source == 'yahoo': from findatapy.market.datavendorweb import DataVendorYahoo data_vendor = DataVendorYahoo() elif source in ['google', 'fred', 'oecd', 'eurostat', 'edgar-index']: from findatapy.market.datavendorweb import DataVendorPandasWeb data_vendor = DataVendorPandasWeb() elif source == 'bitcoincharts': from findatapy.market.datavendorweb import DataVendorBitcoincharts data_vendor = DataVendorBitcoincharts() elif source == 'poloniex': from findatapy.market.datavendorweb import DataVendorPoloniex data_vendor = DataVendorPoloniex() elif source == 'binance': from findatapy.market.datavendorweb import DataVendorBinance data_vendor = DataVendorBinance() elif source == 'bitfinex': from findatapy.market.datavendorweb import DataVendorBitfinex data_vendor = DataVendorBitfinex() elif source == 'gdax': from findatapy.market.datavendorweb import DataVendorGdax data_vendor = DataVendorGdax() elif source == 'kraken': from findatapy.market.datavendorweb import DataVendorKraken data_vendor = DataVendorKraken() elif source == 'bitmex': from findatapy.market.datavendorweb import DataVendorBitmex data_vendor = DataVendorBitmex() elif '.csv' in source or '.h5' in source: from findatapy.market.datavendorweb import DataVendorFlatFile data_vendor = DataVendorFlatFile() elif source == 'alphavantage': from findatapy.market.datavendorweb import DataVendorAlphaVantage data_vendor = DataVendorAlphaVantage() elif source == 'huobi': from findatapy.market.datavendorweb import DataVendorHuobi data_vendor = DataVendorHuobi() # TODO add support for other data sources (like Reuters) return data_vendor def fetch_market_data(self, market_data_request, kill_session=True): """Loads time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ # data_vendor = self.get_data_vendor(market_data_request.data_source) # check if tickers have been specified (if not load all of them for a category) # also handle single tickers/list tickers create_tickers = False if market_data_request.vendor_tickers is not None and market_data_request.tickers is None: market_data_request.tickers = market_data_request.vendor_tickers tickers = market_data_request.tickers if tickers is None: create_tickers = True elif isinstance(tickers, str): if tickers == '': create_tickers = True elif isinstance(tickers, list): if tickers == []: create_tickers = True if create_tickers: market_data_request.tickers = ConfigManager().get_instance( ).get_tickers_list_for_category(market_data_request.category, market_data_request.data_source, market_data_request.freq, market_data_request.cut) # intraday or tick: only one ticker per cache file if (market_data_request.freq in ['intraday', 'tick', 'second', 'hour', 'minute']): data_frame_agg = self.download_intraday_tick(market_data_request) # return data_frame_agg # daily: multiple tickers per cache file - assume we make one API call to vendor library else: data_frame_agg = self.download_daily(market_data_request) if ('internet_load' in market_data_request.cache_algo): self.logger.debug("Internet loading.. ") # signal to data_vendor template to exit session # if data_vendor is not None and kill_session == True: data_vendor.kill_session() if (market_data_request.cache_algo == 'cache_algo'): self.logger.debug( "Only caching data in memory, do not return any time series.") return # only return time series if specified in the algo if 'return' in market_data_request.cache_algo: # special case for events/events-dt which is not indexed like other tables (also same for downloading futures # contracts dates) if market_data_request.category is not None: if 'events' in market_data_request.category: return data_frame_agg # pad columns a second time (is this necessary to do here again?) # TODO only do this for not daily data? try: if data_frame_agg is not None: data_frame_agg = self.filter.filter_time_series(market_data_request, data_frame_agg, pad_columns=True)\ .dropna(how = 'all') # resample data using pandas if specified in the MarketDataRequest if market_data_request.resample is not None: if 'last' in market_data_request.resample_how: data_frame_agg = data_frame_agg.resample( market_data_request.resample).last() elif 'first' in market_data_request.resample_how: data_frame_agg = data_frame_agg.resample( market_data_request.resample).first() if 'dropna' in market_data_request.resample_how: data_frame_agg = data_frame_agg.dropna(how='all') else: self.logger.warn("No data returned for " + str(market_data_request.tickers)) return data_frame_agg except Exception as e: print(str(e)) if data_frame_agg is not None: return data_frame_agg import traceback self.logger.warn("No data returned for " + str(market_data_request.tickers)) return None def create_time_series_hash_key(self, market_data_request, ticker=None): """Creates a hash key for retrieving the time series Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- str """ if (isinstance(ticker, list)): ticker = ticker[0] return self.create_cache_file_name( MarketDataRequest().create_category_key(market_data_request, ticker)) def download_intraday_tick(self, market_data_request): """Loads intraday time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ data_frame_agg = None calcuations = Calculations() ticker_cycle = 0 data_frame_group = [] # single threaded version # handle intraday ticker calls separately one by one if len(market_data_request.tickers) == 1 or DataConstants( ).market_thread_no['other'] == 1: for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 # we downscale into float32, to avoid memory problems in Python (32 bit) # data is stored on disk as float32 anyway # old_finish_date = market_data_request_single.finish_date # # market_data_request_single.finish_date = self.refine_expiry_date(market_data_request) # # if market_data_request_single.finish_date >= market_data_request_single.start_date: # data_frame_single = data_vendor.load_ticker(market_data_request_single) # else: # data_frame_single = None # # market_data_request_single.finish_date = old_finish_date # # data_frame_single = data_vendor.load_ticker(market_data_request_single) data_frame_single = self.fetch_single_time_series( market_data_request) # if the vendor doesn't provide any data, don't attempt to append if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' data_frame_single = data_frame_single.astype('float32') data_frame_group.append(data_frame_single) # # if you call for returning multiple tickers, be careful with memory considerations! # if data_frame_agg is not None: # data_frame_agg = data_frame_agg.join(data_frame_single, how='outer') # else: # data_frame_agg = data_frame_single # key = self.create_category_key(market_data_request, ticker) # fname = self.create_cache_file_name(key) # self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday) # if you call for returning multiple tickers, be careful with memory considerations! if data_frame_group is not None: data_frame_agg = calcuations.pandas_outer_join( data_frame_group) return data_frame_agg else: market_data_request_list = [] # create a list of MarketDataRequests for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 market_data_request_list.append(market_data_request_single) return self.fetch_group_time_series(market_data_request_list) def fetch_single_time_series(self, market_data_request): market_data_request = MarketDataRequest(md_request=market_data_request) # only includes those tickers have not expired yet! start_date = pandas.Timestamp(market_data_request.start_date).date() import datetime current_date = datetime.datetime.utcnow().date() from datetime import timedelta tickers = market_data_request.tickers vendor_tickers = market_data_request.vendor_tickers expiry_date = market_data_request.expiry_date config = ConfigManager().get_instance() # in many cases no expiry is defined so skip them for i in range(0, len(tickers)): try: expiry_date = config.get_expiry_for_ticker( market_data_request.data_source, tickers[i]) except: pass if expiry_date is not None: expiry_date = pandas.Timestamp(expiry_date).date() # use pandas Timestamp, a bit more robust with weird dates (can fail if comparing date vs datetime) # if the expiry is before the start date of our download don't bother downloading this ticker if expiry_date < start_date: tickers[i] = None # special case for futures-contracts which are intraday # avoid downloading if the expiry date is very far in the past # (we need this before there might be odd situations where we run on an expiry date, but still want to get # data right till expiry time) if market_data_request.category == 'futures-contracts' and market_data_request.freq == 'intraday' \ and self.days_expired_intraday_contract_download > 0: if expiry_date + timedelta( days=self.days_expired_intraday_contract_download ) < current_date: tickers[i] = None if vendor_tickers is not None and tickers[i] is None: vendor_tickers[i] = None market_data_request.tickers = [e for e in tickers if e != None] if vendor_tickers is not None: market_data_request.vendor_tickers = [ e for e in vendor_tickers if e != None ] data_frame_single = None if len(market_data_request.tickers) > 0: data_frame_single = self.get_data_vendor( market_data_request.data_source).load_ticker( market_data_request) #print(data_frame_single.head(n=10)) if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' # will fail for dataframes which includes dates/strings (eg. futures contract names) try: data_frame_single = data_frame_single.astype('float32') except: self.logger.warning('Could not convert to float') if market_data_request.freq == "second": data_frame_single = data_frame_single.resample("1s") return data_frame_single def fetch_group_time_series(self, market_data_request_list): data_frame_agg = None thread_no = DataConstants().market_thread_no['other'] if market_data_request_list[0].data_source in DataConstants( ).market_thread_no: thread_no = DataConstants().market_thread_no[ market_data_request_list[0].data_source] if thread_no > 0: pool = SwimPool().create_pool( thread_technique=DataConstants().market_thread_technique, thread_no=thread_no) # open the market data downloads in their own threads and return the results result = pool.map_async(self.fetch_single_time_series, market_data_request_list) data_frame_group = result.get() pool.close() pool.join() else: data_frame_group = [] for md_request in market_data_request_list: data_frame_group.append( self.fetch_single_time_series(md_request)) # collect together all the time series if data_frame_group is not None: data_frame_group = [i for i in data_frame_group if i is not None] # for debugging! # import pickle # import datetime # pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb")) if data_frame_group is not None: try: data_frame_agg = self.calculations.pandas_outer_join( data_frame_group) except Exception as e: self.logger.warning( 'Possible overlap of columns? Have you specifed same ticker several times: ' + str(e)) return data_frame_agg def download_daily(self, market_data_request): """Loads daily time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ key = MarketDataRequest().create_category_key(market_data_request) is_key_overriden = False for k in DataConstants().override_multi_threading_for_categories: if k in key: is_key_overriden = True break # by default use other thread_no = DataConstants().market_thread_no['other'] if market_data_request.data_source in DataConstants().market_thread_no: thread_no = DataConstants().market_thread_no[ market_data_request.data_source] # daily data does not include ticker in the key, as multiple tickers in the same file if thread_no == 1: # data_frame_agg = data_vendor.load_ticker(market_data_request) data_frame_agg = self.fetch_single_time_series(market_data_request) else: market_data_request_list = [] # when trying your example 'equitiesdata_example' I had a -1 result so it went out of the comming loop and I had errors in execution group_size = max( int(len(market_data_request.tickers) / thread_no - 1), 0) if group_size == 0: group_size = 1 # split up tickers into groups related to number of threads to call for i in range(0, len(market_data_request.tickers), group_size): market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = market_data_request.tickers[ i:i + group_size] if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = \ market_data_request.vendor_tickers[i:i + group_size] market_data_request_list.append(market_data_request_single) # special case where we make smaller calls one after the other if is_key_overriden: data_frame_list = [] for md in market_data_request_list: data_frame_list.append(self.fetch_single_time_series(md)) data_frame_agg = self.calculations.pandas_outer_join( data_frame_list) else: data_frame_agg = self.fetch_group_time_series( market_data_request_list) # fname = self.create_cache_file_name(key) # self._time_series_cache[fname] = data_frame_agg # cache in memory (ok for daily data) return data_frame_agg def refine_expiry_date(self, market_data_request): # expiry date if market_data_request.expiry_date is None: ConfigManager().get_instance().get_expiry_for_ticker( market_data_request.data_source, market_data_request.ticker) return market_data_request def create_cache_file_name(self, filename): return DataConstants().folder_time_series_data + "/" + filename
def download_intraday_tick(self, market_data_request): """Loads intraday time series from specified data provider Parameters ---------- market_data_request : MarketDataRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ data_frame_agg = None calcuations = Calculations() ticker_cycle = 0 data_frame_group = [] # single threaded version # handle intraday ticker calls separately one by one if len(market_data_request.tickers) == 1 or DataConstants( ).market_thread_no['other'] == 1: for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 # we downscale into float32, to avoid memory problems in Python (32 bit) # data is stored on disk as float32 anyway # old_finish_date = market_data_request_single.finish_date # # market_data_request_single.finish_date = self.refine_expiry_date(market_data_request) # # if market_data_request_single.finish_date >= market_data_request_single.start_date: # data_frame_single = data_vendor.load_ticker(market_data_request_single) # else: # data_frame_single = None # # market_data_request_single.finish_date = old_finish_date # # data_frame_single = data_vendor.load_ticker(market_data_request_single) data_frame_single = self.fetch_single_time_series( market_data_request) # if the vendor doesn't provide any data, don't attempt to append if data_frame_single is not None: if data_frame_single.empty == False: data_frame_single.index.name = 'Date' data_frame_single = data_frame_single.astype('float32') data_frame_group.append(data_frame_single) # # if you call for returning multiple tickers, be careful with memory considerations! # if data_frame_agg is not None: # data_frame_agg = data_frame_agg.join(data_frame_single, how='outer') # else: # data_frame_agg = data_frame_single # key = self.create_category_key(market_data_request, ticker) # fname = self.create_cache_file_name(key) # self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday) # if you call for returning multiple tickers, be careful with memory considerations! if data_frame_group is not None: data_frame_agg = calcuations.pandas_outer_join( data_frame_group) return data_frame_agg else: market_data_request_list = [] # create a list of MarketDataRequests for ticker in market_data_request.tickers: market_data_request_single = copy.copy(market_data_request) market_data_request_single.tickers = ticker if market_data_request.vendor_tickers is not None: market_data_request_single.vendor_tickers = [ market_data_request.vendor_tickers[ticker_cycle] ] ticker_cycle = ticker_cycle + 1 market_data_request_list.append(market_data_request_single) return self.fetch_group_time_series(market_data_request_list)
def create_tech_ind(self, data_frame_non_nan, name, tech_params, data_frame_non_nan_early=None): self._signal = None self._techind = None if tech_params.fillna: data_frame = data_frame_non_nan.fillna(method="ffill") else: data_frame = data_frame_non_nan if data_frame_non_nan_early is not None: data_frame_early = data_frame_non_nan_early.fillna(method="ffill") if name == "SMA": if (data_frame_non_nan_early is not None): # calculate the lagged sum of the n-1 point if pd.__version__ < '0.17': rolling_sum = pd.rolling_sum( data_frame.shift(1).rolling, window=tech_params.sma_period - 1) else: rolling_sum = data_frame.shift(1).rolling( center=False, window=tech_params.sma_period - 1).sum() # add non-nan one for today rolling_sum = rolling_sum + data_frame_early # calculate average = sum / n self._techind = rolling_sum / tech_params.sma_period narray = np.where(data_frame_early > self._techind, 1, -1) else: if pd.__version__ < '0.17': self._techind = pd.rolling_sum( data_frame, window=tech_params.sma_period) else: self._techind = data_frame.rolling( window=tech_params.sma_period, center=False).mean() narray = np.where(data_frame > self._techind, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.sma_period] = np.nan self._signal.columns = [ x + " SMA Signal" for x in data_frame.columns.values ] self._techind.columns = [ x + " SMA" for x in data_frame.columns.values ] elif name == "EMA": # self._techind = pd.ewma(data_frame, span = tech_params.ema_period) self._techind = data_frame.ewm(ignore_na=False, span=tech_params.ema_period, min_periods=0, adjust=True).mean() narray = np.where(data_frame > self._techind, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.ema_period] = np.nan self._signal.columns = [ x + " EMA Signal" for x in data_frame.columns.values ] self._techind.columns = [ x + " EMA" for x in data_frame.columns.values ] elif name == "ROC": if (data_frame_non_nan_early is not None): self._techind = data_frame_early / \ data_frame.shift(tech_params.roc_period) - 1 else: self._techind = data_frame / \ data_frame.shift(tech_params.roc_period) - 1 narray = np.where(self._techind > 0, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.loc[0:tech_params.roc_period] = np.nan self._signal.columns = [ x + " ROC Signal" for x in data_frame.columns.values ] self._techind.columns = [ x + " ROC" for x in data_frame.columns.values ] elif name == "polarity": self._techind = data_frame narray = np.where(self._techind > 0, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " Polarity Signal" for x in data_frame.columns.values ] self._techind.columns = [ x + " Polarity" for x in data_frame.columns.values ] elif name == "SMA2": sma = data_frame.rolling(window=tech_params.sma_period, center=False).mean() sma2 = data_frame.rolling(window=tech_params.sma2_period, center=False).mean() narray = np.where(sma > sma2, 1, -1) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " SMA2 Signal" for x in data_frame.columns.values ] sma.columns = [x + " SMA" for x in data_frame.columns.values] sma2.columns = [x + " SMA2" for x in data_frame.columns.values] most = max(tech_params.sma_period, tech_params.sma2_period) self._signal.loc[0:most] = np.nan self._techind = pd.concat([sma, sma2], axis=1) elif name in ['RSI']: # delta = data_frame.diff() # # dUp, dDown = delta.copy(), delta.copy() # dUp[dUp < 0] = 0 # dDown[dDown > 0] = 0 # # rolUp = pd.rolling_mean(dUp, tech_params.rsi_period) # rolDown = pd.rolling_mean(dDown, tech_params.rsi_period).abs() # # rsi = rolUp / rolDown # Get the difference in price from previous step delta = data_frame.diff() # Get rid of the first row, which is NaN since it did not have a previous # row to calculate the differences delta = delta[1:] # Make the positive gains (up) and negative gains (down) Series up, down = delta.copy(), delta.copy() up[up < 0] = 0 down[down > 0] = 0 # Calculate the EWMA roll_up1 = pd.stats.moments.ewma(up, tech_params.rsi_period) roll_down1 = pd.stats.moments.ewma(down.abs(), tech_params.rsi_period) # Calculate the RSI based on EWMA RS1 = roll_up1 / roll_down1 RSI1 = 100.0 - (100.0 / (1.0 + RS1)) # Calculate the SMA roll_up2 = up.rolling(window=tech_params.rsi_period, center=False).mean() roll_down2 = down.abs().rolling(window=tech_params.rsi_period, center=False).mean() # Calculate the RSI based on SMA RS2 = roll_up2 / roll_down2 RSI2 = 100.0 - (100.0 / (1.0 + RS2)) self._techind = RSI2 self._techind.columns = [ x + " RSI" for x in data_frame.columns.values ] signal = data_frame.copy() sells = (signal.shift(-1) < tech_params.rsi_lower) & (signal > tech_params.rsi_lower) buys = (signal.shift(-1) > tech_params.rsi_upper) & (signal < tech_params.rsi_upper) # print (buys[buys == True]) # buys signal[buys] = 1 signal[sells] = -1 signal[~(buys | sells)] = np.nan signal = signal.fillna(method='ffill') self._signal = signal self._signal.loc[0:tech_params.rsi_period] = np.nan self._signal.columns = [ x + " RSI Signal" for x in data_frame.columns.values ] elif name in ["BB"]: # calcuate Bollinger bands mid = data_frame.rolling(center=False, window=tech_params.bb_period).mean() mid.columns = [x + " BB Mid" for x in data_frame.columns.values] std_dev = data_frame.rolling(center=False, window=tech_params.bb_period).std() BB_std = tech_params.bb_mult * std_dev lower = pd.DataFrame(data=mid.values - BB_std.values, index=mid.index, columns=data_frame.columns) upper = pd.DataFrame(data=mid.values + BB_std.values, index=mid.index, columns=data_frame.columns) # calculate signals signal = data_frame.copy() buys = signal > upper sells = signal < lower signal[buys] = 1 signal[sells] = -1 signal[~(buys | sells)] = np.nan signal = signal.fillna(method='ffill') self._signal = signal self._signal.loc[0:tech_params.bb_period] = np.nan self._signal.columns = [ x + " " + name + " Signal" for x in data_frame.columns.values ] lower.columns = [ x + " BB Lower" for x in data_frame.columns.values ] upper.columns = [x + " BB Mid" for x in data_frame.columns.values] upper.columns = [ x + " BB Lower" for x in data_frame.columns.values ] self._techind = pd.concat([lower, mid, upper], axis=1) elif name == "long-only": # have +1 signals only self._techind = data_frame # the technical indicator is just "prices" narray = np.ones((len(data_frame.index), len(data_frame.columns))) self._signal = pd.DataFrame(index=data_frame.index, data=narray) self._signal.columns = [ x + " Long Only Signal" for x in data_frame.columns.values ] self._techind.columns = [ x + " Long Only" for x in data_frame.columns.values ] elif name == "ATR": # get all the asset names (assume we have names 'close', 'low', 'high' in the Data) # keep ordering of assets asset_name = list( OrderedDict.fromkeys( [x.split('.')[0] for x in data_frame.columns])) df = [] # can improve the performance of this if vectorise more! for a in asset_name: close = [a + '.close'] low = [a + '.low'] high = [a + '.high'] # if we don't fill NaNs, we need to remove those rows and then # calculate the ATR if not (tech_params.fillna): data_frame_short = data_frame[[close[0], low[0], high[0]]] data_frame_short = data_frame_short.dropna() else: data_frame_short = data_frame prev_close = data_frame_short[close].shift(1) c1 = data_frame_short[high].values - \ data_frame_short[low].values c2 = np.abs(data_frame_short[high].values - prev_close.values) c3 = np.abs(data_frame_short[low].values - prev_close.values) true_range = np.max((c1, c2, c3), axis=0) true_range = pd.DataFrame(index=data_frame_short.index, data=true_range, columns=[close[0] + ' True Range']) # put back NaNs into ATR if necessary if (not (tech_params.fillna)): true_range = true_range.reindex(data_frame.index, fill_value=np.nan) df.append(true_range) calc = Calculations() true_range = calc.pandas_outer_join(df) self._techind = true_range.rolling(window=tech_params.atr_period, center=False).mean() # self._techind = true_range.ewm(ignore_na=False, span=tech_params.atr_period, min_periods=0, adjust=True).mean() self._techind.columns = [x + ".close ATR" for x in asset_name] elif name in ["VWAP"]: asset_name = list( OrderedDict.fromkeys( [x.split('.')[0] for x in data_frame.columns])) df = [] for a in asset_name: high = [a + '.high'] low = [a + '.low'] close = [a + '.close'] volume = [a + '.volume'] if not tech_params.fillna: df_mod = data_frame[[high[0], low[0], close[0], volume[0]]] df_mod.dropna(inplace=True) else: df_mod = data_frame l = df_mod[low].values h = df_mod[high].values c = df_mod[close].values v = df_mod[volume].values vwap = np.cumsum(((h + l + c) / 3) * v) / np.cumsum(v) vwap = pd.DataFrame(index=df_mod.index, data=vwap, columns=[close[0] + ' VWAP']) print(vwap.columns) if not tech_params.fillna: vwap = vwap.reindex(data_frame.index, fill_value=np.nan) df.append(vwap) calc = Calculations() vwap = calc.pandas_outer_join(df) self._techind = vwap self._techind.columns = [x + ".close VWAP" for x in asset_name] self.create_custom_tech_ind(data_frame_non_nan, name, tech_params, data_frame_non_nan_early) # TODO create other indicators if hasattr(tech_params, 'only_allow_longs'): self._signal[self._signal < 0] = 0 # TODO create other indicators if hasattr(tech_params, 'only_allow_shorts'): self._signal[self._signal > 0] = 0 # apply signal multiplier (typically to flip signals) if hasattr(tech_params, 'signal_mult'): self._signal = self._signal * tech_params.signal_mult if hasattr(tech_params, 'strip_signal_name'): if tech_params.strip_signal_name: self._signal.columns = data_frame.columns return self._techind, self._signal
fx_forwards_tenor=fx_forwards_tenors, depo_tenor='1M') print(implied_depo_df) market_cols = [cross + ".close"] for f in fx_forwards_to_print: market_cols.append(cross + f + '.close') market_cols.append("USD1M.close") print(market_df[market_cols]) # Download implied deposits (via FX forwards) for base currency (AUD) from Bloomberg # Note: some implied deposits tenors might have no data in it tickers = [] for t in fx_forwards_tenors: tickers.append(cross[0:3] + "I" + t) md_request = MarketDataRequest(start_date='01 Jan 2020', finish_date='01 Feb 2020', data_source='bloomberg', cut='NYC', tickers=tickers, vendor_tickers= [x + " CMPN Curncy" for x in tickers], cache_algo='cache_algo_return') implied_depo_bbg_df = market.fetch_market(md_request=md_request) chart.plot(calculations.pandas_outer_join([implied_depo_bbg_df, implied_depo_df]))
class FXForwardsCurve(object): """Constructs continuous forwards time series total return indices from underlying forwards contracts. Incomplete! """ def __init__(self, market_data_generator=None, fx_forwards_trading_tenor='1M', roll_date=0, construct_via_currency='no', fx_forwards_tenor=constants.fx_forwards_tenor, base_depos_tenor=constants.base_depos_tenor): self._market_data_generator = market_data_generator self._calculations = Calculations() self._calendar = Calendar() self._fx_forwards_trading_tenor = fx_forwards_trading_tenor self._roll_date = roll_date self._construct_via_currency = construct_via_currency self._fx_forwards_tenor = fx_forwards_tenor self._base_depos_tenor = base_depos_tenor def generate_key(self): from findatapy.market.ioengine import SpeedCache # Don't include any "large" objects in the key return SpeedCache().generate_key( self, ['_market_data_generator', '_calculations', '_calendar']) def fetch_continuous_time_series(self, md_request, market_data_generator, fx_forwards_trading_tenor=None, roll_date=None, construct_via_currency=None, fx_forwards_tenor=None, base_depos_tenor=None): if market_data_generator is None: market_data_generator = self._market_data_generator if fx_forwards_trading_tenor is None: fx_forwards_trading_tenor = self._fx_forwards_trading_tenor if roll_date is None: roll_date = self._roll_date if construct_via_currency is None: construct_via_currency = self._construct_via_currency if fx_forwards_tenor is None: fx_forwards_tenor = self._fx_forwards_tenor if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor # Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient forward data for this) if construct_via_currency == 'no': # Download FX spot, FX forwards points and base depos market = Market(market_data_generator=market_data_generator) md_request_download = MarketDataRequest(md_request=md_request) md_request_download.category = 'fx-forwards-market' md_request_download.fields = 'close' md_request_download.abstract_curve = None md_request_download.fx_forwards_tenor = fx_forwards_tenor md_request_download.base_depos_tenor = base_depos_tenor forwards_market_df = market.fetch_market(md_request_download) return self.construct_total_return_index( md_request.tickers, fx_forwards_trading_tenor, roll_date, forwards_market_df, fx_forwards_tenor=fx_forwards_tenor, base_depos_tenor=base_depos_tenor) else: # eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency # Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns) total_return_indices = [] for tick in md_request.tickers: base = tick[0:3] terms = tick[3:6] md_request_base = MarketDataRequest(md_request=md_request) md_request_base.tickers = base + construct_via_currency md_request_terms = MarketDataRequest(md_request=md_request) md_request_terms.tickers = terms + construct_via_currency base_vals = self.fetch_continuous_time_series( md_request_base, market_data_generator, construct_via_currency='no') terms_vals = self.fetch_continuous_time_series( md_request_terms, market_data_generator, construct_via_currency='no') # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == 'USDUSD': base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + 'USD' == 'USDUSD': cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + 'USD' == 'USDUSD': cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [tick + '-tot.close'] total_return_indices.append(cross_vals) return self._calculations.pandas_outer_join(total_return_indices) def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None): pass def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None, total_return_indices_df=None): pass def get_day_count_conv(self, currency): if currency in ['AUD', 'CAD', 'GBP', 'NZD']: return 365.0 return 360.0 def construct_total_return_index( self, cross_fx, fx_forwards_trading_tenor, roll_date, forwards_market_df, fx_forwards_tenor=constants.fx_forwards_tenor, base_depos_tenor=constants.base_depos_tenor): if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] total_return_index_agg = [] # Remove columns where there is no data (because these points typically aren't quoted) forwards_market_df = forwards_market_df.dropna(axis=1) for cross in cross_fx: # Eg. if we specify USDUSD if cross[0:3] == cross[3:6]: total_return_index_agg.append( pd.DataFrame(100, index=forwards_market_df.index, columns=[cross + "-tot.close"])) else: spot = forwards_market_df[cross + ".close"].to_frame() fx_forwards_tenor_pickout = [] for f in fx_forwards_tenor: if f + ".close" in fx_forwards_tenor: fx_forwards_tenor_pickout.append(f) if f == fx_forwards_trading_tenor: break divisor = 10000.0 if cross[3:6] == 'JPY': divisor = 100.0 forward_pts = forwards_market_df[[cross + x + ".close" for x in fx_forwards_tenor_pickout]].to_frame() \ / divisor outright = spot + forward_pts # Calculate the time difference between each data point spot['index_col'] = spot.index time = spot['index_col'].diff() spot = spot.drop('index_col', 1) total_return_index = pd.DataFrame( index=spot.index, columns=[cross + "-tot.close"]) total_return_index.iloc[0] = 100 time_diff = time.values.astype( float) / 86400000000000.0 # get time difference in days # TODO incomplete forwards calculations total_return_index_agg.append(total_return_index) return self._calculations.pandas_outer_join(total_return_index_agg)
class FXCrossFactory(object): """Generates FX spot time series and FX total return time series (assuming we already have total return indices available from xxxUSD form) from underlying series. Can also produce cross rates from the USD crosses. """ def __init__(self, market_data_generator=None): self.fxconv = FXConv() self.cache = {} self._calculations = Calculations() self._market_data_generator = market_data_generator return def get_fx_cross_tick(self, start, end, cross, cut="NYC", data_source="dukascopy", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['bid', 'ask']): if isinstance(cross, str): cross = [cross] market_data_request = MarketDataRequest( gran_freq="tick", freq_mult=1, freq='tick', cut=cut, fields=['bid', 'ask', 'bidv', 'askv'], cache_algo=cache_algo, environment=environment, start_date=start, finish_date=end, data_source=data_source, category='fx') market_data_generator = self._market_data_generator data_frame_agg = None for cr in cross: if (type == 'spot'): market_data_request.tickers = cr cross_vals = market_data_generator.fetch_market_data( market_data_request) if cross_vals is not None: # If user only wants 'close' calculate that from the bid/ask fields if fields == ['close']: cross_vals = cross_vals[[cr + '.bid', cr + '.ask']].mean(axis=1) cross_vals.columns = [cr + '.close'] else: filter = Filter() filter_columns = [cr + '.' + f for f in fields] cross_vals = filter.filter_time_series_by_columns( filter_columns, cross_vals) if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') if data_frame_agg is not None: # Strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def get_fx_cross(self, start, end, cross, cut="NYC", data_source="bloomberg", freq="intraday", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['close']): if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick': return self.get_fx_cross_tick(start, end, cross, cut=cut, data_source=data_source, cache_algo=cache_algo, type='spot', fields=fields) if isinstance(cross, str): cross = [cross] market_data_request_list = [] freq_list = [] type_list = [] for cr in cross: market_data_request = MarketDataRequest(freq_mult=1, cut=cut, fields=['close'], freq=freq, cache_algo=cache_algo, start_date=start, finish_date=end, data_source=data_source, environment=environment) market_data_request.type = type market_data_request.cross = cr if freq == 'intraday': market_data_request.gran_freq = "minute" # intraday elif freq == 'daily': market_data_request.gran_freq = "daily" # daily market_data_request_list.append(market_data_request) data_frame_agg = [] # Depends on the nature of operation as to whether we should use threading or multiprocessing library if constants.market_thread_technique is "thread": from multiprocessing.dummy import Pool else: # Most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing # must use the multiprocess library otherwise can't pickle objects correctly # note: currently not very stable from multiprocess import Pool thread_no = constants.market_thread_no['other'] if market_data_request_list[ 0].data_source in constants.market_thread_no: thread_no = constants.market_thread_no[ market_data_request_list[0].data_source] # Fudge, issue with multithreading and accessing HDF5 files # if self._market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator': # thread_no = 0 thread_no = 0 if (thread_no > 0): pool = Pool(thread_no) # Open the market data downloads in their own threads and return the results df_list = pool.map_async(self._get_individual_fx_cross, market_data_request_list).get() data_frame_agg = self._calculations.iterative_outer_join(df_list) # data_frame_agg = self._calculations.pandas_outer_join(result.get()) try: pool.close() pool.join() except: pass else: for md_request in market_data_request_list: data_frame_agg.append( self._get_individual_fx_cross(md_request)) data_frame_agg = self._calculations.pandas_outer_join( data_frame_agg) # Strip the nan elements data_frame_agg = data_frame_agg.dropna(how='all') # self.speed_cache.put_dataframe(key, data_frame_agg) return data_frame_agg def _get_individual_fx_cross(self, market_data_request): cr = market_data_request.cross type = market_data_request.type freq = market_data_request.freq base = cr[0:3] terms = cr[3:6] if (type == 'spot'): # Non-USD crosses if base != 'USD' and terms != 'USD': base_USD = self.fxconv.correct_notation('USD' + base) terms_USD = self.fxconv.correct_notation('USD' + terms) # TODO check if the cross exists in the database # Download base USD cross market_data_request.tickers = base_USD market_data_request.category = 'fx' base_vals = self._market_data_generator.fetch_market_data( market_data_request) # Download terms USD cross market_data_request.tickers = terms_USD market_data_request.category = 'fx' terms_vals = self._market_data_generator.fetch_market_data( market_data_request) # If quoted USD/base flip to get USD terms if (base_USD[0:3] == 'USD'): base_vals = 1 / base_vals # If quoted USD/terms flip to get USD terms if (terms_USD[0:3] == 'USD'): terms_vals = 1 / terms_vals base_vals.columns = ['temp'] terms_vals.columns = ['temp'] cross_vals = base_vals.div(terms_vals, axis='index') cross_vals.columns = [cr + '.close'] base_vals.columns = [base_USD + '.close'] terms_vals.columns = [terms_USD + '.close'] else: # if base == 'USD': non_USD = terms # if terms == 'USD': non_USD = base correct_cr = self.fxconv.correct_notation(cr) market_data_request.tickers = correct_cr market_data_request.category = 'fx' cross_vals = self._market_data_generator.fetch_market_data( market_data_request) # Special case for USDUSD! if base + terms == 'USDUSD': if freq == 'daily': cross_vals = pd.DataFrame(1, index=cross_vals.index, columns=cross_vals.columns) filter = Filter() cross_vals = filter.filter_time_series_by_holidays( cross_vals, cal='WEEKDAY') else: # Flip if not convention (eg. JPYUSD) if (correct_cr != cr): cross_vals = 1 / cross_vals # cross_vals = self._market_data_generator.harvest_time_series(market_data_request) cross_vals.columns = [cr + '.close'] elif type[0:3] == "tot": if freq == 'daily': # Download base USD cross market_data_request.tickers = base + 'USD' market_data_request.category = 'fx-' + type if type[0:3] == "tot": base_vals = self._market_data_generator.fetch_market_data( market_data_request) # Download terms USD cross market_data_request.tickers = terms + 'USD' market_data_request.category = 'fx-' + type if type[0:3] == "tot": terms_vals = self._market_data_generator.fetch_market_data( market_data_request) # base_rets = self._calculations.calculate_returns(base_vals) # terms_rets = self._calculations.calculate_returns(terms_vals) # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == 'USDUSD': base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + 'USD' == 'USDUSD': cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + 'USD' == 'USDUSD': cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [cr + '-' + type + '.close'] elif freq == 'intraday': LoggerManager().getLogger(__name__).info( 'Total calculated returns for intraday not implemented yet' ) return None return cross_vals
class FXSpotCurve(object): """Construct total return (spot) indices for FX. In future will also convert assets from local currency to foreign currency denomination and construct indices from forwards series. """ def __init__(self, market_data_generator=None, depo_tenor='ON', construct_via_currency='no'): self._market_data_generator = market_data_generator self._calculations = Calculations() self._depo_tenor = depo_tenor self._construct_via_currency = construct_via_currency def generate_key(self): from findatapy.market.ioengine import SpeedCache # Don't include any "large" objects in the key return SpeedCache().generate_key( self, ['_market_data_generator', '_calculations']) def fetch_continuous_time_series(self, md_request, market_data_generator, construct_via_currency=None): if market_data_generator is None: market_data_generator = self._market_data_generator if construct_via_currency is None: construct_via_currency = self._construct_via_currency # Eg. we construct AUDJPY via AUDJPY directly if construct_via_currency == 'no': base_depo_tickers = [ x[0:3] + self._depo_tenor for x in md_request.tickers ] terms_depo_tickers = [ x[3:6] + self._depo_tenor for x in md_request.tickers ] depo_tickers = list(set(base_depo_tickers + terms_depo_tickers)) market = Market(market_data_generator=market_data_generator) # Deposit data for base and terms currency md_request_download = MarketDataRequest(md_request=md_request) md_request_download.tickers = depo_tickers md_request_download.category = 'base-depos' md_request_download.fields = 'close' md_request_download.abstract_curve = None depo_df = market.fetch_market(md_request_download) # Spot data md_request_download.tickers = md_request.tickers md_request_download.category = 'fx' spot_df = market.fetch_market(md_request_download) return self.construct_total_return_index(md_request.tickers, self._depo_tenor, spot_df, depo_df) else: # eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency # Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns) total_return_indices = [] for tick in md_request.tickers: base = tick[0:3] terms = tick[3:6] md_request_base = MarketDataRequest(md_request=md_request) md_request_base.tickers = base + construct_via_currency md_request_terms = MarketDataRequest(md_request=md_request) md_request_terms.tickers = terms + construct_via_currency base_vals = self.fetch_continuous_time_series( md_request_base, market_data_generator, construct_via_currency='no') terms_vals = self.fetch_continuous_time_series( md_request_terms, market_data_generator, construct_via_currency='no') # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == 'USDUSD': base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + 'USD' == 'USDUSD': cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + 'USD' == 'USDUSD': cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [tick + '-tot.close'] total_return_indices.append(cross_vals) return self._calculations.pandas_outer_join(total_return_indices) def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None): pass def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None, total_return_indices_df=None): pass def get_day_count_conv(self, currency): if currency in ['AUD', 'CAD', 'GBP', 'NZD']: return 365.0 return 360.0 def construct_total_return_index(self, cross_fx, tenor, spot_df, deposit_df): """Creates total return index for selected FX crosses from spot and deposit data Parameters ---------- cross_fx : String Crosses to construct total return indices (can be a list) tenor : String Tenor of deposit rates to use to compute carry (typically ON for spot) spot_df : pd.DataFrame Spot data (must include crosses we select) deposit_df : pd.DataFrame Deposit data Returns ------- pd.DataFrame """ if not (isinstance(cross_fx, list)): cross_fx = [cross_fx] total_return_index_agg = [] for cross in cross_fx: # Get the spot series, base deposit base_deposit = deposit_df[cross[0:3] + tenor + ".close"].to_frame() terms_deposit = deposit_df[cross[3:6] + tenor + ".close"].to_frame() # Eg. if we specify USDUSD if cross[0:3] == cross[3:6]: total_return_index_agg.append( pd.DataFrame(100, index=base_deposit.index, columns=[cross + "-tot.close"])) else: carry = base_deposit.join(terms_deposit, how='inner') spot = spot_df[cross + ".close"].to_frame() base_daycount = self.get_day_count_conv(cross[0:3]) terms_daycount = self.get_day_count_conv(cross[4:6]) # Align the base & terms deposits series to spot spot, carry = spot.align(carry, join='left', axis=0) # Sometimes depo data can be patchy, ok to fill down, given not very volatile (don't do this with spot!) carry = carry.fillna(method='ffill') / 100.0 # In case there are values missing at start of list (fudge for old data!) carry = carry.fillna(method='bfill') spot = spot[cross + ".close"].to_frame() base_deposit = carry[base_deposit.columns] terms_deposit = carry[terms_deposit.columns] # Calculate the time difference between each data point spot['index_col'] = spot.index time = spot['index_col'].diff() spot = spot.drop('index_col', 1) total_return_index = pd.DataFrame( index=spot.index, columns=[cross + "-tot.close"]) total_return_index.iloc[0] = 100 time_diff = time.values.astype( float) / 86400000000000.0 # get time difference in days for i in range(1, len(total_return_index.index)): # TODO vectorise this formulae or use Numba # Calculate total return index as product of yesterday, changes in spot and carry accrued total_return_index.values[i] = total_return_index.values[i - 1] * \ (1 + (1 + base_deposit.values[i] * time_diff[i] / base_daycount) * (spot.values[i] / spot.values[i - 1]) \ - (1 + terms_deposit.values[i] * time_diff[i] / terms_daycount)) total_return_index_agg.append(total_return_index) return self._calculations.pandas_outer_join(total_return_index_agg)