class FXCrossFactory(object): """Generates FX spot time series and FX total return time series (assuming we already have total return indices available from xxxUSD form) from underlying series. Can also produce cross rates from the USD crosses. """ def __init__(self, market_data_generator=None): self.fxconv = FXConv() self.cache = {} self._calculations = Calculations() self._market_data_generator = market_data_generator return def get_fx_cross_tick(self, start, end, cross, cut="NYC", data_source="dukascopy", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['bid', 'ask']): if isinstance(cross, str): cross = [cross] market_data_request = MarketDataRequest( gran_freq="tick", freq_mult=1, freq='tick', cut=cut, fields=['bid', 'ask', 'bidv', 'askv'], cache_algo=cache_algo, environment=environment, start_date=start, finish_date=end, data_source=data_source, category='fx') market_data_generator = self._market_data_generator data_frame_agg = None for cr in cross: if (type == 'spot'): market_data_request.tickers = cr cross_vals = market_data_generator.fetch_market_data( market_data_request) if cross_vals is not None: # If user only wants 'close' calculate that from the bid/ask fields if fields == ['close']: cross_vals = cross_vals[[cr + '.bid', cr + '.ask']].mean(axis=1) cross_vals.columns = [cr + '.close'] else: filter = Filter() filter_columns = [cr + '.' + f for f in fields] cross_vals = filter.filter_time_series_by_columns( filter_columns, cross_vals) if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') if data_frame_agg is not None: # Strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def get_fx_cross(self, start, end, cross, cut="NYC", data_source="bloomberg", freq="intraday", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['close']): if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick': return self.get_fx_cross_tick(start, end, cross, cut=cut, data_source=data_source, cache_algo=cache_algo, type='spot', fields=fields) if isinstance(cross, str): cross = [cross] market_data_request_list = [] freq_list = [] type_list = [] for cr in cross: market_data_request = MarketDataRequest(freq_mult=1, cut=cut, fields=['close'], freq=freq, cache_algo=cache_algo, start_date=start, finish_date=end, data_source=data_source, environment=environment) market_data_request.type = type market_data_request.cross = cr if freq == 'intraday': market_data_request.gran_freq = "minute" # intraday elif freq == 'daily': market_data_request.gran_freq = "daily" # daily market_data_request_list.append(market_data_request) data_frame_agg = [] # Depends on the nature of operation as to whether we should use threading or multiprocessing library if constants.market_thread_technique is "thread": from multiprocessing.dummy import Pool else: # Most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing # must use the multiprocess library otherwise can't pickle objects correctly # note: currently not very stable from multiprocess import Pool thread_no = constants.market_thread_no['other'] if market_data_request_list[ 0].data_source in constants.market_thread_no: thread_no = constants.market_thread_no[ market_data_request_list[0].data_source] # Fudge, issue with multithreading and accessing HDF5 files # if self._market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator': # thread_no = 0 thread_no = 0 if (thread_no > 0): pool = Pool(thread_no) # Open the market data downloads in their own threads and return the results df_list = pool.map_async(self._get_individual_fx_cross, market_data_request_list).get() data_frame_agg = self._calculations.iterative_outer_join(df_list) # data_frame_agg = self._calculations.pandas_outer_join(result.get()) try: pool.close() pool.join() except: pass else: for md_request in market_data_request_list: data_frame_agg.append( self._get_individual_fx_cross(md_request)) data_frame_agg = self._calculations.pandas_outer_join( data_frame_agg) # Strip the nan elements data_frame_agg = data_frame_agg.dropna(how='all') # self.speed_cache.put_dataframe(key, data_frame_agg) return data_frame_agg def _get_individual_fx_cross(self, market_data_request): cr = market_data_request.cross type = market_data_request.type freq = market_data_request.freq base = cr[0:3] terms = cr[3:6] if (type == 'spot'): # Non-USD crosses if base != 'USD' and terms != 'USD': base_USD = self.fxconv.correct_notation('USD' + base) terms_USD = self.fxconv.correct_notation('USD' + terms) # TODO check if the cross exists in the database # Download base USD cross market_data_request.tickers = base_USD market_data_request.category = 'fx' base_vals = self._market_data_generator.fetch_market_data( market_data_request) # Download terms USD cross market_data_request.tickers = terms_USD market_data_request.category = 'fx' terms_vals = self._market_data_generator.fetch_market_data( market_data_request) # If quoted USD/base flip to get USD terms if (base_USD[0:3] == 'USD'): base_vals = 1 / base_vals # If quoted USD/terms flip to get USD terms if (terms_USD[0:3] == 'USD'): terms_vals = 1 / terms_vals base_vals.columns = ['temp'] terms_vals.columns = ['temp'] cross_vals = base_vals.div(terms_vals, axis='index') cross_vals.columns = [cr + '.close'] base_vals.columns = [base_USD + '.close'] terms_vals.columns = [terms_USD + '.close'] else: # if base == 'USD': non_USD = terms # if terms == 'USD': non_USD = base correct_cr = self.fxconv.correct_notation(cr) market_data_request.tickers = correct_cr market_data_request.category = 'fx' cross_vals = self._market_data_generator.fetch_market_data( market_data_request) # Special case for USDUSD! if base + terms == 'USDUSD': if freq == 'daily': cross_vals = pd.DataFrame(1, index=cross_vals.index, columns=cross_vals.columns) filter = Filter() cross_vals = filter.filter_time_series_by_holidays( cross_vals, cal='WEEKDAY') else: # Flip if not convention (eg. JPYUSD) if (correct_cr != cr): cross_vals = 1 / cross_vals # cross_vals = self._market_data_generator.harvest_time_series(market_data_request) cross_vals.columns = [cr + '.close'] elif type[0:3] == "tot": if freq == 'daily': # Download base USD cross market_data_request.tickers = base + 'USD' market_data_request.category = 'fx-' + type if type[0:3] == "tot": base_vals = self._market_data_generator.fetch_market_data( market_data_request) # Download terms USD cross market_data_request.tickers = terms + 'USD' market_data_request.category = 'fx-' + type if type[0:3] == "tot": terms_vals = self._market_data_generator.fetch_market_data( market_data_request) # base_rets = self._calculations.calculate_returns(base_vals) # terms_rets = self._calculations.calculate_returns(terms_vals) # Special case for USDUSD case (and if base or terms USD are USDUSD if base + terms == 'USDUSD': base_rets = self._calculations.calculate_returns(base_vals) cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns) elif base + 'USD' == 'USDUSD': cross_rets = -self._calculations.calculate_returns( terms_vals) elif terms + 'USD' == 'USDUSD': cross_rets = self._calculations.calculate_returns( base_vals) else: base_rets = self._calculations.calculate_returns(base_vals) terms_rets = self._calculations.calculate_returns( terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # First returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self._calculations.create_mult_index(cross_rets) cross_vals.columns = [cr + '-' + type + '.close'] elif freq == 'intraday': LoggerManager().getLogger(__name__).info( 'Total calculated returns for intraday not implemented yet' ) return None return cross_vals
class FXCrossFactory(object): def __init__(self, market_data_generator=None): self.logger = LoggerManager().getLogger(__name__) self.fxconv = FXConv() self.cache = {} self.calculations = Calculations() self.market_data_generator = market_data_generator return def flush_cache(self): self.cache = {} def get_fx_cross_tick(self, start, end, cross, cut="NYC", source="dukascopy", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['bid', 'ask']): if isinstance(cross, str): cross = [cross] market_data_request = MarketDataRequest( gran_freq="tick", freq_mult=1, freq='tick', cut=cut, fields=['bid', 'ask', 'bidv', 'askv'], cache_algo=cache_algo, environment=environment, start_date=start, finish_date=end, data_source=source, category='fx') market_data_generator = self.market_data_generator data_frame_agg = None for cr in cross: if (type == 'spot'): market_data_request.tickers = cr cross_vals = market_data_generator.fetch_market_data( market_data_request) # if user only wants 'close' calculate that from the bid/ask fields if fields == ['close']: cross_vals = cross_vals[[cr + '.bid', cr + '.ask']].mean(axis=1) cross_vals.columns = [cr + '.close'] if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def get_fx_cross(self, start, end, cross, cut="NYC", source="bloomberg", freq="intraday", cache_algo='internet_load_return', type='spot', environment='backtest', fields=['close']): if source == "gain" or source == 'dukascopy' or freq == 'tick': return self.get_fx_cross_tick(start, end, cross, cut=cut, source=source, cache_algo=cache_algo, type='spot', fields=fields) if isinstance(cross, str): cross = [cross] market_data_request_list = [] freq_list = [] type_list = [] for cr in cross: market_data_request = MarketDataRequest(freq_mult=1, cut=cut, fields=['close'], freq=freq, cache_algo=cache_algo, start_date=start, finish_date=end, data_source=source, environment=environment) market_data_request.type = type market_data_request.cross = cr if freq == 'intraday': market_data_request.gran_freq = "minute" # intraday elif freq == 'daily': market_data_request.gran_freq = "daily" # daily market_data_request_list.append(market_data_request) data_frame_agg = [] # depends on the nature of operation as to whether we should use threading or multiprocessing library if DataConstants().market_thread_technique is "thread": from multiprocessing.dummy import Pool else: # most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing # must use the multiprocessing_on_dill library otherwise can't pickle objects correctly # note: currently not very stable from multiprocessing_on_dill import Pool thread_no = DataConstants().market_thread_no['other'] if market_data_request_list[0].data_source in DataConstants( ).market_thread_no: thread_no = DataConstants().market_thread_no[ market_data_request_list[0].data_source] # fudge, issue with multithreading and accessing HDF5 files # if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator': # thread_no = 0 if (thread_no > 0): pool = Pool(thread_no) # open the market data downloads in their own threads and return the results result = pool.map_async(self._get_individual_fx_cross, market_data_request_list) data_frame_agg = self.calculations.iterative_outer_join( result.get()) # data_frame_agg = self.calculations.pandas_outer_join(result.get()) # pool would have already been closed earlier # try: # pool.close() # pool.join() # except: pass else: for md_request in market_data_request_list: data_frame_agg.append( self._get_individual_fx_cross(md_request)) data_frame_agg = self.calculations.pandas_outer_join( data_frame_agg) # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg def _get_individual_fx_cross(self, market_data_request): cr = market_data_request.cross type = market_data_request.type freq = market_data_request.freq base = cr[0:3] terms = cr[3:6] if (type == 'spot'): # non-USD crosses if base != 'USD' and terms != 'USD': base_USD = self.fxconv.correct_notation('USD' + base) terms_USD = self.fxconv.correct_notation('USD' + terms) # TODO check if the cross exists in the database # download base USD cross market_data_request.tickers = base_USD market_data_request.category = 'fx' if base_USD + '.close' in self.cache: base_vals = self.cache[base_USD + '.close'] else: base_vals = self.market_data_generator.fetch_market_data( market_data_request) self.cache[base_USD + '.close'] = base_vals # download terms USD cross market_data_request.tickers = terms_USD market_data_request.category = 'fx' if terms_USD + '.close' in self.cache: terms_vals = self.cache[terms_USD + '.close'] else: terms_vals = self.market_data_generator.fetch_market_data( market_data_request) self.cache[terms_USD + '.close'] = terms_vals # if quoted USD/base flip to get USD terms if (base_USD[0:3] == 'USD'): if 'USD' + base in '.close' in self.cache: base_vals = self.cache['USD' + base + '.close'] else: base_vals = 1 / base_vals self.cache['USD' + base + '.close'] = base_vals # if quoted USD/terms flip to get USD terms if (terms_USD[0:3] == 'USD'): if 'USD' + terms in '.close' in self.cache: terms_vals = self.cache['USD' + terms + '.close'] else: terms_vals = 1 / terms_vals self.cache['USD' + terms + '.close'] = base_vals base_vals.columns = ['temp'] terms_vals.columns = ['temp'] cross_vals = base_vals.div(terms_vals, axis='index') cross_vals.columns = [cr + '.close'] base_vals.columns = [base_USD + '.close'] terms_vals.columns = [terms_USD + '.close'] else: # if base == 'USD': non_USD = terms # if terms == 'USD': non_USD = base correct_cr = self.fxconv.correct_notation(cr) market_data_request.tickers = correct_cr market_data_request.category = 'fx' if correct_cr + '.close' in self.cache: cross_vals = self.cache[correct_cr + '.close'] else: cross_vals = self.market_data_generator.fetch_market_data( market_data_request) # flip if not convention if (correct_cr != cr): if cr + '.close' in self.cache: cross_vals = self.cache[cr + '.close'] else: cross_vals = 1 / cross_vals self.cache[cr + '.close'] = cross_vals self.cache[correct_cr + '.close'] = cross_vals # cross_vals = self.market_data_generator.harvest_time_series(market_data_request) cross_vals.columns.names = [cr + '.close'] elif type[0:3] == "tot": if freq == 'daily': # download base USD cross market_data_request.tickers = base + 'USD' market_data_request.category = 'fx-tot' if type == "tot": base_vals = self.market_data_generator.fetch_market_data( market_data_request) else: x = 0 # download terms USD cross market_data_request.tickers = terms + 'USD' market_data_request.category = 'fx-tot' if type == "tot": terms_vals = self.market_data_generator.fetch_market_data( market_data_request) else: pass base_rets = self.calculations.calculate_returns(base_vals) terms_rets = self.calculations.calculate_returns(terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0) # first returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = self.calculations.create_mult_index(cross_rets) cross_vals.columns = [cr + '-tot.close'] elif freq == 'intraday': self.logger.info( 'Total calculated returns for intraday not implemented yet' ) return None return cross_vals