def get_surprise_against_intraday_moves_over_event( self, data_frame_cross_orig, cross, event_fx, event_name, start, end, offset_list=[1, 5, 30, 60], add_surprise=False, surprise_field='survey-average'): tsf = TimeSeriesFilter() fields = [ 'actual-release', 'survey-median', 'survey-average', 'survey-high', 'survey-low' ] ef_time_frame = self.get_economic_event_date_time_fields( fields, event_fx, event_name) ef_time_frame = tsf.filter_time_series_by_date(start, end, ef_time_frame) return self.get_surprise_against_intraday_moves_over_custom_event( data_frame_cross_orig, ef_time_frame, cross, event_fx, event_name, start, end, offset_list=offset_list, add_surprise=add_surprise, surprise_field=surprise_field)
def get_time_series_cached(self, time_series_request): """ get_time_series_cached - Loads time series from cache (if it exists) Parameters ---------- time_series_request : TimeSeriesRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- pandas.DataFrame """ if (time_series_request.freq == "intraday"): ticker = time_series_request.tickers else: ticker = None fname = self.create_time_series_hash_key(time_series_request, ticker) if (fname in self._time_series_cache): data_frame = self._time_series_cache[fname] tsf = TimeSeriesFilter() return tsf.filter_time_series(time_series_request, data_frame) return None
def get_intraday_moves_over_event(self, data_frame_rets, cross, event_fx, event_name, start, end, vol, mins=3 * 60, min_offset=0, create_index=False, resample=False, freq='minutes'): tsf = TimeSeriesFilter() ef_time_frame = self.get_economic_event_date_time_dataframe( event_fx, event_name) ef_time_frame = tsf.filter_time_series_by_date(start, end, ef_time_frame) return self.get_intraday_moves_over_custom_event( data_frame_rets, ef_time_frame, vol, mins=mins, min_offset=min_offset, create_index=create_index, resample=resample, freq=freq) #, start, end)
def harvest_time_series(self, time_series_request, kill_session = True): """ havest_time_series - Loads time series from specified data provider Parameters ---------- time_series_request : TimeSeriesRequest contains various properties describing time series to fetched, including ticker, start & finish date etc. Returns ------- DataFrame """ tickers = time_series_request.tickers loader = self.get_loader(time_series_request.data_source) # check if tickers have been specified (if not load all of them for a category) # also handle single tickers/list tickers create_tickers = False if tickers is None : create_tickers = True elif isinstance(tickers, str): if tickers == '': create_tickers = True elif isinstance(tickers, list): if tickers == []: create_tickers = True if create_tickers: time_series_request.tickers = self.config.get_tickers_list_for_category( time_series_request.category, time_series_request.source, time_series_request.freq, time_series_request.cut) # intraday or tick: only one ticker per cache file if (time_series_request.freq in ['intraday', 'tick']): data_frame_agg = self.download_intraday_tick(time_series_request, loader) # daily: multiple tickers per cache file - assume we make one API call to vendor library else: data_frame_agg = self.download_daily(time_series_request, loader) if('internet_load' in time_series_request.cache_algo): self.logger.debug("Internet loading.. ") # signal to loader template to exit session if loader is not None and kill_session == True: loader.kill_session() if(time_series_request.cache_algo == 'cache_algo'): self.logger.debug("Only caching data in memory, do not return any time series."); return tsf = TimeSeriesFilter() # only return time series if specified in the algo if 'return' in time_series_request.cache_algo: # special case for events/events-dt which is not indexed like other tables if hasattr(time_series_request, 'category'): if 'events' in time_series_request.category: return data_frame_agg try: return tsf.filter_time_series(time_series_request, data_frame_agg) except: return None
def __init__(self): # self.config = ConfigManager() self.logger = LoggerManager().getLogger(__name__) self.time_series_filter = TimeSeriesFilter() self.time_series_io = TimeSeriesIO() self._bbg_default_api = Constants().bbg_default_api self._intraday_code = -1 return
def compare_strategy_vs_benchmark(self, br, strategy_df, benchmark_df): """ compare_strategy_vs_benchmark - Compares the trading strategy we are backtesting against a benchmark Parameters ---------- br : BacktestRequest Parameters for backtest such as start and finish dates strategy_df : pandas.DataFrame Strategy time series benchmark_df : pandas.DataFrame Benchmark time series """ include_benchmark = False calc_stats = False if hasattr(br, 'include_benchmark'): include_benchmark = br.include_benchmark if hasattr(br, 'calc_stats'): calc_stats = br.calc_stats if include_benchmark: tsd = TimeSeriesDesc() cash_backtest = CashBacktest() ts_filter = TimeSeriesFilter() ts_calcs = TimeSeriesCalcs() # align strategy time series with that of benchmark strategy_df, benchmark_df = strategy_df.align(benchmark_df, join='left', axis = 0) # if necessary apply vol target to benchmark (to make it comparable with strategy) if hasattr(br, 'portfolio_vol_adjust'): if br.portfolio_vol_adjust is True: benchmark_df = cash_backtest.calculate_vol_adjusted_index_from_prices(benchmark_df, br = br) # only calculate return statistics if this has been specified (note when different frequencies of data # might underrepresent vol if calc_stats: benchmark_df = benchmark_df.fillna(method='ffill') tsd.calculate_ret_stats_from_prices(benchmark_df, br.ann_factor) benchmark_df.columns = tsd.summary() # realign strategy & benchmark strategy_benchmark_df = strategy_df.join(benchmark_df, how='inner') strategy_benchmark_df = strategy_benchmark_df.fillna(method='ffill') strategy_benchmark_df = ts_filter.filter_time_series_by_date(br.plot_start, br.finish_date, strategy_benchmark_df) strategy_benchmark_df = ts_calcs.create_mult_index_from_prices(strategy_benchmark_df) self._benchmark_pnl = benchmark_df self._benchmark_tsd = tsd return strategy_benchmark_df return strategy_df
def __init__(self): super(EventStudy, self).__init__() self.config = ConfigManager() self.logger = LoggerManager().getLogger(__name__) self.time_series_filter = TimeSeriesFilter() self.time_series_io = TimeSeriesIO() if (LightEventsFactory._econ_data_frame is None): self.load_economic_events() return
def get_economic_event_ret_over_custom_event_day(self, data_frame_in, event_dates, name, event, start, end, lagged=False, NYC_cutoff=10): time_series_filter = TimeSeriesFilter() event_dates = time_series_filter.filter_time_series_by_date( start, end, event_dates) data_frame = data_frame_in.copy( deep=True) # because we change the dates! time_series_tz = TimeSeriesTimezone() calendar = Calendar() bday = CustomBusinessDay(weekmask='Mon Tue Wed Thu Fri') event_dates_nyc = time_series_tz.convert_index_from_UTC_to_new_york_time( event_dates) average_hour_nyc = numpy.average(event_dates_nyc.index.hour) event_dates = calendar.floor_date(event_dates) # realised is traditionally on later day eg. 3rd Jan realised ON is 2nd-3rd Jan realised # so if Fed meeting is on 2nd Jan later, then we need realised labelled on 3rd (so minus a day) # implied expires on next day eg. 3rd Jan implied ON is 3rd-4th Jan implied # TODO smarter way of adjusting dates, as sometimes events can be before/after 10am NY cut if (lagged and average_hour_nyc >= NYC_cutoff): data_frame.index = data_frame.index - bday elif (not lagged and average_hour_nyc < NYC_cutoff): # ie. implied data_frame.index = data_frame.index + bday # set as New York time and select only those ON vols at the 10am NY cut just before the event data_frame_events = data_frame.ix[event_dates.index] data_frame_events.columns = data_frame.columns.values + '-' + name + ' ' + event return data_frame_events
def bus_day_of_month_seasonality( self, data_frame, month_list=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], cum=True, cal="FX", partition_by_month=True, add_average=False, price_index=False): tsc = TimeSeriesCalcs() tsf = TimeSeriesFilter() if price_index: data_frame = data_frame.resample( 'B') # resample into business days data_frame = tsc.calculate_returns(data_frame) data_frame.index = pandas.to_datetime(data_frame.index) data_frame = tsf.filter_time_series_by_holidays(data_frame, cal) monthly_seasonality = tsc.average_by_month_day_by_bus_day( data_frame, cal) monthly_seasonality = monthly_seasonality.loc[month_list] if partition_by_month: monthly_seasonality = monthly_seasonality.unstack(level=0) if add_average: monthly_seasonality['Avg'] = monthly_seasonality.mean(axis=1) if cum is True: if partition_by_month: monthly_seasonality.loc[0] = numpy.zeros( len(monthly_seasonality.columns)) # monthly_seasonality.index = monthly_seasonality.index + 1 # shifting index monthly_seasonality = monthly_seasonality.sort() monthly_seasonality = tsc.create_mult_index(monthly_seasonality) return monthly_seasonality
def dump_indicators(self): tsf = TimeSeriesFilter() self.logger.info("About to write all indicators to CSV") self.indicator.to_csv(self._csv_indicator_dump, date_format='%d/%m/%Y') if (self._csv_econ_indicator_dump is not None): self.logger.info("About to write economy based indicators to CSV") self.indicator_econ.to_csv(self._csv_econ_indicator_dump, date_format='%d/%m/%Y') self.logger.info("About to write final indicators to CSV") # remove weekends and remove start of series if (self._csv_final_indicator_dump is not None): indicator_final_copy = tsf.filter_time_series_by_holidays( self.indicator_final, cal='WEEKDAY') indicator_final_copy = tsf.filter_time_series_by_date( start_date="01 Jan 2000", finish_date=None, data_frame=indicator_final_copy) indicator_final_copy.to_csv(self._csv_final_indicator_dump, date_format='%d/%m/%Y')
groupby([date_index.month, date_index.day]).mean() def group_by_year(self, data_frame): date_index = data_frame.index return data_frame.\ groupby([date_index.year]) def average_by_day_hour_min_by_bus_day(self, data_frame): date_index = data_frame.index return data_frame.\ groupby([Calendar().get_bus_day_of_month(date_index), date_index.hour, date_index.minute]).mean() def remove_NaN_rows(self, data_frame): return data_frame.dropna() if __name__ == '__main__': tsc = TimeSeriesCalcs() tsf = TimeSeriesFilter() # test rolling ewma date_range = pandas.bdate_range('2014-01-01', '2014-02-28') print(tsc.get_bus_day_of_month(date_range)) foo = pandas.DataFrame(numpy.arange(0.0,13.0)) print(tsc.rolling_ewma(foo, span=3))
# fetch NFP times from Bloomberg time_series_request = TimeSeriesRequest( start_date = start_date, # start date finish_date = finish_date, # finish date category = "events", freq = 'daily', # daily data data_source = 'bloomberg', # use Bloomberg as data source tickers = ['NFP'], fields = ['release-date-time-full'], # which fields to download vendor_tickers = ['NFP TCH Index'], # ticker (Bloomberg) vendor_fields = ['ECO_FUTURE_RELEASE_DATE_LIST'], # which Bloomberg fields to download cache_algo = 'internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() ts_filter = TimeSeriesFilter() df_event_times = ltsf.harvest_time_series(time_series_request) utc_time = pytz.utc df_event_times = pandas.DataFrame(index = df_event_times['NFP.release-date-time-full']) df_event_times.index = df_event_times.index.tz_localize(utc_time) # work in UTC time df_event_times = ts_filter.filter_time_series_by_date(start_date, finish_date, df_event_times) # get last NFP time start_date = df_event_times.index[-1] - timedelta(minutes=1) finish_date = start_date + timedelta(minutes=4) tickers = ['EURUSD', 'JPYUSD', 'GBPUSD'] vendor_tickers = ['EURUSD BGN Curncy', 'USDJPY BGN Curncy', 'GBPUSD BGN Curncy']
def get_intraday_moves_over_custom_event(self, data_frame_rets, ef_time_frame, vol=False, minute_start=5, mins=3 * 60, min_offset=0, create_index=False, resample=False, freq='minutes'): tsf = TimeSeriesFilter() ef_time_frame = tsf.filter_time_series_by_date( data_frame_rets.index[0], data_frame_rets.index[-1], ef_time_frame) ef_time = ef_time_frame.index if freq == 'minutes': ef_time_start = ef_time - timedelta(minutes=minute_start) ef_time_end = ef_time + timedelta(minutes=mins) ann_factor = 252 * 1440 elif freq == 'days': ef_time = ef_time_frame.index.normalize() ef_time_start = ef_time - timedelta(days=minute_start) ef_time_end = ef_time + timedelta(days=mins) ann_factor = 252 ords = range(-minute_start + min_offset, mins + min_offset) # all data needs to be equally spaced if resample: tsf = TimeSeriesFilter() # make sure time series is properly sampled at 1 min intervals data_frame_rets = data_frame_rets.resample('1min') data_frame_rets = data_frame_rets.fillna(value=0) data_frame_rets = tsf.remove_out_FX_out_of_hours(data_frame_rets) data_frame_rets['Ind'] = numpy.nan start_index = data_frame_rets.index.searchsorted(ef_time_start) finish_index = data_frame_rets.index.searchsorted(ef_time_end) # not all observation windows will be same length (eg. last one?) # fill the indices which represent minutes # TODO vectorise this! for i in range(0, len(ef_time_frame.index)): try: data_frame_rets.ix[start_index[i]:finish_index[i], 'Ind'] = ords except: data_frame_rets.ix[start_index[i]:finish_index[i], 'Ind'] = ords[0:(finish_index[i] - start_index[i])] # set the release dates data_frame_rets.ix[start_index, 'Rel'] = ef_time # set entry points data_frame_rets.ix[finish_index + 1, 'Rel'] = numpy.zeros( len(start_index)) # set exit points data_frame_rets['Rel'] = data_frame_rets['Rel'].fillna( method='pad') # fill down signals data_frame_rets = data_frame_rets[pandas.notnull( data_frame_rets['Ind'])] # get rid of other data_frame = data_frame_rets.pivot(index='Ind', columns='Rel', values=data_frame_rets.columns[0]) data_frame.index.names = [None] if create_index: tsc = TimeSeriesCalcs() data_frame.ix[-minute_start + min_offset, :] = numpy.nan data_frame = tsc.create_mult_index(data_frame) else: if vol is True: # annualise (if vol) data_frame = pandas.rolling_std( data_frame, window=5) * math.sqrt(ann_factor) else: data_frame = data_frame.cumsum() return data_frame
def calculate_leverage_factor(self, returns_df, vol_target, vol_max_leverage, vol_periods = 60, vol_obs_in_year = 252, vol_rebalance_freq = 'BM', data_resample_freq = None, data_resample_type = 'mean', returns = True, period_shift = 0): """ calculate_leverage_factor - Calculates the time series of leverage for a specified vol target Parameters ---------- returns_df : DataFrame Asset returns vol_target : float vol target for assets vol_max_leverage : float maximum leverage allowed vol_periods : int number of periods to calculate volatility vol_obs_in_year : int number of observations in the year vol_rebalance_freq : str how often to rebalance vol_resample_type : str do we need to resample the underlying data first? (eg. have we got intraday data?) returns : boolean is this returns time series or prices? period_shift : int should we delay the signal by a number of periods? Returns ------- pandas.Dataframe """ tsc = TimeSeriesCalcs() tsf = TimeSeriesFilter() if data_resample_freq is not None: return # TODO not implemented yet if not returns: returns_df = tsc.calculate_returns(returns_df) roll_vol_df = tsc.rolling_volatility(returns_df, periods = vol_periods, obs_in_year = vol_obs_in_year).shift(period_shift) # calculate the leverage as function of vol target (with max lev constraint) lev_df = vol_target / roll_vol_df lev_df[lev_df > vol_max_leverage] = vol_max_leverage lev_df = tsf.resample_time_series_frequency(lev_df, vol_rebalance_freq, data_resample_type) returns_df, lev_df = returns_df.align(lev_df, join='left', axis = 0) lev_df = lev_df.fillna(method='ffill') lev_df.ix[0:vol_periods] = numpy.nan # ignore the first elements before the vol window kicks in return lev_df