def get_economic_data_history(self, start_date, finish_date, country_group, data_type, source='fred', cache_algo="internet_load_return"): #vendor_country_codes = self.fred_country_codes[country_group] #vendor_pretty_country = self.fred_nice_country_codes[country_group] if isinstance(country_group, list): pretty_country_names = country_group else: # get all the country names in the country_group pretty_country_names = list(self._econ_country_groups[ self._econ_country_groups["Country Group"] == country_group] ['Country']) # construct the pretty tickers pretty_tickers = [x + '-' + data_type for x in pretty_country_names] # get vendor tickers vendor_tickers = [] for pretty_ticker in pretty_tickers: vendor_ticker = list( self._all_econ_tickers[self._all_econ_tickers["Full Code"] == pretty_ticker][source].values) if vendor_ticker == []: vendor_ticker = None self.logger.error('Could not find match for ' + pretty_ticker) else: vendor_ticker = vendor_ticker[0] vendor_tickers.append(vendor_ticker) vendor_fields = ['close'] if source == 'bloomberg': vendor_fields = ['PX_LAST'] time_series_request = TimeSeriesRequest( start_date=start_date, # start date finish_date=finish_date, # finish date category='economic', freq='daily', # intraday data data_source=source, # use Bloomberg as data source cut='LOC', tickers=pretty_tickers, fields=['close'], # which fields to download vendor_tickers=vendor_tickers, vendor_fields=vendor_fields, # which Bloomberg fields to download cache_algo=cache_algo) # how to return data return self.time_series_factory.harvest_time_series( time_series_request)
def fill_assets(self): ##### FILL IN WITH YOUR ASSET DATA # for FX basket full_bkt = [ 'EURUSD', 'USDJPY', 'GBPUSD', 'AUDUSD', 'USDCAD', 'NZDUSD', 'USDCHF', 'USDNOK', 'USDSEK' ] basket_dict = {} for i in range(0, len(full_bkt)): basket_dict[full_bkt[i]] = [full_bkt[i]] basket_dict['Thalesians FX CTA'] = full_bkt br = self.fill_backtest_request() self.logger.info("Loading asset data...") vendor_tickers = [ 'FRED/DEXUSEU', 'FRED/DEXJPUS', 'FRED/DEXUSUK', 'FRED/DEXUSAL', 'FRED/DEXCAUS', 'FRED/DEXUSNZ', 'FRED/DEXSZUS', 'FRED/DEXNOUS', 'FRED/DEXSDUS' ] time_series_request = TimeSeriesRequest( start_date=br.start_date, # start date finish_date=br.finish_date, # finish date freq='daily', # daily data data_source='quandl', # use Quandl as data source tickers=full_bkt, # ticker (Thalesians) fields=['close'], # which fields to download vendor_tickers=vendor_tickers, # ticker (Quandl) vendor_fields=['close'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data asset_df = self.tsfactory.harvest_time_series(time_series_request) # if web connection fails read from CSV if asset_df is None: import pandas asset_df = pandas.read_csv( "d:/fxcta.csv", index_col=0, parse_dates=['Date'], date_parser=lambda x: pandas.datetime.strptime(x, '%Y-%m-%d')) # signalling variables spot_df = asset_df spot_df2 = None # asset_df return asset_df, spot_df, spot_df2, basket_dict
def construct_strategy_benchmark(self): ###### FILL IN WITH YOUR OWN BENCHMARK tsr_indices = TimeSeriesRequest( start_date = '01 Jan 1980', # start date finish_date = datetime.datetime.utcnow(), # finish date freq = 'daily', # intraday data data_source = 'quandl', # use Bloomberg as data source tickers = ["EURUSD"], # tickers to download vendor_tickers=['FRED/DEXUSEU'], fields = ['close'], # which fields to download vendor_fields = ['close'], cache_algo = 'cache_algo_return') # how to return data) df = self.tsfactory.harvest_time_series(tsr_indices) df.columns = [x.split(".")[0] for x in df.columns] return df
# In[ ]: from pythalesians.market.requests.timeseriesrequest import TimeSeriesRequest from datetime import timedelta import datetime tickers = ['S&P500', 'FTSE', 'Nikkei'] vendor_tickers = ['SPX Index', 'UKX Index', 'NKY Index'] time_series_request = TimeSeriesRequest( start_date = datetime.datetime.utcnow() - timedelta(days=180), # start date finish_date = datetime.datetime.utcnow(), # finish date freq = 'daily', # daily data data_source = 'bloomberg', # use Bloomberg as data source tickers = tickers, # ticker (Thalesians) fields = ['close'], # which fields to download vendor_tickers = vendor_tickers, # ticker (Bloomberg) vendor_fields = ['PX_LAST'], # which Bloomberg fields to download cache_algo = 'internet_load_return') # how to return data # We now pass this TimeSeriesRequest object to a LightTimeSeriesFactory, which will return a pandas dataframe from the data source and will also output logger text (we can edit logging.conf to make this less verbose or to change the output of the log to disk, rather than screen, however, I've included it, so we can see which classes are being called). # # By default, PyThalesians will split up the ticker download into several chunks, using multithreading to make several calls to our data source. The number of threads can be changed in the Constants.py file (we can also specify if we would prefer to use multiprocessing as opposed to multithreading, which gets around the Python GIL. In most cases, however, the overhead of doing this is unnecessary). # In[ ]: from pythalesians.market.loaders.lighttimeseriesfactory import LightTimeSeriesFactory ltsf = LightTimeSeriesFactory()
# just change "False" to "True" to run any of the below examples ###### download daily data from Bloomberg for FX, with different threading techniques if False: time_series_request = TimeSeriesRequest( start_date="01 Jan 1999", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='bloomberg', # use Bloomberg as data source tickers=[ 'EURUSD', # ticker (Thalesians) 'GBPUSD', 'USDJPY', 'AUDUSD' ], fields=['close', 'high', 'low'], # which fields to download vendor_tickers=[ 'EURUSD BGN Curncy', # ticker (Bloomberg) 'GBPUSD BGN Curncy', 'USDJPY BGN Curncy', 'AUDUSD BGN Curncy' ], vendor_fields=['PX_LAST', 'PX_HIGH', 'PX_LOW'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() from pythalesians.util.constants import Constants
import datetime from pythalesians.market.loaders.lighttimeseriesfactory import LightTimeSeriesFactory from pythalesians.market.requests.timeseriesrequest import TimeSeriesRequest from pythalesians.timeseries.calcs.timeseriescalcs import TimeSeriesCalcs from pythalesians.graphics.graphs.plotfactory import PlotFactory from pythalesians.graphics.graphs.graphproperties import GraphProperties if True: time_series_request = TimeSeriesRequest( start_date="01 Jan 2013", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='google', # use Bloomberg as data source tickers=['Apple', 'S&P500 ETF'], # ticker (Thalesians) fields=['close'], # which fields to download vendor_tickers=['aapl', 'spy'], # ticker (Google) vendor_fields=['Close'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() tsc = TimeSeriesCalcs() df = tsc.create_mult_index_from_prices( ltsf.harvest_time_series(time_series_request)) gp = GraphProperties() gp.title = "S&P500 vs Apple" # plot first with PyThalesians and then Plotly (via Cufflinks)
def get_fx_cross(self, start, end, cross, cut = "NYC", source = "bloomberg", freq = "intraday", cache_algo='cache_algo_return', type = 'spot'): if source == "gain" or source == 'dukascopy' or freq == 'tick': return self.get_fx_cross_tick(start, end, cross, cut = cut, source = source, cache_algo='cache_algo_return', type = 'spot') if isinstance(cross, str): cross = [cross] time_series_request = TimeSeriesRequest() time_series_factory = self.time_series_factory time_series_calcs = TimeSeriesCalcs() data_frame_agg = None if freq == 'intraday': time_series_request.gran_freq = "minute" # intraday elif freq == 'daily': time_series_request.gran_freq = "daily" # intraday time_series_request.freq_mult = 1 # 1 min time_series_request.cut = cut # NYC/BGN ticker time_series_request.fields = 'close' # close field only time_series_request.cache_algo = cache_algo # cache_algo_only, cache_algo_return, internet_load time_series_request.environment = 'backtest' time_series_request.start_date = start time_series_request.finish_date = end time_series_request.data_source = source for cr in cross: base = cr[0:3] terms = cr[3:6] if (type == 'spot'): # non-USD crosses if base != 'USD' and terms != 'USD': base_USD = self.fxconv.correct_notation('USD' + base) terms_USD = self.fxconv.correct_notation('USD' + terms) # TODO check if the cross exists in the database # download base USD cross time_series_request.tickers = base_USD time_series_request.category = self.fxconv.em_or_g10(base, freq) base_vals = time_series_factory.harvest_time_series(time_series_request) # download terms USD cross time_series_request.tickers = terms_USD time_series_request.category = self.fxconv.em_or_g10(terms, freq) terms_vals = time_series_factory.harvest_time_series(time_series_request) if (base_USD[0:3] == 'USD'): base_vals = 1 / base_vals if (terms_USD[0:3] == 'USD'): terms_vals = 1 / terms_vals base_vals.columns = ['temp'] terms_vals.columns = ['temp'] cross_vals = base_vals.div(terms_vals, axis = 'index') cross_vals.columns = [cr + '.close'] else: if base == 'USD': non_USD = terms if terms == 'USD': non_USD = base correct_cr = self.fxconv.correct_notation(cr) time_series_request.tickers = correct_cr time_series_request.category = self.fxconv.em_or_g10(non_USD, freq) cross_vals = time_series_factory.harvest_time_series(time_series_request) # flip if not convention if(correct_cr != cr): cross_vals = 1 / cross_vals cross_vals.columns.names = [cr + '.close'] elif type[0:3] == "tot": if freq == 'daily': # download base USD cross time_series_request.tickers = base + 'USD' time_series_request.category = self.fxconv.em_or_g10(base, freq) + '-tot' if type == "tot": base_vals = time_series_factory.harvest_time_series(time_series_request) else: x = 0 # download terms USD cross time_series_request.tickers = terms + 'USD' time_series_request.category = self.fxconv.em_or_g10(terms, freq) + '-tot' if type == "tot": terms_vals = time_series_factory.harvest_time_series(time_series_request) else: x = 0 base_rets = time_series_calcs.calculate_returns(base_vals) terms_rets = time_series_calcs.calculate_returns(terms_vals) cross_rets = base_rets.sub(terms_rets.iloc[:,0],axis=0) # first returns of a time series will by NaN, given we don't know previous point cross_rets.iloc[0] = 0 cross_vals = time_series_calcs.create_mult_index(cross_rets) cross_vals.columns = [cr + '-tot.close'] elif freq == 'intraday': self.logger.info('Total calculated returns for intraday not implemented yet') return None if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg
def get_fx_cross_tick(self, start, end, cross, cut = "NYC", source = "gain", cache_algo='cache_algo_return', type = 'spot'): if isinstance(cross, str): cross = [cross] time_series_request = TimeSeriesRequest() time_series_factory = self.time_series_factory data_frame_agg = None time_series_request.gran_freq = "tick" # tick time_series_request.freq_mult = 1 # 1 min time_series_request.cut = cut # NYC/BGN ticker time_series_request.fields = ['bid', 'ask'] # bid/ask field only time_series_request.cache_algo = cache_algo # cache_algo_only, cache_algo_return, internet_load time_series_request.environment = 'backtest' time_series_request.start_date = start time_series_request.finish_date = end time_series_request.data_source = source time_series_request.category = 'fx' for cr in cross: if (type == 'spot'): time_series_request.tickers = cr cross_vals = time_series_factory.harvest_time_series(time_series_request) cross_vals.columns = [cr + '.bid', cr + '.ask'] if data_frame_agg is None: data_frame_agg = cross_vals else: data_frame_agg = data_frame_agg.join(cross_vals, how='outer') # strip the nan elements data_frame_agg = data_frame_agg.dropna() return data_frame_agg
if True: from pythalesians.market.loaders.lighttimeseriesfactory import LightTimeSeriesFactory from pythalesians.market.requests.timeseriesrequest import TimeSeriesRequest from pythalesians.timeseries.techind.techindicator import TechIndicator from pythalesians.timeseries.techind.techparams import TechParams from pythalesians.graphics.graphs.plotfactory import PlotFactory import datetime time_series_request = TimeSeriesRequest( start_date="01 Jan 1970", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='quandl', # use Quandl as data source tickers=[ 'EURUSD', # ticker (Thalesians) 'GBPUSD' ], fields=['close'], # which fields to download vendor_tickers=['FRED/DEXUSEU', 'FRED/DEXUSUK'], # ticker (Quandl) vendor_fields=['close'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() daily_vals = ltsf.harvest_time_series(time_series_request) techind = TechIndicator() tech_params = TechParams() tech_params.sma_period = 20
if True: logger = LoggerManager().getLogger(__name__) import datetime # just change "False" to "True" to run any of the below examples ###### download daily data from quandl for S&P500 and then plot if True: time_series_request = TimeSeriesRequest( start_date = "01 Jan 2014", # start date finish_date = datetime.date.today(), # finish date freq = 'daily', # daily data data_source = 'quandl', # use quandl as data source tickers = ['S&P500'], # ticker (Thalesians) fields = ['close', 'open', 'adjusted-close', 'high'], # which fields to download vendor_tickers = ['YAHOO/INDEX_GSPC'], # ticker (quandl) vendor_fields = ['close', 'open', 'adjusted-close', 'high'], # which quandl fields to download cache_algo = 'internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = None df = ltsf.harvest_time_series(time_series_request) pf = PlotFactory() pf.plot_line_graph(df, adapter = 'pythalesians') ###### download monthly quandl data for Total US nonfarm payrolls if True:
if True: logger = LoggerManager().getLogger(__name__) import datetime # just change "False" to "True" to run any of the below examples ###### download daily data from Bloomberg for USD/JPY ###### download BoJ intervention data from FRED if True: time_series_request = TimeSeriesRequest( start_date="01 Jan 1995", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='bloomberg', # use Bloomberg as data source tickers=['USDJPY'], # ticker (Thalesians) fields=['close'], # which fields to download vendor_tickers=['USDJPY BGN Curncy'], # ticker (Bloomberg) vendor_fields=['PX_LAST'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = None df = ltsf.harvest_time_series(time_series_request) df.columns = [x.replace('.close', '') for x in df.columns.values] time_series_request.tickers = ['USDJPY purchases (bn USD)'] time_series_request.vendor_tickers = ['JPINTDUSDJPY'] time_series_request.data_source = 'fred'
seasonality = Seasonality() tsc = TimeSeriesCalcs() logger = LoggerManager().getLogger(__name__) pf = PlotFactory() # just change "False" to "True" to run any of the below examples ###### calculate seasonal moves in SPX (using Quandl data) if True: time_series_request = TimeSeriesRequest( start_date="01 Jan 1970", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='bloomberg', # use Quandl as data source tickers=['S&P500'], fields=['close'], # which fields to download vendor_tickers=['SPX Index'], # ticker (Quandl) vendor_fields=['PX_LAST'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = ltsf.harvest_time_series(time_series_request) df_ret = tsc.calculate_returns(df) day_of_month_seasonality = seasonality.bus_day_of_month_seasonality( df_ret, partition_by_month=False) day_of_month_seasonality = tsc.convert_month_day_to_date_time( day_of_month_seasonality)
# if write_tick_disk: # tick = open(out_path_list[i], "rb+").read() # else: # tick = tick_list[i] # # time = time_list[i] # data = lzma.decompress(tick) # df_list.append(self.retrieve_df(data, symbol, time)) if __name__ == '__main__': from pythalesians.market.requests.timeseriesrequest import TimeSeriesRequest import datetime dc = LoaderDukasCopy() time_series_request = TimeSeriesRequest( start_date="01 Jun 2015", # start date finish_date="02 Jun 2015", # finish date freq='tick', # tick data data_source='dukascopy', # use Bloomberg as data source tickers=['EURUSD'], # ticker (Thalesians) fields=['bid', 'ask'], # which fields to download vendor_tickers=[ 'EURUSD', # ticker (Bloomberg) 'GBPUSD' ], vendor_fields=['bid', 'ask'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data dc.load_ticker(time_series_request)
import pytz # just change "False" to "True" to run any of the below examples ###### download second data from Bloomberg for EUR/USD, USD/JPY (inverted) and GBP/USD around last payroll if True: finish_date = datetime.datetime.utcnow() start_date = finish_date - timedelta(days=60) # fetch NFP times from Bloomberg time_series_request = TimeSeriesRequest( start_date = start_date, # start date finish_date = finish_date, # finish date category = "events", freq = 'daily', # daily data data_source = 'bloomberg', # use Bloomberg as data source tickers = ['NFP'], fields = ['release-date-time-full'], # which fields to download vendor_tickers = ['NFP TCH Index'], # ticker (Bloomberg) vendor_fields = ['ECO_FUTURE_RELEASE_DATE_LIST'], # which Bloomberg fields to download cache_algo = 'internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() ts_filter = TimeSeriesFilter() df_event_times = ltsf.harvest_time_series(time_series_request) utc_time = pytz.utc df_event_times = pandas.DataFrame(index = df_event_times['NFP.release-date-time-full']) df_event_times.index = df_event_times.index.tz_localize(utc_time) # work in UTC time df_event_times = ts_filter.filter_time_series_by_date(start_date, finish_date, df_event_times)
from pythalesians.market.loaders.lighttimeseriesfactory import LightTimeSeriesFactory import datetime from pythalesians.graphics.graphs.plotfactory import PlotFactory from pythalesians.graphics.graphs.graphproperties import GraphProperties end = datetime.datetime.utcnow() start_date = end.replace(hour=0, minute=0, second=0, microsecond=0) # Returns a copy time_series_request = TimeSeriesRequest( start_date=start_date, # start date finish_date=datetime.datetime.utcnow(), # finish date freq='intraday', # intraday data data_source='bloomberg', # use Bloomberg as data source tickers=['EURUSD'], # ticker (Thalesians) fields=['close'], # which fields to download vendor_tickers=['EURUSD BGN Curncy'], # ticker (Bloomberg) vendor_fields=['close'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = ltsf.harvest_time_series(time_series_request) df.columns = [x.replace('.close', '') for x in df.columns.values] gp = GraphProperties() gp.title = 'EURUSD stuff!' gp.file_output = 'EURUSD.png' gp.source = 'Thalesians/BBG (created with PyThalesians Python library)'
import datetime # just change "False" to "True" to run any of the below examples ###### download daily data from Bloomberg for EUR/USD and GBP/USD spot, then calculate correlation if True: time_series_request = TimeSeriesRequest( start_date="01 Jan 2014", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='bloomberg', # use Bloomberg as data source tickers=[ 'EURUSD', # ticker (Thalesians) 'GBPUSD', 'AUDUSD' ], fields=['close'], # which fields to download vendor_tickers=[ 'EURUSD BGN Curncy', # ticker (Bloomberg) 'GBPUSD BGN Curncy', 'AUDUSD BGN Curncy' ], vendor_fields=['PX_LAST'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = None df = ltsf.harvest_time_series(time_series_request) tsc = TimeSeriesCalcs()
logger = LoggerManager().getLogger(__name__) import datetime ###### download daily data from Bloomberg for EUR/USD and GBP/USD spot and then plot if True: time_series_request = TimeSeriesRequest( start_date="01 Jan 2014", # start date finish_date=datetime.date.today(), # finish date freq='daily', # daily data data_source='bloomberg', # use Bloomberg as data source tickers=[ 'EURUSD', # ticker (Thalesians) 'GBPUSD' ], fields=['close', 'high', 'low'], # which fields to download vendor_tickers=[ 'EURUSD BGN Curncy', # ticker (Bloomberg) 'GBPUSD BGN Curncy' ], vendor_fields=['PX_LAST', 'PX_HIGH', 'PX_LOW'], # which Bloomberg fields to download cache_algo='internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = None df = ltsf.harvest_time_series(time_series_request) pf = PlotFactory()
if True: logger = LoggerManager().getLogger(__name__) import datetime # just change "False" to "True" to run any of the below examples ###### download daily data from Bloomberg for USD/JPY ###### download BoJ intervention data from FRED if True: time_series_request = TimeSeriesRequest( start_date = "01 Jan 1995", # start date finish_date = datetime.date.today(), # finish date freq = 'daily', # daily data data_source = 'bloomberg', # use Bloomberg as data source tickers = ['USDJPY'], # ticker (Thalesians) fields = ['close'], # which fields to download vendor_tickers = ['USDJPY BGN Curncy'], # ticker (Bloomberg) vendor_fields = ['PX_LAST'], # which Bloomberg fields to download cache_algo = 'internet_load_return') # how to return data ltsf = LightTimeSeriesFactory() df = None df = ltsf.harvest_time_series(time_series_request) df.columns = [x.replace('.close', '') for x in df.columns.values] time_series_request.tickers = ['USDJPY purchases (bn USD)'] time_series_request.vendor_tickers = ['JPINTDUSDJPY'] time_series_request.data_source = 'fred'