def linear_reg(): ibm_data = data_retrieval.get_time_series( 'download_yahoo_timeseries', { 'symbol': 'IBM', 'start': datetime.datetime(2003, 11, 11), 'end': datetime.datetime(2013, 11, 11) }) spx_data = data_retrieval.get_time_series( 'download_yahoo_timeseries', { 'symbol': '^GSPC', 'start': datetime.datetime(2003, 11, 11), 'end': datetime.datetime(2013, 11, 11) }) series = zip(*timeseries.common_dates( ibm_data[data_structure.TIMESERIES], spx_data[data_structure.TIMESERIES])) ibm = [float(val) for val in series[1]] spx = [float(val) for val in series[2]] slope, intercept, r_value, _, _ = algorithms.linreg(ibm, spx) print('Beta: {0}'.format(intercept)) print('Slope: {0}'.format(slope)) print('RSq: {0}'.format(r_value ** 2)) series_for_charting = [('IBM', ibm), ('SPX', spx)] charts.line(series[0], series_for_charting) charts.scatter(series_for_charting)
def generate_us_equity_universe(idx_list, start_date, end_date): equity_list_per_date = dict() for index in idx_list: field = '' if index == 'DJI Index': field = 'INDX_MWEIGHT' else: field = 'INDX_MWEIGHT_HIST' loader = 'download_bbg_timeseries'; loader_args = { 'symbol': index, 'start': start_date, 'end': end_date, 'field': field } index_composition_ts = data_retrieval.get_time_series(loader, loader_args) if index_composition_ts: for date, equ_tup in index_composition_ts: if not date in equity_list_per_date: equity_list_per_date[date] = set(equ_tup[1:]) equity_list_per_date[date].update(equ_tup[1:]) return equity_list_per_date
def test_get_time_series_mongo(self): ''' End to end test of the following functions get_from_cache get_time_series clear_cache ''' symbol = 'TGTT' loader = 'download_mock_series' loader_args = { 'symbol': symbol, 'start': datetime.datetime(2012, 11, 11), 'end': datetime.datetime(2013, 11, 11) } # Check we can load a db client instance db_client = data_retrieval.get_db() self.assertIsNotNone(db_client) # Check the id is not in the db to start with id = data_retrieval.get_id(loader, loader_args) if id is not None: db_client.remove(loader, loader_args) id = data_retrieval.get_id(loader, loader_args) self.assertIsNone(id) # Retrieve the series result = data_retrieval.get_time_series(loader, loader_args) # Check the id now exists in the db id = data_retrieval.get_id(loader, loader_args) self.assertIsNotNone(id) self.cache_id = id # Check the contents look ok: expect that the loader args are set # as metadata on the time series, but check # data_loader.download_mock_series for how the test result is # actually created self.assertEqual(loader_args, result[symbol][data_loader.METADATA]) # Check we can now retrieve the id from the cache cached_result = data_retrieval.get_from_cache(id) self.assertEqual(loader_args, cached_result[symbol][data_loader.METADATA]) # Remove it from the cache data_retrieval.clear_cache(id) # Check it's gone from the cache... self.assertFalse(data_retrieval.get_from_cache(id)) # ... and the db db_client.remove(loader, loader_args) id = data_retrieval.get_id(loader, loader_args) self.assertIsNone(id)
def get_series(): args = { 'symbol': 'IBM', 'start': datetime.datetime(2003, 11, 11), 'end': datetime.datetime(2013, 11, 11) } loader = 'download_yahoo_timeseries' ts = data_retrieval.get_time_series(loader, args) dates, values = zip(*ts[data_structure.TIMESERIES]) line_plot(dates, values)
def get_data(date, symbol): args = { 'symbol': symbol, 'start': datetime.datetime(2003, 11, 11), 'end': datetime.datetime(2013, 11, 11) } loader = 'download_yahoo_timeseries' ts = data_retrieval.get_time_series(loader, args) dates, values = zip(*ts[args['symbol']][data_loader.TIMESERIES]) return { 'dates': dates, 'values': [float(v) for v in values] }
def test_get_time_series_mongo(self): """ End to end test of the following functions get_from_cache get_time_series clear_cache """ symbol = 'TGTT' loader = 'download_mock_series' loader_args = { 'symbol': symbol, 'start': datetime.datetime(2012, 11, 11), 'end': datetime.datetime(2013, 11, 11) } # Check we can load a db client instance client = data_retrieval.get_db() self.assertIsNotNone(client) collection = db.TIMESERIES_COLLECTION # Clear out the collection client.remove(collection) # Retrieve the series result = data_retrieval.get_time_series(loader, loader_args) # Check the contents look ok: expect that the loader args are set # as metadata on the time series, but check # data_loader.download_mock_series for how the test result is # actually created self.assertEqual(1, len(result)) self.assertEqual(loader_args, result[0][data_structure.ID]) query = { '{0}.symbol'.format(data_structure.ID): loader_args['symbol'] } # Check the series is in the db series = client.find(collection, query) self.assertEqual(1, len(series)) self.assertEqual(loader_args, series[0][data_structure.ID]) # Remove it from the db client.remove(collection, query) # Check it's gone from the db series = client.find(collection, query) self.assertEqual(0, len(series))
def test_get_time_series(self): ''' End to end test of the following functions get_from_cache get_time_series clear_cache ''' symbol = 'TGTT' loader = 'download_mock_series' loader_args = { 'symbol': symbol, 'start': datetime.datetime(2012, 11, 11), 'end': datetime.datetime(2013, 11, 11) } # Get the id used to store the file in the cache and assign it # so we can delete it in tearDown id = data_retrieval.get_id(loader, loader_args) self.cache_id = id # Check the item isn't in the cache to start with # If the test fails here, just delete the file in the cache folder self.assertFalse(data_retrieval.get_from_cache(id)) # Retrieve the series result = data_retrieval.get_time_series(loader, loader_args) # Check the contents look ok: expect that the loader args are set # as metadata on the time series, but check # data_loader.download_mock_series for how the test result is # actually created self.assertEqual(loader_args, result[symbol][data_loader.METADATA]) # Check we can now retrieve the id from the cache cached_result = data_retrieval.get_from_cache(id) self.assertEqual(loader_args, cached_result[symbol][data_loader.METADATA]) # Remove it from the cache data_retrieval.clear_cache(id) # Check it's gone self.assertFalse(data_retrieval.get_from_cache(id))
def get_bbg_data(): t = datetime.datetime.today() today_date = datetime.datetime(t.year, t.month, t.day) start_date = datetime.datetime(1990, 1, 1) end_date = today_date - datetime.timedelta(1) index_list = ('SPX Index', 'DJI Index', 'RTY Index') equity_set = set() for index in index_list: field = '' if index == 'DJI Index': field = 'INDX_MWEIGHT' else: field = 'INDX_MWEIGHT_HIST' idx_data = data_retrieval.get_time_series( 'download_bbg_timeseries', { 'symbol': index, 'start': start_date, 'end': end_date, 'field' : field }) if idx_data is not None: utils.serialise_csv(idx_data, data_retrieval.get_csv_filename('_'.join((index, 'COMPOSITION')))) for date,equity_list in idx_data: equity_set.update(equity_list) fields = ( 'PX_LAST', 'PX_OFFICIAL_CLOSE', 'PX_OPEN', 'PX_LAST', 'PX_OPEN', 'PX_LOW', 'TOT_RETURN_INDEX_GROSS_DVDS', 'DAY_TO_DAY_TOT_RETURN_GROSS_DVDS', 'EQY_DPS', 'SECURITY_DES', 'TRAIL_12M_GROSS_MARGIN', 'ASSET_TURNOVER', 'BS_SH_OUT', 'TRAIL_12M_CASH_FROM_OPER', 'CF_CASH_FROM_OPER', 'BS_LT_BORROW', 'LT_DEBT_TO_TOT_ASSET', 'LIVE_LT_DEBT_TO_TOTAL_ASSETS', 'CUR_RATIO', 'CASH_RATIO', 'QUICK_RATIO', 'NET_INCOME', 'CF_CASH_FROM_OPER', 'RETURN_ON_ASSET', 'EBIT', 'EBITDA', 'TOT_DEBT_TO_TOT_ASSET', 'TOT_DEBT_TO_COM_EQY', 'BS_TOT_ASSET', 'HISTORICAL_MARKET_CAP') for equity in equity_set: equity = equity + " Equity" for field in fields: equ_data = data_retrieval.get_time_series( 'download_bbg_timeseries', { 'symbol': equity, 'start': start_date, 'end': end_date, 'field' : field }) if equ_data is not None: utils.serialise_csv(equ_data, data_retrieval.get_csv_filename('_'.join((equity, field))))