def linear_reg():
ibm_data = data_retrieval.get_time_series(
'download_yahoo_timeseries',
{
'symbol': 'IBM',
'start': datetime.datetime(2003, 11, 11),
'end': datetime.datetime(2013, 11, 11)
})
spx_data = data_retrieval.get_time_series(
'download_yahoo_timeseries',
{
'symbol': '^GSPC',
'start': datetime.datetime(2003, 11, 11),
'end': datetime.datetime(2013, 11, 11)
})
series = zip(*timeseries.common_dates(
ibm_data[data_structure.TIMESERIES],
spx_data[data_structure.TIMESERIES]))
ibm = [float(val) for val in series[1]]
spx = [float(val) for val in series[2]]
slope, intercept, r_value, _, _ = algorithms.linreg(ibm, spx)
print('Beta: {0}'.format(intercept))
print('Slope: {0}'.format(slope))
print('RSq: {0}'.format(r_value ** 2))
series_for_charting = [('IBM', ibm), ('SPX', spx)]
charts.line(series[0], series_for_charting)
charts.scatter(series_for_charting)
def generate_us_equity_universe(idx_list, start_date, end_date):
equity_list_per_date = dict()
for index in idx_list:
field = ''
if index == 'DJI Index':
field = 'INDX_MWEIGHT'
else:
field = 'INDX_MWEIGHT_HIST'
loader = 'download_bbg_timeseries';
loader_args = {
'symbol': index,
'start': start_date,
'end': end_date,
'field': field
}
index_composition_ts = data_retrieval.get_time_series(loader, loader_args)
if index_composition_ts:
for date, equ_tup in index_composition_ts:
if not date in equity_list_per_date:
equity_list_per_date[date] = set(equ_tup[1:])
equity_list_per_date[date].update(equ_tup[1:])
return equity_list_per_date
def test_get_time_series_mongo(self):
'''
End to end test of the following functions
get_from_cache
get_time_series
clear_cache
'''
symbol = 'TGTT'
loader = 'download_mock_series'
loader_args = {
'symbol': symbol,
'start': datetime.datetime(2012, 11, 11),
'end': datetime.datetime(2013, 11, 11)
}
# Check we can load a db client instance
db_client = data_retrieval.get_db()
self.assertIsNotNone(db_client)
# Check the id is not in the db to start with
id = data_retrieval.get_id(loader, loader_args)
if id is not None:
db_client.remove(loader, loader_args)
id = data_retrieval.get_id(loader, loader_args)
self.assertIsNone(id)
# Retrieve the series
result = data_retrieval.get_time_series(loader, loader_args)
# Check the id now exists in the db
id = data_retrieval.get_id(loader, loader_args)
self.assertIsNotNone(id)
self.cache_id = id
# Check the contents look ok: expect that the loader args are set
# as metadata on the time series, but check
# data_loader.download_mock_series for how the test result is
# actually created
self.assertEqual(loader_args,
result[symbol][data_loader.METADATA])
# Check we can now retrieve the id from the cache
cached_result = data_retrieval.get_from_cache(id)
self.assertEqual(loader_args,
cached_result[symbol][data_loader.METADATA])
# Remove it from the cache
data_retrieval.clear_cache(id)
# Check it's gone from the cache...
self.assertFalse(data_retrieval.get_from_cache(id))
# ... and the db
db_client.remove(loader, loader_args)
id = data_retrieval.get_id(loader, loader_args)
self.assertIsNone(id)
def get_series():
args = {
'symbol': 'IBM',
'start': datetime.datetime(2003, 11, 11),
'end': datetime.datetime(2013, 11, 11)
}
loader = 'download_yahoo_timeseries'
ts = data_retrieval.get_time_series(loader, args)
dates, values = zip(*ts[data_structure.TIMESERIES])
line_plot(dates, values)
def get_data(date, symbol):
args = {
'symbol': symbol,
'start': datetime.datetime(2003, 11, 11),
'end': datetime.datetime(2013, 11, 11)
}
loader = 'download_yahoo_timeseries'
ts = data_retrieval.get_time_series(loader, args)
dates, values = zip(*ts[args['symbol']][data_loader.TIMESERIES])
return { 'dates': dates, 'values': [float(v) for v in values] }
def test_get_time_series_mongo(self):
"""
End to end test of the following functions
get_from_cache
get_time_series
clear_cache
"""
symbol = 'TGTT'
loader = 'download_mock_series'
loader_args = {
'symbol': symbol,
'start': datetime.datetime(2012, 11, 11),
'end': datetime.datetime(2013, 11, 11)
}
# Check we can load a db client instance
client = data_retrieval.get_db()
self.assertIsNotNone(client)
collection = db.TIMESERIES_COLLECTION
# Clear out the collection
client.remove(collection)
# Retrieve the series
result = data_retrieval.get_time_series(loader, loader_args)
# Check the contents look ok: expect that the loader args are set
# as metadata on the time series, but check
# data_loader.download_mock_series for how the test result is
# actually created
self.assertEqual(1, len(result))
self.assertEqual(loader_args, result[0][data_structure.ID])
query = {
'{0}.symbol'.format(data_structure.ID): loader_args['symbol']
}
# Check the series is in the db
series = client.find(collection, query)
self.assertEqual(1, len(series))
self.assertEqual(loader_args, series[0][data_structure.ID])
# Remove it from the db
client.remove(collection, query)
# Check it's gone from the db
series = client.find(collection, query)
self.assertEqual(0, len(series))
def test_get_time_series(self):
'''
End to end test of the following functions
get_from_cache
get_time_series
clear_cache
'''
symbol = 'TGTT'
loader = 'download_mock_series'
loader_args = {
'symbol': symbol,
'start': datetime.datetime(2012, 11, 11),
'end': datetime.datetime(2013, 11, 11)
}
# Get the id used to store the file in the cache and assign it
# so we can delete it in tearDown
id = data_retrieval.get_id(loader, loader_args)
self.cache_id = id
# Check the item isn't in the cache to start with
# If the test fails here, just delete the file in the cache folder
self.assertFalse(data_retrieval.get_from_cache(id))
# Retrieve the series
result = data_retrieval.get_time_series(loader, loader_args)
# Check the contents look ok: expect that the loader args are set
# as metadata on the time series, but check
# data_loader.download_mock_series for how the test result is
# actually created
self.assertEqual(loader_args,
result[symbol][data_loader.METADATA])
# Check we can now retrieve the id from the cache
cached_result = data_retrieval.get_from_cache(id)
self.assertEqual(loader_args,
cached_result[symbol][data_loader.METADATA])
# Remove it from the cache
data_retrieval.clear_cache(id)
# Check it's gone
self.assertFalse(data_retrieval.get_from_cache(id))
def get_bbg_data():
t = datetime.datetime.today()
today_date = datetime.datetime(t.year, t.month, t.day)
start_date = datetime.datetime(1990, 1, 1)
end_date = today_date - datetime.timedelta(1)
index_list = ('SPX Index', 'DJI Index', 'RTY Index')
equity_set = set()
for index in index_list:
field = ''
if index == 'DJI Index':
field = 'INDX_MWEIGHT'
else:
field = 'INDX_MWEIGHT_HIST'
idx_data = data_retrieval.get_time_series(
'download_bbg_timeseries',
{
'symbol': index,
'start': start_date,
'end': end_date,
'field' : field
})
if idx_data is not None:
utils.serialise_csv(idx_data, data_retrieval.get_csv_filename('_'.join((index, 'COMPOSITION'))))
for date,equity_list in idx_data:
equity_set.update(equity_list)
fields = (
'PX_LAST',
'PX_OFFICIAL_CLOSE',
'PX_OPEN',
'PX_LAST',
'PX_OPEN',
'PX_LOW',
'TOT_RETURN_INDEX_GROSS_DVDS',
'DAY_TO_DAY_TOT_RETURN_GROSS_DVDS',
'EQY_DPS',
'SECURITY_DES',
'TRAIL_12M_GROSS_MARGIN',
'ASSET_TURNOVER',
'BS_SH_OUT',
'TRAIL_12M_CASH_FROM_OPER',
'CF_CASH_FROM_OPER',
'BS_LT_BORROW',
'LT_DEBT_TO_TOT_ASSET',
'LIVE_LT_DEBT_TO_TOTAL_ASSETS',
'CUR_RATIO',
'CASH_RATIO',
'QUICK_RATIO',
'NET_INCOME',
'CF_CASH_FROM_OPER',
'RETURN_ON_ASSET',
'EBIT',
'EBITDA',
'TOT_DEBT_TO_TOT_ASSET',
'TOT_DEBT_TO_COM_EQY',
'BS_TOT_ASSET',
'HISTORICAL_MARKET_CAP')
for equity in equity_set:
equity = equity + " Equity"
for field in fields:
equ_data = data_retrieval.get_time_series(
'download_bbg_timeseries',
{
'symbol': equity,
'start': start_date,
'end': end_date,
'field' : field
})
if equ_data is not None:
utils.serialise_csv(equ_data, data_retrieval.get_csv_filename('_'.join((equity, field))))
