def load_from_cache(self,
indexes={},
stocks=[],
start=pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc),
end=datetime.datetime.now().replace(tzinfo=pytz.utc),
):
'''Equivalent to zipline.utils.factory.load_from_yahoo.
'''
assert start <= end, 'start must be before end'
datetime_index = get_trading_days(start=start, end=end)
df = pd.DataFrame(index=datetime_index)
python_datetimes = list(datetime_index.to_pydatetime())
for symbol in stocks:
# there's probably a clever way of avoiding building a dictionary.
try:
date_values = self.get(symbol=symbol, dates=python_datetimes)
values = {date : value for date, value in date_values}
except NoDataForStock:
warnings.warn('No data for {}'.format(symbol))
except ExternalRequestFailed as e:
warnings.warn('Getting data for {} failed {}'.format(symbol,
e.message))
else:
series = pd.Series(values)
df[symbol] = series
for name, ticker in indexes.iteritems():
values = {date : value for date, value in self.get(symbol=ticker,
dates=python_datetimes)}
df[name] = pd.Series(values)
return df
def test_lots_of_dates(self):
'''sqlite can only handle 999 variables.'''
start = datetime.datetime(1990, 1,1, tzinfo=pytz.UTC)
end = datetime.datetime.now(pytz.UTC)
dates = list(get_trading_days(start, end).to_pydatetime())
for price, date in enumerate(dates):
self.connection.execute('INSERT INTO {} (symbol, date, metric, value) VALUES (?, ?, ?, ?)'.format(self.table),
('ABC', date, self.metric, price))
list(self.driver.get(symbol='ABC', dates=dates))
def test_lots_of_dates(self):
'''sqlite can only handle 999 variables.'''
start = datetime.datetime(1990, 1,1, tzinfo=pytz.UTC)
end = datetime.datetime.now(pytz.UTC)
dates = list(get_trading_days(start, end).to_pydatetime())
for price, date in enumerate(dates):
self.connection.execute('INSERT INTO {} (symbol, date, metric, value) VALUES (?, ?, ?, ?)'.format(self.table),
('ABC', date, self.metric, price))
list(self.driver.get(symbol='ABC', dates=dates))
def get_num_days_nxt_month(month, year):
"""
Inputs: today's month number and year number
Output: number of trading days in the following month
"""
nxt_month = month + 1 if month < 12 else 1
_year = year if nxt_month != 1 else year + 1
start = datetime(_year, nxt_month, 1, tzinfo=pytz.utc)
end = datetime(_year if nxt_month != 12 else _year + 1, nxt_month + 1 if nxt_month != 12 else 1, 1, tzinfo=pytz.utc)
return(len(get_trading_days(start, end)))
def test_volume(self):
'''make sure a larger number of records doesn't choke it somehow.'''
symbols = S_P_500_TICKERS[:200]
datetimeindex = get_trading_days(start=datetime.datetime(2012, 1, 1, tzinfo=pytz.UTC),
end=datetime.datetime(2012, 7, 4, tzinfo=pytz.UTC))
dates = [datetime.datetime(d.date().year, d.date().month, d.date().day).replace(tzinfo=pytz.UTC)
for d in datetimeindex]
symbol_date_combos = [(symbol, date) for symbol in symbols for date in dates]
test_vals = self.insert_date_combos(symbol_date_combos)
for symbol in symbols:
cached_values = self.driver.get(symbol=symbol, dates=dates)
cache_dict = {date : price for date, price in cached_values}
self.assertDictEqual(test_vals[symbol], cache_dict)
def test_volume(self):
'''make sure a larger number of records doesn't choke it somehow.'''
symbols = S_P_500_TICKERS[:200]
datetimeindex = get_trading_days(start=datetime.datetime(2012, 1, 1, tzinfo=pytz.UTC),
end=datetime.datetime(2012, 7, 4, tzinfo=pytz.UTC))
dates = [datetime.datetime(d.date().year, d.date().month, d.date().day).replace(tzinfo=pytz.UTC)
for d in datetimeindex]
symbol_date_combos = [(symbol, date) for symbol in symbols for date in dates]
test_vals = self.insert_date_combos(symbol_date_combos)
for symbol in symbols:
cached_values = self.driver.get(symbol=symbol, dates=dates)
cache_dict = {date : price for date, price in cached_values}
self.assertDictEqual(test_vals[symbol], cache_dict)
def get_start_date(iLag, tNow=None):
if tNow == None:
tNow = now_local().date()
# convert business lag into calendar lag, 365/252 = 1.45
# divide by 2 b/c we're using open prices and closing prices
iLagDays = int(iLag*1.5/2.0) + 2
trading_days = get_trading_days(tNow - datetime.timedelta(days=iLagDays), tNow)
# print '----> tNow = %s' % tNow
# print '----> iLagDays = %i' % iLagDays
# print '----> len(trading_days): %i ' % len(trading_days)
return trading_days[0].replace(tzinfo=None)
def get_start_date(iLag, tNow=None):
if tNow == None:
tNow = now_local().date()
# convert business lag into calendar lag, 365/252 = 1.45
# divide by 2 b/c we're using open prices and closing prices
iLagDays = int(iLag * 1.5 / 2.0) + 2
trading_days = get_trading_days(tNow - datetime.timedelta(days=iLagDays),
tNow)
# print '----> tNow = %s' % tNow
# print '----> iLagDays = %i' % iLagDays
# print '----> len(trading_days): %i ' % len(trading_days)
return trading_days[0].replace(tzinfo=None)
def get_trading_dates(start, end, offset=0):
trading_dates = list([])
trading_days= tradingcalendar.get_trading_days(start, end)
month = trading_days[0].month
for i in range(len(trading_days)) :
if trading_days[i].month != month :
try :
trading_dates = trading_dates + list([trading_days[i + offset]])
except :
raise
month = trading_days[i].month
return trading_dates
def endpoints(start, end, period='m'):
dates = tradingcalendar.get_trading_days(start, end)
if isinstance(period, int):
dates = [dates[i] for i in range(0, len(dates), period)]
else:
if period == 'm': months = 1
elif period == 'q': months = 3
elif period == 'b': months = 6
elif period == 'y': months = 12
e_dates = [dates[i - 1] for i in range(1,len(dates))\
if dates[i].month > dates[i-1].month\
or dates[i].year > dates[i-1].year ]+ list([dates[-1]])
dates = [e_dates[i] for i in range(0, len(e_dates), months)]
return dates
def endpoints(start, end, period='m') :
dates = tradingcalendar.get_trading_days(start, end)
if isinstance(period, int) :
dates = [dates[i] for i in range(0, len(dates), period)]
else :
if period == 'm' : months = 1
elif period == 'q' : months = 3
elif period == 'b' : months = 6
elif period == 'y' : months = 12
e_dates = [dates[i - 1] for i in range(1,len(dates))\
if dates[i].month > dates[i-1].month\
or dates[i].year > dates[i-1].year ]+ list([dates[-1]])
dates = [e_dates[i] for i in range(0,len(e_dates),months)]
return dates
def load_from_cache(self,
stocks,
start=pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc),
end=datetime.datetime.now().replace(tzinfo=pytz.utc)):
assert start <= end, 'start must be before end'
datetime_index = get_trading_days(start=start, end=end)
df = pd.DataFrame(index=datetime_index)
python_datetimes = list(datetime_index.to_pydatetime())
for symbol in stocks:
try:
values = self.get(symbol=symbol, dates=python_datetimes)
series = pd.Series(data=values, index=datetime_index)
except (NoDataForStock, NoDataForStockOnDate):
warnings.warn('No data for {}'.format(symbol))
except ExternalRequestFailed as e:
warnings.warn('Getting data for {} failed {}'.format(
symbol, e.message))
else:
df[symbol] = series
return df
def load_from_cache(self,
stocks,
start=pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc),
end=datetime.datetime.now().replace(tzinfo=pytz.utc)):
assert start <= end, 'start must be before end'
datetime_index = get_trading_days(start=start, end=end)
df = pd.DataFrame(index=datetime_index)
python_datetimes = list(datetime_index.to_pydatetime())
for symbol in stocks:
try:
values = self.get(symbol=symbol, dates=python_datetimes)
series = pd.Series(data=values, index=datetime_index)
except (NoDataForStock, NoDataForStockOnDate) as e:
warnings.warn('No data for {}'.format(symbol))
except ExternalRequestFailed as e:
warnings.warn('Getting data for {} failed {}'.format(symbol,
e.message))
else:
df[symbol] = series
return df
def get_trading_dates(start, end, offset=0):
''' to create a list of trading dates (timestamps) for use with Zipline or Quantopian.
offset = 0 -> 1st trading day of month, offset = -1 -> last trading day of month.
start, end are datetime.dates'''
trading_dates = list([])
trading_days= tradingcalendar.get_trading_days(start, end)
month = trading_days[0].month
for i in range(len(trading_days)) :
if trading_days[i].month != month :
try :
trading_dates = trading_dates + list([trading_days[i + offset]])
except :
raise
month = trading_days[i].month
return trading_dates
def get_trading_dates(start, end, offset=0):
''' to create a list of trading dates (timestamps) for use with Zipline or Quantopian.
offset = 0 -> 1st trading day of month, offset = -1 -> last trading day of month.
start, end are datetime.dates'''
trading_dates = list([])
trading_days = tradingcalendar.get_trading_days(start, end)
month = trading_days[0].month
for i in range(len(trading_days)):
if trading_days[i].month != month:
try:
trading_dates = trading_dates + list(
[trading_days[i + offset]])
except:
raise
month = trading_days[i].month
return trading_dates
def load_from_cache(
self,
indexes={},
stocks=[],
start=pd.datetime(1990, 1, 1, 0, 0, 0, 0, pytz.utc),
end=datetime.datetime.now().replace(tzinfo=pytz.utc),
):
'''Equivalent to zipline.utils.factory.load_from_yahoo.
'''
assert start <= end, 'start must be before end'
datetime_index = get_trading_days(start=start, end=end)
df = pd.DataFrame(index=datetime_index)
python_datetimes = list(datetime_index.to_pydatetime())
for symbol in stocks:
# there's probably a clever way of avoiding building a dictionary.
try:
date_values = self.get(symbol=symbol, dates=python_datetimes)
values = {date: value for date, value in date_values}
except NoDataForStock:
warnings.warn('No data for {}'.format(symbol))
except ExternalRequestFailed as e:
warnings.warn('Getting data for {} failed {}'.format(
symbol, e.message))
else:
series = pd.Series(values)
df[symbol] = series
for name, ticker in indexes.iteritems():
values = {
date: value
for date, value in self.get(symbol=ticker,
dates=python_datetimes)
}
df[name] = pd.Series(values)
return df
def get_trading_days_num(start_date, end_date):
return len(get_trading_days(start_date, end_date))
#%%
import pandas as pd
import csv
import numpy as np
from trading_calendars import get_calendar
from zipline.utils.calendars import TradingCalendar
from zipline.utils.tradingcalendar import get_trading_days
from pandas import Timestamp
#%%
#This is the process to get the correct trading days, which will then be use in the Bundle Ingest process to index the days
trading_days = get_trading_days(
start=Timestamp('2005-01-03 00:00:00+0000', tz='UTC'),
end=Timestamp('2020-10-26 02:21:29.456875+0000')).date.astype(str)
# %%
# US Markets Closed December 5, 2018: National Day of Mourning for George H.W. Bush
trading_days = np.delete(trading_days, np.where(trading_days == '2018-12-05'))
trading_days[trading_days == '2018-12-05']
#%%
# Create a CSV file to set the Zipline Bundle Ingest process against
with open('trading_calendar.csv', 'w', newline='') as f:
writer = csv.writer(f)
writer.writerow(trading_days)
# %%
def get_trading_days_num(start_date, end_date):
return len(get_trading_days(start_date, end_date))
