def read(cls, rootdir):
path = cls.metadata_path(rootdir)
with open(path) as fp:
raw_data = json.load(fp)
try:
version = raw_data['version']
except KeyError:
# Version was first written with version 1, assume 0,
# if version does not match.
version = 0
default_ohlc_ratio = raw_data['ohlc_ratio']
if version >= 1:
minutes_per_day = raw_data['minutes_per_day']
else:
# version 0 always assumed US equities.
minutes_per_day = US_EQUITIES_MINUTES_PER_DAY
if version >= 2:
calendar = get_calendar(raw_data['calendar_name'])
start_session = pd.Timestamp(
raw_data['start_session'], tz='UTC')
end_session = pd.Timestamp(raw_data['end_session'], tz='UTC')
else:
# No calendar info included in older versions, so
# default to NYSE.
calendar = get_calendar('XNYS')
start_session = pd.Timestamp(
raw_data['first_trading_day'], tz='UTC')
end_session = calendar.minute_to_session_label(
pd.Timestamp(
raw_data['market_closes'][-1], unit='m', tz='UTC')
)
if version >= 3:
ohlc_ratios_per_sid = raw_data['ohlc_ratios_per_sid']
if ohlc_ratios_per_sid is not None:
ohlc_ratios_per_sid = keymap(int, ohlc_ratios_per_sid)
else:
ohlc_ratios_per_sid = None
return cls(
default_ohlc_ratio,
ohlc_ratios_per_sid,
calendar,
start_session,
end_session,
minutes_per_day,
version=version,
)
def make_equity_info(cls):
out = pd.concat(
[
# 15 assets on each exchange. Each asset lives for 5 days.
# A new asset starts each day.
make_rotating_equity_info(
num_assets=20,
first_start=cls.START_DATE,
frequency=get_calendar(exchange).day,
periods_between_starts=1,
# NOTE: The asset_lifetime parameter name is a bit
# misleading. It determines the number of trading
# days between each asset's start_date and end_date,
# so assets created with this method actual "live"
# for (asset_lifetime + 1) days. But, since pipeline
# doesn't show you an asset the day it IPOs, this
# number matches the number of days that each asset
# should appear in a pipeline output.
asset_lifetime=5,
exchange=exchange,
)
for exchange in cls.EXCHANGE_INFO.exchange
],
ignore_index=True,
)
assert_equal(out.end_date.max(), cls.END_DATE)
return out
def setUpClass(cls):
cls.nyse_calendar = get_calendar("NYSE")
# july 15 is friday, so there are 3 sessions in this range (15, 18, 19)
cls.sessions = cls.nyse_calendar.sessions_in_range(
pd.Timestamp("2016-07-15"),
pd.Timestamp("2016-07-19")
)
trading_o_and_c = cls.nyse_calendar.schedule.ix[cls.sessions]
cls.opens = trading_o_and_c['market_open']
cls.closes = trading_o_and_c['market_close']
def setUp(self):
self.trading_day = get_calendar("NYSE").day
self.nsids = 5
self.ndates = 20
self.sids = Int64Index(range(self.nsids))
self.dates = DatetimeIndex(
start='2014-01-02',
freq=self.trading_day,
periods=self.ndates,
)
self.mask = ones((len(self.dates), len(self.sids)), dtype=bool)
def setUpClass(cls):
# On the AfterOpen and BeforeClose tests, we want ensure that the
# functions are pure, and that running them with the same input will
# provide the same output, regardless of whether the function is run 1
# or N times. (For performance reasons, we cache some internal state
# in AfterOpen and BeforeClose, but we don't want it to affect
# purity). Hence, we use the same before_close and after_open across
# subtests.
cls.before_close = BeforeClose(hours=1, minutes=5)
cls.after_open = AfterOpen(hours=1, minutes=5)
cls.class_ = None # Mark that this is the base class.
cal = get_calendar(cls.CALENDAR_STRING)
cls.before_close.cal = cal
cls.after_open.cal = cal
def sessions(self):
if self._sessions.empty:
outer_dates = pd.read_sql(
'SELECT MIN(day) as min_day, MAX(day) as max_day FROM ohlcv_daily',
self.conn)
start_session = Timestamp(outer_dates['min_day'][0], tz='UTC')
end_session = Timestamp(outer_dates['max_day'][0], tz='UTC')
calendar_name = 'XNYS' # NYSE for POC only
cal = get_calendar(calendar_name)
self._sessions = cal.sessions_in_range(start_session, end_session)
return self._sessions
def setUpClass(cls):
# On the AfterOpen and BeforeClose tests, we want ensure that the
# functions are pure, and that running them with the same input will
# provide the same output, regardless of whether the function is run 1
# or N times. (For performance reasons, we cache some internal state
# in AfterOpen and BeforeClose, but we don't want it to affect
# purity). Hence, we use the same before_close and after_open across
# subtests.
cls.before_close = BeforeClose(hours=1, minutes=5)
cls.after_open = AfterOpen(hours=1, minutes=5)
cls.class_ = None # Mark that this is the base class.
cal = get_calendar(cls.CALENDAR_STRING)
cls.before_close.cal = cal
cls.after_open.cal = cal
def sessions(self):
if 'calendar' in self._table.attrs.attrs:
# backwards compatibility with old formats, will remove
return DatetimeIndex(self._table.attrs['calendar'], tz='UTC')
else:
cal = get_calendar(self._table.attrs['calendar_name'])
start_session_ns = self._table.attrs['start_session_ns']
start_session = Timestamp(start_session_ns, tz='UTC')
end_session_ns = self._table.attrs['end_session_ns']
end_session = Timestamp(end_session_ns, tz='UTC')
sessions = cal.sessions_in_range(start_session, end_session)
return sessions
def sessions(self):
if "calendar" in self._table.attrs.attrs:
# backwards compatibility with old formats, will remove
return DatetimeIndex(self._table.attrs["calendar"], tz="UTC")
else:
cal = get_calendar(self._table.attrs["calendar_name"])
start_session_ns = self._table.attrs["start_session_ns"]
start_session = Timestamp(start_session_ns, tz="UTC")
end_session_ns = self._table.attrs["end_session_ns"]
end_session = Timestamp(end_session_ns, tz="UTC")
sessions = cal.sessions_in_range(start_session, end_session)
return sessions
def sessions(self):
if 'calendar' in self._table.attrs.attrs:
# backwards compatibility with old formats, will remove
return DatetimeIndex(self._table.attrs['calendar'], tz='UTC')
else:
cal = get_calendar(self._table.attrs['calendar_name'])
start_session_ns = self._table.attrs['start_session_ns']
start_session = Timestamp(start_session_ns, tz='UTC')
end_session_ns = self._table.attrs['end_session_ns']
end_session = Timestamp(end_session_ns, tz='UTC')
sessions = cal.sessions_in_range(start_session, end_session)
return sessions
def get_treasury_data(start_date, end_date):
"""读取期间资金成本数据
Parameters
----------
start_date : datetime-like
开始日期
end_date : datetime-like
结束日期
return
----------
DataFrame: DataFrame对象。
Examples
--------
>>> start_date = '2020-05-15'
>>> end_date = '2020-05-25'
>>> df = get_treasury_data(start_date, end_date)
>>> df.iloc[:5, :5]
cash 1month 2month 3month 6month
date
2020-05-15 00:00:00+00:00 0.006838 0.009496 0.009506 0.010076 0.011570
2020-05-18 00:00:00+00:00 0.006838 0.009369 0.009611 0.010414 0.011701
2020-05-19 00:00:00+00:00 0.009838 0.009425 0.010490 0.010307 0.012016
2020-05-20 00:00:00+00:00 0.008188 0.009084 0.010712 0.011012 0.012378
2020-05-21 00:00:00+00:00 0.007028 0.008569 0.010695 0.011032 0.012465
"""
start, end = sanitize_dates(start_date, end_date)
db = get_db()
collection = db['国债利率']
predicate = {'date': {'$gte': start, '$lte': end}}
projection = {'_id': 0}
sort = [('date', 1)]
cursor = collection.find(predicate, projection, sort=sort)
df = pd.DataFrame.from_records(cursor)
# 缺少2年数据，使用简单平均插值
value = (df['y1'] + df['y3']) / 2
df.insert(7, '2year', value)
df.rename(columns=TREASURY_COL_MAPS, inplace=True)
df.set_index('date', inplace=True)
df = df.tz_localize('UTC')
calendar = get_calendar('XSHG')
start = start.tz_localize('UTC')
end = end.tz_localize('UTC')
sessions = calendar.sessions_in_range(start, end)
# 务必与交易日历一致
return df.reindex(sessions).fillna(method='ffill')
def get_exch_time(start=None,end=None,isoCode='XNYS',tz='America/New_York',dbname='ara',tablename='calendar_trading',wmode=None,debugTF=True):
cal = get_calendar(isoCode)
open_time = cal._opens.tz_convert(tz).to_pydatetime()[start:end]
close_time = cal._closes.tz_convert(tz).to_pydatetime()[start:end]
pbdate=[ int(dt2ymd(x)) for x in close_time]
df = pd.DataFrame(data=np.array([open_time,close_time,pbdate]).T,columns=['open_time','close_time','pbdate'])
df['iso_code'] = isoCode
if wmode:
save2pgdb(df,db=dbname,tablename=tablename,wmode=wmode)
pname = "pickle/{tablename}.{isoCode}.pickle".format(**locals())
fp = open(pname,"wb")
pickle.dump(df,fp)
fp.close()
if debugTF:
sys.stderr.write("Calendar Saved to Table:{tablename}: Pickle:{pname}\n".format(**locals()))
return df
def get_market_year_fraction(start_date, end_date, adjustment):
"""Calculate the year fraction until the expiry date of an option in trading minutes.
Parameters
----------
start_date : string
Inclusive start date for the time remaining [MM-DD-YYYY] ie: ('10-18-2020')
end_date : string
Inclusive end date for the time remaining [MM-DD-YYYY] ie: ('10-20-2020')
adjustment : float
[mins] An adjustment factor for handling intraday calculations
"""
mins = 390 * len(
get_calendar('XNYS').sessions_in_range(start_date,
end_date)) + adjustment
return mins / (252 * 390)
def __init__(self,
key_id=None,
secret=None,
base_url=None,
api_version='v1'):
self._key_id = key_id
self._secret = secret
self._base_url = base_url
self._api = tradeapi.REST(key_id,
secret,
base_url,
api_version=api_version)
self._cal = get_calendar('NYSE')
self._open_orders = {}
self._orders_pending_submission = {}
def read_mongo(collector):
data = []
for it in collector.find({}, {"_id": 0}):
it["volume"] = it["volume"]
data.append(it)
df = DataFrame(data)
df["date"] = to_datetime(df["date"], utc=True)
df.set_index("date", inplace=True)
# print(df.index)
sessions = get_calendar('XSHG').sessions_in_range("19900101", "20181231")
# print(sessions)
df = df.reindex(sessions).fillna(method="ffill")
df.dropna(inplace=True)
df["volume"] = list(map(lambda x: int(x), df["volume"]))
# print(df.dtypes)
return df
def __init__(self,
start: Timestamp,
end: Timestamp,
codes: List[str],
window: int = 100):
if not end:
end = Timestamp.now(tz='Asia/Shanghai')
if not (start or window > 0):
raise RuntimeError("必须指定start和end 或者 end和window")
if not codes or len(codes) <= 0:
raise RuntimeError("codes不能为空")
self.start = start
self.end = end
self.codes = codes
self.window = window
self.calendar: TradingCalendar = trading_calendars.get_calendar('NYSE')
def run_example(example_name, environ):
"""
Run an example module from zipline.examples.
"""
mod = EXAMPLE_MODULES[example_name]
register_calendar("YAHOO", get_calendar("NYSE"), force=True)
return run_algorithm(
initialize=getattr(mod, 'initialize', None),
handle_data=getattr(mod, 'handle_data', None),
before_trading_start=getattr(mod, 'before_trading_start', None),
analyze=getattr(mod, 'analyze', None),
bundle='test',
environ=environ,
# Provide a default capital base, but allow the test to override.
**merge({'capital_base': 1e7}, mod._test_args()))
def drop_extra_sessions(df):
cal: TradingCalendar = trading_calendars.get_calendar('NYSE')
sessions = cal.sessions_in_range(df.index[0], df.index[-1])
if len(df.index) == len(sessions):
return df
to_drop = df.index.difference(sessions)
if len(to_drop) == 0:
return df
df.drop(to_drop, inplace=True)
dropped = len(to_drop)
print(f'\nWarning! Dropped {dropped} extra sessions!')
return df
def _make_sure_dates_are_initialized_properly(self, dtbegin, dtend,
granularity):
"""
dates may or may not be specified by the user.
when they do, they are probably don't include NY timezome data
also, when granularity is minute, we want to make sure we get data when
market is opened. so if it doesn't - let's get set end date to be last
known minute with opened market.
this nethod takes care of all these issues.
:param dtbegin:
:param dtend:
:param granularity:
:return:
"""
if not dtend:
dtend = pd.Timestamp('now', tz=NY)
else:
dtend = pd.Timestamp(pytz.timezone(NY).localize(dtend))
if granularity == Granularity.Minute:
calendar = trading_calendars.get_calendar(name='NYSE')
if pd.Timestamp('now', tz=NY).date() == dtend.date():
if calendar.is_open_on_minute(dtend):
# we execute during market open, we don't want today's data
# we will receive it through the websocket
dtend = dtend.replace(hour=15,
minute=59,
second=0,
microsecond=0)
dtend -= timedelta(days=1)
while not calendar.is_open_on_minute(dtend):
dtend = dtend.replace(hour=15,
minute=59,
second=0,
microsecond=0)
dtend -= timedelta(days=1)
if not dtbegin:
days = 30 if granularity == Granularity.Daily else 3
delta = timedelta(days=days)
dtbegin = dtend - delta
else:
dtbegin = pd.Timestamp(pytz.timezone(NY).localize(dtbegin))
while dtbegin > dtend:
# if we start the script during market hours we could get this
# situation. this resolves that.
dtbegin -= timedelta(days=1)
return dtbegin, dtend
def __init__(self): # pylint: disable=super-init-not-called
self._calendar = self.p.calendar # pylint: disable=no-member
if isinstance(self._calendar, string_types):
from trading_calendars import get_calendar
self._calendar = get_calendar(self._calendar)
self.dcache = DatetimeIndex([0.0])
self.idcache = DataFrame(index=DatetimeIndex([0.0]))
self.csize = datetime.timedelta(days=self.p.cachesize) # pylint: disable=no-member
self._tz = self.p.tz # pylint: disable=no-member
if self._tz is None:
self._tz = self._calendar.tz
elif isinstance(self._tz, string_types):
self._tz = tzparse(self._tz)
def df_generator(interval, start, end, divs_splits, assets_to_sids={}):
exchange = 'NYSE'
# get calendar and extend it to 20 days to the future to be able
# to set dividend-pay-date to a valid session
cal: TradingCalendar = trading_calendars.get_calendar('NYSE')
sessions = cal.sessions_in_range(start, end + timedelta(days=20))
asset_list = list_assets()
for symbol in asset_list:
try:
df = av_get_data_for_symbol(symbol, start, end, interval)
sid = assets_to_sids[symbol]
first_traded = df.index[0]
auto_close_date = df.index[-1] + pd.Timedelta(days=1)
if 'split' in df.columns:
split = calc_split(sid, df)
divs_splits['splits'] = divs_splits['splits'].append(split)
if 'dividend' in df.columns:
div = calc_dividend(sid, df, sessions)
divs_splits['divs'] = pd.concat([divs_splits['divs'], div])
yield (
sid, df
), symbol, symbol, start, end, first_traded, auto_close_date, exchange
except KeyboardInterrupt:
exit()
except Exception as e:
# somehow rate-limiting does not work with exceptions, throttle manually
if 'Thank you for using Alpha Vantage! Our standard API call frequency is' in str(
e):
print(
f'\nGot rate-limit on remote-side, retrying symbol {symbol} later'
)
asset_list.append(symbol)
else:
print(f'\nException for symbol {symbol}')
print(e)
def run(self, portfolio, start, end, capital_base=100000):
MovingAverageStrategy._portfolio = portfolio
data, _ = AlphaVantageAPI.get_daily_data_for(portfolio.assets)
data['benchmark'] = data[portfolio.benchmark]
assets_panel = MovingAverageStrategy.to_panel(data)
nyse_calendar = trading_calendars.get_calendar('XNYS')
portfolio.performance = zipline.run_algorithm(
start=start,
end=end,
data_frequency='daily',
capital_base=capital_base,
before_trading_start=MovingAverageStrategy.before_trading_start(),
initialize=MovingAverageStrategy.initialize,
handle_data=MovingAverageStrategy.handle_data,
analyze=MovingAverageStrategy.analyze,
trading_calendar=nyse_calendar,
data=assets_panel)
def get_benchmark_returns(symbol):
"""
Return a data series from 1930 to 2030 with all zero values,
which will effectively sidestep this web call issue,
with no negative impact.
Parameters
----------
symbol : str
Benchmark symbol for which we're getting the returns.
"""
cal = get_calendar('NYSE')
first_date = datetime(1930, 1, 1)
last_date = datetime(2030, 1, 1)
dates = cal.sessions_in_range(first_date, last_date)
data = pd.DataFrame(0.0, index=dates, columns=['close'])
data = data['close']
return data.sort_index().iloc[1:]
def run_example(example_name, environ):
"""
Run an example module from zipline.examples.
"""
mod = EXAMPLE_MODULES[example_name]
register_calendar("YAHOO", get_calendar("NYSE"), force=True)
return run_algorithm(
initialize=getattr(mod, 'initialize', None),
handle_data=getattr(mod, 'handle_data', None),
before_trading_start=getattr(mod, 'before_trading_start', None),
analyze=getattr(mod, 'analyze', None),
bundle='test',
environ=environ,
# Provide a default capital base, but allow the test to override.
**merge({'capital_base': 1e7}, mod._test_args())
)
def run(args: argparse.Namespace):
gauges_value = {}
gauges_latency = {}
gauge_avg = Gauge(f"{args.prefix}_avg_latency", "avg latency")
for query in args.queries:
# USDJPY/1Sec/TICK -> usdjpy_1sec_tick
key = query.replace("/", "_").replace("-", "_").lower()
gauges_value[query] = Gauge(f"{args.prefix}_{key}_value",
"value of {}".format(query))
gauges_latency[query] = Gauge(f"{args.prefix}_{key}_latency",
"latency of {}".format(query))
url = f"http://{args.marketstore_host}:{args.marketstore_port}/rpc"
delta = datetime.timedelta(seconds=args.lookback)
cal = None
if args.market:
cal = tc.get_calendar("XNYS")
while True:
client = pymkts.Client(url)
end_dt = pd.Timestamp.utcnow()
start_dt = end_dt - delta
holiday = False
if cal and cal.is_session(end_dt) is False:
holiday = True
total = 0
for query in args.queries:
if holiday:
value, latency = (0, 0)
else:
(value, latency) = get_value(client, query, args.column,
start_dt, end_dt)
gauges_value[query].set(value)
gauges_latency[query].set(latency)
total += latency
gauge_avg.set(total / len(args.queries))
time.sleep(args.interval)
def test_detect_non_market_minutes(self):
cal = get_calendar('NYSE')
# NOTE: This test is here instead of being on the base class for all
# calendars because some of our calendars are 24/7, which means there
# aren't any non-market minutes to find.
day0 = cal.minutes_for_sessions_in_range(
pd.Timestamp('2013-07-03', tz='UTC'),
pd.Timestamp('2013-07-03', tz='UTC'),
)
for minute in day0:
self.assertTrue(cal.is_open_on_minute(minute))
day1 = cal.minutes_for_sessions_in_range(
pd.Timestamp('2013-07-05', tz='UTC'),
pd.Timestamp('2013-07-05', tz='UTC'),
)
for minute in day1:
self.assertTrue(cal.is_open_on_minute(minute))
def NYSE_timestamp(s):
return pd.Timestamp(s, tz='US/Eastern').tz_convert('UTC')
non_market = [
# After close.
NYSE_timestamp('2013-07-03 16:01'),
# Holiday.
NYSE_timestamp('2013-07-04 10:00'),
# Before open.
NYSE_timestamp('2013-07-05 9:29'),
]
for minute in non_market:
self.assertFalse(cal.is_open_on_minute(minute), minute)
input_ = pd.to_datetime(
np.hstack([day0.values, minute.asm8, day1.values]),
utc=True,
)
with self.assertRaises(ValueError) as e:
cal.minute_index_to_session_labels(input_)
exc_str = str(e.exception)
self.assertIn("First Bad Minute: {}".format(minute), exc_str)
def test_only_currency_converted_data(self, name, domain, calendar_name):
# Test running a pipeline on a domain whose assets are all denominated
# in the same currency.
pipe = Pipeline(
{
"close_USD": EquityPricing.close.fx("USD").latest,
"close_EUR": EquityPricing.close.fx("EUR").latest,
},
domain=domain,
)
start, end = self.daily_bar_sessions[calendar_name][-2:]
result = self.run_pipeline(pipe, start, end)
calendar = get_calendar(calendar_name)
daily_bars = self.daily_bar_data[calendar_name]
currency_codes = self.daily_bar_currency_codes[calendar_name]
for (dt, asset), row in result.iterrows():
# Subtract a day b/c pipeline output on day N should have prior
# day's price.
price_date = dt - calendar.day
expected_close = daily_bars[asset].loc[price_date, "close"]
expected_base = currency_codes.loc[asset]
expected_rate_USD = self.in_memory_fx_rate_reader.get_rate_scalar(
rate="mid",
quote="USD",
base=expected_base,
dt=price_date.asm8,
)
expected_price = expected_close * expected_rate_USD
assert_equal(row.close_USD, expected_price)
expected_rate_EUR = self.in_memory_fx_rate_reader.get_rate_scalar(
rate="mid",
quote="EUR",
base=expected_base,
dt=price_date.asm8,
)
expected_price = expected_close * expected_rate_EUR
assert_equal(row.close_EUR, expected_price)
def ingest(start):
from sharadar.pipeline.engine import load_sharadar_bundle
from zipline.assets import ASSET_DB_VERSION
from sharadar.data.sql_lite_assets import SQLiteAssetDBWriter
from sharadar.data.sql_lite_daily_pricing import SQLiteDailyBarWriter
from trading_calendars import get_calendar
from sharadar.loaders.constant import EXCHANGE_DF
calendar = get_calendar('XNYS')
macro_equities_df = create_macro_equities_df()
macro_prices_df = create_macro_prices_df(start)
output_dir = get_output_dir()
asset_dbpath = os.path.join(output_dir,
("assets-%d.sqlite" % ASSET_DB_VERSION))
asset_db_writer = SQLiteAssetDBWriter(asset_dbpath)
asset_db_writer.write(equities=macro_equities_df, exchanges=EXCHANGE_DF)
prices_dbpath = os.path.join(output_dir, "prices.sqlite")
sql_daily_bar_writer = SQLiteDailyBarWriter(prices_dbpath, calendar)
sql_daily_bar_writer.write(macro_prices_df)
return macro_prices_df.shape[0]
def test_write_attrs(self):
result = self.bcolz_daily_bar_ctable
expected_first_row = {
'1': 0,
'3': 5, # Asset 1 has 5 trading days.
'5': 12, # Asset 3 has 7 trading days.
'7': 33, # Asset 5 has 21 trading days.
'9': 44, # Asset 7 has 11 trading days.
'11': 49, # Asset 9 has 5 trading days.
}
expected_last_row = {
'1': 4,
'3': 11,
'5': 32,
'7': 43,
'9': 48,
'11': 57, # Asset 11 has 9 trading days.
}
expected_calendar_offset = {
'1': 0, # Starts on 6-01, 1st trading day of month.
'3': 15, # Starts on 6-22, 16th trading day of month.
'5': 1, # Starts on 6-02, 2nd trading day of month.
'7': 0, # Starts on 6-01, 1st trading day of month.
'9': 9, # Starts on 6-12, 10th trading day of month.
'11': 10, # Starts on 6-15, 11th trading day of month.
}
self.assertEqual(result.attrs['first_row'], expected_first_row)
self.assertEqual(result.attrs['last_row'], expected_last_row)
self.assertEqual(
result.attrs['calendar_offset'],
expected_calendar_offset,
)
cal = get_calendar(result.attrs['calendar_name'])
first_session = Timestamp(result.attrs['start_session_ns'], tz='UTC')
end_session = Timestamp(result.attrs['end_session_ns'], tz='UTC')
sessions = cal.sessions_in_range(first_session, end_session)
assert_equal(
self.sessions,
sessions
)
def test_write_attrs(self):
result = self.bcolz_daily_bar_ctable
expected_first_row = {
'1': 0,
'3': 5, # Asset 1 has 5 trading days.
'5': 12, # Asset 3 has 7 trading days.
'7': 33, # Asset 5 has 21 trading days.
'9': 44, # Asset 7 has 11 trading days.
'11': 49, # Asset 9 has 5 trading days.
}
expected_last_row = {
'1': 4,
'3': 11,
'5': 32,
'7': 43,
'9': 48,
'11': 57, # Asset 11 has 9 trading days.
}
expected_calendar_offset = {
'1': 0, # Starts on 6-01, 1st trading day of month.
'3': 15, # Starts on 6-22, 16th trading day of month.
'5': 1, # Starts on 6-02, 2nd trading day of month.
'7': 0, # Starts on 6-01, 1st trading day of month.
'9': 9, # Starts on 6-12, 10th trading day of month.
'11': 10, # Starts on 6-15, 11th trading day of month.
}
self.assertEqual(result.attrs['first_row'], expected_first_row)
self.assertEqual(result.attrs['last_row'], expected_last_row)
self.assertEqual(
result.attrs['calendar_offset'],
expected_calendar_offset,
)
cal = get_calendar(result.attrs['calendar_name'])
first_session = Timestamp(result.attrs['start_session_ns'], tz='UTC')
end_session = Timestamp(result.attrs['end_session_ns'], tz='UTC')
sessions = cal.sessions_in_range(first_session, end_session)
assert_equal(
self.sessions,
sessions
)
def __init__(self, start=None, end=None, assets=None, exchange='NYSE'):
self.start = normalize_date(pd.Timestamp(start or '2018-08-13'))
self.end = normalize_date(pd.Timestamp(end or '2018-08-14'))
self._exchange = exchange
self._calendar = get_calendar(exchange)
self.assets = assets or ['asset-0', 'asset-1', 'asset-2']
minutes = self._calendar.minutes_for_sessions_in_range(
self.start, self.end)
self._minutely_bars = {}
for i, asset in enumerate(self.get_equities()):
bars = create_bars(minutes, i)
self._minutely_bars[asset] = bars
days = self._calendar.sessions_in_range(self.start, self.end)
self._daily_bars = {}
for i, asset in enumerate(self.get_equities()):
bars = create_bars(days, i)
self._daily_bars[asset] = bars
def update_daily_prices(start_date, end_date, connection):
tw_calendar = get_calendar('XTAI')
main_df = pd.DataFrame()
for date in pd.date_range(start_date, end_date):
if date not in tw_calendar.opens:
continue
df, in_db = get_daily_prices(date, connection)
if df is not None and not in_db:
main_df = main_df.append(df, sort=False)
print('{} 每日收盤行情抓取完成，等待15秒'.format(date.strftime('%Y%m%d')))
time.sleep(15)
if not main_df.empty:
main_df.to_sql('daily_prices', connection, if_exists='append')
return main_df
def get_balance(self):
us_calendar = get_calendar("XNYS")
history = pd.DataFrame([
{
"date": self.created,
"amount": self.balance
},
*self.transactions.values("date", "amount"),
{
"date": timezone.now(),
"amount": 0
},
])
history.set_index("date", inplace=True)
history = history.resample("d").sum()
history = history.rename(columns={"amount": "close"})
history.index = history.index.date
history["close"] = history.close.cumsum()
history = history.loc[history.apply(
lambda x: us_calendar.is_session(x.name), axis=1)]
return history
def run_example(example_modules,
example_name,
environ,
benchmark_returns=None):
"""
Run an example module from zipline.examples.
"""
mod = example_modules[example_name]
register_calendar("YAHOO", get_calendar("NYSE"), force=True)
return run_algorithm(
initialize=getattr(mod, "initialize", None),
handle_data=getattr(mod, "handle_data", None),
before_trading_start=getattr(mod, "before_trading_start", None),
analyze=getattr(mod, "analyze", None),
bundle="test",
environ=environ,
benchmark_returns=benchmark_returns,
# Provide a default capital base, but allow the test to override.
**merge({"capital_base": 1e7}, mod._test_args()))
def create_simulation_parameters(year=2006,
start=None,
end=None,
capital_base=float("1.0e5"),
num_days=None,
data_frequency='daily',
emission_rate='daily',
trading_calendar=None,
financing_costs: Callable[[Account], float] = None,
manager_fees: Callable[[Account, Portfolio, pd.Timestamp], float] = None):
if not trading_calendar:
trading_calendar = get_calendar("NYSE")
if start is None:
start = pd.Timestamp("{0}-01-01".format(year), tz='UTC')
elif type(start) == datetime:
start = pd.Timestamp(start)
if end is None:
if num_days:
start_index = trading_calendar.all_sessions.searchsorted(start)
end = trading_calendar.all_sessions[start_index + num_days - 1]
else:
end = pd.Timestamp("{0}-12-31".format(year), tz='UTC')
elif type(end) == datetime:
end = pd.Timestamp(end)
sim_params = SimulationParameters(
start_session=start,
end_session=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=emission_rate,
trading_calendar=trading_calendar,
financing_costs=financing_costs,
manager_fees=manager_fees
)
return sim_params
def check_holidays(holiday_key_path, calendar_column, holiday_column,
date_format, delimiter, strip_x_from_cal_name,
answer_key_calendar_name, min_date, calendars):
data = pd.read_csv(holiday_key_path,
sep=delimiter,
parse_dates=[holiday_column],
date_parser=partial(pd.to_datetime,
format=date_format,
utc=True))
for calendar_name in default_calendar_names:
if calendars and calendar_name not in calendars:
continue
cal = get_calendar(calendar_name)
if answer_key_calendar_name:
csv_cal_code = answer_key_calendar_name
elif strip_x_from_cal_name:
# Convert the calendar name to the format expected in the CSV.
csv_cal_code = calendar_name.lstrip('X')
else:
csv_cal_code = calendar_name
holidays = set(
data[holiday_column][data[calendar_column] == csv_cal_code])
if holidays:
start = max(cal.first_session, min(holidays), min_date)
end = min(cal.last_session, max(holidays))
_check_range(start, end, holidays, cal, calendar_name)
else:
click.secho(
'No holidays found for {} in the holiday key.'.format(
calendar_name, ),
fg='yellow',
)
click.echo()
def setUpClass(cls):
super().setUpClass()
cls.class_ = StatelessRule
cls.cal = get_calendar(cls.CALENDAR_STRING)
# First day of 09/2014 is closed whereas that for 10/2014 is open
cls.sept_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-09-01', tz='UTC'),
pd.Timestamp('2014-09-30', tz='UTC'),
)
cls.oct_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-10-01', tz='UTC'),
pd.Timestamp('2014-10-31', tz='UTC'),
)
cls.sept_week = cls.cal.minutes_for_sessions_in_range(
pd.Timestamp("2014-09-22", tz='UTC'),
pd.Timestamp("2014-09-26", tz='UTC'))
cls.HALF_SESSION = None
cls.FULL_SESSION = None
def __init__(
self,
calendar=None,
before_trading_start_minute=(pd.Timestamp('8:45').time(),
'America/New_York'),
minute_emission=True,
time_skew=None,
):
if calendar is None:
calendar = get_calendar('NYSE')
self.calendar = calendar
self.before_trading_start_minute = before_trading_start_minute
self.minute_emission = minute_emission
self._last_emit = None
self._before_trading_start_bar_yielded = False
now = pd.Timestamp.utcnow()
prev_close = calendar.previous_close(now)
current = self._set_before_trading_start(prev_close)
self._current_time = current
self._fake_end = prev_close
def setUpClass(cls):
super(StatelessRulesTests, cls).setUpClass()
cls.class_ = StatelessRule
cls.cal = get_calendar(cls.CALENDAR_STRING)
# First day of 09/2014 is closed whereas that for 10/2014 is open
cls.sept_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-09-01', tz='UTC'),
pd.Timestamp('2014-09-30', tz='UTC'),
)
cls.oct_sessions = cls.cal.sessions_in_range(
pd.Timestamp('2014-10-01', tz='UTC'),
pd.Timestamp('2014-10-31', tz='UTC'),
)
cls.sept_week = cls.cal.minutes_for_sessions_in_range(
pd.Timestamp("2014-09-22", tz='UTC'),
pd.Timestamp("2014-09-26", tz='UTC')
)
cls.HALF_SESSION = None
cls.FULL_SESSION = None
def create_simulation_parameters(year=2006,
start=None,
end=None,
capital_base=float("1.0e5"),
num_days=None,
data_frequency='daily',
emission_rate='daily',
trading_calendar=None):
if not trading_calendar:
trading_calendar = get_calendar("NYSE")
if start is None:
start = pd.Timestamp("{0}-01-01".format(year), tz='UTC')
elif type(start) == datetime:
start = pd.Timestamp(start)
if end is None:
if num_days:
start_index = trading_calendar.all_sessions.searchsorted(start)
end = trading_calendar.all_sessions[start_index + num_days - 1]
else:
end = pd.Timestamp("{0}-12-31".format(year), tz='UTC')
elif type(end) == datetime:
end = pd.Timestamp(end)
sim_params = SimulationParameters(
start_session=start,
end_session=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=emission_rate,
trading_calendar=trading_calendar,
)
return sim_params
def setUpClass(cls):
super(StatefulRulesTests, cls).setUpClass()
cls.class_ = StatefulRule
cls.cal = get_calendar(cls.CALENDAR_STRING)
def run(ctx,
algofile,
algotext,
define,
data_frequency,
capital_base,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
blotter):
"""Run a backtest for the given algorithm.
"""
# check that the start and end dates are passed correctly
if start is None and end is None:
# check both at the same time to avoid the case where a user
# does not pass either of these and then passes the first only
# to be told they need to pass the second argument also
ctx.fail(
"must specify dates with '-s' / '--start' and '-e' / '--end'",
)
if start is None:
ctx.fail("must specify a start date with '-s' / '--start'")
if end is None:
ctx.fail("must specify an end date with '-e' / '--end'")
if (algotext is not None) == (algofile is not None):
ctx.fail(
"must specify exactly one of '-f' / '--algofile' or"
" '-t' / '--algotext'",
)
trading_calendar = get_calendar(trading_calendar)
perf = _run(
initialize=None,
handle_data=None,
before_trading_start=None,
analyze=None,
algofile=algofile,
algotext=algotext,
defines=define,
data_frequency=data_frequency,
capital_base=capital_base,
bundle=bundle,
bundle_timestamp=bundle_timestamp,
start=start,
end=end,
output=output,
trading_calendar=trading_calendar,
print_algo=print_algo,
metrics_set=metrics_set,
local_namespace=local_namespace,
environ=os.environ,
blotter=blotter,
benchmark_returns=None,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def calendar(self):
return get_calendar(self.calendar_name)
def _run(handle_data,
initialize,
before_trading_start,
analyze,
algofile,
algotext,
defines,
data_frequency,
capital_base,
bundle,
bundle_timestamp,
start,
end,
output,
trading_calendar,
print_algo,
metrics_set,
local_namespace,
environ,
blotter,
benchmark_returns):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`zipline.run_algo`.
"""
if benchmark_returns is None:
benchmark_returns, _ = load_market_data(environ=environ)
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
if trading_calendar is None:
trading_calendar = get_calendar('XSHG')
# date parameter validation
if trading_calendar.session_distance(start, end) < 1:
raise _RunAlgoError(
'There are no trading days between %s and %s' % (
start.date(),
end.date(),
),
)
bundle_data = bundles.load(
bundle,
environ,
bundle_timestamp,
)
first_trading_day = \
bundle_data.equity_minute_bar_reader.first_trading_day
data = DataPortal(
bundle_data.asset_finder,
trading_calendar=trading_calendar,
first_trading_day=first_trading_day,
equity_minute_reader=bundle_data.equity_minute_bar_reader,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
pipeline_loader = CNEquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader,
)
def choose_loader(column):
if column in CNEquityPricing.columns:
return pipeline_loader
# # 简单处理
elif type(column) == BoundColumn:
# # 使用实例才能避免KeyError
return global_loader
raise ValueError(
"No PipelineLoader registered for column %s." % column
)
if isinstance(metrics_set, six.string_types):
try:
metrics_set = metrics.load(metrics_set)
except ValueError as e:
raise _RunAlgoError(str(e))
if isinstance(blotter, six.string_types):
try:
blotter = load(Blotter, blotter)
except ValueError as e:
raise _RunAlgoError(str(e))
perf = TradingAlgorithm(
namespace=namespace,
data_portal=data,
get_pipeline_loader=choose_loader,
trading_calendar=trading_calendar,
sim_params=SimulationParameters(
start_session=start,
end_session=end,
trading_calendar=trading_calendar,
capital_base=capital_base,
data_frequency=data_frequency,
),
metrics_set=metrics_set,
blotter=blotter,
benchmark_returns=benchmark_returns,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}
).run()
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the zipline magic not write any data
perf.to_pickle(output)
return perf
def test_ingest(self):
calendar = get_calendar('XNYS')
sessions = calendar.sessions_in_range(self.START_DATE, self.END_DATE)
minutes = calendar.minutes_for_sessions_in_range(
self.START_DATE, self.END_DATE,
)
sids = tuple(range(3))
equities = make_simple_equity_info(
sids,
self.START_DATE,
self.END_DATE,
)
daily_bar_data = make_bar_data(equities, sessions)
minute_bar_data = make_bar_data(equities, minutes)
first_split_ratio = 0.5
second_split_ratio = 0.1
splits = pd.DataFrame.from_records([
{
'effective_date': str_to_seconds('2014-01-08'),
'ratio': first_split_ratio,
'sid': 0,
},
{
'effective_date': str_to_seconds('2014-01-09'),
'ratio': second_split_ratio,
'sid': 1,
},
])
@self.register(
'bundle',
calendar_name='NYSE',
start_session=self.START_DATE,
end_session=self.END_DATE,
)
def bundle_ingest(environ,
asset_db_writer,
minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
start_session,
end_session,
cache,
show_progress,
output_dir):
assert_is(environ, self.environ)
asset_db_writer.write(equities=equities)
minute_bar_writer.write(minute_bar_data)
daily_bar_writer.write(daily_bar_data)
adjustment_writer.write(splits=splits)
assert_is_instance(calendar, TradingCalendar)
assert_is_instance(cache, dataframe_cache)
assert_is_instance(show_progress, bool)
self.ingest('bundle', environ=self.environ)
bundle = self.load('bundle', environ=self.environ)
assert_equal(set(bundle.asset_finder.sids), set(sids))
columns = 'open', 'high', 'low', 'close', 'volume'
actual = bundle.equity_minute_bar_reader.load_raw_arrays(
columns,
minutes[0],
minutes[-1],
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(minutes, sids, equities, colname),
msg=colname,
)
actual = bundle.equity_daily_bar_reader.load_raw_arrays(
columns,
self.START_DATE,
self.END_DATE,
sids,
)
for actual_column, colname in zip(actual, columns):
assert_equal(
actual_column,
expected_bar_values_2d(sessions, sids, equities, colname),
msg=colname,
)
adjustments_for_cols = bundle.adjustment_reader.load_adjustments(
columns,
sessions,
pd.Index(sids),
)
for column, adjustments in zip(columns, adjustments_for_cols[:-1]):
# iterate over all the adjustments but `volume`
assert_equal(
adjustments,
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=second_split_ratio,
)],
},
msg=column,
)
# check the volume, the value should be 1/ratio
assert_equal(
adjustments_for_cols[-1],
{
2: [Float64Multiply(
first_row=0,
last_row=2,
first_col=0,
last_col=0,
value=1 / first_split_ratio,
)],
3: [Float64Multiply(
first_row=0,
last_row=3,
first_col=1,
last_col=1,
value=1 / second_split_ratio,
)],
},
msg='volume',
)
def algo_args(year=2017,
from_date=None,
to_date=None,
capital_base=float('1.0e7'),
num_days=None,
metrics_set=None):
# Constant inputs
benchmark_sid = 3623 # '000300.SH'
bundle = "cndaily"
calendar = 'XSHG'
# With trading calendar
trading_calendar = get_calendar(calendar)
# With simulation parameters
if from_date:
from_date = pd.Timestamp(from_date, tz='utc')
if to_date:
to_date = pd.Timestamp(to_date, tz='utc')
sim_params = create_simulation_parameters(
year=year,
start=from_date,
end=to_date,
capital_base=capital_base,
num_days=num_days,
trading_calendar=trading_calendar
)
# With data portal
bundle_data = load(bundle)
data_portal = DataPortal(
bundle_data.asset_finder,
trading_calendar,
trading_calendar.first_session,
equity_daily_reader=bundle_data.equity_daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader
)
# With pipelineloader
pipeline_loader = EquityPricingLoader(
bundle_data.equity_daily_bar_reader,
bundle_data.adjustment_reader
)
def choose_loader(column):
if column.unspecialize() in EquityPricing.columns:
return pipeline_loader
elif column in global_loader:
return global_loader
raise ValueError("%s is NOT registered in `PipelineLoader`." % column)
return {
'sim_params': sim_params,
'data_portal': data_portal,
'benchmark_sid': benchmark_sid,
'metrics_set': metrics_set,
'get_pipeline_loader': choose_loader,
}
def load_market_data(trading_day=None, trading_days=None, bm_symbol='000300',
environ=None):
"""
Load benchmark returns and treasury yield curves for the given calendar and
benchmark symbol.
Benchmarks are downloaded as a Series from IEX Trading. Treasury curves
are US Treasury Bond rates and are downloaded from 'www.federalreserve.gov'
by default. For Canadian exchanges, a loader for Canadian bonds from the
Bank of Canada is also available.
Results downloaded from the internet are cached in
~/.zipline/data. Subsequent loads will attempt to read from the cached
files before falling back to redownload.
Parameters
----------
trading_day : pandas.CustomBusinessDay, optional
A trading_day used to determine the latest day for which we
expect to have data. Defaults to an NYSE trading day.
trading_days : pd.DatetimeIndex, optional
A calendar of trading days. Also used for determining what cached
dates we should expect to have cached. Defaults to the NYSE calendar.
bm_symbol : str, optional
Symbol for the benchmark index to load. Defaults to 'SPY', the ticker
for the S&P 500, provided by IEX Trading.
Returns
-------
(benchmark_returns, treasury_curves) : (pd.Series, pd.DataFrame)
Notes
-----
Both return values are DatetimeIndexed with values dated to midnight in UTC
of each stored date. The columns of `treasury_curves` are:
'1month', '3month', '6month',
'1year','2year','3year','5year','7year','10year','20year','30year'
"""
if trading_day is None:
trading_day = get_calendar('XSHG').day
if trading_days is None:
trading_days = get_calendar('XSHG').all_sessions
first_date = trading_days[0]
now = pd.Timestamp.utcnow()
# we will fill missing benchmark data through latest trading date
last_date = trading_days[trading_days.get_loc(now, method='ffill')]
br = ensure_benchmark_data(
bm_symbol,
first_date,
last_date,
now,
# We need the trading_day to figure out the close prior to the first
# date so that we can compute returns for the first date.
trading_day,
environ,
)
tc = ensure_treasury_data(
bm_symbol,
first_date,
last_date,
now,
environ,
)
# combine dt indices and reindex using ffill then bfill
all_dt = br.index.union(tc.index)
br = br.reindex(all_dt, method='ffill').fillna(method='bfill')
tc = tc.reindex(all_dt, method='ffill').fillna(method='bfill')
benchmark_returns = br[br.index.slice_indexer(first_date, last_date)]
treasury_curves = tc[tc.index.slice_indexer(first_date, last_date)]
return benchmark_returns, treasury_curves
def init_class_fixtures(cls):
super(TestMinuteBarDataFuturesCalendar, cls).init_class_fixtures()
cls.trading_calendar = get_calendar('CMES')
def trading_calendar(self):
if 'calendar_name' in self._table.attrs.attrs:
return get_calendar(self._table.attrs['calendar_name'])
else:
return None
def init_class_fixtures(cls):
super(TestDateUtils, cls).init_class_fixtures()
cls.calendar = get_calendar('XSHG')
