def read(cls, rootdir): path = cls.metadata_path(rootdir) with open(path) as fp: raw_data = json.load(fp) try: version = raw_data['version'] except KeyError: # Version was first written with version 1, assume 0, # if version does not match. version = 0 default_ohlc_ratio = raw_data['ohlc_ratio'] if version >= 1: minutes_per_day = raw_data['minutes_per_day'] else: # version 0 always assumed US equities. minutes_per_day = US_EQUITIES_MINUTES_PER_DAY if version >= 2: calendar = get_calendar(raw_data['calendar_name']) start_session = pd.Timestamp( raw_data['start_session'], tz='UTC') end_session = pd.Timestamp(raw_data['end_session'], tz='UTC') else: # No calendar info included in older versions, so # default to NYSE. calendar = get_calendar('XNYS') start_session = pd.Timestamp( raw_data['first_trading_day'], tz='UTC') end_session = calendar.minute_to_session_label( pd.Timestamp( raw_data['market_closes'][-1], unit='m', tz='UTC') ) if version >= 3: ohlc_ratios_per_sid = raw_data['ohlc_ratios_per_sid'] if ohlc_ratios_per_sid is not None: ohlc_ratios_per_sid = keymap(int, ohlc_ratios_per_sid) else: ohlc_ratios_per_sid = None return cls( default_ohlc_ratio, ohlc_ratios_per_sid, calendar, start_session, end_session, minutes_per_day, version=version, )
def make_equity_info(cls): out = pd.concat( [ # 15 assets on each exchange. Each asset lives for 5 days. # A new asset starts each day. make_rotating_equity_info( num_assets=20, first_start=cls.START_DATE, frequency=get_calendar(exchange).day, periods_between_starts=1, # NOTE: The asset_lifetime parameter name is a bit # misleading. It determines the number of trading # days between each asset's start_date and end_date, # so assets created with this method actual "live" # for (asset_lifetime + 1) days. But, since pipeline # doesn't show you an asset the day it IPOs, this # number matches the number of days that each asset # should appear in a pipeline output. asset_lifetime=5, exchange=exchange, ) for exchange in cls.EXCHANGE_INFO.exchange ], ignore_index=True, ) assert_equal(out.end_date.max(), cls.END_DATE) return out
def setUpClass(cls): cls.nyse_calendar = get_calendar("NYSE") # july 15 is friday, so there are 3 sessions in this range (15, 18, 19) cls.sessions = cls.nyse_calendar.sessions_in_range( pd.Timestamp("2016-07-15"), pd.Timestamp("2016-07-19") ) trading_o_and_c = cls.nyse_calendar.schedule.ix[cls.sessions] cls.opens = trading_o_and_c['market_open'] cls.closes = trading_o_and_c['market_close']
def setUp(self): self.trading_day = get_calendar("NYSE").day self.nsids = 5 self.ndates = 20 self.sids = Int64Index(range(self.nsids)) self.dates = DatetimeIndex( start='2014-01-02', freq=self.trading_day, periods=self.ndates, ) self.mask = ones((len(self.dates), len(self.sids)), dtype=bool)
def setUpClass(cls): # On the AfterOpen and BeforeClose tests, we want ensure that the # functions are pure, and that running them with the same input will # provide the same output, regardless of whether the function is run 1 # or N times. (For performance reasons, we cache some internal state # in AfterOpen and BeforeClose, but we don't want it to affect # purity). Hence, we use the same before_close and after_open across # subtests. cls.before_close = BeforeClose(hours=1, minutes=5) cls.after_open = AfterOpen(hours=1, minutes=5) cls.class_ = None # Mark that this is the base class. cal = get_calendar(cls.CALENDAR_STRING) cls.before_close.cal = cal cls.after_open.cal = cal
def sessions(self): if self._sessions.empty: outer_dates = pd.read_sql( 'SELECT MIN(day) as min_day, MAX(day) as max_day FROM ohlcv_daily', self.conn) start_session = Timestamp(outer_dates['min_day'][0], tz='UTC') end_session = Timestamp(outer_dates['max_day'][0], tz='UTC') calendar_name = 'XNYS' # NYSE for POC only cal = get_calendar(calendar_name) self._sessions = cal.sessions_in_range(start_session, end_session) return self._sessions
def sessions(self): if 'calendar' in self._table.attrs.attrs: # backwards compatibility with old formats, will remove return DatetimeIndex(self._table.attrs['calendar'], tz='UTC') else: cal = get_calendar(self._table.attrs['calendar_name']) start_session_ns = self._table.attrs['start_session_ns'] start_session = Timestamp(start_session_ns, tz='UTC') end_session_ns = self._table.attrs['end_session_ns'] end_session = Timestamp(end_session_ns, tz='UTC') sessions = cal.sessions_in_range(start_session, end_session) return sessions
def sessions(self): if "calendar" in self._table.attrs.attrs: # backwards compatibility with old formats, will remove return DatetimeIndex(self._table.attrs["calendar"], tz="UTC") else: cal = get_calendar(self._table.attrs["calendar_name"]) start_session_ns = self._table.attrs["start_session_ns"] start_session = Timestamp(start_session_ns, tz="UTC") end_session_ns = self._table.attrs["end_session_ns"] end_session = Timestamp(end_session_ns, tz="UTC") sessions = cal.sessions_in_range(start_session, end_session) return sessions
def get_treasury_data(start_date, end_date): """读取期间资金成本数据 Parameters ---------- start_date : datetime-like 开始日期 end_date : datetime-like 结束日期 return ---------- DataFrame: DataFrame对象。 Examples -------- >>> start_date = '2020-05-15' >>> end_date = '2020-05-25' >>> df = get_treasury_data(start_date, end_date) >>> df.iloc[:5, :5] cash 1month 2month 3month 6month date 2020-05-15 00:00:00+00:00 0.006838 0.009496 0.009506 0.010076 0.011570 2020-05-18 00:00:00+00:00 0.006838 0.009369 0.009611 0.010414 0.011701 2020-05-19 00:00:00+00:00 0.009838 0.009425 0.010490 0.010307 0.012016 2020-05-20 00:00:00+00:00 0.008188 0.009084 0.010712 0.011012 0.012378 2020-05-21 00:00:00+00:00 0.007028 0.008569 0.010695 0.011032 0.012465 """ start, end = sanitize_dates(start_date, end_date) db = get_db() collection = db['国债利率'] predicate = {'date': {'$gte': start, '$lte': end}} projection = {'_id': 0} sort = [('date', 1)] cursor = collection.find(predicate, projection, sort=sort) df = pd.DataFrame.from_records(cursor) # 缺少2年数据,使用简单平均插值 value = (df['y1'] + df['y3']) / 2 df.insert(7, '2year', value) df.rename(columns=TREASURY_COL_MAPS, inplace=True) df.set_index('date', inplace=True) df = df.tz_localize('UTC') calendar = get_calendar('XSHG') start = start.tz_localize('UTC') end = end.tz_localize('UTC') sessions = calendar.sessions_in_range(start, end) # 务必与交易日历一致 return df.reindex(sessions).fillna(method='ffill')
def get_exch_time(start=None,end=None,isoCode='XNYS',tz='America/New_York',dbname='ara',tablename='calendar_trading',wmode=None,debugTF=True): cal = get_calendar(isoCode) open_time = cal._opens.tz_convert(tz).to_pydatetime()[start:end] close_time = cal._closes.tz_convert(tz).to_pydatetime()[start:end] pbdate=[ int(dt2ymd(x)) for x in close_time] df = pd.DataFrame(data=np.array([open_time,close_time,pbdate]).T,columns=['open_time','close_time','pbdate']) df['iso_code'] = isoCode if wmode: save2pgdb(df,db=dbname,tablename=tablename,wmode=wmode) pname = "pickle/{tablename}.{isoCode}.pickle".format(**locals()) fp = open(pname,"wb") pickle.dump(df,fp) fp.close() if debugTF: sys.stderr.write("Calendar Saved to Table:{tablename}: Pickle:{pname}\n".format(**locals())) return df
def get_market_year_fraction(start_date, end_date, adjustment): """Calculate the year fraction until the expiry date of an option in trading minutes. Parameters ---------- start_date : string Inclusive start date for the time remaining [MM-DD-YYYY] ie: ('10-18-2020') end_date : string Inclusive end date for the time remaining [MM-DD-YYYY] ie: ('10-20-2020') adjustment : float [mins] An adjustment factor for handling intraday calculations """ mins = 390 * len( get_calendar('XNYS').sessions_in_range(start_date, end_date)) + adjustment return mins / (252 * 390)
def __init__(self, key_id=None, secret=None, base_url=None, api_version='v1'): self._key_id = key_id self._secret = secret self._base_url = base_url self._api = tradeapi.REST(key_id, secret, base_url, api_version=api_version) self._cal = get_calendar('NYSE') self._open_orders = {} self._orders_pending_submission = {}
def read_mongo(collector): data = [] for it in collector.find({}, {"_id": 0}): it["volume"] = it["volume"] data.append(it) df = DataFrame(data) df["date"] = to_datetime(df["date"], utc=True) df.set_index("date", inplace=True) # print(df.index) sessions = get_calendar('XSHG').sessions_in_range("19900101", "20181231") # print(sessions) df = df.reindex(sessions).fillna(method="ffill") df.dropna(inplace=True) df["volume"] = list(map(lambda x: int(x), df["volume"])) # print(df.dtypes) return df
def __init__(self, start: Timestamp, end: Timestamp, codes: List[str], window: int = 100): if not end: end = Timestamp.now(tz='Asia/Shanghai') if not (start or window > 0): raise RuntimeError("必须指定start和end 或者 end和window") if not codes or len(codes) <= 0: raise RuntimeError("codes不能为空") self.start = start self.end = end self.codes = codes self.window = window self.calendar: TradingCalendar = trading_calendars.get_calendar('NYSE')
def run_example(example_name, environ): """ Run an example module from zipline.examples. """ mod = EXAMPLE_MODULES[example_name] register_calendar("YAHOO", get_calendar("NYSE"), force=True) return run_algorithm( initialize=getattr(mod, 'initialize', None), handle_data=getattr(mod, 'handle_data', None), before_trading_start=getattr(mod, 'before_trading_start', None), analyze=getattr(mod, 'analyze', None), bundle='test', environ=environ, # Provide a default capital base, but allow the test to override. **merge({'capital_base': 1e7}, mod._test_args()))
def drop_extra_sessions(df): cal: TradingCalendar = trading_calendars.get_calendar('NYSE') sessions = cal.sessions_in_range(df.index[0], df.index[-1]) if len(df.index) == len(sessions): return df to_drop = df.index.difference(sessions) if len(to_drop) == 0: return df df.drop(to_drop, inplace=True) dropped = len(to_drop) print(f'\nWarning! Dropped {dropped} extra sessions!') return df
def _make_sure_dates_are_initialized_properly(self, dtbegin, dtend, granularity): """ dates may or may not be specified by the user. when they do, they are probably don't include NY timezome data also, when granularity is minute, we want to make sure we get data when market is opened. so if it doesn't - let's get set end date to be last known minute with opened market. this nethod takes care of all these issues. :param dtbegin: :param dtend: :param granularity: :return: """ if not dtend: dtend = pd.Timestamp('now', tz=NY) else: dtend = pd.Timestamp(pytz.timezone(NY).localize(dtend)) if granularity == Granularity.Minute: calendar = trading_calendars.get_calendar(name='NYSE') if pd.Timestamp('now', tz=NY).date() == dtend.date(): if calendar.is_open_on_minute(dtend): # we execute during market open, we don't want today's data # we will receive it through the websocket dtend = dtend.replace(hour=15, minute=59, second=0, microsecond=0) dtend -= timedelta(days=1) while not calendar.is_open_on_minute(dtend): dtend = dtend.replace(hour=15, minute=59, second=0, microsecond=0) dtend -= timedelta(days=1) if not dtbegin: days = 30 if granularity == Granularity.Daily else 3 delta = timedelta(days=days) dtbegin = dtend - delta else: dtbegin = pd.Timestamp(pytz.timezone(NY).localize(dtbegin)) while dtbegin > dtend: # if we start the script during market hours we could get this # situation. this resolves that. dtbegin -= timedelta(days=1) return dtbegin, dtend
def __init__(self): # pylint: disable=super-init-not-called self._calendar = self.p.calendar # pylint: disable=no-member if isinstance(self._calendar, string_types): from trading_calendars import get_calendar self._calendar = get_calendar(self._calendar) self.dcache = DatetimeIndex([0.0]) self.idcache = DataFrame(index=DatetimeIndex([0.0])) self.csize = datetime.timedelta(days=self.p.cachesize) # pylint: disable=no-member self._tz = self.p.tz # pylint: disable=no-member if self._tz is None: self._tz = self._calendar.tz elif isinstance(self._tz, string_types): self._tz = tzparse(self._tz)
def df_generator(interval, start, end, divs_splits, assets_to_sids={}): exchange = 'NYSE' # get calendar and extend it to 20 days to the future to be able # to set dividend-pay-date to a valid session cal: TradingCalendar = trading_calendars.get_calendar('NYSE') sessions = cal.sessions_in_range(start, end + timedelta(days=20)) asset_list = list_assets() for symbol in asset_list: try: df = av_get_data_for_symbol(symbol, start, end, interval) sid = assets_to_sids[symbol] first_traded = df.index[0] auto_close_date = df.index[-1] + pd.Timedelta(days=1) if 'split' in df.columns: split = calc_split(sid, df) divs_splits['splits'] = divs_splits['splits'].append(split) if 'dividend' in df.columns: div = calc_dividend(sid, df, sessions) divs_splits['divs'] = pd.concat([divs_splits['divs'], div]) yield ( sid, df ), symbol, symbol, start, end, first_traded, auto_close_date, exchange except KeyboardInterrupt: exit() except Exception as e: # somehow rate-limiting does not work with exceptions, throttle manually if 'Thank you for using Alpha Vantage! Our standard API call frequency is' in str( e): print( f'\nGot rate-limit on remote-side, retrying symbol {symbol} later' ) asset_list.append(symbol) else: print(f'\nException for symbol {symbol}') print(e)
def run(self, portfolio, start, end, capital_base=100000): MovingAverageStrategy._portfolio = portfolio data, _ = AlphaVantageAPI.get_daily_data_for(portfolio.assets) data['benchmark'] = data[portfolio.benchmark] assets_panel = MovingAverageStrategy.to_panel(data) nyse_calendar = trading_calendars.get_calendar('XNYS') portfolio.performance = zipline.run_algorithm( start=start, end=end, data_frequency='daily', capital_base=capital_base, before_trading_start=MovingAverageStrategy.before_trading_start(), initialize=MovingAverageStrategy.initialize, handle_data=MovingAverageStrategy.handle_data, analyze=MovingAverageStrategy.analyze, trading_calendar=nyse_calendar, data=assets_panel)
def get_benchmark_returns(symbol): """ Return a data series from 1930 to 2030 with all zero values, which will effectively sidestep this web call issue, with no negative impact. Parameters ---------- symbol : str Benchmark symbol for which we're getting the returns. """ cal = get_calendar('NYSE') first_date = datetime(1930, 1, 1) last_date = datetime(2030, 1, 1) dates = cal.sessions_in_range(first_date, last_date) data = pd.DataFrame(0.0, index=dates, columns=['close']) data = data['close'] return data.sort_index().iloc[1:]
def run_example(example_name, environ): """ Run an example module from zipline.examples. """ mod = EXAMPLE_MODULES[example_name] register_calendar("YAHOO", get_calendar("NYSE"), force=True) return run_algorithm( initialize=getattr(mod, 'initialize', None), handle_data=getattr(mod, 'handle_data', None), before_trading_start=getattr(mod, 'before_trading_start', None), analyze=getattr(mod, 'analyze', None), bundle='test', environ=environ, # Provide a default capital base, but allow the test to override. **merge({'capital_base': 1e7}, mod._test_args()) )
def run(args: argparse.Namespace): gauges_value = {} gauges_latency = {} gauge_avg = Gauge(f"{args.prefix}_avg_latency", "avg latency") for query in args.queries: # USDJPY/1Sec/TICK -> usdjpy_1sec_tick key = query.replace("/", "_").replace("-", "_").lower() gauges_value[query] = Gauge(f"{args.prefix}_{key}_value", "value of {}".format(query)) gauges_latency[query] = Gauge(f"{args.prefix}_{key}_latency", "latency of {}".format(query)) url = f"http://{args.marketstore_host}:{args.marketstore_port}/rpc" delta = datetime.timedelta(seconds=args.lookback) cal = None if args.market: cal = tc.get_calendar("XNYS") while True: client = pymkts.Client(url) end_dt = pd.Timestamp.utcnow() start_dt = end_dt - delta holiday = False if cal and cal.is_session(end_dt) is False: holiday = True total = 0 for query in args.queries: if holiday: value, latency = (0, 0) else: (value, latency) = get_value(client, query, args.column, start_dt, end_dt) gauges_value[query].set(value) gauges_latency[query].set(latency) total += latency gauge_avg.set(total / len(args.queries)) time.sleep(args.interval)
def test_detect_non_market_minutes(self): cal = get_calendar('NYSE') # NOTE: This test is here instead of being on the base class for all # calendars because some of our calendars are 24/7, which means there # aren't any non-market minutes to find. day0 = cal.minutes_for_sessions_in_range( pd.Timestamp('2013-07-03', tz='UTC'), pd.Timestamp('2013-07-03', tz='UTC'), ) for minute in day0: self.assertTrue(cal.is_open_on_minute(minute)) day1 = cal.minutes_for_sessions_in_range( pd.Timestamp('2013-07-05', tz='UTC'), pd.Timestamp('2013-07-05', tz='UTC'), ) for minute in day1: self.assertTrue(cal.is_open_on_minute(minute)) def NYSE_timestamp(s): return pd.Timestamp(s, tz='US/Eastern').tz_convert('UTC') non_market = [ # After close. NYSE_timestamp('2013-07-03 16:01'), # Holiday. NYSE_timestamp('2013-07-04 10:00'), # Before open. NYSE_timestamp('2013-07-05 9:29'), ] for minute in non_market: self.assertFalse(cal.is_open_on_minute(minute), minute) input_ = pd.to_datetime( np.hstack([day0.values, minute.asm8, day1.values]), utc=True, ) with self.assertRaises(ValueError) as e: cal.minute_index_to_session_labels(input_) exc_str = str(e.exception) self.assertIn("First Bad Minute: {}".format(minute), exc_str)
def test_only_currency_converted_data(self, name, domain, calendar_name): # Test running a pipeline on a domain whose assets are all denominated # in the same currency. pipe = Pipeline( { "close_USD": EquityPricing.close.fx("USD").latest, "close_EUR": EquityPricing.close.fx("EUR").latest, }, domain=domain, ) start, end = self.daily_bar_sessions[calendar_name][-2:] result = self.run_pipeline(pipe, start, end) calendar = get_calendar(calendar_name) daily_bars = self.daily_bar_data[calendar_name] currency_codes = self.daily_bar_currency_codes[calendar_name] for (dt, asset), row in result.iterrows(): # Subtract a day b/c pipeline output on day N should have prior # day's price. price_date = dt - calendar.day expected_close = daily_bars[asset].loc[price_date, "close"] expected_base = currency_codes.loc[asset] expected_rate_USD = self.in_memory_fx_rate_reader.get_rate_scalar( rate="mid", quote="USD", base=expected_base, dt=price_date.asm8, ) expected_price = expected_close * expected_rate_USD assert_equal(row.close_USD, expected_price) expected_rate_EUR = self.in_memory_fx_rate_reader.get_rate_scalar( rate="mid", quote="EUR", base=expected_base, dt=price_date.asm8, ) expected_price = expected_close * expected_rate_EUR assert_equal(row.close_EUR, expected_price)
def ingest(start): from sharadar.pipeline.engine import load_sharadar_bundle from zipline.assets import ASSET_DB_VERSION from sharadar.data.sql_lite_assets import SQLiteAssetDBWriter from sharadar.data.sql_lite_daily_pricing import SQLiteDailyBarWriter from trading_calendars import get_calendar from sharadar.loaders.constant import EXCHANGE_DF calendar = get_calendar('XNYS') macro_equities_df = create_macro_equities_df() macro_prices_df = create_macro_prices_df(start) output_dir = get_output_dir() asset_dbpath = os.path.join(output_dir, ("assets-%d.sqlite" % ASSET_DB_VERSION)) asset_db_writer = SQLiteAssetDBWriter(asset_dbpath) asset_db_writer.write(equities=macro_equities_df, exchanges=EXCHANGE_DF) prices_dbpath = os.path.join(output_dir, "prices.sqlite") sql_daily_bar_writer = SQLiteDailyBarWriter(prices_dbpath, calendar) sql_daily_bar_writer.write(macro_prices_df) return macro_prices_df.shape[0]
def test_write_attrs(self): result = self.bcolz_daily_bar_ctable expected_first_row = { '1': 0, '3': 5, # Asset 1 has 5 trading days. '5': 12, # Asset 3 has 7 trading days. '7': 33, # Asset 5 has 21 trading days. '9': 44, # Asset 7 has 11 trading days. '11': 49, # Asset 9 has 5 trading days. } expected_last_row = { '1': 4, '3': 11, '5': 32, '7': 43, '9': 48, '11': 57, # Asset 11 has 9 trading days. } expected_calendar_offset = { '1': 0, # Starts on 6-01, 1st trading day of month. '3': 15, # Starts on 6-22, 16th trading day of month. '5': 1, # Starts on 6-02, 2nd trading day of month. '7': 0, # Starts on 6-01, 1st trading day of month. '9': 9, # Starts on 6-12, 10th trading day of month. '11': 10, # Starts on 6-15, 11th trading day of month. } self.assertEqual(result.attrs['first_row'], expected_first_row) self.assertEqual(result.attrs['last_row'], expected_last_row) self.assertEqual( result.attrs['calendar_offset'], expected_calendar_offset, ) cal = get_calendar(result.attrs['calendar_name']) first_session = Timestamp(result.attrs['start_session_ns'], tz='UTC') end_session = Timestamp(result.attrs['end_session_ns'], tz='UTC') sessions = cal.sessions_in_range(first_session, end_session) assert_equal( self.sessions, sessions )
def __init__(self, start=None, end=None, assets=None, exchange='NYSE'): self.start = normalize_date(pd.Timestamp(start or '2018-08-13')) self.end = normalize_date(pd.Timestamp(end or '2018-08-14')) self._exchange = exchange self._calendar = get_calendar(exchange) self.assets = assets or ['asset-0', 'asset-1', 'asset-2'] minutes = self._calendar.minutes_for_sessions_in_range( self.start, self.end) self._minutely_bars = {} for i, asset in enumerate(self.get_equities()): bars = create_bars(minutes, i) self._minutely_bars[asset] = bars days = self._calendar.sessions_in_range(self.start, self.end) self._daily_bars = {} for i, asset in enumerate(self.get_equities()): bars = create_bars(days, i) self._daily_bars[asset] = bars
def update_daily_prices(start_date, end_date, connection): tw_calendar = get_calendar('XTAI') main_df = pd.DataFrame() for date in pd.date_range(start_date, end_date): if date not in tw_calendar.opens: continue df, in_db = get_daily_prices(date, connection) if df is not None and not in_db: main_df = main_df.append(df, sort=False) print('{} 每日收盤行情抓取完成,等待15秒'.format(date.strftime('%Y%m%d'))) time.sleep(15) if not main_df.empty: main_df.to_sql('daily_prices', connection, if_exists='append') return main_df
def get_balance(self): us_calendar = get_calendar("XNYS") history = pd.DataFrame([ { "date": self.created, "amount": self.balance }, *self.transactions.values("date", "amount"), { "date": timezone.now(), "amount": 0 }, ]) history.set_index("date", inplace=True) history = history.resample("d").sum() history = history.rename(columns={"amount": "close"}) history.index = history.index.date history["close"] = history.close.cumsum() history = history.loc[history.apply( lambda x: us_calendar.is_session(x.name), axis=1)] return history
def run_example(example_modules, example_name, environ, benchmark_returns=None): """ Run an example module from zipline.examples. """ mod = example_modules[example_name] register_calendar("YAHOO", get_calendar("NYSE"), force=True) return run_algorithm( initialize=getattr(mod, "initialize", None), handle_data=getattr(mod, "handle_data", None), before_trading_start=getattr(mod, "before_trading_start", None), analyze=getattr(mod, "analyze", None), bundle="test", environ=environ, benchmark_returns=benchmark_returns, # Provide a default capital base, but allow the test to override. **merge({"capital_base": 1e7}, mod._test_args()))
def create_simulation_parameters(year=2006, start=None, end=None, capital_base=float("1.0e5"), num_days=None, data_frequency='daily', emission_rate='daily', trading_calendar=None, financing_costs: Callable[[Account], float] = None, manager_fees: Callable[[Account, Portfolio, pd.Timestamp], float] = None): if not trading_calendar: trading_calendar = get_calendar("NYSE") if start is None: start = pd.Timestamp("{0}-01-01".format(year), tz='UTC') elif type(start) == datetime: start = pd.Timestamp(start) if end is None: if num_days: start_index = trading_calendar.all_sessions.searchsorted(start) end = trading_calendar.all_sessions[start_index + num_days - 1] else: end = pd.Timestamp("{0}-12-31".format(year), tz='UTC') elif type(end) == datetime: end = pd.Timestamp(end) sim_params = SimulationParameters( start_session=start, end_session=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=emission_rate, trading_calendar=trading_calendar, financing_costs=financing_costs, manager_fees=manager_fees ) return sim_params
def check_holidays(holiday_key_path, calendar_column, holiday_column, date_format, delimiter, strip_x_from_cal_name, answer_key_calendar_name, min_date, calendars): data = pd.read_csv(holiday_key_path, sep=delimiter, parse_dates=[holiday_column], date_parser=partial(pd.to_datetime, format=date_format, utc=True)) for calendar_name in default_calendar_names: if calendars and calendar_name not in calendars: continue cal = get_calendar(calendar_name) if answer_key_calendar_name: csv_cal_code = answer_key_calendar_name elif strip_x_from_cal_name: # Convert the calendar name to the format expected in the CSV. csv_cal_code = calendar_name.lstrip('X') else: csv_cal_code = calendar_name holidays = set( data[holiday_column][data[calendar_column] == csv_cal_code]) if holidays: start = max(cal.first_session, min(holidays), min_date) end = min(cal.last_session, max(holidays)) _check_range(start, end, holidays, cal, calendar_name) else: click.secho( 'No holidays found for {} in the holiday key.'.format( calendar_name, ), fg='yellow', ) click.echo()
def setUpClass(cls): super().setUpClass() cls.class_ = StatelessRule cls.cal = get_calendar(cls.CALENDAR_STRING) # First day of 09/2014 is closed whereas that for 10/2014 is open cls.sept_sessions = cls.cal.sessions_in_range( pd.Timestamp('2014-09-01', tz='UTC'), pd.Timestamp('2014-09-30', tz='UTC'), ) cls.oct_sessions = cls.cal.sessions_in_range( pd.Timestamp('2014-10-01', tz='UTC'), pd.Timestamp('2014-10-31', tz='UTC'), ) cls.sept_week = cls.cal.minutes_for_sessions_in_range( pd.Timestamp("2014-09-22", tz='UTC'), pd.Timestamp("2014-09-26", tz='UTC')) cls.HALF_SESSION = None cls.FULL_SESSION = None
def __init__( self, calendar=None, before_trading_start_minute=(pd.Timestamp('8:45').time(), 'America/New_York'), minute_emission=True, time_skew=None, ): if calendar is None: calendar = get_calendar('NYSE') self.calendar = calendar self.before_trading_start_minute = before_trading_start_minute self.minute_emission = minute_emission self._last_emit = None self._before_trading_start_bar_yielded = False now = pd.Timestamp.utcnow() prev_close = calendar.previous_close(now) current = self._set_before_trading_start(prev_close) self._current_time = current self._fake_end = prev_close
def setUpClass(cls): super(StatelessRulesTests, cls).setUpClass() cls.class_ = StatelessRule cls.cal = get_calendar(cls.CALENDAR_STRING) # First day of 09/2014 is closed whereas that for 10/2014 is open cls.sept_sessions = cls.cal.sessions_in_range( pd.Timestamp('2014-09-01', tz='UTC'), pd.Timestamp('2014-09-30', tz='UTC'), ) cls.oct_sessions = cls.cal.sessions_in_range( pd.Timestamp('2014-10-01', tz='UTC'), pd.Timestamp('2014-10-31', tz='UTC'), ) cls.sept_week = cls.cal.minutes_for_sessions_in_range( pd.Timestamp("2014-09-22", tz='UTC'), pd.Timestamp("2014-09-26", tz='UTC') ) cls.HALF_SESSION = None cls.FULL_SESSION = None
def create_simulation_parameters(year=2006, start=None, end=None, capital_base=float("1.0e5"), num_days=None, data_frequency='daily', emission_rate='daily', trading_calendar=None): if not trading_calendar: trading_calendar = get_calendar("NYSE") if start is None: start = pd.Timestamp("{0}-01-01".format(year), tz='UTC') elif type(start) == datetime: start = pd.Timestamp(start) if end is None: if num_days: start_index = trading_calendar.all_sessions.searchsorted(start) end = trading_calendar.all_sessions[start_index + num_days - 1] else: end = pd.Timestamp("{0}-12-31".format(year), tz='UTC') elif type(end) == datetime: end = pd.Timestamp(end) sim_params = SimulationParameters( start_session=start, end_session=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=emission_rate, trading_calendar=trading_calendar, ) return sim_params
def setUpClass(cls): super(StatefulRulesTests, cls).setUpClass() cls.class_ = StatefulRule cls.cal = get_calendar(cls.CALENDAR_STRING)
def run(ctx, algofile, algotext, define, data_frequency, capital_base, bundle, bundle_timestamp, start, end, output, trading_calendar, print_algo, metrics_set, local_namespace, blotter): """Run a backtest for the given algorithm. """ # check that the start and end dates are passed correctly if start is None and end is None: # check both at the same time to avoid the case where a user # does not pass either of these and then passes the first only # to be told they need to pass the second argument also ctx.fail( "must specify dates with '-s' / '--start' and '-e' / '--end'", ) if start is None: ctx.fail("must specify a start date with '-s' / '--start'") if end is None: ctx.fail("must specify an end date with '-e' / '--end'") if (algotext is not None) == (algofile is not None): ctx.fail( "must specify exactly one of '-f' / '--algofile' or" " '-t' / '--algotext'", ) trading_calendar = get_calendar(trading_calendar) perf = _run( initialize=None, handle_data=None, before_trading_start=None, analyze=None, algofile=algofile, algotext=algotext, defines=define, data_frequency=data_frequency, capital_base=capital_base, bundle=bundle, bundle_timestamp=bundle_timestamp, start=start, end=end, output=output, trading_calendar=trading_calendar, print_algo=print_algo, metrics_set=metrics_set, local_namespace=local_namespace, environ=os.environ, blotter=blotter, benchmark_returns=None, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def calendar(self): return get_calendar(self.calendar_name)
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, bundle, bundle_timestamp, start, end, output, trading_calendar, print_algo, metrics_set, local_namespace, environ, blotter, benchmark_returns): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. """ if benchmark_returns is None: benchmark_returns, _ = load_market_data(environ=environ) if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if trading_calendar is None: trading_calendar = get_calendar('XSHG') # date parameter validation if trading_calendar.session_distance(start, end) < 1: raise _RunAlgoError( 'There are no trading days between %s and %s' % ( start.date(), end.date(), ), ) bundle_data = bundles.load( bundle, environ, bundle_timestamp, ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( bundle_data.asset_finder, trading_calendar=trading_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) pipeline_loader = CNEquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader, ) def choose_loader(column): if column in CNEquityPricing.columns: return pipeline_loader # # 简单处理 elif type(column) == BoundColumn: # # 使用实例才能避免KeyError return global_loader raise ValueError( "No PipelineLoader registered for column %s." % column ) if isinstance(metrics_set, six.string_types): try: metrics_set = metrics.load(metrics_set) except ValueError as e: raise _RunAlgoError(str(e)) if isinstance(blotter, six.string_types): try: blotter = load(Blotter, blotter) except ValueError as e: raise _RunAlgoError(str(e)) perf = TradingAlgorithm( namespace=namespace, data_portal=data, get_pipeline_loader=choose_loader, trading_calendar=trading_calendar, sim_params=SimulationParameters( start_session=start, end_session=end, trading_calendar=trading_calendar, capital_base=capital_base, data_frequency=data_frequency, ), metrics_set=metrics_set, blotter=blotter, benchmark_returns=benchmark_returns, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, } ).run() if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the zipline magic not write any data perf.to_pickle(output) return perf
def test_ingest(self): calendar = get_calendar('XNYS') sessions = calendar.sessions_in_range(self.START_DATE, self.END_DATE) minutes = calendar.minutes_for_sessions_in_range( self.START_DATE, self.END_DATE, ) sids = tuple(range(3)) equities = make_simple_equity_info( sids, self.START_DATE, self.END_DATE, ) daily_bar_data = make_bar_data(equities, sessions) minute_bar_data = make_bar_data(equities, minutes) first_split_ratio = 0.5 second_split_ratio = 0.1 splits = pd.DataFrame.from_records([ { 'effective_date': str_to_seconds('2014-01-08'), 'ratio': first_split_ratio, 'sid': 0, }, { 'effective_date': str_to_seconds('2014-01-09'), 'ratio': second_split_ratio, 'sid': 1, }, ]) @self.register( 'bundle', calendar_name='NYSE', start_session=self.START_DATE, end_session=self.END_DATE, ) def bundle_ingest(environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, start_session, end_session, cache, show_progress, output_dir): assert_is(environ, self.environ) asset_db_writer.write(equities=equities) minute_bar_writer.write(minute_bar_data) daily_bar_writer.write(daily_bar_data) adjustment_writer.write(splits=splits) assert_is_instance(calendar, TradingCalendar) assert_is_instance(cache, dataframe_cache) assert_is_instance(show_progress, bool) self.ingest('bundle', environ=self.environ) bundle = self.load('bundle', environ=self.environ) assert_equal(set(bundle.asset_finder.sids), set(sids)) columns = 'open', 'high', 'low', 'close', 'volume' actual = bundle.equity_minute_bar_reader.load_raw_arrays( columns, minutes[0], minutes[-1], sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(minutes, sids, equities, colname), msg=colname, ) actual = bundle.equity_daily_bar_reader.load_raw_arrays( columns, self.START_DATE, self.END_DATE, sids, ) for actual_column, colname in zip(actual, columns): assert_equal( actual_column, expected_bar_values_2d(sessions, sids, equities, colname), msg=colname, ) adjustments_for_cols = bundle.adjustment_reader.load_adjustments( columns, sessions, pd.Index(sids), ) for column, adjustments in zip(columns, adjustments_for_cols[:-1]): # iterate over all the adjustments but `volume` assert_equal( adjustments, { 2: [Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=first_split_ratio, )], 3: [Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=second_split_ratio, )], }, msg=column, ) # check the volume, the value should be 1/ratio assert_equal( adjustments_for_cols[-1], { 2: [Float64Multiply( first_row=0, last_row=2, first_col=0, last_col=0, value=1 / first_split_ratio, )], 3: [Float64Multiply( first_row=0, last_row=3, first_col=1, last_col=1, value=1 / second_split_ratio, )], }, msg='volume', )
def algo_args(year=2017, from_date=None, to_date=None, capital_base=float('1.0e7'), num_days=None, metrics_set=None): # Constant inputs benchmark_sid = 3623 # '000300.SH' bundle = "cndaily" calendar = 'XSHG' # With trading calendar trading_calendar = get_calendar(calendar) # With simulation parameters if from_date: from_date = pd.Timestamp(from_date, tz='utc') if to_date: to_date = pd.Timestamp(to_date, tz='utc') sim_params = create_simulation_parameters( year=year, start=from_date, end=to_date, capital_base=capital_base, num_days=num_days, trading_calendar=trading_calendar ) # With data portal bundle_data = load(bundle) data_portal = DataPortal( bundle_data.asset_finder, trading_calendar, trading_calendar.first_session, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader ) # With pipelineloader pipeline_loader = EquityPricingLoader( bundle_data.equity_daily_bar_reader, bundle_data.adjustment_reader ) def choose_loader(column): if column.unspecialize() in EquityPricing.columns: return pipeline_loader elif column in global_loader: return global_loader raise ValueError("%s is NOT registered in `PipelineLoader`." % column) return { 'sim_params': sim_params, 'data_portal': data_portal, 'benchmark_sid': benchmark_sid, 'metrics_set': metrics_set, 'get_pipeline_loader': choose_loader, }
def load_market_data(trading_day=None, trading_days=None, bm_symbol='000300', environ=None): """ Load benchmark returns and treasury yield curves for the given calendar and benchmark symbol. Benchmarks are downloaded as a Series from IEX Trading. Treasury curves are US Treasury Bond rates and are downloaded from 'www.federalreserve.gov' by default. For Canadian exchanges, a loader for Canadian bonds from the Bank of Canada is also available. Results downloaded from the internet are cached in ~/.zipline/data. Subsequent loads will attempt to read from the cached files before falling back to redownload. Parameters ---------- trading_day : pandas.CustomBusinessDay, optional A trading_day used to determine the latest day for which we expect to have data. Defaults to an NYSE trading day. trading_days : pd.DatetimeIndex, optional A calendar of trading days. Also used for determining what cached dates we should expect to have cached. Defaults to the NYSE calendar. bm_symbol : str, optional Symbol for the benchmark index to load. Defaults to 'SPY', the ticker for the S&P 500, provided by IEX Trading. Returns ------- (benchmark_returns, treasury_curves) : (pd.Series, pd.DataFrame) Notes ----- Both return values are DatetimeIndexed with values dated to midnight in UTC of each stored date. The columns of `treasury_curves` are: '1month', '3month', '6month', '1year','2year','3year','5year','7year','10year','20year','30year' """ if trading_day is None: trading_day = get_calendar('XSHG').day if trading_days is None: trading_days = get_calendar('XSHG').all_sessions first_date = trading_days[0] now = pd.Timestamp.utcnow() # we will fill missing benchmark data through latest trading date last_date = trading_days[trading_days.get_loc(now, method='ffill')] br = ensure_benchmark_data( bm_symbol, first_date, last_date, now, # We need the trading_day to figure out the close prior to the first # date so that we can compute returns for the first date. trading_day, environ, ) tc = ensure_treasury_data( bm_symbol, first_date, last_date, now, environ, ) # combine dt indices and reindex using ffill then bfill all_dt = br.index.union(tc.index) br = br.reindex(all_dt, method='ffill').fillna(method='bfill') tc = tc.reindex(all_dt, method='ffill').fillna(method='bfill') benchmark_returns = br[br.index.slice_indexer(first_date, last_date)] treasury_curves = tc[tc.index.slice_indexer(first_date, last_date)] return benchmark_returns, treasury_curves
def init_class_fixtures(cls): super(TestMinuteBarDataFuturesCalendar, cls).init_class_fixtures() cls.trading_calendar = get_calendar('CMES')
def trading_calendar(self): if 'calendar_name' in self._table.attrs.attrs: return get_calendar(self._table.attrs['calendar_name']) else: return None
def init_class_fixtures(cls): super(TestDateUtils, cls).init_class_fixtures() cls.calendar = get_calendar('XSHG')