def copy_market_data(src_market_data_dir, dest_root_dir): symbol = 'SPY' filenames = (get_benchmark_filename(symbol), INDEX_MAPPING[symbol][1]) ensure_directory(os.path.join(dest_root_dir, 'data')) for filename in filenames: shutil.copyfile( os.path.join(src_market_data_dir, filename), os.path.join(dest_root_dir, 'data', filename) )
def ingest(name, environ=os.environ, timestamp=None, show_progress=False): """Ingest data for a given bundle. Parameters ---------- name : str The name of the bundle. environ : mapping, optional The environment variables. By default this is os.environ. timestamp : datetime, optional The timestamp to use for the load. By default this is the current time. show_progress : bool, optional Tell the ingest function to display the progress where possible. """ try: bundle = bundles[name] except KeyError: raise UnknownBundle(name) if timestamp is None: timestamp = pd.Timestamp.utcnow() timestamp = timestamp.tz_convert('utc').tz_localize(None) timestr = to_bundle_ingest_dirname(timestamp) cachepath = cache_path(name, environ=environ) pth.ensure_directory(pth.data_path([name, timestr], environ=environ)) pth.ensure_directory(cachepath) with dataframe_cache(cachepath, clean_on_failure=False) as cache, \ ExitStack() as stack: # we use `cleanup_on_failure=False` so that we don't purge the # cache directory if the load fails in the middle if bundle.create_writers: wd = stack.enter_context( working_dir(pth.data_path([], environ=environ))) daily_bars_path = wd.ensure_dir(*daily_equity_relative( name, timestr, environ=environ, )) daily_bar_writer = BcolzDailyBarWriter( daily_bars_path, bundle.calendar, bundle.start_session, bundle.end_session, ) # Do an empty write to ensure that the daily ctables exist # when we create the SQLiteAdjustmentWriter below. The # SQLiteAdjustmentWriter needs to open the daily ctables so # that it can compute the adjustment ratios for the dividends. daily_bar_writer.write(()) minute_bar_writer = BcolzMinuteBarWriter( wd.ensure_dir(*minute_equity_relative( name, timestr, environ=environ)), bundle.calendar, bundle.start_session, bundle.end_session, minutes_per_day=bundle.minutes_per_day, ) asset_db_writer = AssetDBWriter( wd.getpath(*asset_db_relative( name, timestr, environ=environ, ))) adjustment_db_writer = stack.enter_context( SQLiteAdjustmentWriter( wd.getpath(*adjustment_db_relative( name, timestr, environ=environ)), BcolzDailyBarReader(daily_bars_path), bundle.calendar.all_sessions, overwrite=True, )) else: daily_bar_writer = None minute_bar_writer = None asset_db_writer = None adjustment_db_writer = None bundle.ingest( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_db_writer, bundle.calendar, bundle.start_session, bundle.end_session, cache, show_progress, pth.data_path([name, timestr], environ=environ), )
def ingest(name, environ=os.environ, timestamp=None, show_progress=False): """Ingest data for a given bundle. Parameters ---------- name : str The name of the bundle. environ : mapping, optional The environment variables. By default this is os.environ. timestamp : datetime, optional The timestamp to use for the load. By default this is the current time. show_progress : bool, optional Tell the ingest function to display the progress where possible. """ try: bundle = bundles[name] except KeyError: raise UnknownBundle(name) if timestamp is None: timestamp = pd.Timestamp.utcnow() timestamp = timestamp.tz_convert('utc').tz_localize(None) timestr = to_bundle_ingest_dirname(timestamp) cachepath = cache_path(name, environ=environ) pth.ensure_directory(pth.data_path([name, timestr], environ=environ)) pth.ensure_directory(cachepath) with dataframe_cache(cachepath, clean_on_failure=False) as cache, \ ExitStack() as stack: # we use `cleanup_on_failure=False` so that we don't purge the # cache directory if the load fails in the middle if bundle.create_writers: wd = stack.enter_context(working_dir( pth.data_path([], environ=environ)) ) daily_bars_path = wd.ensure_dir( *daily_equity_relative( name, timestr, environ=environ, ) ) daily_bar_writer = BcolzDailyBarWriter( daily_bars_path, nyse_cal, bundle.calendar[0], bundle.calendar[-1] ) # Do an empty write to ensure that the daily ctables exist # when we create the SQLiteAdjustmentWriter below. The # SQLiteAdjustmentWriter needs to open the daily ctables so # that it can compute the adjustment ratios for the dividends. daily_bar_writer.write(()) minute_bar_writer = BcolzMinuteBarWriter( bundle.calendar[0], wd.ensure_dir(*minute_equity_relative( name, timestr, environ=environ) ), bundle.opens, bundle.closes, minutes_per_day=bundle.minutes_per_day, ) asset_db_writer = AssetDBWriter( wd.getpath(*asset_db_relative( name, timestr, environ=environ, )) ) adjustment_db_writer = stack.enter_context( SQLiteAdjustmentWriter( wd.getpath(*adjustment_db_relative( name, timestr, environ=environ)), BcolzDailyBarReader(daily_bars_path), bundle.calendar, overwrite=True, ) ) else: daily_bar_writer = None minute_bar_writer = None asset_db_writer = None adjustment_db_writer = None bundle.ingest( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_db_writer, bundle.calendar, cache, show_progress, pth.data_path([name, timestr], environ=environ), )
def backtest_result_path(): from zipline.utils import paths as pth ret = pth.zipline_path(['backtest']) pth.ensure_directory(ret) return ret
def ingest(name, environ=os.environ, timestamp=None, assets_versions=(), show_progress=False): """Ingest data for a given bundle. Parameters ---------- name : str The name of the bundle. environ : mapping, optional The environment variables. By default this is os.environ. timestamp : datetime, optional The timestamp to use for the load. By default this is the current time. assets_versions : Iterable[int], optional Versions of the assets db to which to downgrade. show_progress : bool, optional Tell the ingest function to display the progress where possible. """ try: bundle = bundles[name] except KeyError: raise UnknownBundle(name) calendar = get_calendar(bundle.calendar_name) start_session = bundle.start_session end_session = bundle.end_session if start_session is None or start_session < calendar.first_session: start_session = calendar.first_session if end_session is None or end_session > calendar.last_session: end_session = calendar.last_session if timestamp is None: timestamp = pd.Timestamp.utcnow() timestamp = timestamp.tz_convert('utc').tz_localize(None) timestr = to_bundle_ingest_dirname(timestamp) cachepath = cache_path(name, environ=environ) pth.ensure_directory(pth.data_path([name, timestr], environ=environ)) pth.ensure_directory(cachepath) with dataframe_cache(cachepath, clean_on_failure=False) as cache, \ ExitStack() as stack: # we use `cleanup_on_failure=False` so that we don't purge the # cache directory if the load fails in the middle if bundle.create_writers: wd = stack.enter_context( working_dir(pth.data_path([], environ=environ))) daily_bars_path = wd.ensure_dir( *daily_equity_relative(name, timestr)) daily_bar_writer = BcolzDailyBarWriter( daily_bars_path, calendar, start_session, end_session, ) # Do an empty write to ensure that the daily ctables exist # when we create the SQLiteAdjustmentWriter below. The # SQLiteAdjustmentWriter needs to open the daily ctables so # that it can compute the adjustment ratios for the dividends. daily_bar_writer.write(()) minute_bar_writer = BcolzMinuteBarWriter( wd.ensure_dir(*minute_equity_relative(name, timestr)), calendar, start_session, end_session, minutes_per_day=bundle.minutes_per_day, ) assets_db_path = wd.getpath(*asset_db_relative(name, timestr)) asset_db_writer = AssetDBWriter(assets_db_path) adjustment_db_writer = stack.enter_context( SQLiteAdjustmentWriter( wd.getpath(*adjustment_db_relative(name, timestr)), BcolzDailyBarReader(daily_bars_path), overwrite=True, )) else: daily_bar_writer = None minute_bar_writer = None asset_db_writer = None adjustment_db_writer = None if assets_versions: raise ValueError('Need to ingest a bundle that creates ' 'writers in order to downgrade the assets' ' db.') log.info("Ingesting {}.", name) bundle.ingest( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_db_writer, calendar, start_session, end_session, cache, show_progress, pth.data_path([name, timestr], environ=environ), ) for version in sorted(set(assets_versions), reverse=True): version_path = wd.getpath(*asset_db_relative( name, timestr, db_version=version, )) with working_file(version_path) as wf: shutil.copy2(assets_db_path, wf.path) downgrade(wf.path, version)
def ingest(name, environ=os.environ, timestamp=None, assets_versions=(), show_progress=False): """Ingest data for a given bundle. Parameters ---------- name : str The name of the bundle. environ : mapping, optional The environment variables. By default this is os.environ. timestamp : datetime, optional The timestamp to use for the load. By default this is the current time. assets_versions : Iterable[int], optional Versions of the assets db to which to downgrade. show_progress : bool, optional Tell the ingest function to display the progress where possible. """ try: bundle = bundles[name] except KeyError: raise UnknownBundle(name) calendar = get_calendar(bundle.calendar_name) start_session = bundle.start_session end_session = bundle.end_session if start_session is None or start_session < calendar.first_session: start_session = calendar.first_session if end_session is None or end_session > calendar.last_session: end_session = calendar.last_session if timestamp is None: timestamp = pd.Timestamp.utcnow() timestamp = timestamp.tz_convert('utc').tz_localize(None) timestr = to_bundle_ingest_dirname(timestamp) cachepath = cache_path(name, environ=environ) pth.ensure_directory(pth.data_path([name, timestr], environ=environ)) pth.ensure_directory(cachepath) with dataframe_cache(cachepath, clean_on_failure=False) as cache, \ ExitStack() as stack: # we use `cleanup_on_failure=False` so that we don't purge the # cache directory if the load fails in the middle if bundle.create_writers: wd = stack.enter_context(working_dir( pth.data_path([], environ=environ)) ) daily_bars_path = wd.ensure_dir( *daily_equity_relative( name, timestr, environ=environ, ) ) daily_bar_writer = BcolzDailyBarWriter( daily_bars_path, calendar, start_session, end_session, ) # Do an empty write to ensure that the daily ctables exist # when we create the SQLiteAdjustmentWriter below. The # SQLiteAdjustmentWriter needs to open the daily ctables so # that it can compute the adjustment ratios for the dividends. daily_bar_writer.write(()) minute_bar_writer = BcolzMinuteBarWriter( wd.ensure_dir(*minute_equity_relative( name, timestr, environ=environ) ), calendar, start_session, end_session, minutes_per_day=bundle.minutes_per_day, ) assets_db_path = wd.getpath(*asset_db_relative( name, timestr, environ=environ, )) asset_db_writer = AssetDBWriter(assets_db_path) adjustment_db_writer = stack.enter_context( SQLiteAdjustmentWriter( wd.getpath(*adjustment_db_relative( name, timestr, environ=environ)), BcolzDailyBarReader(daily_bars_path), calendar.all_sessions, overwrite=True, ) ) else: daily_bar_writer = None minute_bar_writer = None asset_db_writer = None adjustment_db_writer = None if assets_versions: raise ValueError('Need to ingest a bundle that creates ' 'writers in order to downgrade the assets' ' db.') bundle.ingest( environ, asset_db_writer, minute_bar_writer, daily_bar_writer, adjustment_db_writer, calendar, start_session, end_session, cache, show_progress, pth.data_path([name, timestr], environ=environ), ) for version in sorted(set(assets_versions), reverse=True): version_path = wd.getpath(*asset_db_relative( name, timestr, environ=environ, db_version=version, )) with working_file(version_path) as wf: shutil.copy2(assets_db_path, wf.path) downgrade(wf.path, version)
def show_draw_result(title, results_df, bundle): import matplotlib from matplotlib import gridspec import matplotlib.image as mpimg import matplotlib.pyplot as plt from zipline.utils import paths from datetime import datetime plt.style.use('ggplot') red = "#aa4643" blue = "#4572a7" black = "#000000" figsize = (18, 6) f = plt.figure(title, figsize=figsize) gs = gridspec.GridSpec(10, 8) # TODO draw logo # ax = plt.subplot(gs[:3, -1:]) # ax.axis("off") # filename = os.path.join(paths.zipline_root(), 'zipline.png') # img = mpimg.imread(filename) # imgplot = ax.imshow(img, interpolation="nearest") # ax.autoscale_view() # draw risk and portfolio series = results_df.iloc[-1] font_size = 12 value_font_size = 11 label_height, value_height = 0.8, 0.6 label_height2, value_height2 = 0.35, 0.15 fig_data = [ (0.00, label_height, value_height, "Total Returns", "{0:.3%}".format(series.algorithm_period_return), red, black), (0.15, label_height, value_height, "Annual Returns", "{0:.3%}".format(series.annualized_algorithm_return), red, black), (0.00, label_height2, value_height2, "Benchmark Total", "{0:.3%}".format(series.benchmark_period_return), blue, black), (0.15, label_height2, value_height2, "Benchmark Annual", "{0:.3%}".format(series.annualized_benchmark_return), blue, black), (0.30, label_height, value_height, "Alpha", "{0:.4}".format(series.alpha), black, black), (0.40, label_height, value_height, "Beta", "{0:.4}".format(series.beta), black, black), (0.55, label_height, value_height, "Sharpe", "{0:.4}".format(series.sharpe), black, black), (0.70, label_height, value_height, "Sortino", "{0:.4}".format(series.sortino), black, black), (0.85, label_height, value_height, "Information Ratio", "{0:.4}".format(series.information), black, black), (0.30, label_height2, value_height2, "Volatility", "{0:.4}".format(series.algo_volatility), black, black), (0.40, label_height2, value_height2, "MaxDrawdown", "{0:.3%}".format(series.max_drawdown), black, black), # (0.55, label_height2, value_height2, "Tracking Error", "{0:.4}".format(series.tracking_error), black, black), # (0.70, label_height2, value_height2, "Downside Risk", "{0:.4}".format(series.downside_risk), black, black), ] ax = plt.subplot(gs[:3, :-1]) ax.axis("off") for x, y1, y2, label, value, label_color, value_color in fig_data: ax.text(x, y1, label, color=label_color, fontsize=font_size) ax.text(x, y2, value, color=value_color, fontsize=value_font_size) # strategy vs benchmark ax = plt.subplot(gs[4:, :]) ax.get_xaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator()) ax.get_yaxis().set_minor_locator(matplotlib.ticker.AutoMinorLocator()) ax.grid(b=True, which='minor', linewidth=.2) ax.grid(b=True, which='major', linewidth=1) ax.plot(results_df["benchmark_period_return"], label="benchmark", alpha=1, linewidth=2, color=blue) ax.plot(results_df["algorithm_period_return"], label="algorithm", alpha=1, linewidth=2, color=red) # manipulate vals = ax.get_yticks() ax.set_yticklabels(['{:3.2f}%'.format(x * 100) for x in vals]) leg = plt.legend(loc="upper left") leg.get_frame().set_alpha(0.5) plt.show() now = datetime.now() paths.ensure_directory(paths.zipline_path(['perf'])) plt.savefig(filename=os.path.join( paths.zipline_path(['perf']), os.path.basename(title).split('.')[0] + '_' + bundle + '_' + now.strftime('%Y%m%dT%H%M%s') + '.png'))