def _bundle_trading_environment(bundle_data, environ): prefix, connstr = re.split(r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1) if prefix: raise ValueError("invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url)) return TradingEnvironment(asset_db_path=connstr, environ=environ)
def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) if live and exchange is not None: exchange_name = exchange start = pd.Timestamp.utcnow() end = start + timedelta(minutes=1439) portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) exchange_auth = get_exchange_auth(exchange_name) if exchange_name == 'bitfinex': exchange = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchange = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise NotImplementedError('exchange not supported: %s' % exchange_name) open_calendar = get_calendar('OPEN') sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) if live and exchange is not None: env = TradingEnvironment(environ=environ, exchange_tz='UTC', asset_db_path=None) env.asset_finder = AssetFinderExchange(exchange) data = DataPortalExchange(exchange=exchange, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True)) choose_loader = None def fetch_capital_base(attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: sleep(5) return fetch_capital_base(attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) sim_params = create_simulation_parameters( start=start, end=end, capital_base=fetch_capital_base(), emission_rate='minute', data_frequency='minute') elif bundle is not None: bundles = bundle.split(',') def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment( load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data def get_loader_for_bundle(b): bundle_data = load( b, environ, bundle_timestamp, ) if b == 'poloniex': return CryptoPricingLoader( bundle_data, data_frequency, CryptoPricing, ) elif b == 'quandl': return USEquityPricingLoader( bundle_data, data_frequency, USEquityPricing, ) raise ValueError("No PipelineLoader registered for bundle %s." % b) loaders = [get_loader_for_bundle(b) for b in bundles] env, data = get_trading_env_and_data(bundles) def choose_loader(column): for loader in loaders: if column in loader.columns: return loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None TradingAlgorithmClass = (partial(ExchangeTradingAlgorithm, exchange=exchange, algo_namespace=algo_namespace, live_graph=live_graph) if live and exchange else TradingAlgorithm) perf = TradingAlgorithmClass( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, quote_currency, live_graph, analyze_live, simulate_orders, auth_aliases, stats_output): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ # TODO: refactor for more granularity if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) log.info('Catalyst version {}'.format(catalyst.__version__)) if not DISABLE_ALPHA_WARNING: log.warn(ALPHA_WARNING_MESSAGE) # sleep(3) if live: if simulate_orders: mode = 'paper-trading' else: mode = 'live-trading' else: mode = 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') if isinstance(auth_aliases, string_types): aliases = auth_aliases.split(',') if len(aliases) < 2 or len(aliases) % 2 != 0: raise ValueError( 'the `auth_aliases` parameter must contain an even list ' 'of comma-delimited values. For example, ' '"binance,auth2" or "binance,auth2,bittrex,auth2".' ) auth_aliases = dict(zip(aliases[::2], aliases[1::2])) exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for name in exchange_list: if auth_aliases is not None and name in auth_aliases: auth_alias = auth_aliases[name] else: auth_alias = None exchanges[name] = get_exchange( exchange_name=name, quote_currency=quote_currency, must_authenticate=(live and not simulate_orders), skip_init=True, auth_alias=auth_alias, ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial( load_crypto_market_data, environ=environ, start_dt=start, end_dt=end ), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) def choose_loader(column): bound_cols = TradingPairPricing.columns if column in bound_cols: return ExchangePricingLoader(data_frequency) raise ValueError( "No PipelineLoader registered for column %s." % column ) if live: # TODO: fix the start data. # is_start checks if a start date was specified by user # needed for live clock is_start = True if start is None: start = pd.Timestamp.utcnow() is_start = False elif start: assert pd.Timestamp.utcnow() <= start, \ "specified start date is in the past." elif start and end: assert start < end, "start date is later than end date." # TODO: fix the end data. # is_end checks if an end date was specified by user # needed for live clock is_end = True if end is None: end = start + timedelta(hours=8760) is_end = False data = DataPortalExchangeLive( exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True) ) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute' ) # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial( ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph, simulate_orders=simulate_orders, stats_output=stats_output, analyze_live=analyze_live, start=start, is_start=is_start, end=end, is_end=is_end, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. if (start and start != pd.tslib.normalize_date(start)) or \ (end and end != pd.tslib.normalize_date(end)): # todo: add to Sim_Params the option to # start & end at specific times log.warn( "Catalyst currently starts and ends on the start and " "end of the dates specified, respectively. We hope to " "Modify this and support specific times in a future release." ) data = DataPortalExchangeBacktest( exchange_names=[ex_name for ex_name in exchanges], asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end ) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial( ExchangeTradingAlgorithmBacktest, exchanges=exchanges ) elif bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, } ).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: # Looking for the portfolio from the cache first portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) # This corresponds to the json file containing api token info exchange_auth = get_exchange_auth(exchange_name) if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''): raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder( exchange_name, environ), 'auth.json')) if exchange_name == 'bitfinex': exchanges[exchange_name] = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchanges[exchange_name] = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'poloniex': exchanges[exchange_name] = Poloniex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise ExchangeNotFoundError(exchange_name=exchange_name) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = AssetFinderExchange() choose_loader = None # TODO: use the DataPortal for in the algorithm class for this if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) capital_base = 0 for exchange_name in exchanges: exchange = exchanges[exchange_name] capital_base += fetch_capital_base(exchange) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial(ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph) else: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest(exchanges=exchanges, asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def __enter__(self): return TradingEnvironment( load=self._load, asset_db_path=super(tmp_trading_env, self).__enter__().engine, )
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) if bundle is not None: bundles = bundle.split(',') def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data def get_loader_for_bundle(b): bundle_data = load( b, environ, bundle_timestamp, ) if b == 'poloniex': return CryptoPricingLoader( bundle_data, data_frequency, CryptoPricing, ) elif b == 'quandl': return USEquityPricingLoader( bundle_data, data_frequency, USEquityPricing, ) raise ValueError("No PipelineLoader registered for bundle %s." % b) loaders = [get_loader_for_bundle(b) for b in bundles] env, data = get_trading_env_and_data(bundles) def choose_loader(column): for loader in loaders: if column in loader.columns: return loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None perf = TradingAlgorithm( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ), **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _build_algo_and_data(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph, analyze_live, simulate_orders, stats_output): namespace = _build_namespace(algotext, local_namespace, defines) if algotext is not None: algotext = algofile.read() if print_algo: _pretty_print_code(algotext) mode = _mode(simulate_orders, live) log.info('running algo in {mode} mode'.format(mode=mode)) exchanges = _build_exchanges_dict(exchange, live, simulate_orders, base_currency) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None) # We don't need an asset db, we have exchanges env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) choose_loader = partial(_choose_loader, data_frequency) if live: start, end = _get_live_time_range() data_frequency = 'minute' # TODO double check if this is the desired behavior sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate=data_frequency, data_frequency=data_frequency) if algotext is None: algorithm_class_kwargs = { 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze } else: algorithm_class_kwargs = { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext } if live: return _build_live_algo_and_data( sim_params, exchanges, env, open_calendar, simulate_orders, algo_namespace, capital_base, live_graph, stats_output, analyze_live, base_currency, namespace, choose_loader, algorithm_class_kwargs) else: return _build_backtest_algo_and_data(exchanges, bundle, env, environ, bundle_timestamp, open_calendar, start, end, namespace, choose_loader, sim_params, algorithm_class_kwargs)
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph, analyze_live, simulate_orders, stats_output): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'paper-trading' if simulate_orders else 'live-trading' \ if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: exchanges[exchange_name] = get_exchange( exchange_name=exchange_name, base_currency=base_currency, must_authenticate=(live and not simulate_orders), skip_init=True, ) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) def choose_loader(column): bound_cols = TradingPairPricing.columns if column in bound_cols: return ExchangePricingLoader(data_frequency) raise ValueError("No PipelineLoader registered for column %s." % column) if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: base_currency_available = balances[base_currency]['free'] log.info( 'base currency available in the account: {} {}'.format( base_currency_available, base_currency)) return base_currency_available else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) if not simulate_orders: for exchange_name in exchanges: exchange = exchanges[exchange_name] balance = fetch_capital_base(exchange) if balance < capital_base: raise NotEnoughCapitalError( exchange=exchange_name, base_currency=base_currency, balance=balance, capital_base=capital_base, ) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial( ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph, simulate_orders=simulate_orders, stats_output=stats_output, analyze_live=analyze_live, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest( exchange_names=[exchange_name for exchange_name in exchanges], asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) elif bundle is not None: bundle_data = load( bundle, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def generate_minute_test_data(first_day, last_day, starting_open, starting_volume, multipliers_list, path): """ Utility method to generate fake minute-level CSV data. :param first_day: first trading day :param last_day: last trading day :param starting_open: first open value, raw value. :param starting_volume: first volume value, raw value. :param multipliers_list: ordered list of pd.Timestamp -> float, one per day in the range :param path: path to save the CSV :return: None """ full_minutes = BcolzMinuteBarWriter.full_minutes_for_days( first_day, last_day) minutes_count = len(full_minutes) minutes = TradingEnvironment.instance().minutes_for_days_in_range( first_day, last_day) o = np.zeros(minutes_count, dtype=np.uint32) h = np.zeros(minutes_count, dtype=np.uint32) l = np.zeros(minutes_count, dtype=np.uint32) c = np.zeros(minutes_count, dtype=np.uint32) v = np.zeros(minutes_count, dtype=np.uint32) last_open = starting_open * 1000 last_volume = starting_volume for minute in minutes: # ugly, but works idx = full_minutes.searchsorted(minute) new_open = last_open + round((random.random() * 5), 2) o[idx] = new_open h[idx] = new_open + round((random.random() * 10000), 2) l[idx] = new_open - round((random.random() * 10000), 2) c[idx] = (h[idx] + l[idx]) / 2 v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4)) last_open = o[idx] last_volume = v[idx] # now deal with multipliers if len(multipliers_list) > 0: for idx, multiplier_info in enumerate(multipliers_list): start_idx = idx * 390 end_idx = start_idx + 390 # dividing by the multipler because we're going backwards # and generating the original data that will then be adjusted. o[start_idx:end_idx] /= multiplier_info[1] h[start_idx:end_idx] /= multiplier_info[1] l[start_idx:end_idx] /= multiplier_info[1] c[start_idx:end_idx] /= multiplier_info[1] v[start_idx:end_idx] *= multiplier_info[1] df = pd.DataFrame({ "open": o, "high": h, "low": l, "close": c, "volume": v }, columns=[ "open", "high", "low", "close", "volume" ], index=minutes) df.to_csv(path, index_label="minute")
def generate_daily_test_data(first_day, last_day, starting_open, starting_volume, multipliers_list, path): days = TradingEnvironment.instance().days_in_range(first_day, last_day) days_count = len(days) o = np.zeros(days_count, dtype=np.uint32) h = np.zeros(days_count, dtype=np.uint32) l = np.zeros(days_count, dtype=np.uint32) c = np.zeros(days_count, dtype=np.uint32) v = np.zeros(days_count, dtype=np.uint32) last_open = starting_open * 1000 last_volume = starting_volume for idx in range(days_count): new_open = last_open + round((random.random() * 5), 2) o[idx] = new_open h[idx] = new_open + round((random.random() * 10000), 2) l[idx] = new_open - round((random.random() * 10000), 2) c[idx] = (h[idx] + l[idx]) / 2 v[idx] = int(last_volume + (random.randrange(-10, 10) * 1e4)) last_open = o[idx] last_volume = v[idx] # now deal with multipliers if len(multipliers_list) > 0: range_start = 0 for multiplier_info in multipliers_list: range_end = days.searchsorted(multiplier_info[0]) # dividing by the multiplier because we're going backwards # and generating the original data that will then be adjusted. o[range_start:range_end] /= multiplier_info[1] h[range_start:range_end] /= multiplier_info[1] l[range_start:range_end] /= multiplier_info[1] c[range_start:range_end] /= multiplier_info[1] v[range_start:range_end] *= multiplier_info[1] range_start = range_end df = pd.DataFrame({ "open": o, "high": h, "low": l, "close": c, "volume": v }, columns=[ "open", "high", "low", "close", "volume" ], index=days) df.to_csv(path, index_label="day")