def get_reader(self, data_frequency, path=None): """ Get a data writer object, either a new object or from cache :return: BcolzMinuteBarReader or BcolzDailyBarReader """ if path is None: root = get_exchange_folder(self.exchange.name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) if path in self._readers and self._readers[path] is not None: return self._readers[path] try: self._readers[path] = BcolzExchangeBarReader( rootdir=path, data_frequency=data_frequency ) except IOError: self._readers[path] = None return self._readers[path]
def get_writer(self, start_dt, end_dt, data_frequency): """ Get a data writer object, either a new object or from cache :return: BcolzMinuteBarWriter or BcolzDailyBarWriter """ root = get_exchange_folder(self.exchange.name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) if path in self._writers: return self._writers[path] ensure_directory(path) if len(os.listdir(path)) > 0: metadata = BcolzMinuteBarMetadata.read(path) write_metadata = False if start_dt < metadata.start_session: write_metadata = True start_session = start_dt else: start_session = metadata.start_session if end_dt > metadata.end_session: write_metadata = True end_session = end_dt else: end_session = metadata.end_session self._writers[path] = \ BcolzExchangeBarWriter( rootdir=path, start_session=start_session, end_session=end_session, write_metadata=write_metadata, data_frequency=data_frequency ) else: self._writers[path] = BcolzExchangeBarWriter( rootdir=path, start_session=start_dt, end_session=end_dt, write_metadata=True, data_frequency=data_frequency ) return self._writers[path]
def get_exchange(exchange_name, base_currency=None, must_authenticate=False): exchange_auth = get_exchange_auth(exchange_name) has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') if must_authenticate and not has_auth: raise ExchangeAuthEmpty( exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder(exchange_name), 'auth.json' ) ) return CCXT( exchange_name=exchange_name, key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, )
def clean(self, data_frequency): """ Removing the bundle data from the catalyst folder. Parameters ---------- data_frequency: str """ log.debug('cleaning exchange {}, frequency {}'.format( self.exchange_name, data_frequency)) root = get_exchange_folder(self.exchange_name) symbols = os.path.join(root, 'symbols.json') if os.path.isfile(symbols): os.remove(symbols) local_symbols = os.path.join(root, 'symbols_local.json') if os.path.isfile(local_symbols): os.remove(local_symbols) temp_bundles = os.path.join(root, 'temp_bundles') if os.path.isdir(temp_bundles): log.debug('removing folder and content: {}'.format(temp_bundles)) shutil.rmtree(temp_bundles) log.debug('{} removed'.format(temp_bundles)) frequencies = ['daily', 'minute'] if data_frequency is None \ else [data_frequency] for frequency in frequencies: label = '{}_bundle'.format(frequency) frequency_bundle = os.path.join(root, label) if os.path.isdir(frequency_bundle): log.debug( 'removing folder and content: {}'.format(frequency_bundle)) shutil.rmtree(frequency_bundle) log.debug('{} removed'.format(frequency_bundle))
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: # Looking for the portfolio from the cache first portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) # This corresponds to the json file containing api token info exchange_auth = get_exchange_auth(exchange_name) if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''): raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder( exchange_name, environ), 'auth.json')) if exchange_name == 'bitfinex': exchanges[exchange_name] = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchanges[exchange_name] = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'poloniex': exchanges[exchange_name] = Poloniex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise ExchangeNotFoundError(exchange_name=exchange_name) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = AssetFinderExchange() choose_loader = None # TODO: use the DataPortal for in the algorithm class for this if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) capital_base = 0 for exchange_name in exchanges: exchange = exchanges[exchange_name] capital_base += fetch_capital_base(exchange) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial(ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph) else: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest(exchanges=exchanges, asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def test_daily_data_to_minute_table(self): exchange_name = 'poloniex' # Switch between daily and minute for testing data_frequency = 'daily' # data_frequency = 'minute' exchange = get_exchange(exchange_name) assets = [ exchange.get_asset('eth_btc'), exchange.get_asset('etc_btc'), ] start = pd.to_datetime('2017-9-1', utc=True) end = pd.to_datetime('2017-9-30', utc=True) # Preparing the bundle folder root = get_exchange_folder(exchange.name) path = BUNDLE_NAME_TEMPLATE.format( root=root, frequency=data_frequency ) ensure_directory(path) exchange_bundle = ExchangeBundle(exchange) calendar = get_calendar('OPEN') # We are using a BcolzMinuteBarWriter even though the data is daily # Each day has a maximum of one bar # I tried setting the minutes_per_day to 1 will not create # unnecessary bars writer = BcolzExchangeBarWriter( rootdir=path, data_frequency=data_frequency, start_session=start, end_session=end, write_metadata=True ) # This will read the daily data in a bundle created by # the daily writer. It will write to the minute writer which # we are passing. # Ingesting a second asset to ensure that multiple chunks # don't override each other for asset in assets: exchange_bundle.ingest_ctable( asset=asset, data_frequency=data_frequency, period='2017', start_dt=start, end_dt=end, writer=writer, empty_rows_behavior='strip' ) reader = BcolzExchangeBarReader(rootdir=path, data_frequency=data_frequency) # Reading the two assets to ensure that no data was lost for asset in assets: sid = asset.sid daily_values = reader.load_raw_arrays( fields=['open', 'high', 'low', 'close', 'volume'], start_dt=start, end_dt=end, sids=[sid], ) print('found {} rows for last ingestion'.format( len(daily_values[0])) ) pass