def portfolio(self): """ Return the Portfolio :return: """ if self._portfolio is None: self._portfolio = ExchangePortfolio( start_date=pd.Timestamp.utcnow()) self.synchronize_portfolio() return self._portfolio
def portfolio(self): """ The exchange portfolio Returns ------- ExchangePortfolio """ if self._portfolio is None: self._portfolio = ExchangePortfolio( start_date=pd.Timestamp.utcnow()) self.synchronize_portfolio() return self._portfolio
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) if live and exchange is not None: exchange_name = exchange start = pd.Timestamp.utcnow() end = start + timedelta(minutes=1439) portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) exchange_auth = get_exchange_auth(exchange_name) if exchange_name == 'bitfinex': exchange = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchange = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise NotImplementedError('exchange not supported: %s' % exchange_name) open_calendar = get_calendar('OPEN') sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) if live and exchange is not None: env = TradingEnvironment(environ=environ, exchange_tz='UTC', asset_db_path=None) env.asset_finder = AssetFinderExchange(exchange) data = DataPortalExchange(exchange=exchange, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True)) choose_loader = None def fetch_capital_base(attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: sleep(5) return fetch_capital_base(attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) sim_params = create_simulation_parameters( start=start, end=end, capital_base=fetch_capital_base(), emission_rate='minute', data_frequency='minute') elif bundle is not None: bundles = bundle.split(',') def get_trading_env_and_data(bundles): env = data = None b = 'poloniex' if len(bundles) == 0: return env, data elif len(bundles) == 1: b = bundles[0] bundle_data = load( b, environ, bundle_timestamp, ) prefix, connstr = re.split( r'sqlite:///', str(bundle_data.asset_finder.engine.url), maxsplit=1, ) if prefix: raise ValueError( "invalid url %r, must begin with 'sqlite:///'" % str(bundle_data.asset_finder.engine.url), ) env = TradingEnvironment( load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, environ=environ, ) first_trading_day = bundle_data.minute_bar_reader.first_trading_day data = DataPortal( env.asset_finder, open_calendar, first_trading_day=first_trading_day, minute_reader=bundle_data.minute_bar_reader, five_minute_reader=bundle_data.five_minute_bar_reader, daily_reader=bundle_data.daily_bar_reader, adjustment_reader=bundle_data.adjustment_reader, ) return env, data def get_loader_for_bundle(b): bundle_data = load( b, environ, bundle_timestamp, ) if b == 'poloniex': return CryptoPricingLoader( bundle_data, data_frequency, CryptoPricing, ) elif b == 'quandl': return USEquityPricingLoader( bundle_data, data_frequency, USEquityPricing, ) raise ValueError("No PipelineLoader registered for bundle %s." % b) loaders = [get_loader_for_bundle(b) for b in bundles] env, data = get_trading_env_and_data(bundles) def choose_loader(column): for loader in loaders: if column in loader.columns: return loader raise ValueError("No PipelineLoader registered for column %s." % column) else: env = TradingEnvironment(environ=environ) choose_loader = None TradingAlgorithmClass = (partial(ExchangeTradingAlgorithm, exchange=exchange, algo_namespace=algo_namespace, live_graph=live_graph) if live and exchange else TradingAlgorithm) perf = TradingAlgorithmClass( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
def _run(handle_data, initialize, before_trading_start, analyze, algofile, algotext, defines, data_frequency, capital_base, data, bundle, bundle_timestamp, start, end, output, print_algo, local_namespace, environ, live, exchange, algo_namespace, base_currency, live_graph): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa namespace = ip.user_ns else: namespace = {} for assign in defines: try: name, value = assign.split('=', 2) except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % assign, ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( 'failed to execute definition for name %r: %s' % (name, e), ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', "cannot pass '-D' / '--define' without '-t' / '--algotext'", ) else: namespace = {} if algofile is not None: algotext = algofile.read() if print_algo: if PYGMENTS: highlight( algotext, PythonLexer(), TerminalFormatter(), outfile=sys.stdout, ) else: click.echo(algotext) mode = 'live' if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] exchanges = dict() for exchange_name in exchange_list: # Looking for the portfolio from the cache first portfolio = get_algo_object(algo_name=algo_namespace, key='portfolio_{}'.format(exchange_name), environ=environ) if portfolio is None: portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow()) # This corresponds to the json file containing api token info exchange_auth = get_exchange_auth(exchange_name) if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''): raise ExchangeAuthEmpty(exchange=exchange_name.title(), filename=os.path.join( get_exchange_folder( exchange_name, environ), 'auth.json')) if exchange_name == 'bitfinex': exchanges[exchange_name] = Bitfinex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'bittrex': exchanges[exchange_name] = Bittrex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) elif exchange_name == 'poloniex': exchanges[exchange_name] = Poloniex(key=exchange_auth['key'], secret=exchange_auth['secret'], base_currency=base_currency, portfolio=portfolio) else: raise ExchangeNotFoundError(exchange_name=exchange_name) open_calendar = get_calendar('OPEN') env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ, start_dt=start, end_dt=end), environ=environ, exchange_tz='UTC', asset_db_path=None # We don't need an asset db, we have exchanges ) env.asset_finder = AssetFinderExchange() choose_loader = None # TODO: use the DataPortal for in the algorithm class for this if live: start = pd.Timestamp.utcnow() # TODO: fix the end data. end = start + timedelta(hours=8760) data = DataPortalExchangeLive(exchanges=exchanges, asset_finder=env.asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime( 'today', utc=True)) def fetch_capital_base(exchange, attempt_index=0): """ Fetch the base currency amount required to bootstrap the algorithm against the exchange. The algorithm cannot continue without this value. :param exchange: the targeted exchange :param attempt_index: :return capital_base: the amount of base currency available for trading """ try: log.debug('retrieving capital base in {} to bootstrap ' 'exchange {}'.format(base_currency, exchange_name)) balances = exchange.get_balances() except ExchangeRequestError as e: if attempt_index < 20: log.warn('could not retrieve balances on {}: {}'.format( exchange.name, e)) sleep(5) return fetch_capital_base(exchange, attempt_index + 1) else: raise ExchangeRequestErrorTooManyAttempts( attempts=attempt_index, error=e) if base_currency in balances: return balances[base_currency] else: raise BaseCurrencyNotFoundError(base_currency=base_currency, exchange=exchange_name) capital_base = 0 for exchange_name in exchanges: exchange = exchanges[exchange_name] capital_base += fetch_capital_base(exchange) sim_params = create_simulation_parameters(start=start, end=end, capital_base=capital_base, emission_rate='minute', data_frequency='minute') # TODO: use the constructor instead sim_params._arena = 'live' algorithm_class = partial(ExchangeTradingAlgorithmLive, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph) else: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. # I don't think that we should have arbitrary price data bundles # Instead, we should center this data around exchanges. # We still need to support bundles for other misc data, but we # can handle this later. data = DataPortalExchangeBacktest(exchanges=exchanges, asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, last_available_session=end) sim_params = create_simulation_parameters( start=start, end=end, capital_base=capital_base, data_frequency=data_frequency, emission_rate=data_frequency, ) algorithm_class = partial(ExchangeTradingAlgorithmBacktest, exchanges=exchanges) perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, 'before_trading_start': before_trading_start, 'analyze': analyze, } if algotext is None else { 'algo_filename': getattr(algofile, 'name', '<algorithm>'), 'script': algotext, }).run( data, overwrite_sim_params=False, ) if output == '-': click.echo(str(perf)) elif output != os.devnull: # make the catalyst magic not write any data perf.to_pickle(output) return perf
class Exchange: __metaclass__ = ABCMeta def __init__(self): self.name = None self.assets = {} self._portfolio = None self.minute_writer = None self.minute_reader = None self.base_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self) @property def positions(self): return self.portfolio.positions @property def portfolio(self): """ Return the Portfolio :return: """ if self._portfolio is None: self._portfolio = ExchangePortfolio( start_date=pd.Timestamp.utcnow()) self.synchronize_portfolio() return self._portfolio @abstractproperty def account(self): pass @abstractproperty def time_skew(self): pass def ask_request(self): """ Asks permission to issue a request to the exchange. The primary purpose is to avoid hitting rate limits. The application will pause if the maximum requests per minute permitted by the exchange is exceeded. :return boolean: """ now = pd.Timestamp.utcnow() if not self.request_cpt: self.request_cpt = dict() self.request_cpt[now] = 0 return True cpt_date = self.request_cpt.keys()[0] cpt = self.request_cpt[cpt_date] if now > cpt_date + timedelta(minutes=1): self.request_cpt = dict() self.request_cpt[now] = 0 return True if cpt >= self.max_requests_per_minute: delta = now - cpt_date sleep_period = 60 - delta.total_seconds() sleep(sleep_period) now = pd.Timestamp.utcnow() self.request_cpt = dict() self.request_cpt[now] = 0 return True else: self.request_cpt[cpt_date] += 1 def get_symbol(self, asset): """ Get the exchange specific symbol of the given asset. :param asset: Asset :return: symbol: str """ symbol = None for key in self.assets: if not symbol and self.assets[key].symbol == asset.symbol: symbol = key if not symbol: raise ValueError('Currency %s not supported by exchange %s' % (asset['symbol'], self.name.title())) return symbol def get_symbols(self, assets): """ Get a list of symbols corresponding to each given asset. :param assets: Asset[] :return: """ symbols = [] for asset in assets: symbols.append(self.get_symbol(asset)) return symbols def get_assets(self, symbols=None): assets = [] if symbols is not None: for symbol in symbols: asset = self.get_asset(symbol) assets.append(asset) else: for key in self.assets: assets.append(self.assets[key]) return assets def get_asset(self, symbol): """ Find an Asset on the current exchange based on its Catalyst symbol :param symbol: the [target]_[base] currency pair symbol :return: Asset """ asset = None for key in self.assets: if not asset and self.assets[key].symbol.lower() == symbol.lower(): asset = self.assets[key] if not asset: supported_symbols = [ pair.symbol.encode('utf-8') for pair in self.assets.values() ] raise SymbolNotFoundOnExchange(symbol=symbol, exchange=self.name.title(), supported_symbols=supported_symbols) return asset def fetch_symbol_map(self): return get_exchange_symbols(self.name) def load_assets(self): """ Populate the 'assets' attribute with a dictionary of Assets. The key of the resulting dictionary is the exchange specific currency pair symbol. The universal symbol is contained in the 'symbol' attribute of each asset. Notes ----- The sid of each asset is calculated based on a numeric hash of the universal symbol. This simple approach avoids maintaining a mapping of sids. This method can be overridden if an exchange offers equivalent data via its api. """ symbol_map = self.fetch_symbol_map() for exchange_symbol in symbol_map: asset = symbol_map[exchange_symbol] if 'start_date' in asset: start_date = pd.to_datetime(asset['start_date'], utc=True) else: start_date = None if 'end_date' in asset: end_date = pd.to_datetime(asset['end_date'], utc=True) else: end_date = None if 'leverage' in asset: leverage = asset['leverage'] else: leverage = 1.0 if 'asset_name' in asset: asset_name = asset['asset_name'] else: asset_name = None if 'min_trade_size' in asset: min_trade_size = asset['min_trade_size'] else: min_trade_size = 0.0000001 if 'end_daily' in asset and asset['end_daily'] != 'N/A': end_daily = pd.to_datetime(asset['end_daily'], utc=True) else: end_daily = None if 'end_minute' in asset and asset['end_minute'] != 'N/A': end_minute = pd.to_datetime(asset['end_minute'], utc=True) else: end_minute = None trading_pair = TradingPair(symbol=asset['symbol'], exchange=self.name, start_date=start_date, end_date=end_date, leverage=leverage, asset_name=asset_name, min_trade_size=min_trade_size, end_daily=end_daily, end_minute=end_minute, exchange_symbol=exchange_symbol) self.assets[exchange_symbol] = trading_pair def check_open_orders(self): """ Loop through the list of open orders in the Portfolio object. For each executed order found, create a transaction and apply to the Portfolio. :return: transactions: Transaction[] """ transactions = list() if self.portfolio.open_orders: for order_id in list(self.portfolio.open_orders): log.debug('found open order: {}'.format(order_id)) order, executed_price = self.get_order(order_id) log.debug('got updated order {} {}'.format( order, executed_price)) if order.status == ORDER_STATUS.FILLED: transaction = Transaction(asset=order.asset, amount=order.amount, dt=pd.Timestamp.utcnow(), price=executed_price, order_id=order.id, commission=order.commission) transactions.append(transaction) self.portfolio.execute_order(order, transaction) elif order.status == ORDER_STATUS.CANCELLED: self.portfolio.remove_order(order) else: delta = pd.Timestamp.utcnow() - order.dt log.info( 'order {order_id} still open after {delta}'.format( order_id=order_id, delta=delta)) return transactions def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ Public API method that returns a scalar value representing the value of the desired asset's field at either the given dt. Parameters ---------- assets : Asset, ContinuousFuture, or iterable of same. The asset or assets whose data is desired. field : {'open', 'high', 'low', 'close', 'volume', 'price', 'last_traded'} The desired field of the asset. dt : pd.Timestamp The timestamp for the desired value. data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- value : float, int, or pd.Timestamp The spot value of ``field`` for ``asset`` The return type is based on the ``field`` requested. If the field is one of 'open', 'high', 'low', 'close', or 'price', the value will be a float. If the ``field`` is 'volume' the value will be a int. If the ``field`` is 'last_traded' the value will be a Timestamp. Bitfinex timeframes ------------------- Available values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' """ if field not in BASE_FIELDS: raise KeyError('Invalid column: {}'.format(field)) values = [] for asset in assets: value = self.get_single_spot_value(asset, field, data_frequency) values.append(value) return values def get_single_spot_value(self, asset, field, data_frequency): """ Similar to 'get_spot_value' but for a single asset Note ---- We're writing each minute bar to disk using zipline's machinery. This is especially useful when running multiple algorithms concurrently. By using local data when possible, we try to reaching request limits on exchanges. :param asset: :param field: :param data_frequency: :return value: The spot value of the given asset / field """ log.debug('fetching spot value {field} for symbol {symbol}'.format( symbol=asset.symbol, field=field)) ohlc = self.get_candles(data_frequency, asset) if field not in ohlc: raise KeyError('Invalid column: %s' % field) value = ohlc[field] log.debug('got spot value: {}'.format(value)) return value def get_series_from_candles(self, candles, start_dt, end_dt, field, previous_value=None): """ Get a series of field data for the specified candles. :param candles: :param start_dt: :param end_dt: :param field: :param previous_value: :return: """ dates = [candle['last_traded'] for candle in candles] values = [candle[field] for candle in candles] periods = pd.date_range(start_dt, end_dt) series = pd.Series(values, index=dates) series.reindex(periods, method='ffill', fill_value=previous_value) return series def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, ffill=True): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list of catalyst.data.Asset objects The assets whose data is desired. end_dt: not applicable to cryptocurrencies bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- A dataframe containing the requested data. """ freq_match = re.match(r'([0-9].*)(m|M|d|D)', frequency, re.M | re.I) if freq_match: candle_size = int(freq_match.group(1)) unit = freq_match.group(2) else: raise InvalidHistoryFrequencyError(frequency) if unit.lower() == 'd': if data_frequency == 'minute': data_frequency = 'daily' elif unit.lower() == 'm': if data_frequency == 'daily': data_frequency = 'minute' else: raise InvalidHistoryFrequencyError(frequency) adj_bar_count = candle_size * bar_count try: series = self.bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=data_frequency) except PricingDataNotLoadedError: series = dict() for asset in assets: if asset not in series or series[asset].index[-1] < end_dt: # Adding bars too recent to be contained in the consolidated # exchanges bundles. We go directly against the exchange # to retrieve the candles. start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ if asset in series else start_dt trailing_bar_count = \ get_periods(trailing_dt, end_dt, data_frequency) # The get_history method supports multiple asset candles = self.get_candles(data_frequency=data_frequency, assets=asset, bar_count=trailing_bar_count, end_dt=end_dt) last_value = series[asset].iloc(0) if asset in series \ else np.nan candle_series = self.get_series_from_candles( candles=candles, start_dt=trailing_dt, end_dt=end_dt, field=field, previous_value=last_value) if asset in series: series[asset].append(candle_series) else: series[asset] = candle_series df = pd.DataFrame(series) if candle_size > 1: if field == 'open': agg = 'first' elif field == 'high': agg = 'max' elif field == 'low': agg = 'min' elif field == 'close': agg = 'last' elif field == 'volume': agg = 'sum' else: raise ValueError('Invalid field.') df = df.resample('{}T'.format(candle_size)).agg(agg) return df def synchronize_portfolio(self): """ Update the portfolio cash and position balances based on the latest ticker prices. :return: """ log.debug('synchronizing portfolio with exchange {}'.format(self.name)) balances = self.get_balances() base_position_available = balances[self.base_currency] \ if self.base_currency in balances else None if base_position_available is None: raise BaseCurrencyNotFoundError(base_currency=self.base_currency, exchange=self.name.title()) portfolio = self._portfolio portfolio.cash = base_position_available log.debug('found base currency balance: {}'.format(portfolio.cash)) if portfolio.starting_cash is None: portfolio.starting_cash = portfolio.cash if portfolio.positions: assets = portfolio.positions.keys() tickers = self.tickers(assets) portfolio.positions_value = 0.0 for asset in tickers: # TODO: convert if the position is not in the base currency ticker = tickers[asset] position = portfolio.positions[asset] position.last_sale_price = ticker['last_price'] position.last_sale_date = ticker['timestamp'] portfolio.positions_value += \ position.amount * position.last_sale_price portfolio.portfolio_value = \ portfolio.positions_value + portfolio.cash def order(self, asset, amount, limit_price=None, stop_price=None, style=None): """Place an order. Parameters ---------- asset : Asset The asset that this order is for. amount : int The amount of shares to order. If ``amount`` is positive, this is the number of shares to buy or cover. If ``amount`` is negative, this is the number of shares to sell or short. limit_price : float, optional The limit price for the order. stop_price : float, optional The stop price for the order. style : ExecutionStyle, optional The execution style for the order. Returns ------- order_id : str or None The unique identifier for this order, or None if no order was placed. Notes ----- The ``limit_price`` and ``stop_price`` arguments provide shorthands for passing common execution styles. Passing ``limit_price=N`` is equivalent to ``style=LimitOrder(N)``. Similarly, passing ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing ``limit_price=N`` and ``stop_price=M`` is equivalent to ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style`` and ``limit_price`` or ``stop_price``. See Also -------- :class:`catalyst.finance.execution.ExecutionStyle` :func:`catalyst.api.order_value` :func:`catalyst.api.order_percent` """ if amount == 0: log.warn('skipping order amount of 0') return None if asset.base_currency != self.base_currency.lower(): raise MismatchingBaseCurrencies(base_currency=asset.base_currency, algo_currency=self.base_currency) is_buy = (amount > 0) if limit_price is not None and stop_price is not None: style = ExchangeStopLimitOrder(limit_price, stop_price, exchange=self.name) elif limit_price is not None: style = ExchangeLimitOrder(limit_price, exchange=self.name) elif stop_price is not None: style = ExchangeStopOrder(stop_price, exchange=self.name) elif style is not None: raise InvalidOrderStyle(exchange=self.name.title(), style=style.__class__.__name__) else: raise ValueError('Incomplete order data.') display_price = limit_price if limit_price is not None else stop_price log.debug( 'issuing {side} order of {amount} {symbol} for {type}: {price}'. format(side='buy' if is_buy else 'sell', amount=amount, symbol=asset.symbol, type=style.__class__.__name__, price='{}{}'.format(display_price, asset.base_currency))) order = self.create_order(asset, amount, is_buy, style) if order: self._portfolio.create_order(order) return order.id else: return None # The methods below must be implemented for each exchange. @abstractmethod def get_balances(self): """ Retrieve wallet balances for the exchange :return balances: A dict of currency => available balance """ pass @abstractmethod def create_order(self, asset, amount, is_buy, style): """ Place an order on the exchange. :param asset : Asset The asset that this order is for. :param amount : int The amount of shares to order. If ``amount`` is positive, this is the number of shares to buy or cover. If ``amount`` is negative, this is the number of shares to sell or short. :param style : ExecutionStyle The execution style for the order. :param is_buy: boolean Is it a buy order? :return: """ pass @abstractmethod def get_open_orders(self, asset): """Retrieve all of the current open orders. Parameters ---------- asset : Asset If passed and not None, return only the open orders for the given asset instead of all open orders. Returns ------- open_orders : dict[list[Order]] or list[Order] If no asset is passed this will return a dict mapping Assets to a list containing all the open orders for the asset. If an asset is passed then this will return a list of the open orders for this asset. """ pass @abstractmethod def get_order(self, order_id): """Lookup an order based on the order id returned from one of the order functions. Parameters ---------- order_id : str The unique identifier for the order. Returns ------- order : Order The order object. execution_price: float The execution price per share of the order """ pass @abstractmethod def cancel_order(self, order_param): """Cancel an open order. Parameters ---------- order_param : str or Order The order_id or order object to cancel. """ pass @abstractmethod def get_candles(self, data_frequency, assets, bar_count=None, start_dt=None, end_dt=None): """ Retrieve OHLCV candles for the given assets :param data_frequency: The candle frequency: minute or daily :param assets: list[TradingPair] The targeted assets. :param bar_count: The number of bar desired. (default 1) :param end_dt: datetime, optional The last bar date. :param start_dt: datetime, optional The first bar date. :return dict[TradingPair, dict[str, Object]]: OHLCV data A dictionary of OHLCV candles. Each TradingPair instance is mapped to a list of dictionaries with this structure: open: float high: float low: float close: float volume: float last_traded: datetime See definition here: http://www.investopedia.com/terms/o/ohlcchart.asp """ pass @abc.abstractmethod def tickers(self, assets): """ Retrieve current tick data for the given assets :param assets: :return: """ pass @abc.abstractmethod def get_account(self): """ Retrieve the account parameters. :return: """ pass @abc.abstractmethod def get_orderbook(self, asset, order_type): """ Retrieve the the orderbook for the given trading pair. :param asset: TradingPair :param order_type: str The type of orders: bid, ask or all :return: """ pass
class Exchange: __metaclass__ = ABCMeta def __init__(self): self.name = None self.trading_pairs = None self.assets = {} self._portfolio = None self.minute_writer = None self.minute_reader = None self.base_currency = None @property def positions(self): return self.portfolio.positions @property def portfolio(self): """ Return the Portfolio :return: """ if self._portfolio is None: self._portfolio = ExchangePortfolio( start_date=pd.Timestamp.utcnow()) self.synchronize_portfolio() return self._portfolio @abstractproperty def account(self): pass @abstractproperty def time_skew(self): pass def get_symbol(self, asset): """ Get the exchange specific symbol of the given asset. :param asset: Asset :return: symbol: str """ symbol = None for key in self.assets: if not symbol and self.assets[key].symbol == asset.symbol: symbol = key if not symbol: raise ValueError('Currency %s not supported by exchange %s' % (asset['symbol'], self.name)) return symbol def get_symbols(self, assets): """ Get a list of symbols corresponding to each given asset. :param assets: Asset[] :return: """ symbols = [] for asset in assets: symbols.append(self.get_symbol(asset)) return symbols def get_asset(self, symbol): """ Find an Asset on the current exchange based on its Catalyst symbol :param symbol: the [target]_[base] currency pair symbol :return: Asset """ asset = None for key in self.assets: if not asset and self.assets[key].symbol.lower() == symbol.lower(): asset = self.assets[key] if not asset: raise SymbolNotFound(symbol=symbol) return asset def fetch_symbol_map(self): return get_exchange_symbols(self.name) def load_assets(self): """ Populate the 'assets' attribute with a dictionary of Assets. The key of the resulting dictionary is the exchange specific currency pair symbol. The universal symbol is contained in the 'symbol' attribute of each asset. Notes ----- The sid of each asset is calculated based on a numeric hash of the universal symbol. This simple approach avoids maintaining a mapping of sids. This method can be overridden if an exchange offers equivalent data via its api. """ symbol_map = self.fetch_symbol_map() for exchange_symbol in symbol_map: asset = symbol_map[exchange_symbol] if 'start_date' in asset: start_date = pd.to_datetime(asset['start_date'], utc=True) else: start_date = None if 'end_date' in asset: end_date = pd.to_datetime(asset['end_date'], utc=True) else: end_date = None if 'leverage' in asset: leverage = asset['leverage'] else: leverage = 1.0 if 'asset_name' in asset: asset_name = asset['asset_name'] else: asset_name = None trading_pair = TradingPair(symbol=asset['symbol'], exchange=self.name, start_date=start_date, end_date=end_date, leverage=leverage, asset_name=asset_name) self.assets[exchange_symbol] = trading_pair def check_open_orders(self): """ Loop through the list of open orders in the Portfolio object. For each executed order found, create a transaction and apply to the Portfolio. :return: transactions: Transaction[] """ transactions = list() if self.portfolio.open_orders: for order_id in list(self.portfolio.open_orders): log.debug('found open order: {}'.format(order_id)) order, executed_price = self.get_order(order_id) log.debug('got updated order {} {}'.format( order, executed_price)) if order.status == ORDER_STATUS.FILLED: transaction = Transaction(asset=order.asset, amount=order.amount, dt=pd.Timestamp.utcnow(), price=executed_price, order_id=order.id, commission=order.commission) transactions.append(transaction) self.portfolio.execute_order(order, transaction) elif order.status == ORDER_STATUS.CANCELLED: self.portfolio.remove_order(order) else: delta = pd.Timestamp.utcnow() - order.dt log.info( 'order {order_id} still open after {delta}'.format( order_id=order_id, delta=delta)) return transactions def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ Public API method that returns a scalar value representing the value of the desired asset's field at either the given dt. Parameters ---------- assets : Asset, ContinuousFuture, or iterable of same. The asset or assets whose data is desired. field : {'open', 'high', 'low', 'close', 'volume', 'price', 'last_traded'} The desired field of the asset. dt : pd.Timestamp The timestamp for the desired value. data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- value : float, int, or pd.Timestamp The spot value of ``field`` for ``asset`` The return type is based on the ``field`` requested. If the field is one of 'open', 'high', 'low', 'close', or 'price', the value will be a float. If the ``field`` is 'volume' the value will be a int. If the ``field`` is 'last_traded' the value will be a Timestamp. Bitfinex timeframes ------------------- Available values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' """ if field not in BASE_FIELDS: raise KeyError('Invalid column: ' + str(field)) if isinstance(assets, collections.Iterable): values = list() for asset in assets: value = self.get_single_spot_value(asset, field, data_frequency) values.append(value) return values else: return self.get_single_spot_value(assets, field, data_frequency) def get_single_spot_value(self, asset, field, data_frequency): """ Similar to 'get_spot_value' but for a single asset Note ---- We're writing each minute bar to disk using zipline's machinery. This is especially useful when running multiple algorithms concurrently. By using local data when possible, we try to reaching request limits on exchanges. :param asset: :param field: :param data_frequency: :return value: The spot value of the given asset / field """ log.debug('fetching spot value {field} for symbol {symbol}'.format( symbol=asset.symbol, field=field)) if field == 'price': field = 'close' # Don't use a timezone here dt = pd.Timestamp.utcnow().floor('1 min') value = None if self.minute_reader is not None: try: # Slight delay to minimize the chances that multiple algos # might try to hit the cache at the exact same time. sleep_time = random.uniform(0.5, 0.8) sleep(sleep_time) # TODO: This does not always! Why is that? Open an issue with zipline. # See: https://github.com/zipline-live/zipline/issues/26 value = self.minute_reader.get_value(sid=asset.sid, dt=dt, field=field) except Exception as e: log.warn('minute data not found: {}'.format(e)) if value is None or np.isnan(value): ohlc = self.get_candles(data_frequency, asset) if field not in ohlc: raise KeyError('Invalid column: %s' % field) if self.minute_writer is not None: df = pd.DataFrame( [ohlc], index=pd.DatetimeIndex([dt]), columns=['open', 'high', 'low', 'close', 'volume']) try: self.minute_writer.write_sid(sid=asset.sid, df=df) log.debug('wrote minute data: {}'.format(dt)) except Exception as e: log.warn('unable to write minute data: {} {}'.format( dt, e)) value = ohlc[field] log.debug('got spot value: {}'.format(value)) else: log.debug('got spot value from cache: {}'.format(value)) return value def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency, ffill=True): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list of catalyst.data.Asset objects The assets whose data is desired. end_dt: not applicable to cryptocurrencies bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- A dataframe containing the requested data. """ candles = self.get_candles( data_frequency=frequency, assets=assets, bar_count=bar_count, ) series = dict() for asset in assets: asset_candles = candles[asset] values = map(lambda candle: candle[field], asset_candles) dates = map(lambda candle: candle['last_traded'], asset_candles) value_series = pd.Series(values, index=dates) series[asset] = value_series df = pd.concat(series) return df def synchronize_portfolio(self): """ Update the portfolio cash and position balances based on the latest ticker prices. :return: """ log.debug('synchronizing portfolio with exchange {}'.format(self.name)) balances = self.get_balances() base_position_available = balances[self.base_currency] \ if self.base_currency in balances else None if base_position_available is None: raise BaseCurrencyNotFoundError(base_currency=self.base_currency, exchange=self.name) portfolio = self._portfolio portfolio.cash = base_position_available log.debug('found base currency balance: {}'.format(portfolio.cash)) if portfolio.starting_cash is None: portfolio.starting_cash = portfolio.cash if portfolio.positions: assets = portfolio.positions.keys() tickers = self.tickers(assets) portfolio.positions_value = 0.0 for asset in tickers: # TODO: convert if the position is not in the base currency ticker = tickers[asset] position = portfolio.positions[asset] position.last_sale_price = ticker['last_price'] position.last_sale_date = ticker['timestamp'] portfolio.positions_value += \ position.amount * position.last_sale_price portfolio.portfolio_value = \ portfolio.positions_value + portfolio.cash @abstractmethod def get_balances(self): """ Retrieve wallet balances for the exchange :return balances: A dict of currency => available balance """ pass @abstractmethod def create_order(self, asset, amount, is_buy, style): pass def order(self, asset, amount, limit_price=None, stop_price=None, style=None): """Place an order. Parameters ---------- asset : Asset The asset that this order is for. amount : int The amount of shares to order. If ``amount`` is positive, this is the number of shares to buy or cover. If ``amount`` is negative, this is the number of shares to sell or short. limit_price : float, optional The limit price for the order. stop_price : float, optional The stop price for the order. style : ExecutionStyle, optional The execution style for the order. Returns ------- order_id : str or None The unique identifier for this order, or None if no order was placed. Notes ----- The ``limit_price`` and ``stop_price`` arguments provide shorthands for passing common execution styles. Passing ``limit_price=N`` is equivalent to ``style=LimitOrder(N)``. Similarly, passing ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing ``limit_price=N`` and ``stop_price=M`` is equivalent to ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style`` and ``limit_price`` or ``stop_price``. See Also -------- :class:`catalyst.finance.execution.ExecutionStyle` :func:`catalyst.api.order_value` :func:`catalyst.api.order_percent` """ if amount == 0: log.warn('skipping order amount of 0') return None if asset.base_currency != self.base_currency.lower(): raise MismatchingBaseCurrencies(base_currency=asset.base_currency, algo_currency=self.base_currency) is_buy = (amount > 0) if limit_price is not None and stop_price is not None: style = ExchangeStopLimitOrder(limit_price, stop_price, exchange=self.name) elif limit_price is not None: style = ExchangeLimitOrder(limit_price, exchange=self.name) elif stop_price is not None: style = ExchangeStopOrder(stop_price, exchange=self.name) elif style is not None: raise InvalidOrderStyle(exchange=self.name, style=style.__class__.__name__) else: raise ValueError('Incomplete order data.') display_price = limit_price if limit_price is not None else stop_price log.debug( 'issuing {side} order of {amount} {symbol} for {type}: {price}'. format(side='buy' if is_buy else 'sell', amount=amount, symbol=asset.symbol, type=style.__class__.__name__, price='{}{}'.format(display_price, asset.base_currency))) order = self.create_order(asset, amount, is_buy, style) if order: self._portfolio.create_order(order) return order.id else: return None @abstractmethod def get_open_orders(self, asset): """Retrieve all of the current open orders. Parameters ---------- asset : Asset If passed and not None, return only the open orders for the given asset instead of all open orders. Returns ------- open_orders : dict[list[Order]] or list[Order] If no asset is passed this will return a dict mapping Assets to a list containing all the open orders for the asset. If an asset is passed then this will return a list of the open orders for this asset. """ pass @abstractmethod def get_order(self, order_id): """Lookup an order based on the order id returned from one of the order functions. Parameters ---------- order_id : str The unique identifier for the order. Returns ------- order : Order The order object. execution_price: float The execution price per share of the order """ pass @abstractmethod def cancel_order(self, order_param): """Cancel an open order. Parameters ---------- order_param : str or Order The order_id or order object to cancel. """ pass @abstractmethod def get_candles(self, data_frequency, assets, bar_count=None): """ Retrieve OHLCV candles for the given assets :param data_frequency: :param assets: :param end_dt: :param bar_count: :param limit: :return: """ pass @abc.abstractmethod def tickers(self, assets): """ Retrieve current tick data for the given assets :param assets: :return: """ pass @abc.abstractmethod def get_account(self): """ Retrieve the account parameters. :return: """ pass