def bcolz_exchange_daily_write_read(self, exchange_name): start = pd.to_datetime('2017-10-01 00:00') end = pd.to_datetime('today') freq = 'daily' bundle = ExchangeBundle(exchange_name) df = self.generate_df(exchange_name, freq, start, end) print(df.index[0], df.index[-1]) writer = BcolzExchangeBarWriter(rootdir=self.root_dir, start_session=df.index[0], end_session=df.index[-1], data_frequency=freq, write_metadata=True) data = [] data.append((1, df)) writer.write(data) reader = BcolzExchangeBarReader(rootdir=self.root_dir, data_frequency=freq) arrays = reader.load_raw_arrays(self.columns, start, end, [ 1, ]) periods = bundle.get_calendar_periods_range(start, end, freq) dx = get_df_from_arrays(arrays, periods) assert_equals(df.equals(dx), True) pass
def bcolz_exchange_daily_write_read(self, exchange_name): start = pd.to_datetime('2017-10-01 00:00') end = pd.to_datetime('today') freq = 'daily' bundle = ExchangeBundle(exchange_name) df = self.generate_df(exchange_name, freq, start, end) print(df.index[0], df.index[-1]) writer = BcolzExchangeBarWriter( rootdir=self.root_dir, start_session=df.index[0], end_session=df.index[-1], data_frequency=freq, write_metadata=True) data = [] data.append((1, df)) writer.write(data) reader = BcolzExchangeBarReader(rootdir=self.root_dir, data_frequency=freq) arrays = reader.load_raw_arrays(self.columns, start, end, [1, ]) periods = bundle.get_calendar_periods_range( start, end, freq ) dx = get_df_from_arrays(arrays, periods) assert_equals(df.equals(dx), True) pass
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") if not csv and exchange_name not in EXCHANGE_NAMES: ctx.fail("ingest-exchange does not support {}, " "please choose exchange from: {}".format( exchange_name, EXCHANGE_NAMES)) exchange_bundle = ExchangeBundle(exchange_name) click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv)
def chunk_to_df(self, exchange_name, symbol, data_frequency, period): exchange = get_exchange(exchange_name) asset = exchange.get_asset(symbol) filename = get_bcolz_chunk( exchange_name=exchange_name, symbol=symbol, data_frequency=data_frequency, period=period ) reader = BcolzExchangeBarReader(rootdir=filename, data_frequency=data_frequency) # metadata = BcolzMinuteBarMetadata.read(filename) start = reader.first_trading_day end = reader.last_available_dt if data_frequency == 'daily': end = end - pd.Timedelta(hours=23, minutes=59) print(start, end, data_frequency) arrays = reader.load_raw_arrays(self.columns, start, end, [asset.sid, ]) bundle = ExchangeBundle(exchange_name) periods = bundle.get_calendar_periods_range( start, end, data_frequency ) return get_df_from_arrays(arrays, periods)
def chunk_to_df(self, exchange_name, symbol, data_frequency, period): exchange = get_exchange(exchange_name) asset = exchange.get_asset(symbol) filename = get_bcolz_chunk(exchange_name=exchange_name, symbol=symbol, data_frequency=data_frequency, period=period) reader = BcolzExchangeBarReader(rootdir=filename, data_frequency=data_frequency) # metadata = BcolzMinuteBarMetadata.read(filename) start = reader.first_trading_day end = reader.last_available_dt if data_frequency == 'daily': end = end - pd.Timedelta(hours=23, minutes=59) print(start, end, data_frequency) arrays = reader.load_raw_arrays(self.columns, start, end, [ asset.sid, ]) bundle = ExchangeBundle(exchange_name) periods = bundle.get_calendar_periods_range(start, end, data_frequency) return get_df_from_arrays(arrays, periods)
def ingest_exchange_bundles(exchange_name, data_freq, symbols): exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle.ingest( include_symbols=symbols, data_frequency=data_freq, )
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv )
def test_ingest_minute(self): data_frequency = 'minute' exchange_name = 'poloniex' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) assets = [exchange.get_asset('eth_btc')] start = pd.to_datetime('2016-03-01', utc=True) end = pd.to_datetime('2017-11-1', utc=True) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=','.join([asset.symbol for asset in assets]), # include_symbols=None, exclude_symbols=None, start=start, end=end, show_progress=True) reader = exchange_bundle.get_reader(data_frequency) for asset in assets: arrays = reader.load_raw_arrays(sids=[asset.sid], fields=['close'], start_dt=start, end_dt=end) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0])) pass
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv )
def test_validate_data(self): exchange_name = 'bitfinex' data_frequency = 'minute' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) assets = [exchange.get_asset('iot_btc')] end_dt = pd.to_datetime('2017-9-2 1:00', utc=True) bar_count = 60 bundle_series = exchange_bundle.get_history_window_series( assets=assets, end_dt=end_dt, bar_count=bar_count * 5, field='close', data_frequency='minute', ) candles = exchange.get_candles( assets=assets, end_dt=end_dt, bar_count=bar_count, data_frequency='minute' ) start_dt = get_start_dt(end_dt, bar_count, data_frequency) frames = [] for asset in assets: bundle_df = pd.DataFrame( data=dict(bundle_price=bundle_series[asset]), index=bundle_series[asset].index ) exchange_series = exchange.get_series_from_candles( candles=candles[asset], start_dt=start_dt, end_dt=end_dt, data_frequency=data_frequency, field='close' ) exchange_df = pd.DataFrame( data=dict(exchange_price=exchange_series), index=exchange_series.index ) df = exchange_df.join(bundle_df, how='left') df['last_traded'] = df.index df['asset'] = asset.symbol df.set_index(['asset', 'last_traded'], inplace=True) frames.append(df) df = pd.concat(frames) print('\n' + df_to_string(df)) pass
def clean_exchange(ctx, exchange_name, data_frequency): """Clean up bundles from 'ingest-exchange'. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Cleaning exchange bundle {}...'.format(exchange_name)) exchange_bundle.clean(data_frequency=data_frequency, ) click.echo('Done')
def __init__(self): self.name = None self.assets = {} self._portfolio = None self.minute_writer = None self.minute_reader = None self.base_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self)
def __init__(self): self.name = None self.assets = [] self._symbol_maps = [None, None] self.minute_writer = None self.minute_reader = None self.base_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self.name)
def test_ingest_daily(self): exchange_name = 'bitfinex' data_frequency = 'minute' include_symbols = 'neo_btc' # exchange_name = 'poloniex' # data_frequency = 'daily' # include_symbols = 'eth_btc' # start = pd.to_datetime('2017-1-1', utc=True) # end = pd.to_datetime('2017-10-16', utc=True) # periods = get_periods_range(start, end, data_frequency) start = None end = None exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=None, start=start, end=end, show_progress=True ) symbols = include_symbols.split(',') assets = [] for pair_symbol in symbols: assets.append(exchange.get_asset(pair_symbol)) reader = exchange_bundle.get_reader(data_frequency) start_dt = reader.first_trading_day end_dt = reader.last_available_dt if data_frequency == 'daily': end_dt = end_dt - pd.Timedelta(hours=23, minutes=59) for asset in assets: arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], start_dt=start_dt, end_dt=end_dt ) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0]) ) pass
def test_spot_value(self): data_frequency = 'daily' exchange_name = 'poloniex' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) assets = [exchange.get_asset('btc_usdt')] dt = pd.to_datetime('2017-10-14', utc=True) values = exchange_bundle.get_spot_values(assets=assets, field='close', dt=dt, data_frequency=data_frequency) pass
def ingest_exchange(exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, show_progress): """ Ingest data for the given exchange. """ exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress)
def __init__(self): self.name = None self.assets = [] self.active_assets = [] self._symbol_maps = [None, None] self.minute_writer = None self.minute_reader = None self.quote_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self.name) self.low_balance_threshold = None
def clean_exchange(ctx, exchange_name, data_frequency): """Clean up bundles from 'ingest-exchange'. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.clean( data_frequency=data_frequency, ) click.echo('Done', sys.stdout)
def test_ingest_minute_all(self): exchange_name = 'bitfinex' # start = pd.to_datetime('2017-09-01', utc=True) start = pd.to_datetime('2017-10-01', utc=True) end = pd.to_datetime('2017-10-05', utc=True) exchange_bundle = ExchangeBundle(get_exchange(exchange_name)) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest(data_frequency='minute', exclude_symbols=None, start=start, end=end, show_progress=True) pass
def __init__(self, key, secret, base_currency, portfolio=None): self.url = BITFINEX_URL self.key = key self.secret = secret.encode('UTF-8') self.name = 'bitfinex' self.color = 'green' self.assets = dict() self.load_assets() self.local_assets = dict() self.load_assets(is_local=True) self.base_currency = base_currency self._portfolio = portfolio self.minute_writer = None self.minute_reader = None # The candle limit for each request self.num_candles_limit = 1000 # Max is 90 but playing it safe # https://www.bitfinex.com/posts/188 self.max_requests_per_minute = 80 self.request_cpt = dict() self.bundle = ExchangeBundle(self.name)
def _bundle_to_csv(self, asset, exchange_name, data_frequency, filename, path=None, start_dt=None, end_dt=None): bundle = ExchangeBundle(exchange_name) reader = bundle.get_reader(data_frequency, path=path) if start_dt is None: start_dt = reader.first_trading_day if end_dt is None: end_dt = reader.last_available_dt if data_frequency == 'daily': end_dt = end_dt - pd.Timedelta(hours=23, minutes=59) arrays = None try: arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['open', 'high', 'low', 'close', 'volume'], start_dt=start_dt, end_dt=end_dt) except Exception as e: log.warn('skipping ctable for {} from {} to {}: {}'.format( asset.symbol, start_dt, end_dt, e)) periods = bundle.get_calendar_periods_range(start_dt, end_dt, data_frequency) df = get_df_from_arrays(arrays, periods) folder = os.path.join(tempfile.gettempdir(), 'catalyst', exchange_name, asset.symbol) ensure_directory(folder) path = os.path.join(folder, filename + '.csv') log.info('creating csv file: {}'.format(path)) print('HEAD\n{}'.format(df.head(100))) print('TAIL\n{}'.format(df.tail(100))) df.to_csv(path) pass
def schedule_catalyst_ingest(exchange_name, data_frequency, include_symbols=None, start=None, end=None): exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, # exclude_symbols=params['exclude_symbols'], start=start, end=end, # show_progress=params['show_progress'], # show_breakdown=params['show_breakdown'], # show_report=params['show_report'], # csv=params['csv'] )
def test_ingest_daily(self): # exchange_name = 'bitfinex' # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' exchange_name = 'bittrex' data_frequency = 'daily' include_symbols = 'wings_eth' start = pd.to_datetime('2017-1-1', utc=True) end = pd.to_datetime('2017-10-16', utc=True) periods = get_periods_range(start, end, data_frequency) exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=None, start=start, end=end, show_progress=True ) symbols = include_symbols.split(',') assets = [] for pair_symbol in symbols: assets.append(exchange.get_asset(pair_symbol)) reader = exchange_bundle.get_reader(data_frequency) for asset in assets: arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], start_dt=start, end_dt=end ) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0]) ) pass
def __init__(self, *args, **kwargs): super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs) self.exchange_bundles = dict() self.history_loaders = dict() self.minute_history_loaders = dict() for exchange_name in self.exchanges: exchange = self.exchanges[exchange_name] self.exchange_bundles[exchange_name] = ExchangeBundle(exchange)
def test_ingest_exchange(self): # exchange_name = 'bitfinex' # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' exchange_name = 'bitfinex' data_frequency = 'minute' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=None, exclude_symbols=None, start=None, end=None, show_progress=True) pass
def __init__(self, *args, **kwargs): self.exchange_names = kwargs.pop('exchange_names', None) super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs) self.exchange_bundles = dict() self.history_loaders = dict() self.minute_history_loaders = dict() for name in self.exchange_names: self.exchange_bundles[name] = ExchangeBundle(name)
def ingest_from_trade_config(config): "Ingest exchange bundle data for a given strategy time frame" if config.get("EXCHANGE") is None: log.error("must specify an exchange name") exchange_bundle = ExchangeBundle(config["EXCHANGE"]) log.notice("Ingesting {} exchange bundle {} - {}...".format( config["EXCHANGE"], config["START"], config["END"])) exchange_bundle.ingest( data_frequency=config["DATA_FREQ"], include_symbols=config["ASSET"], exclude_symbols=None, start=pd.to_datetime(config["START"], utc=True), end=pd.to_datetime(config["END"], utc=True), show_progress=True, show_breakdown=True, show_report=True, csv=None, )
def ingest_exchange(exchange, symbol=None, start=None, end=None): exchange_bundle = ExchangeBundle(exchange) if symbol is None: log.warn(f"Queuing ingest {exchange} for all symbols") else: log.warn(f"Queuing ingest {exchange} for {symbol}") log.warn(f"Will ingest timeframe {start} - {end}") log.info(f"Ingesting {exchange} daily data") exchange_bundle.ingest( "daily", start=pd.to_datetime(start, utc=True), end=pd.to_datetime(end, utc=True), include_symbols=symbol, show_progress=True, show_breakdown=True, show_report=True, ) log.info(f"Done ingesting daily {exchange} data") log.info(f"Ingesting {exchange} minute data") exchange_bundle.ingest( "minute", start=pd.to_datetime(start, utc=True), end=pd.to_datetime(end, utc=True), include_symbols=symbol, show_progress=True, show_breakdown=True, show_report=True, ) log.info(f"Done ingesting minute {exchange} data") log.info("Ingest completed")
def test_ingest_candles(self): exchange_name = 'bitfinex' data_frequency = 'minute' exchange = get_exchange(exchange_name) bundle = ExchangeBundle(exchange) assets = [exchange.get_asset('iot_btc')] end_dt = pd.to_datetime('2017-10-20', utc=True) bar_count = 100 start_dt = get_start_dt(end_dt, bar_count, data_frequency) candles = exchange.get_candles( assets=assets, start_dt=start_dt, end_dt=end_dt, bar_count=bar_count, freq='1T' ) writer = bundle.get_writer(start_dt, end_dt, data_frequency) for asset in assets: dates = [candle['last_traded'] for candle in candles[asset]] values = dict() for field in ['open', 'high', 'low', 'close', 'volume']: values[field] = [candle[field] for candle in candles[asset]] periods = bundle.get_calendar_periods_range( start_dt, end_dt, data_frequency ) df = pd.DataFrame(values, index=dates) df = df.loc[periods].fillna(method='ffill') # TODO: why do I get an extra bar? bundle.ingest_df( ohlcv_df=df, data_frequency=data_frequency, asset=asset, writer=writer, empty_rows_behavior='raise', duplicates_behavior='raise' ) bundle_series = bundle.get_history_window_series( assets=assets, end_dt=end_dt, bar_count=bar_count, field='close', data_frequency=data_frequency, reset_reader=True ) df = pd.DataFrame(bundle_series) print('\n' + df_to_string(df)) pass
def test_ingest_csv(self): data_frequency = 'minute' exchange_name = 'bittrex' path = '/Users/fredfortier/Dropbox/Enigma/Data/bittrex_bat_eth.csv' exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle.ingest_csv(path, data_frequency) exchange = get_exchange(exchange_name) asset = exchange.get_asset('bat_eth') start_dt = pd.to_datetime('2017-6-3', utc=True) end_dt = pd.to_datetime('2017-8-3 19:24', utc=True) self._bundle_to_csv(asset=asset, exchange_name=exchange.name, data_frequency=data_frequency, filename='{}_{}_{}'.format(exchange_name, data_frequency, asset.symbol), start_dt=start_dt, end_dt=end_dt) pass
def __init__(self): self.name = None self.assets = [] self._symbol_maps = [None, None] self.minute_writer = None self.minute_reader = None self.base_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self.name) self.low_balance_threshold = None
def test_merge_ctables(self): exchange_name = 'bittrex' # Switch between daily and minute for testing # data_frequency = 'daily' data_frequency = 'daily' exchange = get_exchange(exchange_name) assets = [ exchange.get_asset('eth_btc'), exchange.get_asset('etc_btc'), exchange.get_asset('wings_eth'), ] start = pd.to_datetime('2017-9-1', utc=True) end = pd.to_datetime('2017-9-30', utc=True) exchange_bundle = ExchangeBundle(exchange) writer = exchange_bundle.get_writer(start, end, data_frequency) # In the interest of avoiding abstractions, this is writing a chunk # to the ctable. It does not include the logic which creates chunks. for asset in assets: exchange_bundle.ingest_ctable( asset=asset, data_frequency=data_frequency, # period='2017-9', period='2017', # Dont't forget to update if you change your dates start_dt=start, end_dt=end, writer=writer, empty_rows_behavior='strip' ) # In daily mode, this returns an error. It appears that writing # a second asset in the same date range removed the first asset. # In minute mode, the data is there too. This signals that the minute # writer / reader is more powerful. This explains why I did not # encounter these problems as I have been focusing on minute data. reader = exchange_bundle.get_reader(data_frequency) for asset in assets: # Since this pair was loaded last. It should be there in daily mode. arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], start_dt=start, end_dt=end ) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0]) ) pass
def __init__(self, key, secret, base_currency, portfolio=None): self.api = Poloniex_api(key=key, secret=secret.encode('UTF-8')) self.name = 'poloniex' self.assets = {} self.load_assets() self.base_currency = base_currency self._portfolio = portfolio self.minute_writer = None self.minute_reader = None self.transactions = defaultdict(list) self.num_candles_limit = 2000 self.max_requests_per_minute = 60 self.request_cpt = dict() self.bundle = ExchangeBundle(self)
def __init__(self, exchange_name, key, secret, password, quote_currency): log.debug( 'finding {} in CCXT exchanges:\n{}'.format( exchange_name, ccxt.exchanges ) ) try: # Making instantiation as explicit as possible for code tracking. if exchange_name in SUPPORTED_EXCHANGES: exchange_attr = SUPPORTED_EXCHANGES[exchange_name] else: exchange_attr = getattr(ccxt, exchange_name) self.api = exchange_attr({ 'apiKey': key, 'secret': secret, 'password': password, }) self.api.enableRateLimit = True except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) self._symbol_maps = [None, None] self.name = exchange_name self.quote_currency = quote_currency self.transactions = defaultdict(list) self.num_candles_limit = 2000 self.max_requests_per_minute = 60 self.low_balance_threshold = 0.1 self.request_cpt = dict() self._common_symbols = dict() # Operations with retry features self.attempts = dict( missing_order_attempts=5, retry_sleeptime=5, ) self.bundle = ExchangeBundle(self.name) self.markets = None self._is_init = False
def __init__(self, exchange_name, key, secret, base_currency): log.debug('finding {} in CCXT exchanges:\n{}'.format( exchange_name, ccxt.exchanges)) try: # Making instantiation as explicit as possible for code tracking. if exchange_name in SUPPORTED_EXCHANGES: exchange_attr = SUPPORTED_EXCHANGES[exchange_name] else: exchange_attr = getattr(ccxt, exchange_name) self.api = exchange_attr({ 'apiKey': key, 'secret': secret, }) except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) self._symbol_maps = [None, None] try: markets_symbols = self.api.load_markets() log.debug('the markets:\n{}'.format(markets_symbols)) except ExchangeNotAvailable as e: raise ExchangeRequestError(error=e) self.name = exchange_name self.markets = self.api.fetch_markets() self.load_assets() self.base_currency = base_currency self.transactions = defaultdict(list) self.num_candles_limit = 2000 self.max_requests_per_minute = 60 self.request_cpt = dict() self.bundle = ExchangeBundle(self.name)
def generate_df(self, exchange_name, freq, start, end): bundle = ExchangeBundle(exchange_name) index = bundle.get_calendar_periods_range(start, end, freq) df = pd.DataFrame(index=index, columns=self.columns) df.fillna(random.random(), inplace=True) return df
class Exchange: __metaclass__ = ABCMeta def __init__(self): self.name = None self.assets = [] self._symbol_maps = [None, None] self.minute_writer = None self.minute_reader = None self.base_currency = None self.num_candles_limit = None self.max_requests_per_minute = None self.request_cpt = None self.bundle = ExchangeBundle(self.name) self.low_balance_threshold = None @abstractproperty def account(self): pass @abstractproperty def time_skew(self): pass def has_bundle(self, data_frequency): return has_bundle(self.name, data_frequency) def is_open(self, dt): """ Is the exchange open Parameters ---------- dt: Timestamp Returns ------- bool """ # TODO: implement for each exchange. return True def ask_request(self): """ Asks permission to issue a request to the exchange. The primary purpose is to avoid hitting rate limits. The application will pause if the maximum requests per minute permitted by the exchange is exceeded. Returns ------- bool """ now = pd.Timestamp.utcnow() if not self.request_cpt: self.request_cpt = dict() self.request_cpt[now] = 0 return True cpt_date = list(self.request_cpt.keys())[0] cpt = self.request_cpt[cpt_date] if now > cpt_date + timedelta(minutes=1): self.request_cpt = dict() self.request_cpt[now] = 0 return True if cpt >= self.max_requests_per_minute: delta = now - cpt_date sleep_period = 60 - delta.total_seconds() sleep(sleep_period) now = pd.Timestamp.utcnow() self.request_cpt = dict() self.request_cpt[now] = 0 return True else: self.request_cpt[cpt_date] += 1 def get_symbol(self, asset): """ The exchange specific symbol of the specified market. Parameters ---------- asset: TradingPair Returns ------- str """ symbol = None for a in self.assets: if not symbol and a.symbol == asset.symbol: symbol = a.symbol if not symbol: raise ValueError('Currency %s not supported by exchange %s' % (asset['symbol'], self.name.title())) return symbol def get_symbols(self, assets): """ Get a list of symbols corresponding to each given asset. Parameters ---------- assets: list[TradingPair] Returns ------- list[str] """ symbols = [] for asset in assets: symbols.append(self.get_symbol(asset)) return symbols def get_assets(self, symbols=None, data_frequency=None, is_exchange_symbol=False, is_local=None, quote_currency=None): """ The list of markets for the specified symbols. Parameters ---------- symbols: list[str] data_frequency: str is_exchange_symbol: bool is_local: bool Returns ------- list[TradingPair] A list of asset objects. Notes ----- See get_asset for details of each parameter. """ if symbols is None: # Make a distinct list of all symbols symbols = list(set([asset.symbol for asset in self.assets])) symbols.sort() if quote_currency is not None: for symbol in symbols[:]: suffix = '_{}'.format(quote_currency.lower()) if not symbol.endswith(suffix): symbols.remove(symbol) is_exchange_symbol = False assets = [] for symbol in symbols: try: asset = self.get_asset( symbol, data_frequency, is_exchange_symbol, is_local ) assets.append(asset) except SymbolNotFoundOnExchange: log.debug( 'skipping non-existent market {} {}'.format( self.name, symbol ) ) return assets def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False, is_local=None): """ The market for the specified symbol. Parameters ---------- symbol: str The Catalyst or exchange symbol. data_frequency: str Check for asset corresponding to the specified data_frequency. The same asset might exist in the Catalyst repository or locally (following a CSV ingestion). Filtering by data_frequency picks the right asset. is_exchange_symbol: bool Whether the symbol uses the Catalyst or exchange convention. is_local: bool For the local or Catalyst asset. Returns ------- TradingPair The asset object. """ asset = None # TODO: temp mapping, fix to use a single symbol convention og_symbol = symbol symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol log.debug( 'searching assets for: {} {}'.format( self.name, symbol ) ) # TODO: simplify and loose the loop for a in self.assets: if asset is not None: break if is_local is not None: data_source = 'local' if is_local else 'catalyst' applies = (a.data_source == data_source) elif data_frequency is not None: applies = ( ( data_frequency == 'minute' and a.end_minute is not None) or ( data_frequency == 'daily' and a.end_daily is not None) ) else: applies = True # The symbol provided may use the Catalyst or the exchange # convention key = a.exchange_symbol if \ is_exchange_symbol else self.get_symbol(a) if not asset and key.lower() == symbol.lower(): if applies: asset = a else: raise NoDataAvailableOnExchange( symbol=key, exchange=self.name, data_frequency=data_frequency, ) if asset is None: supported_symbols = sorted([a.symbol for a in self.assets]) raise SymbolNotFoundOnExchange( symbol=og_symbol, exchange=self.name.title(), supported_symbols=supported_symbols ) log.debug('found asset: {}'.format(asset)) return asset def fetch_symbol_map(self, is_local=False): index = 1 if is_local else 0 if self._symbol_maps[index] is not None: return self._symbol_maps[index] else: symbol_map = get_exchange_symbols(self.name, is_local) self._symbol_maps[index] = symbol_map return symbol_map @abstractmethod def init(self): """ Load the asset list from the network. Returns ------- """ @abstractmethod def load_assets(self, is_local=False): """ Populate the 'assets' attribute with a dictionary of Assets. The key of the resulting dictionary is the exchange specific currency pair symbol. The universal symbol is contained in the 'symbol' attribute of each asset. Notes ----- The sid of each asset is calculated based on a numeric hash of the universal symbol. This simple approach avoids maintaining a mapping of sids. This method can be omerridden if an exchange offers equivalent data via its api. """ pass def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ Public API method that returns a scalar value representing the value of the desired asset's field at either the given dt. Parameters ---------- assets : Asset, ContinuousFuture, or iterable of same. The asset or assets whose data is desired. field : {'open', 'high', 'low', 'close', 'volume', 'price', 'last_traded'} The desired field of the asset. dt : pd.Timestamp The timestamp for the desired value. data_frequency : str The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars Returns ------- value : float, int, or pd.Timestamp The spot value of ``field`` for ``asset`` The return type is based on the ``field`` requested. If the field is one of 'open', 'high', 'low', 'close', or 'price', the value will be a float. If the ``field`` is 'volume' the value will be a int. If the ``field`` is 'last_traded' the value will be a Timestamp. Bitfinex timeframes ------------------- Available values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' """ if field not in BASE_FIELDS: raise KeyError('Invalid column: {}'.format(field)) tickers = self.tickers(assets) if field == 'close' or field == 'price': return [tickers[asset]['last'] for asset in tickers] elif field == 'volume': return [tickers[asset]['volume'] for asset in tickers] else: raise NoValueForField(field=field) def get_single_spot_value(self, asset, field, data_frequency): """ Similar to 'get_spot_value' but for a single asset Notes ----- We're writing each minute bar to disk using zipline's machinery. This is especially useful when running multiple algorithms concurrently. By using local data when possible, we try to reaching request limits on exchanges. Parameters ---------- asset: TradingPair field: str data_frequency: str Returns ------- float The spot value of the given asset / field """ log.debug( 'fetching spot value {field} for symbol {symbol}'.format( symbol=asset.symbol, field=field ) ) freq = '1T' if data_frequency == 'minute' else '1D' ohlc = self.get_candles(freq, asset) if field not in ohlc: raise KeyError('Invalid column: %s' % field) value = ohlc[field] log.debug('got spot value: {}'.format(value)) return value # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df def get_series_from_candles(self, candles, start_dt, end_dt, data_frequency, field, previous_value=None): """ Get a series of field data for the specified candles. Parameters ---------- candles: list[dict[str, float]] start_dt: datetime end_dt: datetime data_frequency: str field: str previous_value: float Returns ------- Series """ dates = [candle['last_traded'] for candle in candles] values = [candle[field] for candle in candles] series = pd.Series(values, index=dates) periods = get_periods_range( start_dt=start_dt, end_dt=end_dt, freq=data_frequency ) # TODO: ensure that this working as expected, if not use fillna series = series.reindex( periods, method='ffill', fill_value=previous_value, ) series.sort_index(inplace=True) return series def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, is_current=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. is_current: bool Skip date filters when current data is requested (last few bars until now). Notes ----- Catalysts requires an end data with bar count both CCXT wants a start data with bar count. Since we have to make calculations here, we ensure that the last candle match the end_dt parameter. Returns ------- DataFrame A dataframe containing the requested data. """ freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, bar_count=bar_count, end_dt=end_dt if not is_current else None, ) series = dict() for asset in candles: first_candle = candles[asset][0] asset_series = self.get_series_from_candles( candles=candles[asset], start_dt=first_candle['last_traded'], end_dt=end_dt, data_frequency=frequency, field=field, ) # Checking to make sure that the dates match delta = get_delta(candle_size, data_frequency) adj_end_dt = end_dt - delta last_traded = asset_series.index[-1] if last_traded < adj_end_dt: raise LastCandleTooEarlyError( last_traded=last_traded, end_dt=adj_end_dt, exchange=self.name, ) series[asset] = asset_series df = pd.DataFrame(series) df.dropna(inplace=True) return df def get_history_window_with_bundle(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, ffill=True, force_auto_ingest=False): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list[TradingPair] The assets whose data is desired. end_dt: datetime The date of the last bar. bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- DataFrame A dataframe containing the requested data. """ # TODO: this function needs some work, we're currently using it just for benchmark data freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) adj_bar_count = candle_size * bar_count try: series = self.bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=data_frequency, force_auto_ingest=force_auto_ingest ) except (PricingDataNotLoadedError, NoDataAvailableOnExchange): series = dict() for asset in assets: if asset not in series or series[asset].index[-1] < end_dt: # Adding bars too recent to be contained in the consolidated # exchanges bundles. We go directly against the exchange # to retrieve the candles. start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize # the size of result sets trailing_bars = get_periods( trailing_dt, end_dt, freq ) candles = self.get_candles( freq=freq, assets=asset, end_dt=end_dt, bar_count=trailing_bars if trailing_bars < 500 else 500, ) last_value = series[asset].iloc(0) if asset in series \ else np.nan # Create a series with the common data_frequency, ffill # missing values candle_series = self.get_series_from_candles( candles=candles, start_dt=trailing_dt, end_dt=end_dt, data_frequency=data_frequency, field=field, previous_value=last_value ) if asset in series: series[asset].append(candle_series) else: series[asset] = candle_series df = resample_history_df(pd.DataFrame(series), freq, field) # TODO: consider this more carefully df.dropna(inplace=True) return df def _check_low_balance(self, currency, balances, amount): free = balances[currency]['free'] if currency in balances else 0.0 if free < amount: return free, True else: return free, False def sync_positions(self, positions, cash=None, check_balances=False): """ Update the portfolio cash and position balances based on the latest ticker prices. Parameters ---------- positions: The positions to synchronize. check_balances: Check balances amounts against the exchange. """ free_cash = 0.0 if check_balances: log.debug('fetching {} balances'.format(self.name)) balances = self.get_balances() log.debug( 'got free balances for {} currencies'.format( len(balances) ) ) if cash is not None: free_cash, is_lower = self._check_low_balance( currency=self.base_currency, balances=balances, amount=cash, ) if is_lower: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, free=free_cash, cash=cash, ) positions_value = 0.0 if positions: assets = list(set([position.asset for position in positions])) tickers = self.tickers(assets) for position in positions: asset = position.asset if asset not in tickers: raise TickerNotFoundError( symbol=asset.symbol, exchange=self.name, ) ticker = tickers[asset] log.debug( 'updating {symbol} position, last traded on {dt} for ' '{price}{currency}'.format( symbol=asset.symbol, dt=ticker['last_traded'], price=ticker['last_price'], currency=asset.quote_currency, ) ) position.last_sale_price = ticker['last_price'] position.last_sale_date = ticker['last_traded'] positions_value += \ position.amount * position.last_sale_price if check_balances: free, is_lower = self._check_low_balance( currency=asset.base_currency, balances=balances, amount=position.amount, ) if is_lower: log.warn( 'detected lower balance for {} on {}: {} < {}, ' 'updating position amount'.format( asset.symbol, self.name, free, position.amount ) ) position.amount = free return free_cash, positions_value def order(self, asset, amount, style): """Place an order. Parameters ---------- asset : TradingPair The asset that this order is for. amount : int The amount of shares to order. If ``amount`` is positive, this is the number of shares to buy or cover. If ``amount`` is negative, this is the number of shares to sell or short. limit_price : float, optional The limit price for the order. stop_price : float, optional The stop price for the order. style : ExecutionStyle, optional The execution style for the order. Returns ------- order_id : str or None The unique identifier for this order, or None if no order was placed. Notes ----- The ``limit_price`` and ``stop_price`` arguments provide shorthands for passing common execution styles. Passing ``limit_price=N`` is equivalent to ``style=LimitOrder(N)``. Similarly, passing ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing ``limit_price=N`` and ``stop_price=M`` is equivalent to ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style`` and ``limit_price`` or ``stop_price``. See Also -------- :class:`catalyst.finance.execution.ExecutionStyle` :func:`catalyst.api.order_value` :func:`catalyst.api.order_percent` """ if amount == 0: log.warn('skipping order amount of 0') return None if self.base_currency is None: raise ValueError('no base_currency defined for this exchange') if asset.quote_currency != self.base_currency.lower(): raise MismatchingBaseCurrencies( base_currency=asset.quote_currency, algo_currency=self.base_currency ) is_buy = (amount > 0) display_price = style.get_limit_price(is_buy) log.debug( 'issuing {side} order of {amount} {symbol} for {type}:' ' {price}'.format( side='buy' if is_buy else 'sell', amount=amount, symbol=asset.symbol, type=style.__class__.__name__, price='{}{}'.format(display_price, asset.quote_currency) ) ) return self.create_order(asset, amount, is_buy, style) # The methods below must be implemented for each exchange. @abstractmethod def get_balances(self): """ Retrieve wallet balances for the exchange. Returns ------- dict[TradingPair, float] """ pass @abstractmethod def create_order(self, asset, amount, is_buy, style): """ Place an order on the exchange. Parameters ---------- asset: TradingPair The target market. amount: float The amount of shares to order. If ``amount`` is positive, this is the number of shares to buy or cover. If ``amount`` is negative, this is the number of shares to sell or short. is_buy: bool Is it a buy order? style: ExecutionStyle Returns ------- Order """ pass @abstractmethod def get_open_orders(self, asset): """Retrieve all of the current open orders. Parameters ---------- asset : Asset If passed and not None, return only the open orders for the given asset instead of all open orders. Returns ------- open_orders : dict[list[Order]] or list[Order] If no asset is passed this will return a dict mapping Assets to a list containing all the open orders for the asset. If an asset is passed then this will return a list of the open orders for this asset. """ pass @abstractmethod def get_order(self, order_id, symbol_or_asset=None): """Lookup an order based on the order id returned from one of the order functions. Parameters ---------- order_id : str The unique identifier for the order. symbol_or_asset: str|TradingPair The catalyst symbol, some exchanges need this Returns ------- order : Order The order object. execution_price: float The execution price per share of the order """ pass @abstractmethod def process_order(self, order): """ Similar to get_order but looks only for executed orders. Parameters ---------- order: Order Returns ------- float Avg execution price """ @abstractmethod def cancel_order(self, order_param, symbol_or_asset=None): """Cancel an open order. Parameters ---------- order_param : str or Order The order_id or order object to cancel. symbol_or_asset: str|TradingPair The catalyst symbol, some exchanges need this """ pass @abstractmethod def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None): """ Retrieve OHLCV candles for the given assets Parameters ---------- freq: str The frequency alias per convention: http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases assets: list[TradingPair] The targeted assets. bar_count: int The number of bar desired. (default 1) end_dt: datetime, optional The last bar date. start_dt: datetime, optional The first bar date. Returns ------- dict[TradingPair, dict[str, Object]] A dictionary of OHLCV candles. Each TradingPair instance is mapped to a list of dictionaries with this structure: open: float high: float low: float close: float volume: float last_traded: datetime See definition here: http://www.investopedia.com/terms/o/ohlcchart.asp """ pass @abc.abstractmethod def tickers(self, assets, on_ticker_error='raise'): """ Retrieve current tick data for the given assets Parameters ---------- assets: list[TradingPair] on_ticker_error: str [raise|warn] How to handle an error when retrieving a single ticker. Returns ------- list[dict[str, float] """ pass @abc.abstractmethod def get_account(self): """ Retrieve the account parameters. """ pass @abc.abstractmethod def get_orderbook(self, asset, order_type, limit): """ Retrieve the orderbook for the given trading pair. Parameters ---------- asset: TradingPair order_type: str The type of orders: bid, ask or all limit: int Returns ------- list[dict[str, float] """ pass @abc.abstractmethod def get_trades(self, asset, my_trades, start_dt, limit): """