def ingest_exchange_bundles(exchange_name, data_freq, symbols): exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle.ingest( include_symbols=symbols, data_frequency=data_freq, )
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") if not csv and exchange_name not in EXCHANGE_NAMES: ctx.fail("ingest-exchange does not support {}, " "please choose exchange from: {}".format( exchange_name, EXCHANGE_NAMES)) exchange_bundle = ExchangeBundle(exchange_name) click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv)
def test_ingest_minute(self): data_frequency = 'minute' exchange_name = 'poloniex' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) assets = [exchange.get_asset('eth_btc')] start = pd.to_datetime('2016-03-01', utc=True) end = pd.to_datetime('2017-11-1', utc=True) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=','.join([asset.symbol for asset in assets]), # include_symbols=None, exclude_symbols=None, start=start, end=end, show_progress=True) reader = exchange_bundle.get_reader(data_frequency) for asset in assets: arrays = reader.load_raw_arrays(sids=[asset.sid], fields=['close'], start_dt=start, end_dt=end) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0])) pass
def ingest_exchange(exchange, symbol=None, start=None, end=None): exchange_bundle = ExchangeBundle(exchange) if symbol is None: log.warn(f"Queuing ingest {exchange} for all symbols") else: log.warn(f"Queuing ingest {exchange} for {symbol}") log.warn(f"Will ingest timeframe {start} - {end}") log.info(f"Ingesting {exchange} daily data") exchange_bundle.ingest( "daily", start=pd.to_datetime(start, utc=True), end=pd.to_datetime(end, utc=True), include_symbols=symbol, show_progress=True, show_breakdown=True, show_report=True, ) log.info(f"Done ingesting daily {exchange} data") log.info(f"Ingesting {exchange} minute data") exchange_bundle.ingest( "minute", start=pd.to_datetime(start, utc=True), end=pd.to_datetime(end, utc=True), include_symbols=symbol, show_progress=True, show_breakdown=True, show_report=True, ) log.info(f"Done ingesting minute {exchange} data") log.info("Ingest completed")
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv )
def ingest_exchange(ctx, exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, csv, show_progress, verbose, validate): """ Ingest data for the given exchange. """ if exchange_name is None: ctx.fail("must specify an exchange name '-x'") exchange_bundle = ExchangeBundle(exchange_name) click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress, show_breakdown=verbose, show_report=validate, csv=csv )
def test_ingest_daily(self): exchange_name = 'bitfinex' data_frequency = 'minute' include_symbols = 'neo_btc' # exchange_name = 'poloniex' # data_frequency = 'daily' # include_symbols = 'eth_btc' # start = pd.to_datetime('2017-1-1', utc=True) # end = pd.to_datetime('2017-10-16', utc=True) # periods = get_periods_range(start, end, data_frequency) start = None end = None exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=None, start=start, end=end, show_progress=True ) symbols = include_symbols.split(',') assets = [] for pair_symbol in symbols: assets.append(exchange.get_asset(pair_symbol)) reader = exchange_bundle.get_reader(data_frequency) start_dt = reader.first_trading_day end_dt = reader.last_available_dt if data_frequency == 'daily': end_dt = end_dt - pd.Timedelta(hours=23, minutes=59) for asset in assets: arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], start_dt=start_dt, end_dt=end_dt ) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0]) ) pass
def ingest_exchange(exchange_name, data_frequency, start, end, include_symbols, exclude_symbols, show_progress): """ Ingest data for the given exchange. """ exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) click.echo('Ingesting exchange bundle {}...'.format(exchange_name)) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=exclude_symbols, start=start, end=end, show_progress=show_progress)
def test_ingest_minute_all(self): exchange_name = 'bitfinex' # start = pd.to_datetime('2017-09-01', utc=True) start = pd.to_datetime('2017-10-01', utc=True) end = pd.to_datetime('2017-10-05', utc=True) exchange_bundle = ExchangeBundle(get_exchange(exchange_name)) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest(data_frequency='minute', exclude_symbols=None, start=start, end=end, show_progress=True) pass
def schedule_catalyst_ingest(exchange_name, data_frequency, include_symbols=None, start=None, end=None): exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, # exclude_symbols=params['exclude_symbols'], start=start, end=end, # show_progress=params['show_progress'], # show_breakdown=params['show_breakdown'], # show_report=params['show_report'], # csv=params['csv'] )
def test_ingest_daily(self): # exchange_name = 'bitfinex' # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' exchange_name = 'bittrex' data_frequency = 'daily' include_symbols = 'wings_eth' start = pd.to_datetime('2017-1-1', utc=True) end = pd.to_datetime('2017-10-16', utc=True) periods = get_periods_range(start, end, data_frequency) exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( data_frequency=data_frequency, include_symbols=include_symbols, exclude_symbols=None, start=start, end=end, show_progress=True ) symbols = include_symbols.split(',') assets = [] for pair_symbol in symbols: assets.append(exchange.get_asset(pair_symbol)) reader = exchange_bundle.get_reader(data_frequency) for asset in assets: arrays = reader.load_raw_arrays( sids=[asset.sid], fields=['close'], start_dt=start, end_dt=end ) print('found {} rows for {} ingestion\n{}'.format( len(arrays[0]), asset.symbol, arrays[0]) ) pass
def test_ingest_exchange(self): # exchange_name = 'bitfinex' # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' exchange_name = 'bitfinex' data_frequency = 'minute' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest(data_frequency=data_frequency, include_symbols=None, exclude_symbols=None, start=None, end=None, show_progress=True) pass
def ingest_from_trade_config(config): "Ingest exchange bundle data for a given strategy time frame" if config.get("EXCHANGE") is None: log.error("must specify an exchange name") exchange_bundle = ExchangeBundle(config["EXCHANGE"]) log.notice("Ingesting {} exchange bundle {} - {}...".format( config["EXCHANGE"], config["START"], config["END"])) exchange_bundle.ingest( data_frequency=config["DATA_FREQ"], include_symbols=config["ASSET"], exclude_symbols=None, start=pd.to_datetime(config["START"], utc=True), end=pd.to_datetime(config["END"], utc=True), show_progress=True, show_breakdown=True, show_report=True, csv=None, )