def ingest_exchange_bundles(exchange_name, data_freq, symbols):
exchange_bundle = ExchangeBundle(exchange_name)
exchange_bundle.ingest(
include_symbols=symbols,
data_frequency=data_freq,
)
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
include_symbols, exclude_symbols, csv, show_progress,
verbose, validate):
"""
Ingest data for the given exchange.
"""
if exchange_name is None:
ctx.fail("must specify an exchange name '-x'")
if not csv and exchange_name not in EXCHANGE_NAMES:
ctx.fail("ingest-exchange does not support {}, "
"please choose exchange from: {}".format(
exchange_name, EXCHANGE_NAMES))
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
sys.stdout)
exchange_bundle.ingest(data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=exclude_symbols,
start=start,
end=end,
show_progress=show_progress,
show_breakdown=verbose,
show_report=validate,
csv=csv)
def test_ingest_minute(self):
data_frequency = 'minute'
exchange_name = 'poloniex'
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
assets = [exchange.get_asset('eth_btc')]
start = pd.to_datetime('2016-03-01', utc=True)
end = pd.to_datetime('2017-11-1', utc=True)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=','.join([asset.symbol for asset in assets]),
# include_symbols=None,
exclude_symbols=None,
start=start,
end=end,
show_progress=True)
reader = exchange_bundle.get_reader(data_frequency)
for asset in assets:
arrays = reader.load_raw_arrays(sids=[asset.sid],
fields=['close'],
start_dt=start,
end_dt=end)
print('found {} rows for {} ingestion\n{}'.format(
len(arrays[0]), asset.symbol, arrays[0]))
pass
def ingest_exchange(exchange, symbol=None, start=None, end=None):
exchange_bundle = ExchangeBundle(exchange)
if symbol is None:
log.warn(f"Queuing ingest {exchange} for all symbols")
else:
log.warn(f"Queuing ingest {exchange} for {symbol}")
log.warn(f"Will ingest timeframe {start} - {end}")
log.info(f"Ingesting {exchange} daily data")
exchange_bundle.ingest(
"daily",
start=pd.to_datetime(start, utc=True),
end=pd.to_datetime(end, utc=True),
include_symbols=symbol,
show_progress=True,
show_breakdown=True,
show_report=True,
)
log.info(f"Done ingesting daily {exchange} data")
log.info(f"Ingesting {exchange} minute data")
exchange_bundle.ingest(
"minute",
start=pd.to_datetime(start, utc=True),
end=pd.to_datetime(end, utc=True),
include_symbols=symbol,
show_progress=True,
show_breakdown=True,
show_report=True,
)
log.info(f"Done ingesting minute {exchange} data")
log.info("Ingest completed")
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
include_symbols, exclude_symbols, csv, show_progress,
verbose, validate):
"""
Ingest data for the given exchange.
"""
if exchange_name is None:
ctx.fail("must specify an exchange name '-x'")
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
sys.stdout)
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=exclude_symbols,
start=start,
end=end,
show_progress=show_progress,
show_breakdown=verbose,
show_report=validate,
csv=csv
)
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
include_symbols, exclude_symbols, csv, show_progress,
verbose, validate):
"""
Ingest data for the given exchange.
"""
if exchange_name is None:
ctx.fail("must specify an exchange name '-x'")
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
sys.stdout)
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=exclude_symbols,
start=start,
end=end,
show_progress=show_progress,
show_breakdown=verbose,
show_report=validate,
csv=csv
)
def test_ingest_daily(self):
exchange_name = 'bitfinex'
data_frequency = 'minute'
include_symbols = 'neo_btc'
# exchange_name = 'poloniex'
# data_frequency = 'daily'
# include_symbols = 'eth_btc'
# start = pd.to_datetime('2017-1-1', utc=True)
# end = pd.to_datetime('2017-10-16', utc=True)
# periods = get_periods_range(start, end, data_frequency)
start = None
end = None
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=None,
start=start,
end=end,
show_progress=True
)
symbols = include_symbols.split(',')
assets = []
for pair_symbol in symbols:
assets.append(exchange.get_asset(pair_symbol))
reader = exchange_bundle.get_reader(data_frequency)
start_dt = reader.first_trading_day
end_dt = reader.last_available_dt
if data_frequency == 'daily':
end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)
for asset in assets:
arrays = reader.load_raw_arrays(
sids=[asset.sid],
fields=['close'],
start_dt=start_dt,
end_dt=end_dt
)
print('found {} rows for {} ingestion\n{}'.format(
len(arrays[0]), asset.symbol, arrays[0])
)
pass
def ingest_exchange(exchange_name, data_frequency, start, end, include_symbols,
exclude_symbols, show_progress):
"""
Ingest data for the given exchange.
"""
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
exchange_bundle.ingest(data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=exclude_symbols,
start=start,
end=end,
show_progress=show_progress)
def test_ingest_minute_all(self):
exchange_name = 'bitfinex'
# start = pd.to_datetime('2017-09-01', utc=True)
start = pd.to_datetime('2017-10-01', utc=True)
end = pd.to_datetime('2017-10-05', utc=True)
exchange_bundle = ExchangeBundle(get_exchange(exchange_name))
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(data_frequency='minute',
exclude_symbols=None,
start=start,
end=end,
show_progress=True)
pass
def schedule_catalyst_ingest(exchange_name,
data_frequency,
include_symbols=None,
start=None,
end=None):
exchange_bundle = ExchangeBundle(exchange_name)
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
# exclude_symbols=params['exclude_symbols'],
start=start,
end=end,
# show_progress=params['show_progress'],
# show_breakdown=params['show_breakdown'],
# show_report=params['show_report'],
# csv=params['csv']
)
def test_ingest_daily(self):
# exchange_name = 'bitfinex'
# data_frequency = 'daily'
# include_symbols = 'neo_btc,bch_btc,eth_btc'
exchange_name = 'bittrex'
data_frequency = 'daily'
include_symbols = 'wings_eth'
start = pd.to_datetime('2017-1-1', utc=True)
end = pd.to_datetime('2017-10-16', utc=True)
periods = get_periods_range(start, end, data_frequency)
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
exclude_symbols=None,
start=start,
end=end,
show_progress=True
)
symbols = include_symbols.split(',')
assets = []
for pair_symbol in symbols:
assets.append(exchange.get_asset(pair_symbol))
reader = exchange_bundle.get_reader(data_frequency)
for asset in assets:
arrays = reader.load_raw_arrays(
sids=[asset.sid],
fields=['close'],
start_dt=start,
end_dt=end
)
print('found {} rows for {} ingestion\n{}'.format(
len(arrays[0]), asset.symbol, arrays[0])
)
pass
def test_ingest_exchange(self):
# exchange_name = 'bitfinex'
# data_frequency = 'daily'
# include_symbols = 'neo_btc,bch_btc,eth_btc'
exchange_name = 'bitfinex'
data_frequency = 'minute'
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(data_frequency=data_frequency,
include_symbols=None,
exclude_symbols=None,
start=None,
end=None,
show_progress=True)
pass
def ingest_from_trade_config(config):
"Ingest exchange bundle data for a given strategy time frame"
if config.get("EXCHANGE") is None:
log.error("must specify an exchange name")
exchange_bundle = ExchangeBundle(config["EXCHANGE"])
log.notice("Ingesting {} exchange bundle {} - {}...".format(
config["EXCHANGE"], config["START"], config["END"]))
exchange_bundle.ingest(
data_frequency=config["DATA_FREQ"],
include_symbols=config["ASSET"],
exclude_symbols=None,
start=pd.to_datetime(config["START"], utc=True),
end=pd.to_datetime(config["END"], utc=True),
show_progress=True,
show_breakdown=True,
show_report=True,
csv=None,
)
