def ingest(self,
data_frequency,
include_symbols=None,
exclude_symbols=None,
start=None,
end=None,
csv=None,
show_progress=True,
show_breakdown=True,
show_report=True,
from_exchange=False,
exclude_current_month=False):
"""
Inject data based on specified parameters.
Parameters
----------
data_frequency: str
include_symbols: str
exclude_symbols: str
start: pd.Timestamp
end: pd.Timestamp
show_progress: bool
environ:
"""
if from_exchange:
log.warning("Ingesting data directly from the exchange: '{}'",
self.exchange_name)
if csv is not None:
self.ingest_csv(csv, data_frequency)
else:
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.utils.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
assets = get_assets(self.exchange, include_symbols,
exclude_symbols)
self.update_symbols_file(get_assets(self.exchange, None, None))
for frequency in data_frequency.split(','):
self.ingest_assets(assets=assets,
data_frequency=frequency,
start_dt=start,
end_dt=end,
show_progress=show_progress,
show_breakdown=show_breakdown,
show_report=show_report,
from_exchange=from_exchange,
exclude_current_month=exclude_current_month)
def ingest(self, data_frequency, include_symbols=None,
exclude_symbols=None, start=None, end=None, csv=None,
show_progress=True, show_breakdown=True, show_report=True):
"""
Inject data based on specified parameters.
Parameters
----------
data_frequency: str
include_symbols: str
exclude_symbols: str
start: pd.Timestamp
end: pd.Timestamp
show_progress: bool
environ:
"""
# NOTE: Seems to be an overwrite all process, so must append to csv
# with multiple files or only imports last one
if csv is not None:
if csv == 'create':
cached_data = self.check_ohlcv_data(data_frequency, start, end)
if cached_data is None:
csv = self.create_csv_from_ccxt(
data_frequency,
include_symbols=include_symbols,
exclude_symbols=exclude_symbols,
start=start,
end=end
)
else:
csv = cached_data
# csv = '/root/.catalyst/data/csv/cryptopia-minute.csv'
self.ingest_csv(csv, data_frequency)
self.store_ohlcv_data(csv)
else:
self.ingest_csv(csv, data_frequency)
else:
if self.exchange is None:
# Avoid circular dependencies
from .factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
assets = get_assets(
self.exchange, include_symbols, exclude_symbols
)
for frequency in data_frequency.split(','):
self.ingest_assets(
assets=assets,
data_frequency=frequency,
start_dt=start,
end_dt=end,
show_progress=show_progress,
show_breakdown=show_breakdown,
show_report=show_report
)
def ingest(self,
data_frequency,
include_symbols=None,
exclude_symbols=None,
start=None,
end=None,
csv=None,
show_progress=True,
show_breakdown=True,
show_report=True):
"""
Inject data based on specified parameters.
Parameters
----------
data_frequency: str
include_symbols: str
exclude_symbols: str
start: pd.Timestamp
end: pd.Timestamp
show_progress: bool
environ:
"""
if csv is not None:
self.ingest_csv(csv, data_frequency)
else:
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.utils.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
assets = get_assets(self.exchange, include_symbols,
exclude_symbols)
for frequency in data_frequency.split(','):
self.ingest_assets(assets=assets,
data_frequency=frequency,
start_dt=start,
end_dt=end,
show_progress=show_progress,
show_breakdown=show_breakdown,
show_report=show_report)