def get_reader(self, data_frequency, path=None):
"""
Get a data writer object, either a new object or from cache
:return: BcolzMinuteBarReader or BcolzDailyBarReader
"""
if path is None:
root = get_exchange_folder(self.exchange.name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
if path in self._readers and self._readers[path] is not None:
return self._readers[path]
try:
self._readers[path] = BcolzExchangeBarReader(
rootdir=path,
data_frequency=data_frequency
)
except IOError:
self._readers[path] = None
return self._readers[path]
def get_writer(self, start_dt, end_dt, data_frequency):
"""
Get a data writer object, either a new object or from cache
:return: BcolzMinuteBarWriter or BcolzDailyBarWriter
"""
root = get_exchange_folder(self.exchange.name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
if path in self._writers:
return self._writers[path]
ensure_directory(path)
if len(os.listdir(path)) > 0:
metadata = BcolzMinuteBarMetadata.read(path)
write_metadata = False
if start_dt < metadata.start_session:
write_metadata = True
start_session = start_dt
else:
start_session = metadata.start_session
if end_dt > metadata.end_session:
write_metadata = True
end_session = end_dt
else:
end_session = metadata.end_session
self._writers[path] = \
BcolzExchangeBarWriter(
rootdir=path,
start_session=start_session,
end_session=end_session,
write_metadata=write_metadata,
data_frequency=data_frequency
)
else:
self._writers[path] = BcolzExchangeBarWriter(
rootdir=path,
start_session=start_dt,
end_session=end_dt,
write_metadata=True,
data_frequency=data_frequency
)
return self._writers[path]
def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
exchange_auth = get_exchange_auth(exchange_name)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
return CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
)
def clean(self, data_frequency):
"""
Removing the bundle data from the catalyst folder.
Parameters
----------
data_frequency: str
"""
log.debug('cleaning exchange {}, frequency {}'.format(
self.exchange_name, data_frequency))
root = get_exchange_folder(self.exchange_name)
symbols = os.path.join(root, 'symbols.json')
if os.path.isfile(symbols):
os.remove(symbols)
local_symbols = os.path.join(root, 'symbols_local.json')
if os.path.isfile(local_symbols):
os.remove(local_symbols)
temp_bundles = os.path.join(root, 'temp_bundles')
if os.path.isdir(temp_bundles):
log.debug('removing folder and content: {}'.format(temp_bundles))
shutil.rmtree(temp_bundles)
log.debug('{} removed'.format(temp_bundles))
frequencies = ['daily', 'minute'] if data_frequency is None \
else [data_frequency]
for frequency in frequencies:
label = '{}_bundle'.format(frequency)
frequency_bundle = os.path.join(root, label)
if os.path.isdir(frequency_bundle):
log.debug(
'removing folder and content: {}'.format(frequency_bundle))
shutil.rmtree(frequency_bundle)
log.debug('{} removed'.format(frequency_bundle))
def _run(handle_data, initialize, before_trading_start, analyze, algofile,
algotext, defines, data_frequency, capital_base, data, bundle,
bundle_timestamp, start, end, output, print_algo, local_namespace,
environ, live, exchange, algo_namespace, base_currency, live_graph):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
"""
if algotext is not None:
if local_namespace:
ip = get_ipython() # noqa
namespace = ip.user_ns
else:
namespace = {}
for assign in defines:
try:
name, value = assign.split('=', 2)
except ValueError:
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign, )
try:
# evaluate in the same namespace so names may refer to
# eachother
namespace[name] = eval(value, namespace)
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' %
(name, e), )
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
"cannot pass '-D' / '--define' without '-t' / '--algotext'",
)
else:
namespace = {}
if algofile is not None:
algotext = algofile.read()
if print_algo:
if PYGMENTS:
highlight(
algotext,
PythonLexer(),
TerminalFormatter(),
outfile=sys.stdout,
)
else:
click.echo(algotext)
mode = 'live' if live else 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
exchange_name = exchange
if exchange_name is None:
raise ValueError('Please specify at least one exchange.')
exchange_list = [x.strip().lower() for x in exchange.split(',')]
exchanges = dict()
for exchange_name in exchange_list:
# Looking for the portfolio from the cache first
portfolio = get_algo_object(algo_name=algo_namespace,
key='portfolio_{}'.format(exchange_name),
environ=environ)
if portfolio is None:
portfolio = ExchangePortfolio(start_date=pd.Timestamp.utcnow())
# This corresponds to the json file containing api token info
exchange_auth = get_exchange_auth(exchange_name)
if live and (exchange_auth['key'] == ''
or exchange_auth['secret'] == ''):
raise ExchangeAuthEmpty(exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(
exchange_name, environ),
'auth.json'))
if exchange_name == 'bitfinex':
exchanges[exchange_name] = Bitfinex(key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio)
elif exchange_name == 'bittrex':
exchanges[exchange_name] = Bittrex(key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio)
elif exchange_name == 'poloniex':
exchanges[exchange_name] = Poloniex(key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
load=partial(load_crypto_market_data,
environ=environ,
start_dt=start,
end_dt=end),
environ=environ,
exchange_tz='UTC',
asset_db_path=None # We don't need an asset db, we have exchanges
)
env.asset_finder = AssetFinderExchange()
choose_loader = None # TODO: use the DataPortal for in the algorithm class for this
if live:
start = pd.Timestamp.utcnow()
# TODO: fix the end data.
end = start + timedelta(hours=8760)
data = DataPortalExchangeLive(exchanges=exchanges,
asset_finder=env.asset_finder,
trading_calendar=open_calendar,
first_trading_day=pd.to_datetime(
'today', utc=True))
def fetch_capital_base(exchange, attempt_index=0):
"""
Fetch the base currency amount required to bootstrap
the algorithm against the exchange.
The algorithm cannot continue without this value.
:param exchange: the targeted exchange
:param attempt_index:
:return capital_base: the amount of base currency available for
trading
"""
try:
log.debug('retrieving capital base in {} to bootstrap '
'exchange {}'.format(base_currency, exchange_name))
balances = exchange.get_balances()
except ExchangeRequestError as e:
if attempt_index < 20:
log.warn('could not retrieve balances on {}: {}'.format(
exchange.name, e))
sleep(5)
return fetch_capital_base(exchange, attempt_index + 1)
else:
raise ExchangeRequestErrorTooManyAttempts(
attempts=attempt_index, error=e)
if base_currency in balances:
return balances[base_currency]
else:
raise BaseCurrencyNotFoundError(base_currency=base_currency,
exchange=exchange_name)
capital_base = 0
for exchange_name in exchanges:
exchange = exchanges[exchange_name]
capital_base += fetch_capital_base(exchange)
sim_params = create_simulation_parameters(start=start,
end=end,
capital_base=capital_base,
emission_rate='minute',
data_frequency='minute')
# TODO: use the constructor instead
sim_params._arena = 'live'
algorithm_class = partial(ExchangeTradingAlgorithmLive,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph)
else:
# Removed the existing Poloniex fork to keep things simple
# We can add back the complexity if required.
# I don't think that we should have arbitrary price data bundles
# Instead, we should center this data around exchanges.
# We still need to support bundles for other misc data, but we
# can handle this later.
data = DataPortalExchangeBacktest(exchanges=exchanges,
asset_finder=None,
trading_calendar=open_calendar,
first_trading_day=start,
last_available_session=end)
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
)
algorithm_class = partial(ExchangeTradingAlgorithmBacktest,
exchanges=exchanges)
perf = algorithm_class(
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=sim_params,
**{
'initialize': initialize,
'handle_data': handle_data,
'before_trading_start': before_trading_start,
'analyze': analyze,
} if algotext is None else {
'algo_filename': getattr(algofile, 'name', '<algorithm>'),
'script': algotext,
}).run(
data,
overwrite_sim_params=False,
)
if output == '-':
click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output)
return perf
def test_daily_data_to_minute_table(self):
exchange_name = 'poloniex'
# Switch between daily and minute for testing
data_frequency = 'daily'
# data_frequency = 'minute'
exchange = get_exchange(exchange_name)
assets = [
exchange.get_asset('eth_btc'),
exchange.get_asset('etc_btc'),
]
start = pd.to_datetime('2017-9-1', utc=True)
end = pd.to_datetime('2017-9-30', utc=True)
# Preparing the bundle folder
root = get_exchange_folder(exchange.name)
path = BUNDLE_NAME_TEMPLATE.format(
root=root,
frequency=data_frequency
)
ensure_directory(path)
exchange_bundle = ExchangeBundle(exchange)
calendar = get_calendar('OPEN')
# We are using a BcolzMinuteBarWriter even though the data is daily
# Each day has a maximum of one bar
# I tried setting the minutes_per_day to 1 will not create
# unnecessary bars
writer = BcolzExchangeBarWriter(
rootdir=path,
data_frequency=data_frequency,
start_session=start,
end_session=end,
write_metadata=True
)
# This will read the daily data in a bundle created by
# the daily writer. It will write to the minute writer which
# we are passing.
# Ingesting a second asset to ensure that multiple chunks
# don't override each other
for asset in assets:
exchange_bundle.ingest_ctable(
asset=asset,
data_frequency=data_frequency,
period='2017',
start_dt=start,
end_dt=end,
writer=writer,
empty_rows_behavior='strip'
)
reader = BcolzExchangeBarReader(rootdir=path,
data_frequency=data_frequency)
# Reading the two assets to ensure that no data was lost
for asset in assets:
sid = asset.sid
daily_values = reader.load_raw_arrays(
fields=['open', 'high', 'low', 'close', 'volume'],
start_dt=start,
end_dt=end,
sids=[sid],
)
print('found {} rows for last ingestion'.format(
len(daily_values[0]))
)
pass