def get_dataset(self, data_source_name, start=None, end=None):
if self._marketplace is None:
self._marketplace = Marketplace()
return self._marketplace.get_dataset(
data_source_name, start, end,
)
def ls(ctx):
"""List all available datasets.
"""
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
marketplace.list()
def subscribe(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
marketplace = Marketplace()
marketplace.subscribe(dataset)
def ingest(ctx, dataset, data_frequency, start, end):
if dataset is None:
ctx.fail("must specify a dataset to clean with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
def clean(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to ingest with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
marketplace = Marketplace()
marketplace.clean(dataset)
click.echo('Done', sys.stdout)
def publish(ctx, dataset, datadir, watch):
marketplace = Marketplace()
if dataset is None:
ctx.fail("must specify a dataset to publish data for "
" with '--dataset'\n")
if datadir is None:
ctx.fail("must specify a datadir where to find the files to publish "
" with '--datadir'\n")
marketplace.publish(dataset, datadir, watch)
def test_clean(self):
marketplace = Marketplace()
marketplace.clean('marketcap')
pass
def test_publish(self):
marketplace = Marketplace()
datadir = '/Users/fredfortier/Downloads/marketcap_test_single'
marketplace.publish('marketcap1234', datadir, False)
pass
def test_ingest(self):
marketplace = Marketplace()
ds_def = marketplace.ingest('marketcap')
pass
def clean(ctx, dataset):
"""Clean/Remove local data for a given dataset.
"""
marketplace = Marketplace()
marketplace.clean(dataset)
def subscribe(ctx, dataset):
"""Subscribe to an existing dataset.
"""
marketplace = Marketplace()
marketplace.subscribe(dataset)
def get_withdraw_amount(ctx, dataset):
"""Get withdraw amount owner is entitled to.
"""
marketplace = Marketplace()
marketplace.get_withdraw_amount(dataset)
def test_ingest(self):
marketplace = Marketplace()
ds_def = marketplace.ingest('marketcap1234')
pass
def test_subscribe(self):
marketplace = Marketplace()
marketplace.subscribe('marketcap2222')
pass
def test_register(self):
marketplace = Marketplace()
marketplace.register()
pass
def register(ctx):
marketplace = Marketplace()
marketplace.register()
def test_list(self):
marketplace = Marketplace()
marketplace.list()
pass
def test_publish(self):
marketplace = Marketplace()
datadir = '/Users/fredfortier/Downloads/marketcap_test_single'
marketplace.publish('marketcap1234', datadir, False)
pass
class ExchangeTradingAlgorithmBase(TradingAlgorithm):
def __init__(self, *args, **kwargs):
self.exchanges = kwargs.pop('exchanges', None)
self.simulate_orders = kwargs.pop('simulate_orders', None)
super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
self.current_day = None
if self.simulate_orders is None and \
self.sim_params.arena == 'backtest':
self.simulate_orders = True
# Operations with retry features
self.attempts = dict(
get_transactions_attempts=5,
order_attempts=5,
synchronize_portfolio_attempts=5,
get_order_attempts=5,
get_open_orders_attempts=5,
cancel_order_attempts=5,
get_spot_value_attempts=5,
get_history_window_attempts=5,
retry_sleeptime=5,
)
self.blotter = ExchangeBlotter(
data_frequency=self.data_frequency,
# Default to NeverCancel in catalyst
cancel_policy=self.cancel_policy,
simulate_orders=self.simulate_orders,
exchanges=self.exchanges,
attempts=self.attempts,
)
self._marketplace = None
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
Helper method for converting deprecated limit_price and stop_price
arguments into ExecutionStyle instances.
This function assumes that either style == None or (limit_price,
stop_price) == (None, None).
"""
if stop_price:
raise OrderTypeNotSupported(order_type='stop')
if style:
if limit_price is not None:
raise ValueError(
'An order style and a limit price was included in the '
'order. Please pick one to avoid any possible conflict.')
# Currently limiting order types or limit and market to
# be in-line with CXXT and many exchanges. We'll consider
# adding more order types in the future.
if not isinstance(style, ExchangeLimitOrder) or \
not isinstance(style, MarketOrder):
raise OrderTypeNotSupported(
order_type=style.__class__.__name__)
return style
if limit_price:
return ExchangeLimitOrder(limit_price)
else:
return MarketOrder()
@api_method
def set_commission(self, maker=None, taker=None):
"""Sets the maker and taker fees of the commission model for the simulation.
Parameters
----------
maker : float
The taker fee - taking from the order book.
taker : float
The maker fee - adding to the order book.
"""
key = list(self.blotter.commission_models.keys())[0]
if maker is not None:
self.blotter.commission_models[key].maker = maker
if taker is not None:
self.blotter.commission_models[key].taker = taker
@api_method
def set_slippage(self, spread=None):
"""Set the spread of the slippage model for the simulation.
Parameters
----------
spread : float
The spread to be set.
"""
key = list(self.blotter.slippage_models.keys())[0]
if spread is not None:
self.blotter.slippage_models[key].spread = spread
def _calculate_order(self,
asset,
amount,
limit_price=None,
stop_price=None,
style=None):
# Raises a ZiplineError if invalid parameters are detected.
self.validate_order_params(asset, amount, limit_price, stop_price,
style)
# Convert deprecated limit_price and stop_price parameters to use
# ExecutionStyle objects.
style = self.__convert_order_params_for_blotter(
limit_price, stop_price, style)
return amount, style
def _calculate_order_target_amount(self, asset, target):
"""
removes order amounts so we won't run into issues
when two orders are placed one after the other.
it then proceeds to removing positions amount at TradingAlgorithm
:param asset:
:param target:
:return: target
"""
if asset in self.blotter.open_orders:
for open_order in self.blotter.open_orders[asset]:
current_amount = open_order.amount
target -= current_amount
target = super(ExchangeTradingAlgorithmBase, self). \
_calculate_order_target_amount(asset, target)
return target
def round_order(self, amount, asset):
"""
We need fractions with cryptocurrencies
:param amount:
:return:
"""
return round_nearest(amount, asset.min_trade_size)
@api_method
def get_dataset(self, data_source_name, start=None, end=None):
if self._marketplace is None:
self._marketplace = Marketplace()
return self._marketplace.get_dataset(
data_source_name,
start,
end,
)
@api_method
@preprocess(symbol_str=ensure_upper_case)
def symbol(self, symbol_str, exchange_name=None):
"""Lookup a Trading pair by its ticker symbol.
Catalyst defines its own set of "universal" symbols to reference
trading pairs across exchanges. This is required because exchanges
are not adhering to a universal symbolism. For example, Bitfinex
uses the BTC symbol for Bitcon while Kraken uses XBT. In addition,
pairs are sometimes presented differently. For example, Bitfinex
puts the market currency before the base currency without a
separator, Bittrex puts the quote currency first and uses a dash
seperator.
Here is the Catalyst convention: [Base Currency]_[Quote Currency]
For example: btc_usd, eth_btc, neo_eth, ltc_eur.
The symbol for each currency (e.g. btc, eth, ltc) is generally
aligned with the Bittrex exchange.
Parameters
----------
symbol_str : str
The ticker symbol for the TradingPair to lookup.
exchange_name: str
The name of the exchange containing the symbol
Returns
-------
tradingPair : TradingPair
The TradingPair that held the ticker symbol on the current
symbol lookup date.
Raises
------
SymbolNotFound
Raised when the symbols was not held on the current lookup date.
"""
# If the user has not set the symbol lookup date,
# use the end_session as the date for sybmol->sid resolution.
_lookup_date = self._symbol_lookup_date \
if self._symbol_lookup_date is not None \
else self.sim_params.end_session
if exchange_name is None:
exchange = list(self.exchanges.values())[0]
else:
exchange = self.exchanges[exchange_name]
data_frequency = self.data_frequency \
if self.sim_params.arena == 'backtest' else None
return self.asset_finder.lookup_symbol(symbol=symbol_str,
exchange=exchange,
data_frequency=data_frequency,
as_of_date=_lookup_date)
def prepare_period_stats(self, start_dt, end_dt):
"""
Creates a dictionary representing the state of the tracker.
Parameters
----------
start_dt: datetime
end_dt: datetime
Notes
-----
I rewrote this in an attempt to better control the stats.
I don't want things to happen magically through complex logic
pertaining to backtesting.
"""
tracker = self.perf_tracker
cum = tracker.cumulative_performance
pos_stats = cum.position_tracker.stats()
period_stats = calc_period_stats(pos_stats, cum.ending_cash)
stats = dict(
period_start=tracker.period_start,
period_end=tracker.period_end,
capital_base=tracker.capital_base,
progress=tracker.progress,
ending_value=cum.ending_value,
ending_exposure=cum.ending_exposure,
capital_used=cum.cash_flow,
starting_value=cum.starting_value,
starting_exposure=cum.starting_exposure,
starting_cash=cum.starting_cash,
ending_cash=cum.ending_cash,
portfolio_value=cum.ending_cash + cum.ending_value,
pnl=cum.pnl,
returns=cum.returns,
period_open=start_dt,
period_close=end_dt,
gross_leverage=period_stats.gross_leverage,
net_leverage=period_stats.net_leverage,
short_exposure=pos_stats.short_exposure,
long_exposure=pos_stats.long_exposure,
short_value=pos_stats.short_value,
long_value=pos_stats.long_value,
longs_count=pos_stats.longs_count,
shorts_count=pos_stats.shorts_count,
)
# Merging cumulative risk
stats.update(tracker.cumulative_risk_metrics.to_dict())
# Merging latest recorded variables
stats.update(self.recorded_vars)
period = tracker.todays_performance
stats['positions'] = period.position_tracker.get_positions_list()
# we want the key to be absent, not just empty
# Only include transactions for given dt
stats['transactions'] = []
for date in period.processed_transactions:
if start_dt <= date < end_dt:
transactions = period.processed_transactions[date]
for t in transactions:
stats['transactions'].append(t.to_dict())
stats['orders'] = []
for date in period.orders_by_modified:
if start_dt <= date < end_dt:
orders = period.orders_by_modified[date]
for order in orders:
stats['orders'].append(orders[order].to_dict())
return stats
def run(self, data=None, overwrite_sim_params=True):
data.attempts = self.attempts
return super(ExchangeTradingAlgorithmBase,
self).run(data, overwrite_sim_params)
def withdraw(ctx, dataset):
"""Withdraw amount you are entitled to.
"""
marketplace = Marketplace()
marketplace.withdraw(dataset)
def test_clean(self):
marketplace = Marketplace()
marketplace.clean('marketcap')
pass
def test_list(self):
marketplace = Marketplace()
marketplace.list()
pass
def test_register(self):
marketplace = Marketplace()
marketplace.register()
pass
def ingest(ctx, dataset, data_frequency, start, end):
"""Ingest a dataset (requires subscription).
"""
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
def test_subscribe(self):
marketplace = Marketplace()
marketplace.subscribe('marketcap')
pass
def register(ctx):
"""Register a new dataset.
"""
marketplace = Marketplace()
marketplace.register()
def ls(ctx):
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
marketplace.list()
