def _get_averages(self, market: Market) -> Tuple[Decimal, Decimal]:
now = self._datetime_factory.now()
long_average = self._candle_storage.mean(
market.name, self._strategy_run.pair, CANDLE_STORAGE_FIELD_CLOSE,
DateTimeInterval(now - self._long_average_interval, now))
short_average = self._candle_storage.mean(
market.name, self._strategy_run.pair, CANDLE_STORAGE_FIELD_CLOSE,
DateTimeInterval(now - self._short_average_interval, now))
return long_average, short_average
def first_tick_initialize_strategy_data(self, markets: List[Market]) -> None:
market = self.get_market(markets)
current_time = self._datetime_factory.now()
interval = DateTimeInterval(current_time - 4 * self._candle_size.get_as_time_delta(), current_time)
candles = self._candle_storage.find_by(
market_name=market.name,
pair=self._strategy_run.pair,
interval=interval,
candle_size=self._candle_size
)
if len(candles) not in [4, 5]:
raise SkipTickException(
'Expected to get at 4 or 5 candles, but only {} given. Do you have enough data?'.format(len(candles))
)
if len(candles) == 5: # First and last candle can be cut in half, we dont need the first half-candle.
candles.pop(0)
first_candle = create_initial_heikin_ashi_candle(candles[0])
self._second_previous_candle = candle_to_heikin_ashi(candles[1], first_candle)
self._first_previous_candle = candle_to_heikin_ashi(candles[2], self._second_previous_candle)
self._current_unfinished_candle = candle_to_heikin_ashi(candles[3], self._first_previous_candle)
def find_by(self) -> List[StrategyRun]:
self._connection.begin()
cursor = self._connection.cursor()
cursor.execute('SELECT * FROM `strategy_runs` ORDER BY `run_at` DESC')
result = []
for row in cursor.fetchall():
result.append(StrategyRun(
UUID(row[0]),
datetime.datetime.fromtimestamp(row[1], tz=datetime.timezone.utc),
deserialize_pair(row[2]),
deserialize_strategy_run_markets(json.loads(row[3])),
row[4],
json.loads(row[5]),
DateTimeInterval(
datetime.datetime.fromtimestamp(row[6], tz=datetime.timezone.utc),
datetime.datetime.fromtimestamp(row[7], tz=datetime.timezone.utc) \
if row[7] is not None else None,
),
row[8],
row[9],
))
cursor.close()
self._connection.commit()
return result
def find_by(
self,
market_name: str,
pair: Pair,
interval: DateTimeInterval = DateTimeInterval(None, None),
candle_size: CandleSize = CandleSize(CANDLE_SIZE_UNIT_MINUTE, 1)
) -> List[Candle]:
parameters = {
CANDLE_STORAGE_FIELD_MARKET: "= '{}'".format(market_name),
CANDLE_STORAGE_FIELD_PAIR: "= '{}'".format(serialize_pair(pair)),
}
if interval.since is not None:
parameters['"time" >'] = "'{}'".format(interval.since.isoformat())
if interval.till is not None:
parameters['"time" <'] = "'{}'".format(interval.till.isoformat())
select = '*'
if not candle_size.is_one_minute():
select = 'FIRST("open") AS "open", MAX("high") AS "high", MIN("low") AS "low", LAST("close") AS "close"'
sql = 'SELECT {} FROM "{}" WHERE '.format(select,
MEASUREMENT_CANDLES_NAME)
where = []
for key, value in parameters.items():
where.append('{} {}'.format(key, value))
sql += ' AND '.join(where)
sql += self._get_group_by(candle_size)
result: ResultSet = self._client.query(sql)
data = list(result.get_points())
return self._parse_db_result_into_candles(data, market_name, pair,
candle_size)
def test_mean(influx_database: InfluxDBClient, expected_mean: int,
minute_interval: Tuple[int, int]):
storage = CandleInnoDbStorage(influx_database)
storage.write_candles(
[_create_dummy_candle(10, 8000),
_create_dummy_candle(20, 8300)])
interval = DateTimeInterval(
datetime.datetime(2017,
7,
2,
0,
minute_interval[0],
0,
tzinfo=datetime.timezone.utc),
datetime.datetime(2017,
7,
2,
0,
minute_interval[1],
0,
tzinfo=datetime.timezone.utc))
mean = storage.mean(DUMMY_MARKET, BTC_USD_PAIR, CANDLE_STORAGE_FIELD_CLOSE,
interval)
assert Decimal(expected_mean) == mean
def mean(
self,
market: str,
pair: Pair,
field: str,
interval: DateTimeInterval = DateTimeInterval(None, None)
) -> Decimal:
pass
def find_by(
self,
market_name: str,
pair: Pair,
interval: DateTimeInterval = DateTimeInterval(None, None),
candle_size: CandleSize = CandleSize(CANDLE_SIZE_UNIT_MINUTE, 1)
) -> List[Candle]:
raise NotImplementedError()
def mean(
self,
market: str,
pair: Pair,
field: str,
interval: DateTimeInterval = DateTimeInterval(None, None)
) -> Decimal:
raise NotImplementedError()
def test_mean_no_data_raise_exception(influx_database: InfluxDBClient):
"""We want to raise exception to prevent invalid signal by dropping some price to 0."""
storage = CandleInnoDbStorage(influx_database)
interval = DateTimeInterval(
datetime.datetime(2017, 7, 2, 0, 0, 0, tzinfo=datetime.timezone.utc),
datetime.datetime(2017, 7, 2, 0, 30, 0, tzinfo=datetime.timezone.utc))
with pytest.raises(NoCandlesForMarketInStorageException):
storage.mean(DUMMY_MARKET, BTC_USD_PAIR, CANDLE_STORAGE_FIELD_CLOSE,
interval)
def test_immutability():
date0 = datetime.datetime(2017,
12,
31,
0,
0,
0,
tzinfo=datetime.timezone.utc)
date1 = datetime.datetime(2018,
1,
1,
0,
0,
0,
tzinfo=datetime.timezone.utc)
date2 = datetime.datetime(2018,
1,
2,
0,
0,
0,
tzinfo=datetime.timezone.utc)
date3 = datetime.datetime(2018,
1,
3,
0,
0,
0,
tzinfo=datetime.timezone.utc)
interval = DateTimeInterval(date1, date2)
assert str(
interval) == '[2018-01-01T00:00:00+00:00, 2018-01-02T00:00:00+00:00]'
new_till_interval = interval.with_till(date3)
assert str(new_till_interval
) == '[2018-01-01T00:00:00+00:00, 2018-01-03T00:00:00+00:00]'
assert interval != new_till_interval
new_since_interval = interval.with_since(date0)
assert str(new_since_interval
) == '[2017-12-31T00:00:00+00:00, 2018-01-02T00:00:00+00:00]'
assert interval != new_since_interval
def find_by(self,
market_name: str,
pair: Pair,
status: Union[str, None] = None,
direction: Union[str, None] = None,
interval: DateTimeInterval = DateTimeInterval(None, None),
strategy_run_id: Union[str, None] = None) -> List[Order]:
return self._order_storage.find_by(market_name, pair, status,
direction, interval,
strategy_run_id)
def _find_last_candle_for_subscription(
self, candle_storage: CandleStorage,
subscription: LastCandleSubscription) -> Candle:
since = self._datetime_factory.now(
) - SAFETY_MULTIPLIER * subscription.candle_size.get_as_time_delta()
interval = DateTimeInterval(since, None)
candles = candle_storage.find_by(market_name=subscription.market_name,
pair=subscription.pair,
interval=interval,
candle_size=subscription.candle_size)
assert candles != [], 'Its expected to found candles after getting candle-added event. But none found.'
return candles[-1]
def export_to_file(self,
filename: str,
market_name: str,
pair: Pair,
interval: DateTimeInterval = DateTimeInterval(
None, None)):
candles = self._candle_storage.find_by(market_name=market_name,
pair=pair,
interval=interval)
data = serialize_candles(candles)
with open(filename, 'w') as outfile:
json.dump(data, outfile)
def test_save_strategy_run_and_find_it(
mysql_connection: MySQLdb.Connection) -> None:
storage = StrategyRunStorage(mysql_connection)
strategy_run_to_save = StrategyRun(
UUID('637f46a2-d008-48ba-9899-322abb56b425'), DUMMY_DATE,
Pair('USD', 'BTC'),
[StrategyRunMarket('yolo_market', 'yolo_plugin_name', {'foo': 'BAR'})],
'strategy_dummy_name', {'gandalf': 'Gandalf the Gray'},
DateTimeInterval(DUMMY_DATE,
None), 'candle_dummy_storage', 'order_dummy_storage')
storage.insert(strategy_run_to_save)
strategy_runs = storage.find_by()
assert len(strategy_runs) == 1
assert str(strategy_runs[0].strategy_run_id) == str(
strategy_run_to_save.strategy_run_id)
def export_orders(
ctx: Context,
market_name,
pair: Tuple[str, str],
interval: Tuple[str, str],
output_file: str,
order_storage: str
) -> None:
storage = di_container.order_storage_plugins.get_order_storage(order_storage)
pair_obj = Pair(pair[0], pair[1])
interval_obj = DateTimeInterval(
dateutil.parser.parse(interval[0]).replace(tzinfo=datetime.timezone.utc),
dateutil.parser.parse(interval[1]).replace(tzinfo=datetime.timezone.utc)
)
exporter = OrderExporter(storage)
exporter.export_to_file(output_file, market_name, pair_obj, interval_obj)
def run_strategy(
ctx: Context,
strategy_name: str,
pair: Tuple[str, str],
market_names: Tuple[str],
configuration_file: Union[str, None],
candle_storage: str,
order_storage: str,
market_plugin: str
) -> None:
pair_obj = Pair(pair[0], pair[1])
strategy_configuration: Dict = {}
if configuration_file is not None:
strategy_configuration = load_configuration_from_file(configuration_file)
strategy_run_at = di_container.datetime_factory.now()
strategy_run = StrategyRun(
uuid.uuid4(),
strategy_run_at,
pair_obj,
[StrategyRunMarket(market_plugin, market_name, {}) for market_name in market_names],
strategy_name,
strategy_configuration,
DateTimeInterval(strategy_run_at, None),
candle_storage,
order_storage
)
di_container.strategy_run_storage.insert(strategy_run)
di_container.event_emitter.emit_new_strategy_run(strategy_run)
try:
di_container.strategy_standard_runner.run(strategy_run)
except StrategyNotProvidedByAnyPluginException as e:
_terminate_strategy_run(strategy_run)
print_error_and_terminate(str(e))
except ForEndUserException as e:
_terminate_strategy_run(strategy_run)
print_error_and_terminate(str(e))
except KeyboardInterrupt:
_terminate_strategy_run(strategy_run)
pass
def mean(
self,
market_name: str,
pair: Pair,
field: str,
interval: DateTimeInterval = DateTimeInterval(None, None)
) -> Decimal:
sql = '''
SELECT MEAN("{}") AS "field_mean"
FROM "{}" WHERE "time" > '{}' AND "time" < '{}' AND "pair"='{}' AND "market"='{}'
GROUP BY "{}"
'''.format(field, MEASUREMENT_CANDLES_NAME, interval.since.isoformat(),
interval.till.isoformat(), serialize_pair(pair),
market_name, field)
result: ResultSet = self._client.query(sql)
self._validate_result_has_some_data(market_name, result)
mean = list(result.items()[0][1])[0]['field_mean']
return Decimal(mean)
def get_interval_for_datetime(self,
time: datetime.datetime) -> DateTimeInterval:
time = time.replace(second=0)
unit_range = UNIT_RANGES[self.unit]
current_value = self._get_date_value_by_unit(time)
number_of_buckets = unit_range / self.size
bucket_number = int(math.floor(current_value / self.size))
if bucket_number > number_of_buckets:
raise ValueError(
'Bucket {} is higher than max number of buckets [{} ')
value = bucket_number * self.size
since = self._set_value_by_unit(time, value)
till = since + self.get_as_time_delta()
return DateTimeInterval(since, till)
def create_strategy_to_test(candle_storage: CandleStorage,
order_storage: OrderStorage,
emitter_mock: EventEmitter,
datetime_factory: DateTimeFactory):
return DoubleCrossoverStrategy(
candle_storage,
OrderFacade(order_storage, create_portfolio_snapshot_mock(),
emitter_mock), datetime_factory,
StrategyRun(
STRATEGY_RUN_ID,
datetime.datetime(2017,
12,
2,
14,
0,
0,
tzinfo=datetime.timezone.utc), BTC_USD_PAIR, [],
'', {
'long_average_interval': 60 * 60,
'short_average_interval': 15 * 60,
}, DateTimeInterval(None, None), '', ''))
def test_serialize_strategy() -> None:
strategy_run_market = StrategyRun(
UUID('8b3213c8-2c07-4283-8197-a9babfcc1ec8'),
DUMMY_DATE,
Pair('USD', 'BTC'),
[],
'strategy_xyz',
{},
DateTimeInterval(),
'c',
'o'
)
assert serialize_strategy_run(strategy_run_market) == {
'candle_storage_name': 'c',
'interval': {'since': None, 'till': None},
'markets': [],
'order_storage_name': 'o',
'pair': 'USD_BTC',
'run_at': '2017-11-26T10:11:12+00:00',
'strategy_configuration': {},
'strategy_name': 'strategy_xyz',
'strategy_run_id': '8b3213c8-2c07-4283-8197-a9babfcc1ec8',
}
def _tick(self, markets: List[Market]) -> None:
market = self.get_market(markets)
current_time = self._datetime_factory.now()
interval = DateTimeInterval(current_time - 2 * self._candle_size.get_as_time_delta(), current_time)
candles = self._candle_storage.find_by(
market_name=market.name,
pair=self._strategy_run.pair,
interval=interval,
candle_size=self._candle_size
)
if len(candles) not in [2, 3]:
raise SkipTickException(
'Expected to get at 2 or 3 candles, but only {} given. Do you have enough data?'.format(len(candles))
)
if len(candles) == 3: # First and last candle can be cut in half, we don't need the first half-candle.
candles.pop(0)
self.update_trend()
if candles[0].time == self._current_unfinished_candle.time:
self._second_previous_candle = self._first_previous_candle
self._first_previous_candle = candle_to_heikin_ashi(candles[0], self._first_previous_candle)
self._current_unfinished_candle = candle_to_heikin_ashi(candles[1], self._first_previous_candle)
self.log_tick()
try:
self.check_for_buy_or_sell(market)
except NotEnoughBalanceToPerformOrderException as e:
# Intentionally, this strategy does not need state of order,
# just ignores buy/sell and waits for next signal.
logger.warning(str(e))
def find_by(self,
market_name: str,
pair: Pair,
status: Union[str, None] = None,
direction: Union[str, None] = None,
interval: DateTimeInterval = DateTimeInterval(None, None),
strategy_run_id: Union[str, None] = None) -> List[Order]:
assert status in POSSIBLE_ORDER_STATUSES or status is None, 'Invalid status: "{}"'.format(
status)
parameters = {
ORDER_FIELD_MARKET: "= '{}'".format(market_name),
ORDER_FIELD_PAIR: "= '{}'".format(serialize_pair(pair)),
}
if status is not None:
parameters[ORDER_FIELD_STATUS] = "= '{}'".format(status)
if direction is not None:
parameters[ORDER_FIELD_DIRECTION] = "= '{}'".format(direction)
if interval.since is not None:
parameters['"time" >'] = "'{}'".format(interval.since.isoformat())
if interval.till is not None:
parameters['"time" <'] = "'{}'".format(interval.till.isoformat())
if strategy_run_id is not None:
parameters[ORDER_FIELD_STRATEGY_RUN_ID] = "= '{}'".format(
strategy_run_id)
sql = 'SELECT * FROM "{}" WHERE '.format(self._measurement_name)
where = []
for key, value in parameters.items():
where.append('{} {}'.format(key, value))
sql += ' AND '.join(where)
result: ResultSet = self._client.query(sql)
data = list(result.get_points())
return [self._create_order_from_serialized(row) for row in data]
def test_subscription_can_be_found():
storage = SubscriptionStorage()
last_candle_subscription = LastCandleSubscription(
'1',
'foo_storage',
'bar_market',
Pair('USD', 'BTC'),
CandleSize(CANDLE_SIZE_UNIT_MINUTE, 1)
)
storage.subscribe(last_candle_subscription)
subscription_interval = DateTimeInterval(
datetime.datetime(2018, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc),
datetime.datetime(2018, 1, 2, 0, 0, 0, tzinfo=datetime.timezone.utc)
)
date_in_interval = datetime.datetime(2018, 1, 1, 12, 0, 0, tzinfo=datetime.timezone.utc)
date_outside_interval = datetime.datetime(2018, 1, 3, 0, 0, tzinfo=datetime.timezone.utc)
new_order_subscription = NewOrderSubscription(
'1',
'foo_storage',
'bar_market',
Pair('USD', 'BTC'),
subscription_interval
)
storage.subscribe(new_order_subscription)
assert storage.find_subscriptions_for_event('unknown_event_name') == []
candle_suitable_for_subscription = Candle(
'bar_market',
Pair('USD', 'BTC'),
date_in_interval,
Decimal('11000'),
Decimal('11000'),
Decimal('11000'),
Decimal('11000')
)
assert storage.find_subscriptions_for_event(EVENT_LAST_CANDLE_UPDATED)[0] == last_candle_subscription
assert storage.find_subscriptions_for_event(
EVENT_LAST_CANDLE_UPDATED,
{'storage': 'foo_storage', 'candle': serialize_candle(candle_suitable_for_subscription)}
)[0] == last_candle_subscription
assert storage.find_subscriptions_for_event(
EVENT_LAST_CANDLE_UPDATED,
{'storage': 'gandalf', 'candle': serialize_candle(candle_suitable_for_subscription)}
) == []
assert storage.find_subscriptions_for_event(
EVENT_LAST_CANDLE_UPDATED,
{
'storage': 'foo_storage',
'candle': serialize_candle(
Candle(
'bar_market',
Pair('OMG', 'WTF'),
date_in_interval,
Decimal('11000'),
Decimal('11000'),
Decimal('11000'),
Decimal('11000')
)
)
}
) == []
assert storage.find_subscriptions_for_event(
EVENT_LAST_CANDLE_UPDATED,
{
'storage': 'foo_storage',
'candle': serialize_candle(
Candle(
'wtf_market',
Pair('USD', 'BTC'),
date_in_interval,
Decimal('11000'),
Decimal('11000'),
Decimal('11000'),
Decimal('11000')
)
)
}
) == []
order_suitable_for_subscription = _crate_serialized_order(Pair('USD', 'BTC'), 'bar_market', date_in_interval)
assert storage.find_subscriptions_for_event(EVENT_NEW_ORDER)[0] == new_order_subscription
assert storage.find_subscriptions_for_event(
EVENT_NEW_ORDER,
{'storage': 'foo_storage', 'order': order_suitable_for_subscription}
)[0] == new_order_subscription
assert storage.find_subscriptions_for_event(
EVENT_NEW_ORDER,
{'storage': 'gandalf', 'order': order_suitable_for_subscription}
) == []
assert storage.find_subscriptions_for_event(
EVENT_NEW_ORDER,
{
'storage': 'foo_storage',
'order': _crate_serialized_order(Pair('USD', 'BTC'), 'bar_market', date_outside_interval)
}
) == []
assert storage.find_subscriptions_for_event(
EVENT_NEW_ORDER,
{
'storage': 'foo_storage',
'order': _crate_serialized_order(Pair('OMG', 'WTF'), 'bar_market', date_in_interval)
}
) == []
assert storage.find_subscriptions_for_event(
EVENT_NEW_ORDER,
{
'storage': 'foo_storage',
'order': _crate_serialized_order(Pair('USD', 'BTC'), 'wtf_market', date_in_interval)
}
) == []
storage.unsubscribe(EVENT_NEW_ORDER, '2')
assert storage.find_subscriptions_for_event(EVENT_NEW_ORDER)[0] == new_order_subscription
storage.unsubscribe(EVENT_NEW_ORDER, '1')
assert storage.find_subscriptions_for_event(EVENT_NEW_ORDER) == []
storage.unsubscribe(EVENT_LAST_CANDLE_UPDATED, '2')
assert storage.find_subscriptions_for_event(EVENT_LAST_CANDLE_UPDATED)[0] == last_candle_subscription
storage.unsubscribe(EVENT_LAST_CANDLE_UPDATED, '1')
assert storage.find_subscriptions_for_event(EVENT_LAST_CANDLE_UPDATED) == []
BTC_USD_PAIR, ORDER_TYPE_LIMIT, Decimal('1'), Decimal('8000'),
'aaa-id-from-market', ORDER_STATUS_CLOSED,
datetime.datetime(2017, 11, 26, 10, 11, 12, tzinfo=datetime.timezone.utc))
DUMMY_OPEN_ORDER = Order(
UUID('16fd2706-8baf-433b-82eb-8c7fada847db'), STRATEGY_RUN_ID,
DUMMY_MARKET_NAME, DIRECTION_BUY,
datetime.datetime(2017, 11, 26, 10, 11, 12,
tzinfo=datetime.timezone.utc), BTC_USD_PAIR,
ORDER_TYPE_LIMIT, Decimal('1'), Decimal('8000'), 'aaa-id-from-market')
STRATEGY_RUN = StrategyRun(
STRATEGY_RUN_ID,
datetime.datetime(2017, 11, 26, 10, 11, 12, tzinfo=datetime.timezone.utc),
BTC_USD_PAIR, [], '', {
'long_average_interval': 60 * 60,
'short_average_interval': 15 * 60,
}, DateTimeInterval(None, None), '', '')
CLOSED_ORDER_INFO = OrderMarketInfo(order=DUMMY_OPEN_ORDER,
is_open=False,
closed_at=datetime.datetime(
2017,
11,
26,
10,
11,
12,
tzinfo=datetime.timezone.utc),
quantity_remaining=Decimal('0'))
STILL_OPEN_ORDER_INFO = OrderMarketInfo(order=DUMMY_OPEN_ORDER,
is_open=True,