def from_history_base_entry(cls,
event: HistoryBaseEntry) -> 'StakingEvent':
"""
Read staking event from a history base entry.
May raise:
- DeserializationError
"""
return StakingEvent(
event_type=event.event_subtype,
asset=event.asset,
balance=event.balance,
timestamp=ts_ms_to_sec(event.timestamp),
location=event.location,
)
def serialize(self) -> Dict[str, Any]:
return {
'identifier': self.identifier,
'event_identifier': self.event_identifier,
'sequence_index': self.sequence_index,
'timestamp':
ts_ms_to_sec(self.timestamp), # serialize to api in seconds MS
'location': str(self.location),
'asset': self.asset.identifier,
'balance': self.balance.serialize(),
'event_type': self.event_type.serialize(),
'event_subtype': self.event_subtype.serialize_or_none(),
'location_label': self.location_label,
'notes': self.notes,
'counterparty': self.counterparty,
}
def rows_missing_prices_in_base_entries(
self,
filter_query: HistoryEventFilterQuery,
) -> List[Tuple[str, FVal, Asset, Timestamp]]:
"""
Get missing prices for history base entries based on filter query
"""
query, bindings = filter_query.prepare()
query = 'SELECT identifier, amount, asset, timestamp FROM history_events ' + query
result = []
cursor = self.db.conn.cursor()
cursor.execute(query, bindings)
for identifier, amount_raw, asset_name, timestamp in cursor:
try:
amount = deserialize_fval(
value=amount_raw,
name='historic base entry usd_value query',
location='query_missing_prices',
)
result.append(
(
identifier,
amount,
Asset(asset_name),
ts_ms_to_sec(TimestampMS(timestamp)),
),
)
except DeserializationError as e:
log.error(
f'Failed to read value from historic base entry {identifier} '
f'with amount. {str(e)}',
)
except UnknownAsset as e:
log.error(
f'Failed to read asset from historic base entry {identifier} '
f'with asset identifier {asset_name}. {str(e)}',
)
return result
def process_kraken_events_for_trade(
self,
trade_parts: List[HistoryBaseEntry],
adjustments: List[HistoryBaseEntry],
) -> Optional[Trade]:
"""Processes events from trade parts to a trade. If it's an adjustment
adds it to a separate list"""
if trade_parts[0].event_type == HistoryEventType.ADJUSTMENT:
adjustments.append(trade_parts[0])
return None # skip as they don't have same refid
event_id = trade_parts[0].event_identifier
is_spend_receive = False
trade_assets = []
spend_part, receive_part, fee_part, kfee_part = None, None, None, None
for trade_part in trade_parts:
if trade_part.event_type == HistoryEventType.RECEIVE:
is_spend_receive = True
receive_part = trade_part
elif trade_part.event_type == HistoryEventType.SPEND:
if trade_part.event_subtype == HistoryEventSubType.FEE:
fee_part = trade_part
else:
is_spend_receive = True
spend_part = trade_part
elif trade_part.event_type == HistoryEventType.TRADE:
if trade_part.event_subtype == HistoryEventSubType.FEE:
fee_part = trade_part
elif trade_part.asset == A_KFEE:
kfee_part = trade_part
elif trade_part.balance.amount < ZERO:
spend_part = trade_part
else:
receive_part = trade_part
if (trade_part.balance.amount != ZERO
and trade_part.event_subtype != HistoryEventSubType.FEE):
trade_assets.append(trade_part.asset)
if is_spend_receive and len(trade_parts) < 2:
log.warning(
f'Found kraken spend/receive events {event_id} with '
f'less than 2 parts. {trade_parts}', )
self.msg_aggregator.add_warning(
f'Found kraken spend/receive events {event_id} with '
f'less than 2 parts. Skipping...', )
return None
timestamp = ts_ms_to_sec(trade_parts[0].timestamp)
exchange_uuid = (str(event_id) + str(timestamp))
if len(trade_assets) != 2:
# This can happen some times (for lefteris 5 times since start of kraken usage)
# when the other part of a trade is so small it's 0. So it's either a
# receive event with no counterpart or a spend event with no counterpart.
# This happens for really really small amounts. So we add rate 0 trades
if spend_part is not None:
base_asset = spend_part.asset
trade_type = TradeType.SELL
amount = spend_part.balance.amount * -1
elif receive_part is not None:
base_asset = receive_part.asset
trade_type = TradeType.BUY
amount = receive_part.balance.amount
else:
log.warning(
f'Found historic trade entries with no counterpart {trade_parts}'
)
return None
trade = Trade(
timestamp=timestamp,
location=Location.KRAKEN,
base_asset=base_asset,
quote_asset=A_USD, # whatever
trade_type=trade_type,
amount=AssetAmount(amount),
rate=Price(ZERO),
fee=None,
fee_currency=None,
link=exchange_uuid,
)
return trade
if spend_part is None or receive_part is None:
log.error(
f'Failed to process {event_id}. Couldnt find spend/receive parts {trade_parts}',
)
self.msg_aggregator.add_error(
f'Failed to read trades for event {event_id}. '
f'More details are available at the logs', )
return None
spend_asset = spend_part.asset
receive_asset = receive_part.asset
if spend_asset.is_fiat() or trade_parts[0] == receive_part:
trade_type = TradeType.BUY
base_asset = receive_asset
quote_asset = spend_asset
amount = receive_part.balance.amount
if amount == ZERO:
self.msg_aggregator.add_warning(
f'Rate for kraken trade couldnt be calculated. Base amount is ZERO '
f'for event {event_id}. Skipping event', )
return None
rate = Price((spend_part.balance.amount / amount) * -1)
else:
trade_type = TradeType.SELL
base_asset = spend_asset
quote_asset = receive_asset
amount = -1 * spend_part.balance.amount
if amount == ZERO:
self.msg_aggregator.add_warning(
f'Rate for kraken trade couldnt be calculated. Base amount is ZERO '
f'for event {event_id}. Skipping event', )
return None
rate = Price((receive_part.balance.amount / amount))
# If kfee was found we use it as the fee for the trade
if kfee_part is not None and fee_part is None:
fee = Fee(kfee_part.balance.amount)
fee_asset = A_KFEE
elif (None, None) == (fee_part, kfee_part):
fee = None
fee_asset = None
elif fee_part is not None:
fee = Fee(fee_part.balance.amount)
fee_asset = fee_part.asset
trade = Trade(
timestamp=timestamp,
location=Location.KRAKEN,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(amount),
rate=rate,
fee=fee,
fee_currency=fee_asset,
link=exchange_uuid,
)
return trade
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
self.query_kraken_ledgers(start_ts=start_ts, end_ts=end_ts)
filter_query = HistoryEventFilterQuery.make(
from_ts=Timestamp(start_ts),
to_ts=Timestamp(end_ts),
event_types=[
HistoryEventType.DEPOSIT,
HistoryEventType.WITHDRAWAL,
],
location=Location.KRAKEN,
location_label=self.name,
)
events = self.history_events_db.get_history_events(
filter_query=filter_query,
has_premium=True,
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(events))
movements = []
get_attr = operator.attrgetter('event_identifier')
# Create a list of lists where each sublist has the events for the same event identifier
grouped_events = [
list(g) for k, g in itertools.groupby(sorted(events, key=get_attr),
get_attr)
] # noqa: E501
for movement_events in grouped_events:
if len(movement_events) == 2:
if movement_events[0].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[0].balance.amount)
movement = movement_events[1]
elif movement_events[
1].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[1].balance.amount)
movement = movement_events[0]
else:
self.msg_aggregator.add_error(
f'Failed to process deposit/withdrawal. {grouped_events}. Ignoring ...',
)
continue
else:
movement = movement_events[0]
fee = Fee(ZERO)
amount = movement.balance.amount
if movement.event_type == HistoryEventType.WITHDRAWAL:
amount = amount * -1
try:
asset = movement.asset
movement_type = movement.event_type
movements.append(
AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(
movement_type),
timestamp=ts_ms_to_sec(movement.timestamp),
address=None, # no data from kraken ledger endpoint
transaction_id=
None, # no data from kraken ledger endpoint
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=movement.event_identifier,
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement_events,
error=msg,
)
continue
return movements
def process_kraken_trades(
self,
raw_data: List[HistoryBaseEntry],
) -> Tuple[List[Trade], Timestamp]:
"""
Given a list of history events we process them to create Trade objects. The valid
History events type are
- Trade
- Receive
- Spend
- Adjustment
A pair of receive and spend events can be a trade and kraken uses this kind of event
for instant trades and trades made from the phone app. What we do in order to verify
that it is a trade is to check if we can find a pair with the same event id.
Also in some rare occasions Kraken may forcibly adjust something for you.
Example would be delisting of DAO token and forcible exchange to ETH.
Returns:
- The list of trades processed
- The biggest timestamp of all the trades processed
May raise:
- RemoteError if the pairs couldn't be correctly queried
"""
trades = []
max_ts = 0
get_attr = operator.attrgetter('event_identifier')
adjustments: List[HistoryBaseEntry] = []
# Create a list of lists where each sublist has the events for the same event identifier
grouped_events = [
list(g) for k, g in itertools.groupby(
sorted(raw_data, key=get_attr), get_attr)
] # noqa: E501
for trade_parts in grouped_events:
trade = self.process_kraken_events_for_trade(
trade_parts, adjustments)
if trade is None:
continue
trades.append(trade)
max_ts = max(max_ts, trade.timestamp)
adjustments.sort(key=lambda x: x.timestamp)
if len(adjustments) % 2 == 0:
for a1, a2 in pairwise(adjustments):
if a1.event_subtype is None or a2.event_subtype is None:
log.warning(
f'Found two kraken adjustment entries without a subtype: {a1} {a2}',
)
continue
if a1.event_subtype == HistoryEventSubType.SPEND and a2.event_subtype == HistoryEventSubType.RECEIVE: # noqa: E501
spend_event = a1
receive_event = a2
elif a2.event_subtype == HistoryEventSubType.SPEND and a2.event_subtype == HistoryEventSubType.RECEIVE: # noqa: E501
spend_event = a2
receive_event = a1
else:
log.warning(
f'Found two kraken adjustment with unmatching subtype {a1} {a2}',
)
continue
rate = Price(
abs(receive_event.balance.amount /
spend_event.balance.amount))
trade = Trade(
timestamp=ts_ms_to_sec(a1.timestamp),
location=Location.KRAKEN,
base_asset=receive_event.asset,
quote_asset=spend_event.asset,
trade_type=TradeType.BUY,
amount=AssetAmount(receive_event.balance.amount),
rate=rate,
fee=None,
fee_currency=None,
link='adjustment' + a1.event_identifier +
a2.event_identifier,
)
trades.append(trade)
else:
log.warning(
f'Got even number of kraken adjustment historic entries. '
f'Skipping reading them. {adjustments}', )
return trades, Timestamp(max_ts)
def get_timestamp_in_sec(self) -> Timestamp:
return ts_ms_to_sec(self.timestamp)