def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the base, quote, fee asset string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = TradeType.deserialize(data['trade_type'])
location = Location.deserialize(data['location'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=location,
base_asset=Asset(data['base_asset']),
quote_asset=Asset(data['quote_asset']),
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
# Contrary to the Coinbase documentation we will use created_at, and never
# payout_at. It seems like payout_at is not actually the time where the trade is settled.
# Reports generated by Coinbase use created_at as well
if raw_trade.get('created_at') is not None:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = TradeType.deserialize(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible
that some keys don't exist in a trade. The required fields are here:
https://bittrex.github.io/api/v3#definition-Order
May raise:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(bittrex_trade['fillQuantity'])
timestamp = deserialize_timestamp_from_date(
date=bittrex_trade['closedAt'], # we only check closed orders
formatstr='iso8601',
location='bittrex',
)
if 'limit' in bittrex_trade:
rate = deserialize_price(bittrex_trade['limit'])
else:
rate = Price(
deserialize_asset_amount(bittrex_trade['proceeds']) /
deserialize_asset_amount(bittrex_trade['fillQuantity']),
)
order_type = TradeType.deserialize(bittrex_trade['direction'])
fee = deserialize_fee(bittrex_trade['commission'])
base_asset, quote_asset = bittrex_pair_to_world(bittrex_trade['marketSymbol'])
log.debug(
'Processing bittrex Trade',
amount=amount,
rate=rate,
order_type=order_type,
fee=fee,
bittrex_pair=bittrex_trade['marketSymbol'],
base_asset=base_asset,
quote_asset=quote_asset,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_asset,
link=str(bittrex_trade['id']),
)
def deserialize_from_db(cls, entry: TradeDBTuple) -> 'Trade':
"""May raise:
- DeserializationError
- UnknownAsset
"""
return Trade(
timestamp=deserialize_timestamp(entry[1]),
location=Location.deserialize_from_db(entry[2]),
base_asset=Asset(entry[3]),
quote_asset=Asset(entry[4]),
trade_type=TradeType.deserialize_from_db(entry[5]),
amount=deserialize_asset_amount(entry[6]),
rate=deserialize_price(entry[7]),
fee=deserialize_optional(entry[8], deserialize_fee),
fee_currency=deserialize_optional(entry[9], Asset),
link=entry[10],
notes=entry[11],
)
def trade_from_bitcoinde(raw_trade: Dict) -> Trade:
"""Convert bitcoin.de raw data to a trade
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
try:
timestamp = deserialize_timestamp_from_date(
raw_trade['successfully_finished_at'],
'iso8601',
'bitcoinde',
)
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = deserialize_timestamp_from_date(
raw_trade['trade_marked_as_paid_at'],
'iso8601',
'bitcoinde',
)
trade_type = TradeType.deserialize(raw_trade['type'])
tx_amount = deserialize_asset_amount(raw_trade['amount_currency_to_trade'])
native_amount = deserialize_asset_amount(raw_trade['volume_currency_to_pay'])
tx_asset, native_asset = bitcoinde_pair_to_world(raw_trade['trading_pair'])
amount = tx_amount
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee_currency_to_pay'])
fee_asset = A_EUR
return Trade(
timestamp=timestamp,
location=Location.BITCOINDE,
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['trade_id']),
)
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns an FTX transaction into a rotki Trade.
May raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected key
"""
# In the case of perpetuals and futures this fields can be None
if raw_trade.get('baseCurrency', None) is None:
return None
if raw_trade.get('quoteCurrency', None) is None:
return None
timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601',
'FTX')
trade_type = TradeType.deserialize(raw_trade['side'])
base_asset = asset_from_ftx(raw_trade['baseCurrency'])
quote_asset = asset_from_ftx(raw_trade['quoteCurrency'])
amount = deserialize_asset_amount(raw_trade['size'])
rate = deserialize_price(raw_trade['price'])
fee_currency = asset_from_ftx(raw_trade['feeCurrency'])
fee = deserialize_fee(raw_trade['fee'])
return Trade(
timestamp=timestamp,
location=Location.FTX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(raw_trade['id']),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
self.first_connection()
trades = []
# first get all orders, to see which product ids we need to query fills for
orders = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
orders.extend(batch)
queried_product_ids = set()
for order_entry in orders:
product_id = order_entry.get('product_id', None)
if product_id is None:
msg = (
'Skipping coinbasepro trade since it lacks a product_id. '
'Check logs for details')
self.msg_aggregator.add_error(msg)
log.error(
'Error processing a coinbasepro order.',
raw_trade=order_entry,
error=msg,
)
continue
if product_id in queried_product_ids or product_id not in self.available_products:
continue # already queried this product id or delisted product id
# Now let's get all fills for this product id
queried_product_ids.add(product_id)
fills = []
for batch in self._paginated_query(
endpoint='fills',
query_options={'product_id': product_id},
):
fills.extend(batch)
try:
base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
for fill_entry in fills:
try:
timestamp = coinbasepro_deserialize_timestamp(
fill_entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.deserialize(
fill_entry['side']),
amount=deserialize_asset_amount(
fill_entry['size']),
rate=deserialize_price(fill_entry['price']),
fee=deserialize_fee(fill_entry['fee']),
fee_currency=quote_asset,
link=str(fill_entry['trade_id']) + '_' +
fill_entry['order_id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro fill.',
raw_trade=fill_entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
base, quote = gemini_symbol_to_base_quote(symbol)
trades.append(Trade(
timestamp=timestamp,
location=Location.GEMINI,
base_asset=base,
quote_asset=quote,
trade_type=TradeType.deserialize(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=asset_from_gemini(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
def test_deserialize_trade_type():
assert TradeType.deserialize('buy') == TradeType.BUY
assert TradeType.deserialize('LIMIT_BUY') == TradeType.BUY
assert TradeType.deserialize('BUY') == TradeType.BUY
assert TradeType.deserialize('Buy') == TradeType.BUY
assert TradeType.deserialize('sell') == TradeType.SELL
assert TradeType.deserialize('LIMIT_SELL') == TradeType.SELL
assert TradeType.deserialize('SELL') == TradeType.SELL
assert TradeType.deserialize('Sell') == TradeType.SELL
assert TradeType.deserialize('settlement buy') == TradeType.SETTLEMENT_BUY
assert TradeType.deserialize('settlement_buy') == TradeType.SETTLEMENT_BUY
assert TradeType.deserialize(
'settlement sell') == TradeType.SETTLEMENT_SELL
assert TradeType.deserialize(
'settlement_sell') == TradeType.SETTLEMENT_SELL
assert len(list(TradeType)) == 4
with pytest.raises(DeserializationError):
TradeType.deserialize('dsad')
with pytest.raises(DeserializationError):
TradeType.deserialize(None)
with pytest.raises(DeserializationError):
TradeType.deserialize(1)
def update_trades(self, cursor: 'Cursor') -> None:
"""Upgrades the trades table to use base/quote asset instead of a pair
Also upgrades all asset ids if they are ethereum tokens
And also makes sure the new primary key id matches the rules used in the app
"""
# Get all old data and transform it to the new schema
query = cursor.execute(
'SELECT id, '
' time, '
' location, '
' pair, '
' type, '
' amount, '
' rate, '
' fee, '
' fee_currency, '
' link, '
' notes from trades; ', )
new_trade_tuples = []
for entry in query:
try:
base, quote = pair_get_asset_ids(entry[3])
except ValueError as e:
self.msg_aggregator.add_warning(
f'During v24 -> v25 DB upgrade {str(e)}. This should not have happened.'
f' Removing the trade with id {entry[0]} at timestamp {entry[1]} '
f'and location {str(Location.deserialize_from_db(entry[2]))} that '
f'contained the offending pair from the DB.', )
continue
new_id = self.get_new_asset_identifier(base)
new_base = new_id if new_id else base
new_id = self.get_new_asset_identifier(quote)
new_quote = new_id if new_id else quote
new_id = self.get_new_asset_identifier(entry[8])
new_fee_currency = new_id if new_id else entry[8]
timestamp = entry[1]
amount = entry[5]
rate = entry[6]
old_link = entry[9]
link = None if old_link == '' else old_link
notes = None if entry[10] == '' else entry[10]
# Copy the identifier() functionality. This identifier does not sound like a good idea
new_trade_id_string = (
str(Location.deserialize_from_db(entry[2])) + str(timestamp) +
str(TradeType.deserialize_from_db(entry[4])) + new_base +
new_quote + amount + rate + old_link)
new_trade_id = hash_id(new_trade_id_string)
new_trade_tuples.append((
new_trade_id,
entry[1], # time
entry[2], # location
new_base,
new_quote,
entry[4], # type
amount,
rate,
entry[7], # fee
new_fee_currency,
link,
notes,
))
# Upgrade the table
cursor.execute('DROP TABLE IF EXISTS trades;')
cursor.execute("""
CREATE TABLE IF NOT EXISTS trades (
id TEXT PRIMARY KEY NOT NULL,
time INTEGER NOT NULL,
location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location),
base_asset TEXT NOT NULL,
quote_asset TEXT NOT NULL,
type CHAR(1) NOT NULL DEFAULT ('A') REFERENCES trade_type(type),
amount TEXT NOT NULL,
rate TEXT NOT NULL,
fee TEXT,
fee_currency TEXT,
link TEXT,
notes TEXT
);
""")
# Insert the new data
executestr = """
INSERT INTO trades(
id,
time,
location,
base_asset,
quote_asset,
type,
amount,
rate,
fee,
fee_currency,
link,
notes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
cursor.executemany(executestr, new_trade_tuples)
def _deserialize_trade(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[Trade]:
"""Deserializes a bitpanda trades result entry to a Trade
Returns None and logs error is there is a problem or simpy None if
it's not a type of trade we are interested in
"""
try:
if entry['type'] != 'trade' or entry['attributes'][
'status'] != 'finished':
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda trade',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
cryptocoin_id = entry['attributes']['cryptocoin_id']
crypto_asset = self.cryptocoin_map.get(cryptocoin_id)
if crypto_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda cryptocoin '
f'with id {cryptocoin_id} in the mapping. Skipping trade.',
)
return None
fiat_id = entry['attributes']['fiat_id']
fiat_asset = self.fiat_map.get(fiat_id)
if fiat_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda fiat '
f'with id {fiat_id} in the mapping. Skipping trade.', )
return None
trade_type = TradeType.deserialize(entry['attributes']['type'])
if trade_type in (TradeType.BUY, TradeType.SELL):
# you buy crypto with fiat and sell it for fiat
base_asset = crypto_asset
quote_asset = fiat_asset
amount = deserialize_asset_amount(
entry['attributes']['amount_cryptocoin'])
price = deserialize_price(entry['attributes']['price'])
else:
self.msg_aggregator.add_error(
'Found bitpanda trade with unknown trade type {trade_type}'
) # noqa: E501
return None
trade_id = entry['id']
fee = Fee(ZERO)
fee_asset = A_BEST
if entry['attributes']['bfc_used'] is True:
fee = deserialize_fee(
entry['attributes']['best_fee_collection']['attributes']
['wallet_transaction']['attributes']['fee'], # noqa: E501
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for trade entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda trade due to {msg}')
log.error(
'Error processing bitpanda trade entry',
error=msg,
entry=entry,
)
return None
return Trade(
timestamp=time,
location=Location.BITPANDA,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=price,
fee=fee,
fee_currency=fee_asset,
link=trade_id,
)