def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
if cost != amount * rate:
log.warning(
'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
location='kraken',
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
)
def _deserialize(self, value, attr, data, **kwargs): # pylint: disable=unused-argument
try:
price = deserialize_price(value)
except DeserializationError as e:
raise ValidationError(str(e))
return price
def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the fee_currency string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
pair = data['pair']
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = deserialize_trade_type(data['trade_type'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=data['location'],
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def _deserialize_trade(self, raw_result: List[Any]) -> Trade:
"""Process a trade result from Bitfinex and deserialize it.
The base and quote assets are instantiated using the `fee_currency_symbol`
(from raw_result[10]) over the pair (from raw_result[1]).
Known pairs format: 'tETHUST', 'tETH:UST'.
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-trades
"""
amount = deserialize_asset_amount(raw_result[4])
trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL
bfx_pair = self._process_bfx_pair(raw_result[1])
if bfx_pair not in self.pair_bfx_symbols_map:
raise DeserializationError(
f'Could not deserialize bitfinex trade pair {raw_result[1]}. '
f'Raw trade: {raw_result}', )
bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[
bfx_pair]
try:
base_asset = asset_from_bitfinex(
bitfinex_name=bfx_base_asset_symbol,
currency_map=self.currency_map,
)
quote_asset = asset_from_bitfinex(
bitfinex_name=bfx_quote_asset_symbol,
currency_map=self.currency_map,
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[10],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(
e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing trade pair due to: {str(e)}',
) from e
trade = Trade(
timestamp=Timestamp(int(raw_result[2] / 1000)),
location=Location.BITFINEX,
pair=TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}'),
trade_type=trade_type,
amount=AssetAmount(abs(amount)),
rate=deserialize_price(raw_result[5]),
fee=Fee(abs(deserialize_fee(raw_result[9]))),
fee_currency=fee_asset,
link=str(raw_result[0]),
notes='',
)
return trade
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
Throws:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = AssetAmount(
deserialize_asset_amount(bittrex_trade['Quantity']) -
deserialize_asset_amount(bittrex_trade['QuantityRemaining']), )
timestamp = deserialize_timestamp_from_bittrex_date(
bittrex_trade['TimeStamp'])
rate = deserialize_price(bittrex_trade['PricePerUnit'])
order_type = deserialize_trade_type(bittrex_trade['OrderType'])
bittrex_price = deserialize_price(bittrex_trade['Price'])
fee = deserialize_fee(bittrex_trade['Commission'])
pair = bittrex_pair_to_world(bittrex_trade['Exchange'])
quote_currency = get_pair_position_asset(pair, 'second')
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
price=bittrex_price,
fee=fee,
bittrex_pair=bittrex_trade['Exchange'],
pair=pair,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
pair=pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=str(bittrex_trade['OrderUuid']),
)
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible
that some keys don't exist in a trade. The required fields are here:
https://bittrex.github.io/api/v3#definition-Order
Throws:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(bittrex_trade['fillQuantity'])
timestamp = deserialize_timestamp_from_date(
date=bittrex_trade['closedAt'], # we only check closed orders
formatstr='iso8601',
location='bittrex',
)
if 'limit' in bittrex_trade:
rate = deserialize_price(bittrex_trade['limit'])
else:
rate = Price(
FVal(bittrex_trade['proceeds']) /
FVal(bittrex_trade['fillQuantity']))
order_type = deserialize_trade_type(bittrex_trade['direction'])
fee = deserialize_fee(bittrex_trade['commission'])
pair = bittrex_pair_to_world(bittrex_trade['marketSymbol'])
quote_currency = get_pair_position_asset(pair, 'second')
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
fee=fee,
bittrex_pair=bittrex_trade['marketSymbol'],
pair=pair,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
pair=pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=str(bittrex_trade['id']),
)
def _deserialize(
self,
value: str,
attr: Optional[str], # pylint: disable=unused-argument
data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument
**_kwargs: Any,
) -> Price:
try:
price = deserialize_price(value)
except DeserializationError as e:
raise ValidationError(str(e))
return price
def _read_trades(self, filepath: str) -> List[Trade]:
"""Reads a csv fill type report and extracts the Trades"""
with open(filepath, newline='') as csvfile:
reader = csv.DictReader(csvfile)
trades = []
for row in reader:
try:
timestamp = deserialize_timestamp_from_date(
row['created at'],
'iso8601',
'coinbasepro',
)
trades.append(Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
pair=coinbasepro_to_worldpair(row['product']),
trade_type=deserialize_trade_type(row['side']),
amount=deserialize_asset_amount(row['size']),
rate=deserialize_price(row['price']),
fee=deserialize_fee(row['fee']),
fee_currency=Asset(row['price/fee/total unit']),
link=row['trade id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a coinbasepro trade.',
raw_trade=row,
error=msg,
)
continue
return trades
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=deserialize_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=deserialize_ethereum_address(event_tuple[5]),
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def deserialize_token_day_data(
raw_token_day_data: Dict[str, Any], ) -> Tuple[ChecksumEthAddress, Price]:
"""May raise DeserializationError"""
try:
token_address = raw_token_day_data['token']['id']
usd_price = deserialize_price(raw_token_day_data['priceUSD'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
try:
token_address = to_checksum_address(token_address)
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {token_address} in token day data.',
) from e
return token_address, usd_price
def _deserialize_trade(
self,
raw_trade: Dict[str, Any],
) -> Trade:
"""Process a trade user transaction from Bitstamp and deserialize it.
Can raise DeserializationError.
"""
timestamp = deserialize_timestamp_from_bitstamp_date(raw_trade['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_trade)
base_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.quote_asset_symbol],
)
rate = deserialize_price(raw_trade[trade_pair_data.pair])
fee_currency = trade_pair_data.quote_asset
if base_asset_amount >= ZERO:
trade_type = TradeType.BUY
else:
if quote_asset_amount < 0:
raise DeserializationError(
f'Unexpected bitstamp trade format. Both base and quote '
f'amounts are negative: {raw_trade}',
)
trade_type = TradeType.SELL
trade = Trade(
timestamp=timestamp,
location=Location.BITSTAMP,
pair=trade_pair_data.trade_pair,
trade_type=trade_type,
amount=AssetAmount(abs(base_asset_amount)),
rate=rate,
fee=deserialize_fee(raw_trade['fee']),
fee_currency=fee_currency,
link=str(raw_trade['id']),
notes='',
)
return trade
def _deserialize_trade(
self,
raw_trade: Dict[str, Any],
) -> Trade:
"""Process a trade user transaction from Bitstamp and deserialize it.
Can raise DeserializationError.
"""
timestamp = deserialize_timestamp_from_bitstamp_date(raw_trade['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_trade)
base_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.quote_asset_symbol],
)
if base_asset_amount >= ZERO:
trade_type = TradeType.BUY
amount = base_asset_amount
fee_currency = trade_pair_data.quote_asset
else:
trade_type = TradeType.SELL
amount = quote_asset_amount
fee_currency = trade_pair_data.base_asset
trade = Trade(
timestamp=timestamp,
location=Location.BITSTAMP,
pair=trade_pair_data.trade_pair,
trade_type=trade_type,
amount=amount,
rate=deserialize_price(raw_trade[trade_pair_data.pair]),
fee=deserialize_fee(raw_trade['fee']),
fee_currency=fee_currency,
link=str(raw_trade['id']),
notes='',
)
return trade
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
# pylint does not seem to see that Price is essentially FVal
if not cost.is_close(amount * rate): # pylint: disable=no-member
log.warning(
f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
# Kraken trades can have the same ordertxid and postxid for different trades ..
# Also note postxid is optional and can be missing
# The only thing that could differentiate them is timestamps in the milliseconds range
# For example here are parts of two different kraken_trade:
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
#
# In order to counter this for the unique exchange trade link we are going
# to use a concatenation of the above
exchange_uuid = (
str(kraken_trade['ordertxid']) +
str(kraken_trade.get('postxid', '')) + # postxid is optional
str(kraken_trade['time']))
return Trade(
timestamp=timestamp,
location=Location.KRAKEN,
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=exchange_uuid,
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
raw_trades = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
raw_trades.extend(batch)
for entry in raw_trades:
timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
try:
pair = coinbasepro_to_worldpair(entry['product_id'])
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
_, quote_asset = pair_get_assets(pair)
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
pair=coinbasepro_to_worldpair(entry['product_id']),
trade_type=deserialize_trade_type(entry['side']),
amount=deserialize_asset_amount(entry['size']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fill_fees']),
fee_currency=quote_asset,
link=entry['id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def trade_from_binance(
binance_trade: Dict,
binance_symbols_to_pair: Dict[str, BinancePair],
) -> Trade:
"""Turn a binance trade returned from trade history to our common trade
history format
From the official binance api docs (01/09/18):
https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology
base asset refers to the asset that is the quantity of a symbol.
quote asset refers to the asset that is the price of a symbol.
Throws:
- UnsupportedAsset due to asset_from_binance
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(binance_trade['qty'])
rate = deserialize_price(binance_trade['price'])
if binance_trade['symbol'] not in binance_symbols_to_pair:
raise DeserializationError(
f'Error reading a binance trade. Could not find '
f'{binance_trade["symbol"]} in binance_symbols_to_pair',
)
binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
timestamp = deserialize_timestamp_from_binance(binance_trade['time'])
base_asset = asset_from_binance(binance_pair.binance_base_asset)
quote_asset = asset_from_binance(binance_pair.binance_quote_asset)
if binance_trade['isBuyer']:
order_type = TradeType.BUY
# e.g. in RDNETH we buy RDN by paying ETH
else:
order_type = TradeType.SELL
fee_currency = asset_from_binance(binance_trade['commissionAsset'])
fee = deserialize_fee(binance_trade['commission'])
log.debug(
'Processing binance Trade',
sensitive_log=True,
amount=amount,
rate=rate,
timestamp=timestamp,
pair=binance_trade['symbol'],
base_asset=base_asset,
quote=quote_asset,
order_type=order_type,
commision_asset=binance_trade['commissionAsset'],
fee=fee,
)
return Trade(
timestamp=timestamp,
location=Location.BINANCE,
pair=trade_pair_from_assets(base_asset, quote_asset),
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(binance_trade['id']),
)
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
trades.append(
Trade(
timestamp=timestamp,
location=Location.GEMINI,
pair=gemini_symbol_to_pair(symbol),
trade_type=deserialize_trade_type(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=Asset(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - is_token0_unknown
7 - token0_address
8 - token0_symbol
9 - token0_name
10 - token0_decimals
11 - is_token1_unknown
12 - token1_address
13 - token1_symbol
14 - token1_name
15 - token1_decimals
16 - amount0
17 - amount1
18 - usd_price
19 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
is_token0_unknown = event_tuple[6]
is_token1_unknown = event_tuple[11]
token0: Union[EthereumToken, UnknownEthereumToken]
token1: Union[EthereumToken, UnknownEthereumToken]
if is_token0_unknown:
token0 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[7],
symbol=event_tuple[8],
name=event_tuple[9],
decimals=event_tuple[10],
)
else:
token0 = deserialize_ethereum_token_from_db(
identifier=event_tuple[8])
if is_token1_unknown:
token1 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[12],
symbol=event_tuple[13],
name=event_tuple[14],
decimals=event_tuple[15],
)
else:
token1 = deserialize_ethereum_token_from_db(
identifier=event_tuple[13])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=deserialize_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=deserialize_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[16]),
amount1=deserialize_asset_amount(event_tuple[17]),
usd_price=deserialize_price(event_tuple[18]),
lp_amount=deserialize_asset_amount(event_tuple[19]),
)
- UnsupportedAsset
"""
try:
timestamp = deserialize_timestamp(raw_result['createdAt'])
trade_type = TradeType.BUY if raw_result[
'side'] == 'buy' else TradeType.SELL
# amount_key = 'size' if case == KucoinCase.TRADES else 'amount'
# amount = deserialize_asset_amount(raw_result['size'])
fee = deserialize_fee(raw_result['fee'])
trade_pair_symbol = raw_result['symbol']
trade_pair = deserialize_trade_pair(trade_pair_symbol)
if case == KucoinCase.TRADES:
fee_currency_symbol = raw_result['feeCurrency']
fee_currency = asset_from_kucoin(fee_currency_symbol)
amount = deserialize_asset_amount(raw_result['size'])
rate = deserialize_price(raw_result['price'])
# new trades have the timestamp in ms
timestamp = Timestamp(int(timestamp / 1000))
trade_id = raw_result['tradeId']
else: # old v1 trades
amount = deserialize_asset_amount(raw_result['amount'])
base_asset, quote_asset = pair_get_assets(trade_pair)
fee_currency = quote_asset if trade_type == TradeType.SELL else base_asset
rate = deserialize_price(raw_result['dealPrice'])
trade_id = raw_result['id']
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
trade = Trade(
timestamp=timestamp,
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
# Not catching DeserializationError here since we have asserts for the data
# being strings right above
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = create_timestamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location=Location.EXTERNAL,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''
