def _get_cached_forex_data(
date: str,
from_currency: Asset,
to_currency: Asset,
) -> Optional[Price]:
instance = Inquirer()
rate = None
if date in instance._cached_forex_data:
if from_currency in instance._cached_forex_data[date]:
rate = instance._cached_forex_data[date][from_currency].get(
to_currency)
if rate:
log.debug(
'Got cached forex rate',
from_currency=from_currency.identifier,
to_currency=to_currency.identifier,
rate=rate,
)
try:
rate = deserialize_price(rate)
except DeserializationError as e:
log.error(
f'Could not read cached forex entry due to {str(e)}'
)
return rate
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns an FTX transaction into a rotki Trade.
May raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected key
"""
# In the case of perpetuals and futures this fields can be None
if raw_trade.get('baseCurrency', None) is None:
return None
if raw_trade.get('quoteCurrency', None) is None:
return None
timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601', 'FTX')
trade_type = deserialize_trade_type(raw_trade['side'])
base_asset = asset_from_ftx(raw_trade['baseCurrency'])
quote_asset = asset_from_ftx(raw_trade['quoteCurrency'])
amount = deserialize_asset_amount(raw_trade['size'])
rate = deserialize_price(raw_trade['price'])
fee_currency = asset_from_ftx(raw_trade['feeCurrency'])
fee = deserialize_fee(raw_trade['fee'])
return Trade(
timestamp=timestamp,
location=Location.FTX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(raw_trade['id']),
)
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address_token
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
pool_address_token = EthereumToken.from_identifier(
event_tuple[5],
form_with_incomplete_data=
True, # since some may not have decimals input correctly
)
if pool_address_token is None:
raise DeserializationError(
f'Balancer event pool token: {event_tuple[5]} not found in the DB.',
)
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address_token=pool_address_token,
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the base, quote, fee asset string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = TradeType.deserialize(data['trade_type'])
location = Location.deserialize(data['location'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=location,
base_asset=Asset(data['base_asset']),
quote_asset=Asset(data['quote_asset']),
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def get_bisq_market_price(asset: Asset) -> Price:
"""
Get price for pair at bisq marketplace. Price is returned against BTC.
Can raise:
- RemoteError: If the market doesn't exists or request fails
- DeserializationError: If the data returned is not a valid price
"""
url = PRICE_API_URL.format(symbol=asset.symbol)
try:
response = requests.get(url, timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'bisq.markets request {url} failed due to {str(e)}') from e
try:
data = response.json()
except json.decoder.JSONDecodeError as e:
raise RemoteError(
f'Failed to read json response from bisq.markets. {response.text}. {str(e)}',
) from e
if 'error' in data:
raise RemoteError(
f'Request data from bisq.markets {url} is not valid {data["error"]}'
)
try:
price = data['last']
except KeyError as e:
raise DeserializationError(
f'Response from bisq.markets didnt contain expected key "last". {data}',
) from e
return deserialize_price(price)
def _deserialize_trade(self, raw_result: List[Any]) -> Trade:
"""Process a trade result from Bitfinex and deserialize it.
The base and quote assets are instantiated using the `fee_currency_symbol`
(from raw_result[10]) over the pair (from raw_result[1]).
Known pairs format: 'tETHUST', 'tETH:UST'.
Can raise DeserializationError.
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-trades
"""
amount = deserialize_asset_amount(raw_result[4])
trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL
bfx_pair = self._process_bfx_pair(raw_result[1])
if bfx_pair not in self.pair_bfx_symbols_map:
raise DeserializationError(
f'Could not deserialize bitfinex trade pair {raw_result[1]}. '
f'Raw trade: {raw_result}', )
bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[
bfx_pair]
try:
base_asset = asset_from_bitfinex(
bitfinex_name=bfx_base_asset_symbol,
currency_map=self.currency_map,
)
quote_asset = asset_from_bitfinex(
bitfinex_name=bfx_quote_asset_symbol,
currency_map=self.currency_map,
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[10],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'Unknown' if isinstance(
e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing trade pair due to: {str(e)}',
) from e
trade = Trade(
timestamp=Timestamp(int(raw_result[2] / 1000)),
location=Location.BITFINEX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(amount)),
rate=deserialize_price(raw_result[5]),
fee=Fee(abs(deserialize_fee(raw_result[9]))),
fee_currency=fee_asset,
link=str(raw_result[0]),
notes='',
)
return trade
def deserialize_token_day_data(
raw_token_day_data: Dict[str, Any], ) -> Tuple[ChecksumEthAddress, Price]:
"""May raise DeserializationError"""
try:
token_address = raw_token_day_data['token']['id']
usd_price = deserialize_price(raw_token_day_data['priceUSD'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(token_address)
return token_address, usd_price
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible
that some keys don't exist in a trade. The required fields are here:
https://bittrex.github.io/api/v3#definition-Order
May raise:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(bittrex_trade['fillQuantity'])
timestamp = deserialize_timestamp_from_date(
date=bittrex_trade['closedAt'], # we only check closed orders
formatstr='iso8601',
location='bittrex',
)
if 'limit' in bittrex_trade:
rate = deserialize_price(bittrex_trade['limit'])
else:
rate = Price(
deserialize_asset_amount(bittrex_trade['proceeds']) /
deserialize_asset_amount(bittrex_trade['fillQuantity']),
)
order_type = deserialize_trade_type(bittrex_trade['direction'])
fee = deserialize_fee(bittrex_trade['commission'])
base_asset, quote_asset = bittrex_pair_to_world(bittrex_trade['marketSymbol'])
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
fee=fee,
bittrex_pair=bittrex_trade['marketSymbol'],
base_asset=base_asset,
quote_asset=quote_asset,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_asset,
link=str(bittrex_trade['id']),
)
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - token0_identifier
7 - token1_identifier
8 - amount0
9 - amount1
10 - usd_price
11 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6])
token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[8]),
amount1=deserialize_asset_amount(event_tuple[9]),
usd_price=deserialize_price(event_tuple[10]),
lp_amount=deserialize_asset_amount(event_tuple[11]),
)
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType,
event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(
f'Unexpected event type: {event_tuple_type}.') from e
amounts: List[AssetAmount] = [
deserialize_asset_amount(item) for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def _deserialize_transaction(grant_id: int, rawtx: Dict[str,
Any]) -> LedgerAction:
"""May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
timestamp = deserialize_timestamp_from_date(
date=rawtx['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
asset = get_gitcoin_asset(symbol=rawtx['asset'],
token_address=rawtx['token_address'])
raw_amount = deserialize_int_from_str(symbol=rawtx['amount'],
location='gitcoin api')
amount = asset_normalized_value(raw_amount, asset)
if amount == ZERO:
raise ZeroGitcoinAmount()
# let's use gitcoin's calculated rate for now since they include it in the response
usd_value = Price(
ZERO) if rawtx['usd_value'] is None else deserialize_price(
rawtx['usd_value']) # noqa: E501
rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount)
raw_txid = rawtx['tx_hash']
tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx)
# until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501
clr_round = _calculate_clr_round(timestamp, rawtx)
notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501
return LedgerAction(
identifier=1, # whatever -- does not end up in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=AssetAmount(amount),
asset=asset,
rate=rate,
rate_asset=A_USD,
link=raw_txid,
notes=notes,
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=clr_round,
tx_type=tx_type,
),
)
def deserialize_from_db(cls, entry: TradeDBTuple) -> 'Trade':
"""May raise:
- DeserializationError
- UnknownAsset
"""
return Trade(
timestamp=deserialize_timestamp(entry[1]),
location=Location.deserialize_from_db(entry[2]),
base_asset=Asset(entry[3]),
quote_asset=Asset(entry[4]),
trade_type=TradeType.deserialize_from_db(entry[5]),
amount=deserialize_asset_amount(entry[6]),
rate=deserialize_price(entry[7]),
fee=deserialize_optional(entry[8], deserialize_fee),
fee_currency=deserialize_optional(entry[9], Asset),
link=entry[10],
notes=entry[11],
)
def _deserialize_trade(
self,
raw_trade: Dict[str, Any],
) -> Trade:
"""Process a trade user transaction from Bitstamp and deserialize it.
Can raise DeserializationError.
"""
timestamp = deserialize_timestamp_from_bitstamp_date(raw_trade['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_trade)
base_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.quote_asset_symbol],
)
rate = deserialize_price(raw_trade[trade_pair_data.pair])
fee_currency = trade_pair_data.quote_asset
if base_asset_amount >= ZERO:
trade_type = TradeType.BUY
else:
if quote_asset_amount < 0:
raise DeserializationError(
f'Unexpected bitstamp trade format. Both base and quote '
f'amounts are negative: {raw_trade}',
)
trade_type = TradeType.SELL
trade = Trade(
timestamp=timestamp,
location=Location.BITSTAMP,
base_asset=trade_pair_data.base_asset,
quote_asset=trade_pair_data.quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(base_asset_amount)),
rate=rate,
fee=deserialize_fee(raw_trade['fee']),
fee_currency=fee_currency,
link=str(raw_trade['id']),
notes='',
)
return trade
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - token0_identifier
7 - token1_identifier
8 - amount0
9 - amount1
10 - usd_price
11 - lp_amount
"""
event_type = EventType.deserialize_from_db(event_tuple[4])
token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6])
token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[8]),
amount1=deserialize_asset_amount(event_tuple[9]),
usd_price=deserialize_price(event_tuple[10]),
lp_amount=deserialize_asset_amount(event_tuple[11]),
)
def deserialize_from_db(cls: Type[T], timestamp: Timestamp, stringified_json: str) -> T:
"""May raise:
- DeserializationError if something is wrong with reading this from the DB
"""
try:
data = json.loads(stringified_json)
except json.decoder.JSONDecodeError as e:
raise DeserializationError(
f'Could not decode processed accounting event json from the DB due to {str(e)}',
) from e
try:
pnl_taxable = deserialize_fval(data['pnl_taxable'], name='pnl_taxable', location='processed event decoding') # noqa: E501
pnl_free = deserialize_fval(data['pnl_free'], name='pnl_free', location='processed event decoding') # noqa: E501
if data['cost_basis'] is None:
cost_basis = None
else:
cost_basis = CostBasisInfo.deserialize(data['cost_basis'])
event = cls(
type=AccountingEventType.deserialize(data['type']),
notes=data['notes'],
location=Location.deserialize(data['location']),
timestamp=timestamp,
asset=Asset(data['asset']),
free_amount=deserialize_fval(data['free_amount'], name='free_amount', location='processed event decoding'), # noqa: E501
taxable_amount=deserialize_fval(data['taxable_amount'], name='taxable_amount', location='processed event decoding'), # noqa: E501
price=deserialize_price(data['price']),
pnl=PNL(free=pnl_free, taxable=pnl_taxable),
cost_basis=cost_basis,
index=data['index'],
extra_data=data['extra_data'],
)
event.count_cost_basis_pnl = data['count_cost_basis_pnl']
event.count_entire_amount_spend = data['count_entire_amount_spend']
return event
except KeyError as e:
raise DeserializationError(f'Could not decode processed accounting event json from the DB due to missing key {str(e)}') from e # noqa: E501
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
self.first_connection()
trades = []
# first get all orders, to see which product ids we need to query fills for
orders = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
orders.extend(batch)
queried_product_ids = set()
for order_entry in orders:
product_id = order_entry.get('product_id', None)
if product_id is None:
msg = (
'Skipping coinbasepro trade since it lacks a product_id. '
'Check logs for details')
self.msg_aggregator.add_error(msg)
log.error(
'Error processing a coinbasepro order.',
raw_trade=order_entry,
error=msg,
)
continue
if product_id in queried_product_ids or product_id not in self.available_products:
continue # already queried this product id or delisted product id
# Now let's get all fills for this product id
queried_product_ids.add(product_id)
fills = []
for batch in self._paginated_query(
endpoint='fills',
query_options={'product_id': product_id},
):
fills.extend(batch)
try:
base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
for fill_entry in fills:
try:
timestamp = coinbasepro_deserialize_timestamp(
fill_entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.deserialize(
fill_entry['side']),
amount=deserialize_asset_amount(
fill_entry['size']),
rate=deserialize_price(fill_entry['price']),
fee=deserialize_fee(fill_entry['fee']),
fee_currency=quote_asset,
link=str(fill_entry['trade_id']) + '_' +
fill_entry['order_id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro fill.',
raw_trade=fill_entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - is_token0_unknown
7 - token0_address
8 - token0_symbol
9 - token0_name
10 - token0_decimals
11 - is_token1_unknown
12 - token1_address
13 - token1_symbol
14 - token1_name
15 - token1_decimals
16 - amount0
17 - amount1
18 - usd_price
19 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
is_token0_unknown = event_tuple[6]
is_token1_unknown = event_tuple[11]
token0: Union[EthereumToken, UnknownEthereumToken]
token1: Union[EthereumToken, UnknownEthereumToken]
if is_token0_unknown:
token0 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[7],
symbol=event_tuple[8],
name=event_tuple[9],
decimals=event_tuple[10],
)
else:
token0 = deserialize_ethereum_token_from_db(
identifier=event_tuple[8])
if is_token1_unknown:
token1 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[12],
symbol=event_tuple[13],
name=event_tuple[14],
decimals=event_tuple[15],
)
else:
token1 = deserialize_ethereum_token_from_db(
identifier=event_tuple[13])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[16]),
amount1=deserialize_asset_amount(event_tuple[17]),
usd_price=deserialize_price(event_tuple[18]),
lp_amount=deserialize_asset_amount(event_tuple[19]),
)
def trade_from_poloniex(poloniex_trade: Dict[str, Any],
pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(
get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(
get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(
poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
) from e
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(
f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
timestamp=timestamp,
order_type=trade_type,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
base_asset=quote_currency,
quote_asset=base_currency,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def _scrape_xratescom_exchange_rates(url: str) -> Dict[Asset, Price]:
"""
Scrapes x-rates.com website for the exchange rates tables
May raise:
- RemoteError if we can't query x-rates.com
"""
log.debug(f'Querying x-rates.com stats: {url}')
prices = {}
try:
response = requests.get(url=url, timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(f'x-rates.com request {url} failed due to {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'x-rates.com request {url} failed with code: {response.status_code}'
f' and response: {response.text}',
)
soup = BeautifulSoup(
response.text,
'html.parser',
parse_only=SoupStrainer('table', {'class': 'tablesorter ratesTable'}),
)
if soup is None:
raise RemoteError('Could not find <table> while parsing x-rates stats page')
try:
tr = soup.table.tbody.tr
except AttributeError as e:
raise RemoteError('Could not find first <tr> while parsing x-rates.com page') from e
while tr is not None:
secondtd = tr.select('td:nth-of-type(2)')[0]
try:
href = secondtd.a['href']
except (AttributeError, KeyError) as e:
raise RemoteError('Could not find a href of 2nd td while parsing x-rates.com page') from e # noqa: E501
parts = href.split('to=')
if len(parts) != 2:
raise RemoteError(f'Could not find to= in {href} while parsing x-rates.com page')
try:
to_asset = Asset(parts[1])
if not to_asset.is_fiat():
raise ValueError
except (UnknownAsset, ValueError):
log.debug(f'Skipping {parts[1]} asset because its not a known fiat asset while parsing x-rates.com page') # noqa: E501
tr = tr.find_next_sibling()
continue
try:
price = deserialize_price(secondtd.a.text)
except DeserializationError as e:
log.debug(f'Could not parse x-rates.com rate of {to_asset.identifier} due to {str(e)}. Skipping ...') # noqa: E501
tr = tr.find_next_sibling()
continue
prices[to_asset] = price
tr = tr.find_next_sibling()
return prices
def trade_from_binance(
binance_trade: Dict,
binance_symbols_to_pair: Dict[str, BinancePair],
location: Location,
) -> Trade:
"""Turn a binance trade returned from trade history to our common trade
history format
From the official binance api docs (01/09/18):
https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology
base asset refers to the asset that is the quantity of a symbol.
quote asset refers to the asset that is the price of a symbol.
Throws:
- UnsupportedAsset due to asset_from_binance
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(binance_trade['qty'])
rate = deserialize_price(binance_trade['price'])
if binance_trade['symbol'] not in binance_symbols_to_pair:
raise DeserializationError(
f'Error reading a {str(location)} trade. Could not find '
f'{binance_trade["symbol"]} in binance_symbols_to_pair', )
binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
timestamp = deserialize_timestamp_from_binance(binance_trade['time'])
base_asset = asset_from_binance(binance_pair.binance_base_asset)
quote_asset = asset_from_binance(binance_pair.binance_quote_asset)
if binance_trade['isBuyer']:
order_type = TradeType.BUY
# e.g. in RDNETH we buy RDN by paying ETH
else:
order_type = TradeType.SELL
fee_currency = asset_from_binance(binance_trade['commissionAsset'])
fee = deserialize_fee(binance_trade['commission'])
log.debug(
f'Processing {str(location)} Trade',
amount=amount,
rate=rate,
timestamp=timestamp,
pair=binance_trade['symbol'],
base_asset=base_asset,
quote=quote_asset,
order_type=order_type,
commision_asset=binance_trade['commissionAsset'],
fee=fee,
)
return Trade(
timestamp=timestamp,
location=location,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(binance_trade['id']),
)
def _deserialize_trade(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[Trade]:
"""Deserializes a bitpanda trades result entry to a Trade
Returns None and logs error is there is a problem or simpy None if
it's not a type of trade we are interested in
"""
try:
if entry['type'] != 'trade' or entry['attributes'][
'status'] != 'finished':
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda trade',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
cryptocoin_id = entry['attributes']['cryptocoin_id']
crypto_asset = self.cryptocoin_map.get(cryptocoin_id)
if crypto_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda cryptocoin '
f'with id {cryptocoin_id} in the mapping. Skipping trade.',
)
return None
fiat_id = entry['attributes']['fiat_id']
fiat_asset = self.fiat_map.get(fiat_id)
if fiat_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda fiat '
f'with id {fiat_id} in the mapping. Skipping trade.', )
return None
trade_type = TradeType.deserialize(entry['attributes']['type'])
if trade_type in (TradeType.BUY, TradeType.SELL):
# you buy crypto with fiat and sell it for fiat
base_asset = crypto_asset
quote_asset = fiat_asset
amount = deserialize_asset_amount(
entry['attributes']['amount_cryptocoin'])
price = deserialize_price(entry['attributes']['price'])
else:
self.msg_aggregator.add_error(
'Found bitpanda trade with unknown trade type {trade_type}'
) # noqa: E501
return None
trade_id = entry['id']
fee = Fee(ZERO)
fee_asset = A_BEST
if entry['attributes']['bfc_used'] is True:
fee = deserialize_fee(
entry['attributes']['best_fee_collection']['attributes']
['wallet_transaction']['attributes']['fee'], # noqa: E501
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for trade entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda trade due to {msg}')
log.error(
'Error processing bitpanda trade entry',
error=msg,
entry=entry,
)
return None
return Trade(
timestamp=time,
location=Location.BITPANDA,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=price,
fee=fee,
fee_currency=fee_asset,
link=trade_id,
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
base, quote = gemini_symbol_to_base_quote(symbol)
trades.append(Trade(
timestamp=timestamp,
location=Location.GEMINI,
base_asset=base,
quote_asset=quote,
trade_type=TradeType.deserialize(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=asset_from_gemini(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades, (start_ts, end_ts)
- UnsupportedAsset
"""
try:
timestamp = deserialize_timestamp(raw_result['createdAt'])
trade_type = TradeType.BUY if raw_result[
'side'] == 'buy' else TradeType.SELL
# amount_key = 'size' if case == KucoinCase.TRADES else 'amount'
# amount = deserialize_asset_amount(raw_result['size'])
fee = deserialize_fee(raw_result['fee'])
trade_pair_symbol = raw_result['symbol']
base_asset, quote_asset = deserialize_trade_pair(trade_pair_symbol)
if case == KucoinCase.TRADES:
fee_currency_symbol = raw_result['feeCurrency']
fee_currency = asset_from_kucoin(fee_currency_symbol)
amount = deserialize_asset_amount(raw_result['size'])
rate = deserialize_price(raw_result['price'])
# new trades have the timestamp in ms
timestamp = Timestamp(int(timestamp / 1000))
trade_id = raw_result['tradeId']
else: # old v1 trades
amount = deserialize_asset_amount(raw_result['amount'])
fee_currency = quote_asset if trade_type == TradeType.SELL else base_asset
rate = deserialize_price(raw_result['dealPrice'])
trade_id = raw_result['id']
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
trade = Trade(
timestamp=timestamp,
location=Location.KUCOIN,
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
base_asset, quote_asset = kraken_to_world_pair(kraken_trade['pair'])
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
# pylint does not seem to see that Price is essentially FVal
if not cost.is_close(amount * rate): # pylint: disable=no-member
log.warning(f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade')
log.debug(
'Processing kraken Trade',
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
base_asset=base_asset,
quote_asset=quote_asset,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
# Kraken trades can have the same ordertxid and postxid for different trades ..
# Also note postxid is optional and can be missing
# The only thing that could differentiate them is timestamps in the milliseconds range
# For example here are parts of two different kraken_trade:
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
#
# In order to counter this for the unique exchange trade link we are going
# to use a concatenation of the above
exchange_uuid = (
str(kraken_trade['ordertxid']) +
str(kraken_trade.get('postxid', '')) + # postxid is optional
str(kraken_trade['time'])
)
return Trade(
timestamp=timestamp,
location=Location.KRAKEN,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_asset,
link=exchange_uuid,
)
def query_and_store_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> None:
"""
Get historical hour price data from cryptocompare and populate the global DB
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical hour price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
now_ts = ts_now()
# save time at start of the query, in case the query does not complete due to rate limit
self.last_histohour_query_ts = now_ts
range_result = GlobalDBHandler().get_historical_price_range(
from_asset=from_asset,
to_asset=to_asset,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
)
if range_result is not None:
first_cached_ts, last_cached_ts = range_result
if timestamp > last_cached_ts:
# We have a cache but the requested timestamp does not hit it
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=last_cached_ts,
)
else:
# only other possibility, timestamp < cached start_time
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=first_cached_ts,
to_timestamp=Timestamp(0),
)
else:
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=Timestamp(0),
)
calculated_history = list(new_data)
if len(calculated_history) == 0:
return
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# Turn them into the format we will enter in the DB
prices = []
for entry in calculated_history:
try:
price = Price((deserialize_price(entry['high']) + deserialize_price(entry['low'])) / 2) # noqa: E501
if price == Price(ZERO):
continue # don't write zero prices
prices.append(HistoricalPrice(
from_asset=from_asset,
to_asset=to_asset,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(entry['time']),
price=price,
))
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
log.error(
f'{msg}. Error getting price entry from cryptocompare histohour '
f'price results. Skipping entry.',
)
continue
GlobalDBHandler().add_historical_prices(prices)
self.last_histohour_query_ts = ts_now() # also save when last query finished