def get_blocknumber_by_time(self, ts: Timestamp) -> int:
"""Performs the etherscan api call to get the blocknumber by a specific timestamp
May raise:
- RemoteError if there are any problems with reaching Etherscan or if
an unexpected response is returned
"""
if ts < 1438269989:
return 0 # etherscan does not handle timestamps close and before genesis well
options = {'timestamp': ts, 'closest': 'before'}
result = self._query(
module='block',
action='getblocknobytime',
options=options,
)
try:
number = deserialize_int_from_str(result,
'etherscan getblocknobytime')
except DeserializationError as e:
raise RemoteError(
f'Could not read blocknumber from etherscan getblocknobytime '
f'result {result}', ) from e
return number
def _deserialize_asset_movement(
raw_movement: Dict[str, Any], ) -> AssetMovement:
"""Process a deposit/withdrawal user transaction from Bitstamp and
deserialize it.
Can raise DeserializationError.
From Bitstamp documentation, deposits/withdrawals can have a fee
(the amount is expected to be in the currency involved)
https://www.bitstamp.net/fee-schedule/
Endpoint docs:
https://www.bitstamp.net/api/#user-transactions
"""
type_ = deserialize_int_from_str(raw_movement['type'],
'bitstamp asset movement')
category: AssetMovementCategory
if type_ == 0:
category = AssetMovementCategory.DEPOSIT
elif type_ == 1:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(
f'Unexpected Bitstamp asset movement case: {type_}.')
timestamp = deserialize_timestamp_from_bitstamp_date(
raw_movement['datetime'])
amount: FVal
fee_asset: Asset
for symbol in BITSTAMP_ASSET_MOVEMENT_SYMBOLS:
amount = deserialize_asset_amount(raw_movement.get(symbol, '0'))
if amount != ZERO:
fee_asset = asset_from_bitstamp(symbol)
break
if amount == ZERO:
raise DeserializationError(
'Could not deserialize Bitstamp asset movement from user transaction. '
f'Unexpected asset amount combination found in: {raw_movement}.',
)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.BITSTAMP,
category=category,
address=None, # requires query "crypto_transactions" endpoint
transaction_id=None, # requires query "crypto_transactions" endpoint
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=deserialize_fee(raw_movement['fee']),
link=str(raw_movement['id']),
)
return asset_movement
def get_account_balances(self, account_id: int) -> Dict[Asset, Balance]:
"""Get the loopring balances of a given account id
May Raise:
- RemotError if there is a problem querying the loopring api or if the format
of the response does not match expectations
"""
response = self._api_query('user/balances', {'accountId': account_id})
balances = {}
for balance_entry in response:
try:
token_id = balance_entry['tokenId']
total = deserialize_int_from_str(balance_entry['total'],
'loopring_balances')
except KeyError as e:
raise RemoteError(
f'Failed to query loopring balances because a balance entry '
f'{balance_entry} did not contain key {str(e)}', ) from e
except DeserializationError as e:
raise RemoteError(
f'Failed to query loopring balances because a balance entry '
f'amount could not be deserialized {balance_entry}',
) from e
if total == ZERO:
continue
asset = TOKENID_TO_ASSET.get(token_id, None)
if asset is None:
self.msg_aggregator.add_warning(
f'Ignoring loopring balance of unsupported token with id {token_id}',
)
continue
# not checking for UnsupportedAsset since this should not happen thanks
# to the mapping above
amount = asset_normalized_value(amount=total, asset=asset)
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing loopring balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
balances[asset] = Balance(amount=amount,
usd_value=amount * usd_price)
return balances
def _deserialize_transaction(grant_id: int, rawtx: Dict[str,
Any]) -> LedgerAction:
"""May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
timestamp = deserialize_timestamp_from_date(
date=rawtx['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
asset = get_gitcoin_asset(symbol=rawtx['asset'],
token_address=rawtx['token_address'])
raw_amount = deserialize_int_from_str(symbol=rawtx['amount'],
location='gitcoin api')
amount = asset_normalized_value(raw_amount, asset)
if amount == ZERO:
raise ZeroGitcoinAmount()
# let's use gitcoin's calculated rate for now since they include it in the response
usd_value = Price(
ZERO) if rawtx['usd_value'] is None else deserialize_price(
rawtx['usd_value']) # noqa: E501
rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount)
raw_txid = rawtx['tx_hash']
tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx)
# until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501
clr_round = _calculate_clr_round(timestamp, rawtx)
notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501
return LedgerAction(
identifier=1, # whatever -- does not end up in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=AssetAmount(amount),
asset=asset,
rate=rate,
rate_asset=A_USD,
link=raw_txid,
notes=notes,
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=clr_round,
tx_type=tx_type,
),
)
def _check_node_synchronization(self, node_interface: SubstrateInterface) -> BlockNumber:
"""Check the node synchronization comparing the last block obtained via
the node interface against the last block obtained via Subscan API.
Return the last block obtained via the node interface.
May raise:
- RemoteError: the last block/chain metadata requests fail or
there is an error deserializing the chain metadata.
"""
# Last block via node interface
last_block = self._get_last_block(node_interface=node_interface)
# Last block via Subscan API
try:
chain_metadata = self._request_chain_metadata()
except RemoteError:
self.msg_aggregator.add_warning(
f'Unable to verify that {self.chain} node at endpoint {node_interface.url} '
f'is synced with the chain. Balances and other queries may be incorrect.',
)
return last_block
# Check node synchronization
try:
metadata_last_block = BlockNumber(
deserialize_int_from_str(
symbol=chain_metadata['data']['blockNum'],
location='subscan api',
),
)
except (KeyError, DeserializationError) as e:
message = f'{self.chain} failed to deserialize the chain metadata response: {str(e)}.'
log.error(message, chain_metadata=chain_metadata)
raise RemoteError(message) from e
log.debug(
f'{self.chain} subscan API metadata last block',
metadata_last_block=metadata_last_block,
)
if metadata_last_block - last_block > self.chain.blocks_threshold():
self.msg_aggregator.add_warning(
f'Found that {self.chain} node at endpoint {node_interface.url} '
f'is not synced with the chain. Node last block is {last_block}, '
f'expected last block is {metadata_last_block}. '
f'Balances and other queries may be incorrect.',
)
return last_block
msg = f'Kucoin {case} returned an invalid JSON response: {response.text}.'
log.error(msg)
if case in (KucoinCase.API_KEY, KucoinCase.BALANCES):
return False, msg
if case in PAGINATED_CASES:
self.msg_aggregator.add_error(
f'Got remote error while querying Kucoin {case}: {msg}', )
return []
raise AssertionError(f'Unexpected case: {case}') from e
try:
error_code = response_dict.get('code', None)
if error_code is not None:
error_code = deserialize_int_from_str(
error_code, 'kucoin response parsing')
except DeserializationError as e:
raise RemoteError(
f'Could not read Kucoin error code {error_code} as an int'
) from e
if error_code in API_KEY_ERROR_CODE_ACTION:
msg = API_KEY_ERROR_CODE_ACTION[error_code]
else:
reason = response_dict.get('msg', None) or response.text
msg = (
f'Kucoin query responded with error status code: {response.status_code} '
f'and text: {reason}.')
log.error(msg)
if case in (KucoinCase.BALANCES, KucoinCase.API_KEY):
response_list = jsonloads_list(response.text)
except JSONDecodeError:
msg = f'Bitstamp returned invalid JSON response: {response.text}.'
log.error(msg)
self.msg_aggregator.add_error(
f'Got remote error while querying Bistamp trades: {msg}', )
no_results: Union[List[Trade],
List[AssetMovement]] = [] # type: ignore
return no_results
has_results = False
is_result_timestamp_gt_end_ts = False
result: Union[Trade, AssetMovement]
for raw_result in response_list:
try:
entry_type = deserialize_int_from_str(
raw_result['type'], 'bitstamp event')
if entry_type not in raw_result_type_filter:
log.debug(
f'Skipping entry {raw_result} due to type mismatch'
)
continue
result_timestamp = deserialize_timestamp_from_bitstamp_date(
raw_result['datetime'], )
if result_timestamp > end_ts:
is_result_timestamp_gt_end_ts = True # prevent extra request
break
log.debug(
f'Attempting to deserialize bitstamp {case_pretty}: {raw_result}'
)
def _deserialize_trade(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[Trade]:
"""Deserializes a bitpanda trades result entry to a Trade
Returns None and logs error is there is a problem or simpy None if
it's not a type of trade we are interested in
"""
try:
if entry['type'] != 'trade' or entry['attributes'][
'status'] != 'finished':
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda trade',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
cryptocoin_id = entry['attributes']['cryptocoin_id']
crypto_asset = self.cryptocoin_map.get(cryptocoin_id)
if crypto_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda cryptocoin '
f'with id {cryptocoin_id} in the mapping. Skipping trade.',
)
return None
fiat_id = entry['attributes']['fiat_id']
fiat_asset = self.fiat_map.get(fiat_id)
if fiat_asset is None:
self.msg_aggregator.add_error(
f'While deserializing a trade, could not find bitpanda fiat '
f'with id {fiat_id} in the mapping. Skipping trade.', )
return None
trade_type = TradeType.deserialize(entry['attributes']['type'])
if trade_type in (TradeType.BUY, TradeType.SELL):
# you buy crypto with fiat and sell it for fiat
base_asset = crypto_asset
quote_asset = fiat_asset
amount = deserialize_asset_amount(
entry['attributes']['amount_cryptocoin'])
price = deserialize_price(entry['attributes']['price'])
else:
self.msg_aggregator.add_error(
'Found bitpanda trade with unknown trade type {trade_type}'
) # noqa: E501
return None
trade_id = entry['id']
fee = Fee(ZERO)
fee_asset = A_BEST
if entry['attributes']['bfc_used'] is True:
fee = deserialize_fee(
entry['attributes']['best_fee_collection']['attributes']
['wallet_transaction']['attributes']['fee'], # noqa: E501
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for trade entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda trade due to {msg}')
log.error(
'Error processing bitpanda trade entry',
error=msg,
entry=entry,
)
return None
return Trade(
timestamp=time,
location=Location.BITPANDA,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=price,
fee=fee,
fee_currency=fee_asset,
link=trade_id,
)
def _deserialize_wallettx(
self,
entry: Dict[str, Any],
from_ts: Timestamp,
to_ts: Timestamp,
) -> Optional[AssetMovement]:
"""Deserializes a bitpanda fiatwallets/transactions or wallets/transactions
entry to a deposit/withdrawal
Returns None and logs error is there is a problem or simpy None if
it's not a type of entry we are interested in
"""
try:
transaction_type = entry['type']
if (transaction_type
not in ('fiat_wallet_transaction', 'wallet_transaction')
or entry['attributes']['status'] != 'finished'):
return None
time = Timestamp(
deserialize_int_from_str(
symbol=entry['attributes']['time']['unix'],
location='bitpanda wallet transaction',
))
if time < from_ts or time > to_ts:
# should we also stop querying from calling method?
# Probably yes but docs don't mention anything about results
# being ordered by time so let's be conservative
return None
try:
movement_category = deserialize_asset_movement_category(
entry['attributes']['type']) # noqa: E501
except DeserializationError:
return None # not a deposit/withdrawal
if transaction_type == 'fiat_wallet_transaction':
asset_id = entry['attributes']['fiat_id']
asset = self.fiat_map.get(asset_id)
else:
asset_id = entry['attributes']['cryptocoin_id']
asset = self.cryptocoin_map.get(asset_id)
if asset is None:
self.msg_aggregator.add_error(
f'While deserializing Bitpanda fiat transaction, could not find '
f'bitpanda asset with id {asset_id} in the mapping', )
return None
amount = deserialize_asset_amount(entry['attributes']['amount'])
fee = deserialize_fee(entry['attributes']['fee'])
tx_id = entry['id']
transaction_id = entry['attributes'].get('tx_id')
address = entry['attributes'].get('recipient')
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key {msg} for wallet transaction entry'
self.msg_aggregator.add_error(
f'Error processing bitpanda wallet transaction entry due to {msg}'
) # noqa: E501
log.error(
'Error processing bitpanda wallet transaction entry',
error=msg,
entry=entry,
)
return None
return AssetMovement(
location=Location.BITPANDA,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=time,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=tx_id,
)
