def get_blocknumber_by_time(self, ts: Timestamp) -> int: """Performs the etherscan api call to get the blocknumber by a specific timestamp May raise: - RemoteError if there are any problems with reaching Etherscan or if an unexpected response is returned """ if ts < 1438269989: return 0 # etherscan does not handle timestamps close and before genesis well options = {'timestamp': ts, 'closest': 'before'} result = self._query( module='block', action='getblocknobytime', options=options, ) try: number = deserialize_int_from_str(result, 'etherscan getblocknobytime') except DeserializationError as e: raise RemoteError( f'Could not read blocknumber from etherscan getblocknobytime ' f'result {result}', ) from e return number
def _deserialize_asset_movement( raw_movement: Dict[str, Any], ) -> AssetMovement: """Process a deposit/withdrawal user transaction from Bitstamp and deserialize it. Can raise DeserializationError. From Bitstamp documentation, deposits/withdrawals can have a fee (the amount is expected to be in the currency involved) https://www.bitstamp.net/fee-schedule/ Endpoint docs: https://www.bitstamp.net/api/#user-transactions """ type_ = deserialize_int_from_str(raw_movement['type'], 'bitstamp asset movement') category: AssetMovementCategory if type_ == 0: category = AssetMovementCategory.DEPOSIT elif type_ == 1: category = AssetMovementCategory.WITHDRAWAL else: raise AssertionError( f'Unexpected Bitstamp asset movement case: {type_}.') timestamp = deserialize_timestamp_from_bitstamp_date( raw_movement['datetime']) amount: FVal fee_asset: Asset for symbol in BITSTAMP_ASSET_MOVEMENT_SYMBOLS: amount = deserialize_asset_amount(raw_movement.get(symbol, '0')) if amount != ZERO: fee_asset = asset_from_bitstamp(symbol) break if amount == ZERO: raise DeserializationError( 'Could not deserialize Bitstamp asset movement from user transaction. ' f'Unexpected asset amount combination found in: {raw_movement}.', ) asset_movement = AssetMovement( timestamp=timestamp, location=Location.BITSTAMP, category=category, address=None, # requires query "crypto_transactions" endpoint transaction_id=None, # requires query "crypto_transactions" endpoint asset=fee_asset, amount=abs(amount), fee_asset=fee_asset, fee=deserialize_fee(raw_movement['fee']), link=str(raw_movement['id']), ) return asset_movement
def get_account_balances(self, account_id: int) -> Dict[Asset, Balance]: """Get the loopring balances of a given account id May Raise: - RemotError if there is a problem querying the loopring api or if the format of the response does not match expectations """ response = self._api_query('user/balances', {'accountId': account_id}) balances = {} for balance_entry in response: try: token_id = balance_entry['tokenId'] total = deserialize_int_from_str(balance_entry['total'], 'loopring_balances') except KeyError as e: raise RemoteError( f'Failed to query loopring balances because a balance entry ' f'{balance_entry} did not contain key {str(e)}', ) from e except DeserializationError as e: raise RemoteError( f'Failed to query loopring balances because a balance entry ' f'amount could not be deserialized {balance_entry}', ) from e if total == ZERO: continue asset = TOKENID_TO_ASSET.get(token_id, None) if asset is None: self.msg_aggregator.add_warning( f'Ignoring loopring balance of unsupported token with id {token_id}', ) continue # not checking for UnsupportedAsset since this should not happen thanks # to the mapping above amount = asset_normalized_value(amount=total, asset=asset) try: usd_price = Inquirer().find_usd_price(asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing loopring balance entry due to inability to ' f'query USD price: {str(e)}. Skipping balance entry', ) continue balances[asset] = Balance(amount=amount, usd_value=amount * usd_price) return balances
def _deserialize_transaction(grant_id: int, rawtx: Dict[str, Any]) -> LedgerAction: """May raise: - DeserializationError - KeyError - UnknownAsset """ timestamp = deserialize_timestamp_from_date( date=rawtx['timestamp'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) asset = get_gitcoin_asset(symbol=rawtx['asset'], token_address=rawtx['token_address']) raw_amount = deserialize_int_from_str(symbol=rawtx['amount'], location='gitcoin api') amount = asset_normalized_value(raw_amount, asset) if amount == ZERO: raise ZeroGitcoinAmount() # let's use gitcoin's calculated rate for now since they include it in the response usd_value = Price( ZERO) if rawtx['usd_value'] is None else deserialize_price( rawtx['usd_value']) # noqa: E501 rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount) raw_txid = rawtx['tx_hash'] tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx) # until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501 clr_round = _calculate_clr_round(timestamp, rawtx) notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501 return LedgerAction( identifier=1, # whatever -- does not end up in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=AssetAmount(amount), asset=asset, rate=rate, rate_asset=A_USD, link=raw_txid, notes=notes, extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=clr_round, tx_type=tx_type, ), )
def _check_node_synchronization(self, node_interface: SubstrateInterface) -> BlockNumber: """Check the node synchronization comparing the last block obtained via the node interface against the last block obtained via Subscan API. Return the last block obtained via the node interface. May raise: - RemoteError: the last block/chain metadata requests fail or there is an error deserializing the chain metadata. """ # Last block via node interface last_block = self._get_last_block(node_interface=node_interface) # Last block via Subscan API try: chain_metadata = self._request_chain_metadata() except RemoteError: self.msg_aggregator.add_warning( f'Unable to verify that {self.chain} node at endpoint {node_interface.url} ' f'is synced with the chain. Balances and other queries may be incorrect.', ) return last_block # Check node synchronization try: metadata_last_block = BlockNumber( deserialize_int_from_str( symbol=chain_metadata['data']['blockNum'], location='subscan api', ), ) except (KeyError, DeserializationError) as e: message = f'{self.chain} failed to deserialize the chain metadata response: {str(e)}.' log.error(message, chain_metadata=chain_metadata) raise RemoteError(message) from e log.debug( f'{self.chain} subscan API metadata last block', metadata_last_block=metadata_last_block, ) if metadata_last_block - last_block > self.chain.blocks_threshold(): self.msg_aggregator.add_warning( f'Found that {self.chain} node at endpoint {node_interface.url} ' f'is not synced with the chain. Node last block is {last_block}, ' f'expected last block is {metadata_last_block}. ' f'Balances and other queries may be incorrect.', ) return last_block
msg = f'Kucoin {case} returned an invalid JSON response: {response.text}.' log.error(msg) if case in (KucoinCase.API_KEY, KucoinCase.BALANCES): return False, msg if case in PAGINATED_CASES: self.msg_aggregator.add_error( f'Got remote error while querying Kucoin {case}: {msg}', ) return [] raise AssertionError(f'Unexpected case: {case}') from e try: error_code = response_dict.get('code', None) if error_code is not None: error_code = deserialize_int_from_str( error_code, 'kucoin response parsing') except DeserializationError as e: raise RemoteError( f'Could not read Kucoin error code {error_code} as an int' ) from e if error_code in API_KEY_ERROR_CODE_ACTION: msg = API_KEY_ERROR_CODE_ACTION[error_code] else: reason = response_dict.get('msg', None) or response.text msg = ( f'Kucoin query responded with error status code: {response.status_code} ' f'and text: {reason}.') log.error(msg) if case in (KucoinCase.BALANCES, KucoinCase.API_KEY):
response_list = jsonloads_list(response.text) except JSONDecodeError: msg = f'Bitstamp returned invalid JSON response: {response.text}.' log.error(msg) self.msg_aggregator.add_error( f'Got remote error while querying Bistamp trades: {msg}', ) no_results: Union[List[Trade], List[AssetMovement]] = [] # type: ignore return no_results has_results = False is_result_timestamp_gt_end_ts = False result: Union[Trade, AssetMovement] for raw_result in response_list: try: entry_type = deserialize_int_from_str( raw_result['type'], 'bitstamp event') if entry_type not in raw_result_type_filter: log.debug( f'Skipping entry {raw_result} due to type mismatch' ) continue result_timestamp = deserialize_timestamp_from_bitstamp_date( raw_result['datetime'], ) if result_timestamp > end_ts: is_result_timestamp_gt_end_ts = True # prevent extra request break log.debug( f'Attempting to deserialize bitstamp {case_pretty}: {raw_result}' )
def _deserialize_trade( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[Trade]: """Deserializes a bitpanda trades result entry to a Trade Returns None and logs error is there is a problem or simpy None if it's not a type of trade we are interested in """ try: if entry['type'] != 'trade' or entry['attributes'][ 'status'] != 'finished': return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda trade', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None cryptocoin_id = entry['attributes']['cryptocoin_id'] crypto_asset = self.cryptocoin_map.get(cryptocoin_id) if crypto_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda cryptocoin ' f'with id {cryptocoin_id} in the mapping. Skipping trade.', ) return None fiat_id = entry['attributes']['fiat_id'] fiat_asset = self.fiat_map.get(fiat_id) if fiat_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda fiat ' f'with id {fiat_id} in the mapping. Skipping trade.', ) return None trade_type = TradeType.deserialize(entry['attributes']['type']) if trade_type in (TradeType.BUY, TradeType.SELL): # you buy crypto with fiat and sell it for fiat base_asset = crypto_asset quote_asset = fiat_asset amount = deserialize_asset_amount( entry['attributes']['amount_cryptocoin']) price = deserialize_price(entry['attributes']['price']) else: self.msg_aggregator.add_error( 'Found bitpanda trade with unknown trade type {trade_type}' ) # noqa: E501 return None trade_id = entry['id'] fee = Fee(ZERO) fee_asset = A_BEST if entry['attributes']['bfc_used'] is True: fee = deserialize_fee( entry['attributes']['best_fee_collection']['attributes'] ['wallet_transaction']['attributes']['fee'], # noqa: E501 ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for trade entry' self.msg_aggregator.add_error( f'Error processing bitpanda trade due to {msg}') log.error( 'Error processing bitpanda trade entry', error=msg, entry=entry, ) return None return Trade( timestamp=time, location=Location.BITPANDA, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=amount, rate=price, fee=fee, fee_currency=fee_asset, link=trade_id, )
def _deserialize_wallettx( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[AssetMovement]: """Deserializes a bitpanda fiatwallets/transactions or wallets/transactions entry to a deposit/withdrawal Returns None and logs error is there is a problem or simpy None if it's not a type of entry we are interested in """ try: transaction_type = entry['type'] if (transaction_type not in ('fiat_wallet_transaction', 'wallet_transaction') or entry['attributes']['status'] != 'finished'): return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda wallet transaction', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None try: movement_category = deserialize_asset_movement_category( entry['attributes']['type']) # noqa: E501 except DeserializationError: return None # not a deposit/withdrawal if transaction_type == 'fiat_wallet_transaction': asset_id = entry['attributes']['fiat_id'] asset = self.fiat_map.get(asset_id) else: asset_id = entry['attributes']['cryptocoin_id'] asset = self.cryptocoin_map.get(asset_id) if asset is None: self.msg_aggregator.add_error( f'While deserializing Bitpanda fiat transaction, could not find ' f'bitpanda asset with id {asset_id} in the mapping', ) return None amount = deserialize_asset_amount(entry['attributes']['amount']) fee = deserialize_fee(entry['attributes']['fee']) tx_id = entry['id'] transaction_id = entry['attributes'].get('tx_id') address = entry['attributes'].get('recipient') except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for wallet transaction entry' self.msg_aggregator.add_error( f'Error processing bitpanda wallet transaction entry due to {msg}' ) # noqa: E501 log.error( 'Error processing bitpanda wallet transaction entry', error=msg, entry=entry, ) return None return AssetMovement( location=Location.BITPANDA, category=movement_category, address=address, transaction_id=transaction_id, timestamp=time, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=tx_id, )