def query_timed_balances_data(self, given_asset: str, start_ts: int, end_ts: int) -> Dict: try: from_ts = deserialize_timestamp(start_ts) to_ts = deserialize_timestamp(end_ts) asset = Asset(given_asset) except (UnknownAsset, DeserializationError) as e: return {'result': False, 'message': str(e)} res = self.rotkehlchen.data.db.query_timed_balances( from_ts=from_ts, to_ts=to_ts, asset=asset, ) result = {'result': res, 'message': ''} return process_result(result)
def deserialize_from_db( cls, data: LedgerActionDBTupleWithIdentifier, given_gitcoin_map: Optional[Dict[int, GitcoinEventDataDB]] = None, ) -> 'LedgerAction': """May raise: - DeserializationError - UnknownAsset """ extra_data = None gitcoin_map = {} if not given_gitcoin_map else given_gitcoin_map gitcoin_data = gitcoin_map.get(data[0], None) if gitcoin_data is not None: extra_data = GitcoinEventData.deserialize_from_db( data=gitcoin_data) return cls( identifier=data[0], timestamp=deserialize_timestamp(data[1]), action_type=LedgerActionType.deserialize_from_db(data[2]), location=Location.deserialize_from_db(data[3]), amount=deserialize_asset_amount(data[4]), asset=Asset(data[5]), rate=deserialize_optional(data[6], deserialize_price), rate_asset=deserialize_optional(data[7], Asset), link=data[8], notes=data[9], extra_data=extra_data, )
def asset_movements_from_dictlist( given_data: List[Dict[str, Any]], start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """ Gets a list of dict asset movements, most probably read from the json files and a time period. Returns it as a list of the AssetMovement tuples that are inside the time period May raise: - DeserializationError: If the given_data dict contains data in an unexpected format - KeyError: If the given_data dict contains data in an unexpected format """ returned_movements = list() for movement in given_data: timestamp = deserialize_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: break category = deserialize_asset_movement_category(movement['category']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) returned_movements.append(AssetMovement( exchange=deserialize_exchange_name(movement['exchange']), category=category, timestamp=timestamp, asset=Asset(movement['asset']), amount=amount, fee=fee, )) return returned_movements
def deserialize_transaction_from_etherscan( data: Dict[str, Any], internal: bool, ) -> EthereumTransaction: """Reads dict data of a transaction from etherscan and deserializes it Can raise DeserializationError if something is wrong """ try: # internal tx list contains no gasprice gas_price = -1 if internal else read_integer(data, 'gasPrice') tx_hash = read_hash(data, 'hash') input_data = read_hash(data, 'input') timestamp = deserialize_timestamp(data['timeStamp']) block_number = read_integer(data, 'blockNumber') nonce = -1 if internal else read_integer(data, 'nonce') return EthereumTransaction( timestamp=timestamp, block_number=block_number, tx_hash=tx_hash, from_address=to_checksum_address(data['from']), to_address=to_checksum_address(data['to']) if data['to'] != '' else None, value=read_integer(data, 'value'), gas=read_integer(data, 'gas'), gas_price=gas_price, gas_used=read_integer(data, 'gasUsed'), input_data=input_data, nonce=nonce, ) except KeyError as e: raise DeserializationError(f'Etherscan ethereum transaction missing expected key {str(e)}')
def _deserialize(self, value, attr, data, **kwargs): # pylint: disable=unused-argument try: timestamp = deserialize_timestamp(value) except DeserializationError as e: raise ValidationError(str(e)) return timestamp
def deserialize_from_db( cls, event_tuple: BalancerEventDBTuple, ) -> 'BalancerEvent': """May raise DeserializationError Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address_token 6 - lp_amount 7 - usd_value 8 - amount0 9 - amount1 10 - amount2 11 - amount3 12 - amount4 13 - amount5 14 - amount6 15 - amount7 """ event_tuple_type = event_tuple[4] try: event_type = getattr(BalancerBPTEventType, event_tuple_type.upper()) except AttributeError as e: raise DeserializationError( f'Unexpected event type: {event_tuple_type}.') from e pool_address_token = EthereumToken.from_identifier( event_tuple[5], form_with_incomplete_data= True, # since some may not have decimals input correctly ) if pool_address_token is None: raise DeserializationError( f'Balancer event pool token: {event_tuple[5]} not found in the DB.', ) amounts: List[AssetAmount] = [ deserialize_asset_amount(item) for item in event_tuple[8:16] if item is not None ] return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address_token=pool_address_token, lp_balance=Balance( amount=deserialize_asset_amount(event_tuple[6]), usd_value=deserialize_price(event_tuple[7]), ), amounts=amounts, )
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self._get_paginated_query( endpoint='transfers', start_ts=start_ts, end_ts=end_ts, ) movements = [] for entry in result: try: timestamp = deserialize_timestamp(entry['timestampms']) timestamp = Timestamp(int(timestamp / 1000)) asset = Asset(entry['currency']) movement = AssetMovement( location=Location.GEMINI, category=deserialize_asset_movement_category( entry['type']), address=deserialize_asset_movement_address( entry, 'destination', asset), transaction_id=get_key_if_has_val(entry, 'txHash'), timestamp=timestamp, asset=asset, amount=deserialize_asset_amount_force_positive( entry['amount']), fee_asset=asset, # Gemini does not include withdrawal fees neither in the API nor in their UI fee=Fee(ZERO), link=str(entry['eid']), ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found gemini deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found gemini deposit/withdrawal with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Error processing a gemini deposit/withdrawal. Check logs ' 'for details. Ignoring it.', ) log.error( 'Error processing a gemini deposit_withdrawal', asset_movement=entry, error=msg, ) continue movements.append(movement) return movements
def trades_from_dictlist( given_trades: List[Dict[str, Any]], start_ts: Timestamp, end_ts: Timestamp, location: str, msg_aggregator: MessagesAggregator, ) -> List[Trade]: """ Gets a list of dict trades, most probably read from the json files and a time period. Returns it as a list of the Trade tuples that are inside the time period Can raise: - KeyError: If a trade dict does not have a key as we expect it - DeserializationError: If a trade dict entry is of an unexpected format """ returned_trades = list() for given_trade in given_trades: timestamp = deserialize_timestamp(given_trade['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: break try: returned_trades.append(deserialize_trade(given_trade)) except UnknownAsset as e: msg_aggregator.add_warning( f'When processing {location} trades found a trade containing unknown ' f'asset {e.asset_name}. Ignoring it.') continue return returned_trades
def deserialize_invest_event( raw_event: Dict[str, Any], event_type: Literal[BalancerInvestEventType.ADD_LIQUIDITY, BalancerInvestEventType.REMOVE_LIQUIDITY, ], ) -> BalancerInvestEvent: """May raise DeserializationError""" try: tx_hash, log_index = deserialize_transaction_id(raw_event['id']) timestamp = deserialize_timestamp(raw_event['timestamp']) raw_user_address = raw_event['userAddress']['id'] raw_pool_address = raw_event['poolAddress']['id'] if event_type == BalancerInvestEventType.ADD_LIQUIDITY: raw_token_address = raw_event['tokenIn']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountIn']) elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY: raw_token_address = raw_event['tokenOut']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountOut']) else: raise AssertionError(f'Unexpected event type: {event_type}.') except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e try: user_address = to_checksum_address(raw_user_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_user_address} in userAddress.', ) from e try: pool_address = to_checksum_address(raw_pool_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_pool_address} in poolAddress.', ) from e try: token_address = to_checksum_address(raw_token_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_token_address} in tokenIn.', ) from e invest_event = BalancerInvestEvent( tx_hash=tx_hash, log_index=log_index, address=user_address, timestamp=timestamp, event_type=event_type, pool_address=pool_address, token_address=token_address, amount=amount, ) return invest_event
def get_ethereum_transactions( self, filter_: ETHTransactionsFilterQuery, ) -> Tuple[List[EthereumTransaction], int]: """Returns a tuple with 2 entries. First entry is a list of ethereum transactions optionally filtered by time and/or from address and pagination. Second is the number of entries found for the current filter ignoring pagination. This function can raise: - pysqlcipher3.dbapi2.OperationalError if the SQL query fails due to invalid filtering arguments. """ cursor = self.db.conn.cursor() query, bindings = filter_.prepare() query = 'SELECT * FROM ethereum_transactions ' + query results = cursor.execute(query, bindings) ethereum_transactions = [] for result in results: try: tx = EthereumTransaction( tx_hash=result[0], timestamp=deserialize_timestamp(result[1]), block_number=result[2], from_address=result[3], to_address=result[4], value=int(result[5]), gas=int(result[6]), gas_price=int(result[7]), gas_used=int(result[8]), input_data=result[9], nonce=result[10], ) except DeserializationError as e: self.db.msg_aggregator.add_error( f'Error deserializing ethereum transaction from the DB. ' f'Skipping it. Error was: {str(e)}', ) continue ethereum_transactions.append(tx) if filter_.pagination is not None: no_pagination_filter = deepcopy(filter_) no_pagination_filter.pagination = None query, bindings = no_pagination_filter.prepare() query = 'SELECT COUNT(*) FROM ethereum_transactions ' + query results = cursor.execute(query, bindings).fetchone() total_filter_count = results[0] else: total_filter_count = len(ethereum_transactions) return ethereum_transactions, total_filter_count
def get_token_transaction_hashes( self, account: ChecksumEthAddress, from_ts: Optional[Timestamp] = None, to_ts: Optional[Timestamp] = None, ) -> Iterator[List[str]]: options = {'address': str(account), 'sort': 'asc'} if from_ts is not None: from_block = self.get_blocknumber_by_time(from_ts) options['startBlock'] = str(from_block) if to_ts is not None: to_block = self.get_blocknumber_by_time(to_ts) options['endBlock'] = str(to_block) hashes: Set[Tuple[str, Timestamp]] = set() while True: result = self._query(module='account', action='tokentx', options=options) last_ts = deserialize_timestamp( result[0]['timeStamp']) if len(result) != 0 else None # noqa: E501 pylint: disable=unsubscriptable-object for entry in result: gevent.sleep(0) timestamp = deserialize_timestamp(entry['timeStamp']) if timestamp > last_ts and len( hashes) >= TRANSACTIONS_BATCH_NUM: # type: ignore yield _hashes_tuple_to_list(hashes) hashes = set() last_ts = timestamp hashes.add((entry['hash'], timestamp)) if len(result) != ETHERSCAN_TX_QUERY_LIMIT: break # else we hit the limit. Query once more with startBlock being the last # block we got. There may be duplicate entries if there are more than one # transactions for that last block but they should be filtered # out when we input all of these in the DB last_block = result[-1]['blockNumber'] # pylint: disable=unsubscriptable-object options['startBlock'] = last_block yield _hashes_tuple_to_list(hashes)
def deserialize_from_db(cls, entry: MarginPositionDBTuple) -> 'MarginPosition': """May raise: - DeserializationError - UnknownAsset """ if entry[2] == 0: open_time = None else: open_time = deserialize_timestamp(entry[2]) return MarginPosition( location=Location.deserialize_from_db(entry[1]), open_time=open_time, close_time=deserialize_timestamp(entry[3]), profit_loss=deserialize_asset_amount(entry[4]), pl_currency=Asset(entry[5]), fee=deserialize_fee(entry[6]), fee_currency=Asset(entry[7]), link=entry[8], notes=entry[9], )
def _deserialize_bond( self, raw_event: Dict[str, Any], identity_address_map: Dict[ChecksumAddress, ChecksumAddress], ) -> Bond: """Deserialize a bond event. May raise DeserializationError. """ try: adex_event = self._deserialize_adex_staking_event( raw_event=raw_event, identity_address_map=identity_address_map, case='bond', ) amount_int = int(raw_event['amount']) amount = FVal(raw_event['amount']) / ADX_AMOUNT_MANTISSA pool_id = raw_event['poolId'] nonce = int(raw_event['nonce']) bond_id = self._get_bond_id( identity_address=adex_event.identity_address, amount=amount_int, pool_id=pool_id, nonce=nonce, ) slashed_at = deserialize_timestamp(raw_event['slashedAtStart']) except (DeserializationError, KeyError, ValueError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key in event: {msg}.' log.error( 'Failed to deserialize an AdEx bond event', error=msg, raw_event=raw_event, identity_address_map=identity_address_map, ) raise DeserializationError( 'Failed to deserialize an AdEx bond event. Check logs for more details', ) from e return Bond( tx_hash=adex_event.tx_hash, address=adex_event.address, identity_address=adex_event.identity_address, timestamp=adex_event.timestamp, bond_id=bond_id, value=Balance(amount=amount), pool_id=pool_id, nonce=nonce, slashed_at=slashed_at, )
def _deserialize( self, value: str, attr: Optional[str], # pylint: disable=unused-argument data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument **_kwargs: Any, ) -> Timestamp: try: timestamp = deserialize_timestamp(value) except DeserializationError as e: raise ValidationError(str(e)) return timestamp
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - token0_identifier 7 - token1_identifier 8 - amount0 9 - amount1 10 - usd_price 11 - lp_amount """ db_event_type = event_tuple[4] if db_event_type not in {str(event_type) for event_type in EventType}: raise DeserializationError( f'Failed to deserialize event type. Unknown event: {db_event_type}.', ) if db_event_type == str(EventType.MINT): event_type = EventType.MINT elif db_event_type == str(EventType.BURN): event_type = EventType.BURN else: raise ValueError(f'Unexpected event type case: {db_event_type}.') token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6]) token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[8]), amount1=deserialize_asset_amount(event_tuple[9]), usd_price=deserialize_price(event_tuple[10]), lp_amount=deserialize_asset_amount(event_tuple[11]), )
def _deserialize_asset_movement( self, movement_type: AssetMovementCategory, movement_data: Dict[str, Any], ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from polo and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if movement_type == AssetMovementCategory.DEPOSIT: fee = Fee(ZERO) uid_key = 'depositNumber' else: fee = deserialize_fee(movement_data['fee']) uid_key = 'withdrawalNumber' asset = asset_from_poloniex(movement_data['currency']) return AssetMovement( location=Location.POLONIEX, category=movement_type, timestamp=deserialize_timestamp(movement_data['timestamp']), asset=asset, amount=deserialize_asset_amount(movement_data['amount']), fee_asset=asset, fee=fee, link=str(movement_data[uid_key]), ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unsupported poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unknown poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a poloniex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of poloniex ' f'{str(movement_type)}: {movement_data}. Error was: {str(e)}', ) return None
def deserialize_from_db( cls, event_tuple: BalancerEventDBTuple, ) -> 'BalancerEvent': """May raise DeserializationError Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - lp_amount 7 - usd_value 8 - amount0 9 - amount1 10 - amount2 11 - amount3 12 - amount4 13 - amount5 14 - amount6 15 - amount7 """ event_tuple_type = event_tuple[4] try: event_type = getattr(BalancerBPTEventType, event_tuple_type.upper()) except AttributeError as e: raise DeserializationError( f'Unexpected event type: {event_tuple_type}.') from e amounts: List[AssetAmount] = [ deserialize_asset_amount(item) for item in event_tuple[8:16] if item is not None ] return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), lp_balance=Balance( amount=deserialize_asset_amount(event_tuple[6]), usd_value=deserialize_price(event_tuple[7]), ), amounts=amounts, )
def get_ledger_actions( self, from_ts: Optional[Timestamp], to_ts: Optional[Timestamp], location: Optional[Location], ) -> List[LedgerAction]: cursor = self.db.conn.cursor() query = ( 'SELECT identifier,' ' timestamp,' ' type,' ' location,' ' amount,' ' asset,' ' link,' ' notes FROM ledger_actions ' ) if location is not None: query += f'WHERE location="{location.serialize_for_db()}" ' query, bindings = form_query_to_filter_timestamps(query, 'timestamp', from_ts, to_ts) results = cursor.execute(query, bindings) actions = [] for result in results: try: action = LedgerAction( identifier=result[0], timestamp=deserialize_timestamp(result[1]), action_type=deserialize_ledger_action_type_from_db(result[2]), location=deserialize_location_from_db(result[3]), amount=deserialize_asset_amount(result[4]), asset=Asset(result[5]), link=result[6], notes=result[7], ) except DeserializationError as e: self.msg_aggregator.add_error( f'Error deserializing Ledger Action from the DB. Skipping it.' f'Error was: {str(e)}', ) continue except UnknownAsset as e: self.msg_aggregator.add_error( f'Error deserializing Ledger Action from the DB. Skipping it. ' f'Unknown asset {e.asset_name} found', ) continue actions.append(action) return actions
def import_snapshot( self, balances_snapshot_file: Path, location_data_snapshot_file: Path, ) -> Tuple[bool, str]: """ Converts the snapshot files to list to dictionaries. Performs a series of validation checks on the list before importing. """ balances_list = self._csv_to_dict(balances_snapshot_file) location_data_list = self._csv_to_dict(location_data_snapshot_file) # check if the headers match the type stored in the db has_invalid_headers = ( tuple(balances_list[0].keys()) != ('timestamp', 'category', 'asset_identifier', 'amount', 'usd_value') or # noqa: E501 tuple(location_data_list[0].keys()) != ('timestamp', 'location', 'usd_value') # noqa: E501 ) if has_invalid_headers: return False, 'csv file has invalid headers' # check if all timestamps are the same. balances_timestamps = [ int(entry['timestamp']) for entry in balances_list ] location_data_timestamps = [ int(entry['timestamp']) for entry in location_data_list ] has_different_timestamps = ( balances_timestamps.count(balances_timestamps[0]) != len(balances_timestamps) or location_data_timestamps.count( location_data_timestamps[0]) != len(location_data_timestamps) or # noqa: E501 balances_timestamps[0] != location_data_timestamps[0]) if has_different_timestamps: return False, 'csv file has different timestamps' # check if the timestamp can be converted to int try: _ = deserialize_timestamp(balances_list[0]['timestamp']) except DeserializationError: return False, 'csv file contains invalid timestamp format' return self._import_snapshot( balances_list=balances_list, location_data_list=location_data_list, )
def get_ethereum_transactions( self, filter_: ETHTransactionsFilterQuery, has_premium: bool, ) -> List[EthereumTransaction]: """Returns a list of ethereum transactions optionally filtered by the given filter query This function can raise: - pysqlcipher3.dbapi2.OperationalError if the SQL query fails due to invalid filtering arguments. """ cursor = self.db.conn.cursor() query, bindings = filter_.prepare() if has_premium: query = 'SELECT DISTINCT ethereum_transactions.tx_hash, timestamp, block_number, from_address, to_address, value, gas, gas_price, gas_used, input_data, nonce FROM ethereum_transactions ' + query # noqa: E501 results = cursor.execute(query, bindings) else: query = 'SELECT DISTINCT ethereum_transactions.tx_hash, timestamp, block_number, from_address, to_address, value, gas, gas_price, gas_used, input_data, nonce FROM (SELECT * from ethereum_transactions ORDER BY timestamp DESC LIMIT ?) ethereum_transactions ' + query # noqa: E501 results = cursor.execute(query, [FREE_ETH_TX_LIMIT] + bindings) ethereum_transactions = [] for result in results: try: tx = EthereumTransaction( tx_hash=make_evm_tx_hash(result[0]), timestamp=deserialize_timestamp(result[1]), block_number=result[2], from_address=result[3], to_address=result[4], value=int(result[5]), gas=int(result[6]), gas_price=int(result[7]), gas_used=int(result[8]), input_data=result[9], nonce=result[10], ) except DeserializationError as e: self.db.msg_aggregator.add_error( f'Error deserializing ethereum transaction from the DB. ' f'Skipping it. Error was: {str(e)}', ) continue ethereum_transactions.append(tx) return ethereum_transactions
def deserialize_invest_event( raw_event: Dict[str, Any], event_type: Literal[ BalancerInvestEventType.ADD_LIQUIDITY, BalancerInvestEventType.REMOVE_LIQUIDITY, ], ) -> BalancerInvestEvent: """May raise DeserializationError""" try: tx_hash, log_index = deserialize_transaction_id(raw_event['id']) timestamp = deserialize_timestamp(raw_event['timestamp']) raw_user_address = raw_event['userAddress']['id'] raw_pool_address = raw_event['poolAddress']['id'] if event_type == BalancerInvestEventType.ADD_LIQUIDITY: raw_token_address = raw_event['tokenIn']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountIn']) elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY: raw_token_address = raw_event['tokenOut']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountOut']) else: raise AssertionError(f'Unexpected event type: {event_type}.') except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e user_address = deserialize_ethereum_address(raw_user_address) pool_address = deserialize_ethereum_address(raw_pool_address) try: pool_address_token = EthereumToken(pool_address) except UnknownAsset as e: raise DeserializationError( f'Balancer pool token with address {pool_address} should have been in the DB', ) from e token_address = deserialize_ethereum_address(raw_token_address) invest_event = BalancerInvestEvent( tx_hash=tx_hash, log_index=log_index, address=user_address, timestamp=timestamp, event_type=event_type, pool_address_token=pool_address_token, token_address=token_address, amount=amount, ) return invest_event
def deserialize_from_db( cls, trade_tuple: AMMSwapDBTuple, ) -> 'AMMSwap': """Turns a tuple read from DB into an appropriate Swap. May raise a DeserializationError if something is wrong with the DB data Trade_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - from_address 4 - to_address 5 - timestamp 6 - location 7 - token0_identifier 8 - token1_identifier 9 - amount0_in 10 - amount1_in 11 - amount0_out 12 - amount1_out """ address = deserialize_ethereum_address(trade_tuple[2]) from_address = deserialize_ethereum_address(trade_tuple[3]) to_address = deserialize_ethereum_address(trade_tuple[4]) token0 = deserialize_ethereum_token_from_db(identifier=trade_tuple[7]) token1 = deserialize_ethereum_token_from_db(identifier=trade_tuple[8]) return cls( tx_hash=trade_tuple[0], log_index=trade_tuple[1], address=address, from_address=from_address, to_address=to_address, timestamp=deserialize_timestamp(trade_tuple[5]), location=Location.deserialize_from_db(trade_tuple[6]), token0=token0, token1=token1, amount0_in=deserialize_asset_amount(trade_tuple[9]), amount1_in=deserialize_asset_amount(trade_tuple[10]), amount0_out=deserialize_asset_amount(trade_tuple[11]), amount1_out=deserialize_asset_amount(trade_tuple[12]), )
def deserialize_from_db( cls, data: LedgerActionDBTupleWithIdentifier) -> 'LedgerAction': """May raise: - DeserializationError - UnknownAsset """ return cls( identifier=data[0], timestamp=deserialize_timestamp(data[1]), action_type=LedgerActionType.deserialize_from_db(data[2]), location=Location.deserialize_from_db(data[3]), amount=deserialize_asset_amount(data[4]), asset=Asset(data[5]), rate=deserialize_optional(data[6], deserialize_price), rate_asset=deserialize_optional(data[7], Asset), link=data[8], notes=data[9], )
def deserialize_from_db(cls, entry: TradeDBTuple) -> 'Trade': """May raise: - DeserializationError - UnknownAsset """ return Trade( timestamp=deserialize_timestamp(entry[1]), location=Location.deserialize_from_db(entry[2]), base_asset=Asset(entry[3]), quote_asset=Asset(entry[4]), trade_type=TradeType.deserialize_from_db(entry[5]), amount=deserialize_asset_amount(entry[6]), rate=deserialize_price(entry[7]), fee=deserialize_optional(entry[8], deserialize_fee), fee_currency=deserialize_optional(entry[9], Asset), link=entry[10], notes=entry[11], )
def _deserialize_unbond_request( self, raw_event: Dict[str, Any], identity_address_map: Dict[ChecksumAddress, ChecksumAddress], ) -> UnbondRequest: """Deserialize an unbond request event. It may raise KeyError. """ try: adex_event = self._deserialize_adex_staking_event( raw_event=raw_event, identity_address_map=identity_address_map, case='unbond_request', ) bond_id = raw_event['bondId'] unlock_at = deserialize_timestamp(raw_event['willUnlock']) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key in event: {msg}.' log.error( 'Failed to deserialize an AdEx unbond request event', error=msg, raw_event=raw_event, identity_address_map=identity_address_map, ) raise DeserializationError( 'Failed to deserialize an AdEx unbond request event. Check logs for more details', ) from e return UnbondRequest( tx_hash=adex_event.tx_hash, address=adex_event.address, identity_address=adex_event.identity_address, timestamp=adex_event.timestamp, bond_id=bond_id, value=Balance(), unlock_at=unlock_at, )
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - token0_identifier 7 - token1_identifier 8 - amount0 9 - amount1 10 - usd_price 11 - lp_amount """ event_type = EventType.deserialize_from_db(event_tuple[4]) token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6]) token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[8]), amount1=deserialize_asset_amount(event_tuple[9]), usd_price=deserialize_price(event_tuple[10]), lp_amount=deserialize_asset_amount(event_tuple[11]), )
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: """Queries gemini for trades """ log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts) trades = [] gemini_trades = [] for symbol in self.symbols: gemini_trades = self._get_trades_for_symbol( symbol=symbol, start_ts=start_ts, end_ts=end_ts, ) for entry in gemini_trades: try: timestamp = deserialize_timestamp(entry['timestamp']) if timestamp > end_ts: break trades.append( Trade( timestamp=timestamp, location=Location.GEMINI, pair=gemini_symbol_to_pair(symbol), trade_type=deserialize_trade_type(entry['type']), amount=deserialize_asset_amount(entry['amount']), rate=deserialize_price(entry['price']), fee=deserialize_fee(entry['fee_amount']), fee_currency=Asset(entry['fee_currency']), link=str(entry['tid']), notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Gemini asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a gemini trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a gemini trade.', raw_trade=entry, error=msg, ) continue return trades
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - is_token0_unknown 7 - token0_address 8 - token0_symbol 9 - token0_name 10 - token0_decimals 11 - is_token1_unknown 12 - token1_address 13 - token1_symbol 14 - token1_name 15 - token1_decimals 16 - amount0 17 - amount1 18 - usd_price 19 - lp_amount """ db_event_type = event_tuple[4] if db_event_type not in {str(event_type) for event_type in EventType}: raise DeserializationError( f'Failed to deserialize event type. Unknown event: {db_event_type}.', ) if db_event_type == str(EventType.MINT): event_type = EventType.MINT elif db_event_type == str(EventType.BURN): event_type = EventType.BURN else: raise ValueError(f'Unexpected event type case: {db_event_type}.') is_token0_unknown = event_tuple[6] is_token1_unknown = event_tuple[11] token0: Union[EthereumToken, UnknownEthereumToken] token1: Union[EthereumToken, UnknownEthereumToken] if is_token0_unknown: token0 = deserialize_unknown_ethereum_token_from_db( ethereum_address=event_tuple[7], symbol=event_tuple[8], name=event_tuple[9], decimals=event_tuple[10], ) else: token0 = deserialize_ethereum_token_from_db( identifier=event_tuple[8]) if is_token1_unknown: token1 = deserialize_unknown_ethereum_token_from_db( ethereum_address=event_tuple[12], symbol=event_tuple[13], name=event_tuple[14], decimals=event_tuple[15], ) else: token1 = deserialize_ethereum_token_from_db( identifier=event_tuple[13]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[16]), amount1=deserialize_asset_amount(event_tuple[17]), usd_price=deserialize_price(event_tuple[18]), lp_amount=deserialize_asset_amount(event_tuple[19]), )
def deserialize_swap(raw_swap: Dict[str, Any]) -> AMMSwap: """May raise DeserializationError""" try: tx_hash, log_index = deserialize_transaction_id(raw_swap['id']) timestamp = deserialize_timestamp(raw_swap['timestamp']) raw_tokens = raw_swap['poolAddress']['tokens'] token0_symbol = raw_swap['tokenInSym'] token1_symbol = raw_swap['tokenOutSym'] amount0_in = deserialize_asset_amount(raw_swap['tokenAmountIn']) amount1_out = deserialize_asset_amount(raw_swap['tokenAmountOut']) raw_user_address = raw_swap['userAddress']['id'] # address raw_caller_address = raw_swap['caller'] # from_address raw_pool_address = raw_swap['poolAddress']['id'] # to_address raw_token_in_address = raw_swap['tokenIn'] # token0_address raw_token_out_address = raw_swap['tokenOut'] # token1_address except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e if amount0_in == ZERO: # Prevent a division by zero error when creating the trade raise DeserializationError('TokenAmountIn balance is zero.') # Checksum addresses try: user_address = to_checksum_address(raw_user_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_user_address} in userAddress.id.', ) from e try: caller_address = to_checksum_address(raw_caller_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_caller_address} in caller.', ) from e try: pool_address = to_checksum_address(raw_pool_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_pool_address} in poolAddress.id.', ) from e try: token_in_address = to_checksum_address(raw_token_in_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_token_in_address} in tokenIn.', ) from e try: token_out_address = to_checksum_address(raw_token_out_address) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {raw_token_out_address} in tokenOut.', ) from e # Get token0 and token1 # When the controller removes all the tokens from a pool, `raw_tokens` will # be an empty list. Therefore it won't be possible to get their names and # decimals. In case of having to instantiate an UnknownEthereumToken both # params will be None. if len(raw_tokens) != 0: try: raw_address_tokens = { raw_token['address']: raw_token for raw_token in raw_tokens } raw_token0 = raw_address_tokens[raw_token_in_address] raw_token1 = raw_address_tokens[raw_token_out_address] token0_name = raw_token0['name'] token0_decimals = raw_token0['decimals'] token1_name = raw_token1['name'] token1_decimals = raw_token1['decimals'] except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e else: token0_name = None token0_decimals = None token1_name = None token1_decimals = None token0 = get_ethereum_token( symbol=token0_symbol, ethereum_address=token_in_address, name=token0_name, decimals=token0_decimals, ) token1 = get_ethereum_token( symbol=token1_symbol, ethereum_address=token_out_address, name=token1_name, decimals=token1_decimals, ) amm_swap = AMMSwap( tx_hash=tx_hash, log_index=log_index, address=user_address, from_address=caller_address, to_address=pool_address, timestamp=timestamp, location=Location.BALANCER, token0=token0, token1=token1, amount0_in=amount0_in, amount1_in=AssetAmount(ZERO), amount0_out=AssetAmount(ZERO), amount1_out=amount1_out, ) return amm_swap
"""Process an asset movement result and deserialize it May raise: - DeserializationError - UnknownAsset - UnsupportedAsset """ if case == KucoinCase.DEPOSITS: category = AssetMovementCategory.DEPOSIT elif case == KucoinCase.WITHDRAWALS: category = AssetMovementCategory.WITHDRAWAL else: raise AssertionError(f'Unexpected case: {case}') try: timestamp_ms = deserialize_timestamp(raw_result['createdAt']) timestamp = Timestamp(int(timestamp_ms / 1000)) address = raw_result['address'] # The transaction id can have an @ which we should just get rid of transaction_id = raw_result['walletTxId'].split('@')[0] amount = deserialize_asset_amount(raw_result['amount']) fee = deserialize_fee(raw_result['fee']) fee_currency_symbol = raw_result['currency'] link_id = raw_result.get('id', '') # NB: id only exists for withdrawals except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e fee_asset = asset_from_kucoin(fee_currency_symbol) asset_movement = AssetMovement(