def deserialize_timestamp_from_kraken( time: Union[str, FVal, int, float]) -> Timestamp: """Deserializes a timestamp from a kraken api query result entry Kraken has timestamps in floating point strings. Example: '1561161486.3056'. If the dictionary has passed through rlk_jsonloads the entry can also be an Fval Can throw DeserializationError if the data is not as expected """ if not time: raise DeserializationError( 'Failed to deserialize a timestamp entry from a null entry in kraken', ) if isinstance(time, int): return Timestamp(time) if isinstance(time, (float, str)): try: return Timestamp(convert_to_int(time, accept_only_exact=False)) except ConversionError as e: raise DeserializationError( f'Failed to deserialize {time} kraken timestamp entry from {type(time)}', ) from e if isinstance(time, FVal): try: return Timestamp(time.to_int(exact=False)) except ConversionError as e: raise DeserializationError( f'Failed to deserialize {time} kraken timestamp entry from an FVal', ) from e # else raise DeserializationError( f'Failed to deserialize a timestamp entry from a {type(time)} entry in kraken', )
def deserialize_int_from_hex_or_int(symbol: Union[str, int], location: str) -> int: """Takes a symbol which can either be an int or a hex string and turns it into an integer May Raise: - DeserializationError if the given data are in an unexpected format. """ if isinstance(symbol, int): result = symbol elif isinstance(symbol, str): if symbol == '0x': return 0 try: result = int(symbol, 16) except ValueError as e: raise DeserializationError( f'Could not turn string "{symbol}" into an integer {location}', ) from e else: raise DeserializationError( f'Unexpected type {type(symbol)} given to ' f'deserialize_int_from_hex_or_int() for {location}', ) return result
def _parse_asset_data(self, insert_text: str) -> ParsedAssetData: match = self.assets_re.match(insert_text) if match is None: raise DeserializationError( f'At asset DB update could not parse asset data out of {insert_text}', ) if len(match.groups()) != 9: raise DeserializationError( f'At asset DB update could not parse asset data out of {insert_text}', ) raw_type = self._parse_str(match.group(2), 'asset type', insert_text) asset_type = AssetType.deserialize_from_db(raw_type) raw_started = self._parse_optional_int(match.group(5), 'started', insert_text) started = Timestamp(raw_started) if raw_started else None return ParsedAssetData( identifier=self._parse_str(match.group(1), 'identifier', insert_text), asset_type=asset_type, name=self._parse_str(match.group(3), 'name', insert_text), symbol=self._parse_str(match.group(4), 'symbol', insert_text), started=started, swapped_for=self._parse_optional_str(match.group(6), 'swapped_for', insert_text), coingecko=self._parse_optional_str(match.group(7), 'coingecko', insert_text), cryptocompare=self._parse_optional_str(match.group(8), 'cryptocompare', insert_text), )
def deserialize_asset_movement_category( value: Union[str, HistoryEventType], ) -> AssetMovementCategory: """Takes a string and determines whether to accept it as an asset movement category Can throw DeserializationError if value is not as expected """ if isinstance(value, str): if value.lower() == 'deposit': return AssetMovementCategory.DEPOSIT if value.lower() in ('withdraw', 'withdrawal'): return AssetMovementCategory.WITHDRAWAL raise DeserializationError( f'Failed to deserialize asset movement category symbol. Unknown {value}', ) if isinstance(value, HistoryEventType): if value == HistoryEventType.DEPOSIT: return AssetMovementCategory.DEPOSIT if value == HistoryEventType.WITHDRAWAL: return AssetMovementCategory.WITHDRAWAL raise DeserializationError( f'Failed to deserialize asset movement category from ' f'HistoryEventType and {value} entry', ) raise DeserializationError( f'Failed to deserialize asset movement category from {type(value)} entry', )
def deserialize_hex_color_code(symbol: str) -> HexColorCode: """Takes a string either from the API or the DB and deserializes it into a hexadecimal color code. Can throw DeserializationError if the symbol is not as expected """ if not isinstance(symbol, str): raise DeserializationError( f'Failed to deserialize color code from {type(symbol).__name__} entry', ) try: color_value = int(symbol, 16) except ValueError as e: raise DeserializationError( f'The given color code value "{symbol}" could not be processed as a hex color value', ) from e if color_value < 0 or color_value > 16777215: raise DeserializationError( f'The given color code value "{symbol}" is out of range for a normal color field', ) if len(symbol) != 6: raise DeserializationError( f'The given color code value "{symbol}" does not have 6 hexadecimal digits', ) return HexColorCode(symbol)
def deserialize_from_db( cls, event_tuple: BalancerEventDBTuple, ) -> 'BalancerEvent': """May raise DeserializationError Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address_token 6 - lp_amount 7 - usd_value 8 - amount0 9 - amount1 10 - amount2 11 - amount3 12 - amount4 13 - amount5 14 - amount6 15 - amount7 """ event_tuple_type = event_tuple[4] try: event_type = getattr(BalancerBPTEventType, event_tuple_type.upper()) except AttributeError as e: raise DeserializationError(f'Unexpected event type: {event_tuple_type}.') from e pool_address_token = EthereumToken.from_identifier( event_tuple[5], form_with_incomplete_data=True, # since some may not have decimals input correctly ) if pool_address_token is None: raise DeserializationError( f'Balancer event pool token: {event_tuple[5]} not found in the DB.', ) amounts: List[AssetAmount] = [ deserialize_asset_amount(item) for item in event_tuple[8:16] if item is not None ] return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address_token=pool_address_token, lp_balance=Balance( amount=deserialize_asset_amount(event_tuple[6]), usd_value=deserialize_price(event_tuple[7]), ), amounts=amounts, )
def deserialize_trade_pair(pair: str) -> TradePair: """Takes a trade pair string, makes sure it's valid, wraps it in proper type and returns it""" try: pair_get_assets(TradePair(pair)) except UnprocessableTradePair as e: raise DeserializationError(str(e)) from e except UnknownAsset as e: raise DeserializationError( f'Unknown asset {e.asset_name} found while processing trade pair', ) from e return TradePair(pair)
def deserialize_from_db(cls: Type[T], value: str) -> T: """May raise a DeserializationError if something is wrong with the DB data""" if not isinstance(value, str): raise DeserializationError( f'Failed to deserialize {cls.__name__} DB value from non string value: {value}', ) number = ord(value) if number < 65 or number > list(cls)[-1].value + 64: # type: ignore raise DeserializationError( f'Failed to deserialize {cls.__name__} DB value {value}') return cls(number - 64)
def deserialize_int_from_str(symbol: str, location: str) -> int: if not isinstance(symbol, str): raise DeserializationError( f'Expected a string but got {type(symbol)} at {location}') try: result = int(symbol) except ValueError as e: raise DeserializationError( f'Could not turn string "{symbol}" into an integer at {location}', ) from e return result
def deserialize(cls: Type[T], value: str) -> T: """May raise DeserializationError if the given value can't be deserialized""" if not isinstance(value, str): raise DeserializationError( f'Failed to deserialize {cls.__name__} value from non string value: {value}', ) upper_value = value.replace(' ', '_').upper() try: return getattr(cls, upper_value) except AttributeError as e: raise DeserializationError( f'Failed to deserialize {cls.__name__} value {value}' ) from e # noqa: E501
def asset_from_bitfinex( bitfinex_name: str, currency_map: Dict[str, str], is_currency_map_updated: bool = True, ) -> Asset: """May raise: - DeserializationError - UnsupportedAsset - UnknownAsset Currency map coming from `<Bitfinex>._query_currency_map()` is already updated with BITFINEX_TO_WORLD (prevent updating it on each call) """ if not isinstance(bitfinex_name, str): raise DeserializationError( f'Got non-string type {type(bitfinex_name)} for bitfinex asset') if bitfinex_name in UNSUPPORTED_BITFINEX_ASSETS: raise UnsupportedAsset(bitfinex_name) if is_currency_map_updated is False: currency_map.update(BITFINEX_TO_WORLD) symbol = currency_map.get(bitfinex_name, bitfinex_name) return symbol_to_asset_or_token(symbol)
def asset_from_kraken(kraken_name: str) -> Asset: """May raise: - DeserializationError - UnknownAsset """ if not isinstance(kraken_name, str): raise DeserializationError( f'Got non-string type {type(kraken_name)} for kraken asset') if kraken_name.endswith('.S') or kraken_name.endswith('.M'): # this is a staked coin. For now since we don't show staked coins # consider it as the normal version. In the future we may perhaps # differentiate between them in the balances https://github.com/rotki/rotki/issues/569 kraken_name = kraken_name[:-2] if kraken_name.endswith('.HOLD'): kraken_name = kraken_name[:-5] # Some names are not in the map since kraken can have multiple representations # depending on the pair for the same asset. For example XXBT and XBT, XETH and ETH, # ZUSD and USD if kraken_name == 'SETH': name = 'ETH2' elif kraken_name == 'XBT': name = 'BTC' elif kraken_name == 'XDG': name = 'DOGE' elif kraken_name in ('ETH', 'EUR', 'USD', 'GBP', 'CAD', 'JPY', 'KRW', 'CHF', 'AUD'): name = kraken_name else: name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name) return symbol_to_asset_or_token(name)
def convert_transaction_from_covalent( data: Dict[str, Any], ) -> CovalentTransaction: """Reads dict data of a transaction from Covalent and deserializes it Can raise DeserializationError if something is wrong """ try: timestamp = create_timestamp( datestr=data['block_signed_at'], formatstr=DATE_FORMAT_COVALENT, ) # TODO input and nonce is decoded in Covalent api, encoded in future return CovalentTransaction( timestamp=timestamp, block_number=data['block_height'], tx_hash=data['tx_hash'], from_address=data['from_address'], to_address=data['to_address'], value=read_integer(data, 'value', DEFAULT_API), gas=read_integer(data, 'gas_offered', DEFAULT_API), gas_price=read_integer(data, 'gas_price', DEFAULT_API), gas_used=read_integer(data, 'gas_spent', DEFAULT_API), input_data='0x', nonce=0, ) except KeyError as e: raise DeserializationError( f'Covalent avalanche transaction missing expected key {str(e)}', ) from e
def iso8601ts_to_timestamp(datestr: str) -> Timestamp: """Requires python 3.7 due to fromisoformat() But this is still a rather not good function and requires a few tricks to make it work. So TODO: perhaps switch to python-dateutil package? Dateutils package info: https://stackoverflow.com/questions/127803/how-do-i-parse-an-iso-8601-formatted-date Required tricks for fromisoformat: https://stackoverflow.com/questions/127803/how-do-i-parse-an-iso-8601-formatted-date/49784038#49784038 """ # Required due to problems with fromisoformat recognizing the ZULU mark datestr = datestr.replace("Z", "+00:00") # The following function does not always properly handle fractions of a second # so let's just remove it and round to the nearest second since we only deal # with seconds in the rotkehlchen timestamps match = FRACTION_SECS_RE.search(datestr) add_a_second = False if match: fraction_str = match.group(1) datestr = datestr.replace('.' + fraction_str, '') num = int(fraction_str) / int('1' + '0' * len(fraction_str)) if num > 0.5: add_a_second = True try: ts = Timestamp( int(datetime.datetime.fromisoformat(datestr).timestamp())) except ValueError as e: raise DeserializationError( f'Couldnt read {datestr} as iso8601ts timestamp') from e return Timestamp(ts + 1) if add_a_second else ts
def deserialize_transaction_id(raw_tx_id: str) -> Tuple[str, int]: try: tx_hash, raw_log_index = raw_tx_id.split('-') log_index = int(raw_log_index) except ValueError as e: raise DeserializationError(f'Unexpected transaction id: {raw_tx_id}.') from e return tx_hash, log_index
def _parse_str(self, value: str, name: str, insert_text: str) -> str: result = self._parse_value(value) if not isinstance(result, str): raise DeserializationError( f'At asset DB update got invalid {name} {value} from {insert_text}', ) return result
def asset_from_coinbase(cb_name: str, time: Optional[Timestamp] = None) -> Asset: """May raise: - DeserializationError - UnknownAsset """ # During the transition from DAI(SAI) to MCDAI(DAI) coinbase introduced an MCDAI # wallet for the new DAI during the transition period. We should be able to handle this # https://support.coinbase.com/customer/portal/articles/2982947 if cb_name == 'MCDAI': return A_DAI if cb_name == 'DAI': # If it's dai and it's queried from the exchange before the end of the upgrade if not time: time = ts_now() if time < COINBASE_DAI_UPGRADE_END_TS: # Then it should be the single collateral version return A_SAI return A_DAI if not isinstance(cb_name, str): raise DeserializationError( f'Got non-string type {type(cb_name)} for coinbase asset') name = COINBASE_TO_WORLD.get(cb_name, cb_name) return symbol_to_asset_or_token(name)
def hex_or_bytes_to_address(value: Union[bytes, str]) -> ChecksumEthAddress: """Turns a 32bit bytes/HexBytes or a hexstring into an address May raise: - DeserializationError if it can't convert a value to an int or if an unexpected type is given. """ try: hexstr = hex_or_bytes_to_str(value) except ConversionError as e: raise DeserializationError( f'Could not turn {value!r} to an ethereum address') from e try: return ChecksumEthAddress(to_checksum_address('0x' + hexstr[24:])) except ValueError as e: raise DeserializationError( f'Invalid ethereum address: {hexstr[24:]}', ) from e
def deserialize_invest_event( raw_event: Dict[str, Any], event_type: Literal[ BalancerInvestEventType.ADD_LIQUIDITY, BalancerInvestEventType.REMOVE_LIQUIDITY, ], ) -> BalancerInvestEvent: """May raise DeserializationError""" try: tx_hash, log_index = deserialize_transaction_id(raw_event['id']) timestamp = deserialize_timestamp(raw_event['timestamp']) raw_user_address = raw_event['userAddress']['id'] raw_pool_address = raw_event['poolAddress']['id'] if event_type == BalancerInvestEventType.ADD_LIQUIDITY: raw_token_address = raw_event['tokenIn']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountIn']) elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY: raw_token_address = raw_event['tokenOut']['address'] amount = deserialize_asset_amount(raw_event['tokenAmountOut']) else: raise AssertionError(f'Unexpected event type: {event_type}.') except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e user_address = deserialize_ethereum_address(raw_user_address) pool_address = deserialize_ethereum_address(raw_pool_address) try: pool_address_token = EthereumToken(pool_address) except UnknownAsset as e: raise DeserializationError( f'Balancer pool token with address {pool_address} should have been in the DB', ) from e token_address = deserialize_ethereum_address(raw_token_address) invest_event = BalancerInvestEvent( tx_hash=tx_hash, log_index=log_index, address=user_address, timestamp=timestamp, event_type=event_type, pool_address_token=pool_address_token, token_address=token_address, amount=amount, ) return invest_event
def _parse_optional_int(self, value: str, name: str, insert_text: str) -> Optional[int]: result = self._parse_value(value) if result is not None and not isinstance(result, int): raise DeserializationError( f'At asset DB update got invalid {name} {value} from {insert_text}', ) return result
def deserialize_asset_amount(amount: AcceptableFValInitInput) -> AssetAmount: try: result = AssetAmount(FVal(amount)) except ValueError as e: raise DeserializationError( f'Failed to deserialize an amount entry: {str(e)}') from e return result
def deserialize_price(amount: AcceptableFValInitInput) -> Price: try: result = Price(FVal(amount)) except ValueError as e: raise DeserializationError( f'Failed to deserialize a price/rate entry: {str(e)}') from e return result
def read_boolean(value: Union[str, bool]) -> bool: if isinstance(value, bool): return value if isinstance(value, str): return str_to_bool(value) # else raise DeserializationError( f'Failed to read a boolean from {value} which is of type {type(value)}', )
def deserialize_ethereum_token_from_db(identifier: str) -> EthereumToken: """Takes an identifier and returns the <EthereumToken>""" ethereum_token = EthereumToken.from_identifier(identifier=identifier) if ethereum_token is None: raise DeserializationError( f'Could not initialize an ethereum token with identifier {identifier}', ) return ethereum_token
def deserialize(cls: Type['MatchedAcquisition'], data: Dict[str, Any]) -> 'MatchedAcquisition': """May raise DeserializationError""" try: event = AssetAcquisitionEvent.deserialize(data['event']) amount = FVal(data['amount']) # TODO: deserialize_fval taxable = data['taxable'] except KeyError as e: raise DeserializationError(f'Missing key {str(e)}') from e return MatchedAcquisition(amount=amount, event=event, taxable=taxable)
def read_integer(data: Dict[str, Any], key: str, api: str = DEFAULT_API) -> int: try: result = convert_to_int(data[key]) except ConversionError as e: raise DeserializationError( f'Failed to read {key} as an integer during {api} transaction query', ) from e return result
def bitcoinde_pair_to_world(pair: str) -> Tuple[Asset, Asset]: if len(pair) == 6: tx_asset = bitcoinde_asset(pair[:3]) native_asset = bitcoinde_asset(pair[3:]) elif len(pair) in (7, 8): tx_asset = bitcoinde_asset(pair[:4]) native_asset = bitcoinde_asset(pair[4:]) else: raise DeserializationError(f'Could not parse pair: {pair}') return tx_asset, native_asset
def deserialize(value: str) -> 'XpubType': if value == 'p2pkh': return XpubType.P2PKH if value == 'p2sh_p2wpkh': return XpubType.P2SH_P2WPKH if value == 'wpkh': return XpubType.WPKH # else raise DeserializationError( f'Unknown xpub type {value} found at deserialization')
def _get_trade_pair_data_from_transaction( raw_result: Dict[str, Any]) -> TradePairData: """Given a user transaction that contains the base and quote assets' symbol as keys, return the Bitstamp trade pair data (raw pair str, base/quote assets raw symbols, and TradePair). NB: any custom pair conversion (e.g. from Bitstamp asset symbol to world) should happen here. Can raise DeserializationError. """ try: pair = [ key for key in raw_result.keys() if '_' in key and key != 'order_id' ][0] except IndexError as e: raise DeserializationError( 'Could not deserialize Bitstamp trade pair from user transaction. ' f'Trade pair not found in: {raw_result}.', ) from e # NB: `pair_get_assets()` is not used for simplifying the calls and # storing the raw pair strings. base_asset_symbol, quote_asset_symbol = pair.split('_') try: base_asset = asset_from_bitstamp(base_asset_symbol) quote_asset = asset_from_bitstamp(quote_asset_symbol) except (UnknownAsset, UnsupportedAsset) as e: log.error(str(e)) asset_tag = 'Unknown' if isinstance( e, UnknownAsset) else 'Unsupported' raise DeserializationError( f'{asset_tag} {e.asset_name} found while processing trade pair.', ) from e return TradePairData( pair=pair, base_asset_symbol=base_asset_symbol, quote_asset_symbol=quote_asset_symbol, base_asset=base_asset, quote_asset=quote_asset, )
def deserialize(cls: Type['AssetAcquisitionEvent'], data: Dict[str, Any]) -> 'AssetAcquisitionEvent': # noqa: E501 """May raise DeserializationError""" try: return cls( amount=data['full_amount'], timestamp=data['timestamp'], rate=data['rate'], index=data['index'], ) except KeyError as e: raise DeserializationError(f'Missing key {str(e)}') from e