def deserialize_timestamp_from_kraken(
time: Union[str, FVal, int, float]) -> Timestamp:
"""Deserializes a timestamp from a kraken api query result entry
Kraken has timestamps in floating point strings. Example: '1561161486.3056'.
If the dictionary has passed through rlk_jsonloads the entry can also be an Fval
Can throw DeserializationError if the data is not as expected
"""
if not time:
raise DeserializationError(
'Failed to deserialize a timestamp entry from a null entry in kraken',
)
if isinstance(time, int):
return Timestamp(time)
if isinstance(time, (float, str)):
try:
return Timestamp(convert_to_int(time, accept_only_exact=False))
except ConversionError as e:
raise DeserializationError(
f'Failed to deserialize {time} kraken timestamp entry from {type(time)}',
) from e
if isinstance(time, FVal):
try:
return Timestamp(time.to_int(exact=False))
except ConversionError as e:
raise DeserializationError(
f'Failed to deserialize {time} kraken timestamp entry from an FVal',
) from e
# else
raise DeserializationError(
f'Failed to deserialize a timestamp entry from a {type(time)} entry in kraken',
)
def deserialize_int_from_hex_or_int(symbol: Union[str, int],
location: str) -> int:
"""Takes a symbol which can either be an int or a hex string and
turns it into an integer
May Raise:
- DeserializationError if the given data are in an unexpected format.
"""
if isinstance(symbol, int):
result = symbol
elif isinstance(symbol, str):
if symbol == '0x':
return 0
try:
result = int(symbol, 16)
except ValueError as e:
raise DeserializationError(
f'Could not turn string "{symbol}" into an integer {location}',
) from e
else:
raise DeserializationError(
f'Unexpected type {type(symbol)} given to '
f'deserialize_int_from_hex_or_int() for {location}', )
return result
def _parse_asset_data(self, insert_text: str) -> ParsedAssetData:
match = self.assets_re.match(insert_text)
if match is None:
raise DeserializationError(
f'At asset DB update could not parse asset data out of {insert_text}',
)
if len(match.groups()) != 9:
raise DeserializationError(
f'At asset DB update could not parse asset data out of {insert_text}',
)
raw_type = self._parse_str(match.group(2), 'asset type', insert_text)
asset_type = AssetType.deserialize_from_db(raw_type)
raw_started = self._parse_optional_int(match.group(5), 'started',
insert_text)
started = Timestamp(raw_started) if raw_started else None
return ParsedAssetData(
identifier=self._parse_str(match.group(1), 'identifier',
insert_text),
asset_type=asset_type,
name=self._parse_str(match.group(3), 'name', insert_text),
symbol=self._parse_str(match.group(4), 'symbol', insert_text),
started=started,
swapped_for=self._parse_optional_str(match.group(6), 'swapped_for',
insert_text),
coingecko=self._parse_optional_str(match.group(7), 'coingecko',
insert_text),
cryptocompare=self._parse_optional_str(match.group(8),
'cryptocompare',
insert_text),
)
def deserialize_asset_movement_category(
value: Union[str, HistoryEventType], ) -> AssetMovementCategory:
"""Takes a string and determines whether to accept it as an asset movement category
Can throw DeserializationError if value is not as expected
"""
if isinstance(value, str):
if value.lower() == 'deposit':
return AssetMovementCategory.DEPOSIT
if value.lower() in ('withdraw', 'withdrawal'):
return AssetMovementCategory.WITHDRAWAL
raise DeserializationError(
f'Failed to deserialize asset movement category symbol. Unknown {value}',
)
if isinstance(value, HistoryEventType):
if value == HistoryEventType.DEPOSIT:
return AssetMovementCategory.DEPOSIT
if value == HistoryEventType.WITHDRAWAL:
return AssetMovementCategory.WITHDRAWAL
raise DeserializationError(
f'Failed to deserialize asset movement category from '
f'HistoryEventType and {value} entry', )
raise DeserializationError(
f'Failed to deserialize asset movement category from {type(value)} entry',
)
def deserialize_hex_color_code(symbol: str) -> HexColorCode:
"""Takes a string either from the API or the DB and deserializes it into
a hexadecimal color code.
Can throw DeserializationError if the symbol is not as expected
"""
if not isinstance(symbol, str):
raise DeserializationError(
f'Failed to deserialize color code from {type(symbol).__name__} entry',
)
try:
color_value = int(symbol, 16)
except ValueError as e:
raise DeserializationError(
f'The given color code value "{symbol}" could not be processed as a hex color value',
) from e
if color_value < 0 or color_value > 16777215:
raise DeserializationError(
f'The given color code value "{symbol}" is out of range for a normal color field',
)
if len(symbol) != 6:
raise DeserializationError(
f'The given color code value "{symbol}" does not have 6 hexadecimal digits',
)
return HexColorCode(symbol)
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address_token
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType, event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(f'Unexpected event type: {event_tuple_type}.') from e
pool_address_token = EthereumToken.from_identifier(
event_tuple[5],
form_with_incomplete_data=True, # since some may not have decimals input correctly
)
if pool_address_token is None:
raise DeserializationError(
f'Balancer event pool token: {event_tuple[5]} not found in the DB.',
)
amounts: List[AssetAmount] = [
deserialize_asset_amount(item)
for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address_token=pool_address_token,
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def deserialize_trade_pair(pair: str) -> TradePair:
"""Takes a trade pair string, makes sure it's valid, wraps it in proper type and returns it"""
try:
pair_get_assets(TradePair(pair))
except UnprocessableTradePair as e:
raise DeserializationError(str(e)) from e
except UnknownAsset as e:
raise DeserializationError(
f'Unknown asset {e.asset_name} found while processing trade pair',
) from e
return TradePair(pair)
def deserialize_from_db(cls: Type[T], value: str) -> T:
"""May raise a DeserializationError if something is wrong with the DB data"""
if not isinstance(value, str):
raise DeserializationError(
f'Failed to deserialize {cls.__name__} DB value from non string value: {value}',
)
number = ord(value)
if number < 65 or number > list(cls)[-1].value + 64: # type: ignore
raise DeserializationError(
f'Failed to deserialize {cls.__name__} DB value {value}')
return cls(number - 64)
def deserialize_int_from_str(symbol: str, location: str) -> int:
if not isinstance(symbol, str):
raise DeserializationError(
f'Expected a string but got {type(symbol)} at {location}')
try:
result = int(symbol)
except ValueError as e:
raise DeserializationError(
f'Could not turn string "{symbol}" into an integer at {location}',
) from e
return result
def deserialize(cls: Type[T], value: str) -> T:
"""May raise DeserializationError if the given value can't be deserialized"""
if not isinstance(value, str):
raise DeserializationError(
f'Failed to deserialize {cls.__name__} value from non string value: {value}',
)
upper_value = value.replace(' ', '_').upper()
try:
return getattr(cls, upper_value)
except AttributeError as e:
raise DeserializationError(
f'Failed to deserialize {cls.__name__} value {value}'
) from e # noqa: E501
def asset_from_bitfinex(
bitfinex_name: str,
currency_map: Dict[str, str],
is_currency_map_updated: bool = True,
) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
Currency map coming from `<Bitfinex>._query_currency_map()` is already
updated with BITFINEX_TO_WORLD (prevent updating it on each call)
"""
if not isinstance(bitfinex_name, str):
raise DeserializationError(
f'Got non-string type {type(bitfinex_name)} for bitfinex asset')
if bitfinex_name in UNSUPPORTED_BITFINEX_ASSETS:
raise UnsupportedAsset(bitfinex_name)
if is_currency_map_updated is False:
currency_map.update(BITFINEX_TO_WORLD)
symbol = currency_map.get(bitfinex_name, bitfinex_name)
return symbol_to_asset_or_token(symbol)
def asset_from_kraken(kraken_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnknownAsset
"""
if not isinstance(kraken_name, str):
raise DeserializationError(
f'Got non-string type {type(kraken_name)} for kraken asset')
if kraken_name.endswith('.S') or kraken_name.endswith('.M'):
# this is a staked coin. For now since we don't show staked coins
# consider it as the normal version. In the future we may perhaps
# differentiate between them in the balances https://github.com/rotki/rotki/issues/569
kraken_name = kraken_name[:-2]
if kraken_name.endswith('.HOLD'):
kraken_name = kraken_name[:-5]
# Some names are not in the map since kraken can have multiple representations
# depending on the pair for the same asset. For example XXBT and XBT, XETH and ETH,
# ZUSD and USD
if kraken_name == 'SETH':
name = 'ETH2'
elif kraken_name == 'XBT':
name = 'BTC'
elif kraken_name == 'XDG':
name = 'DOGE'
elif kraken_name in ('ETH', 'EUR', 'USD', 'GBP', 'CAD', 'JPY', 'KRW',
'CHF', 'AUD'):
name = kraken_name
else:
name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name)
return symbol_to_asset_or_token(name)
def convert_transaction_from_covalent(
data: Dict[str, Any],
) -> CovalentTransaction:
"""Reads dict data of a transaction from Covalent and deserializes it
Can raise DeserializationError if something is wrong
"""
try:
timestamp = create_timestamp(
datestr=data['block_signed_at'],
formatstr=DATE_FORMAT_COVALENT,
)
# TODO input and nonce is decoded in Covalent api, encoded in future
return CovalentTransaction(
timestamp=timestamp,
block_number=data['block_height'],
tx_hash=data['tx_hash'],
from_address=data['from_address'],
to_address=data['to_address'],
value=read_integer(data, 'value', DEFAULT_API),
gas=read_integer(data, 'gas_offered', DEFAULT_API),
gas_price=read_integer(data, 'gas_price', DEFAULT_API),
gas_used=read_integer(data, 'gas_spent', DEFAULT_API),
input_data='0x',
nonce=0,
)
except KeyError as e:
raise DeserializationError(
f'Covalent avalanche transaction missing expected key {str(e)}',
) from e
def iso8601ts_to_timestamp(datestr: str) -> Timestamp:
"""Requires python 3.7 due to fromisoformat()
But this is still a rather not good function and requires a few tricks to make
it work. So TODO: perhaps switch to python-dateutil package?
Dateutils package info:
https://stackoverflow.com/questions/127803/how-do-i-parse-an-iso-8601-formatted-date
Required tricks for fromisoformat:
https://stackoverflow.com/questions/127803/how-do-i-parse-an-iso-8601-formatted-date/49784038#49784038
"""
# Required due to problems with fromisoformat recognizing the ZULU mark
datestr = datestr.replace("Z", "+00:00")
# The following function does not always properly handle fractions of a second
# so let's just remove it and round to the nearest second since we only deal
# with seconds in the rotkehlchen timestamps
match = FRACTION_SECS_RE.search(datestr)
add_a_second = False
if match:
fraction_str = match.group(1)
datestr = datestr.replace('.' + fraction_str, '')
num = int(fraction_str) / int('1' + '0' * len(fraction_str))
if num > 0.5:
add_a_second = True
try:
ts = Timestamp(
int(datetime.datetime.fromisoformat(datestr).timestamp()))
except ValueError as e:
raise DeserializationError(
f'Couldnt read {datestr} as iso8601ts timestamp') from e
return Timestamp(ts + 1) if add_a_second else ts
def deserialize_transaction_id(raw_tx_id: str) -> Tuple[str, int]:
try:
tx_hash, raw_log_index = raw_tx_id.split('-')
log_index = int(raw_log_index)
except ValueError as e:
raise DeserializationError(f'Unexpected transaction id: {raw_tx_id}.') from e
return tx_hash, log_index
def _parse_str(self, value: str, name: str, insert_text: str) -> str:
result = self._parse_value(value)
if not isinstance(result, str):
raise DeserializationError(
f'At asset DB update got invalid {name} {value} from {insert_text}',
)
return result
def asset_from_coinbase(cb_name: str,
time: Optional[Timestamp] = None) -> Asset:
"""May raise:
- DeserializationError
- UnknownAsset
"""
# During the transition from DAI(SAI) to MCDAI(DAI) coinbase introduced an MCDAI
# wallet for the new DAI during the transition period. We should be able to handle this
# https://support.coinbase.com/customer/portal/articles/2982947
if cb_name == 'MCDAI':
return A_DAI
if cb_name == 'DAI':
# If it's dai and it's queried from the exchange before the end of the upgrade
if not time:
time = ts_now()
if time < COINBASE_DAI_UPGRADE_END_TS:
# Then it should be the single collateral version
return A_SAI
return A_DAI
if not isinstance(cb_name, str):
raise DeserializationError(
f'Got non-string type {type(cb_name)} for coinbase asset')
name = COINBASE_TO_WORLD.get(cb_name, cb_name)
return symbol_to_asset_or_token(name)
def hex_or_bytes_to_address(value: Union[bytes, str]) -> ChecksumEthAddress:
"""Turns a 32bit bytes/HexBytes or a hexstring into an address
May raise:
- DeserializationError if it can't convert a value to an int or if an unexpected
type is given.
"""
try:
hexstr = hex_or_bytes_to_str(value)
except ConversionError as e:
raise DeserializationError(
f'Could not turn {value!r} to an ethereum address') from e
try:
return ChecksumEthAddress(to_checksum_address('0x' + hexstr[24:]))
except ValueError as e:
raise DeserializationError(
f'Invalid ethereum address: {hexstr[24:]}', ) from e
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[
BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY,
],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
try:
pool_address_token = EthereumToken(pool_address)
except UnknownAsset as e:
raise DeserializationError(
f'Balancer pool token with address {pool_address} should have been in the DB',
) from e
token_address = deserialize_ethereum_address(raw_token_address)
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address_token=pool_address_token,
token_address=token_address,
amount=amount,
)
return invest_event
def _parse_optional_int(self, value: str, name: str,
insert_text: str) -> Optional[int]:
result = self._parse_value(value)
if result is not None and not isinstance(result, int):
raise DeserializationError(
f'At asset DB update got invalid {name} {value} from {insert_text}',
)
return result
def deserialize_asset_amount(amount: AcceptableFValInitInput) -> AssetAmount:
try:
result = AssetAmount(FVal(amount))
except ValueError as e:
raise DeserializationError(
f'Failed to deserialize an amount entry: {str(e)}') from e
return result
def deserialize_price(amount: AcceptableFValInitInput) -> Price:
try:
result = Price(FVal(amount))
except ValueError as e:
raise DeserializationError(
f'Failed to deserialize a price/rate entry: {str(e)}') from e
return result
def read_boolean(value: Union[str, bool]) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return str_to_bool(value)
# else
raise DeserializationError(
f'Failed to read a boolean from {value} which is of type {type(value)}',
)
def deserialize_ethereum_token_from_db(identifier: str) -> EthereumToken:
"""Takes an identifier and returns the <EthereumToken>"""
ethereum_token = EthereumToken.from_identifier(identifier=identifier)
if ethereum_token is None:
raise DeserializationError(
f'Could not initialize an ethereum token with identifier {identifier}',
)
return ethereum_token
def deserialize(cls: Type['MatchedAcquisition'], data: Dict[str, Any]) -> 'MatchedAcquisition':
"""May raise DeserializationError"""
try:
event = AssetAcquisitionEvent.deserialize(data['event'])
amount = FVal(data['amount']) # TODO: deserialize_fval
taxable = data['taxable']
except KeyError as e:
raise DeserializationError(f'Missing key {str(e)}') from e
return MatchedAcquisition(amount=amount, event=event, taxable=taxable)
def read_integer(data: Dict[str, Any],
key: str,
api: str = DEFAULT_API) -> int:
try:
result = convert_to_int(data[key])
except ConversionError as e:
raise DeserializationError(
f'Failed to read {key} as an integer during {api} transaction query',
) from e
return result
def bitcoinde_pair_to_world(pair: str) -> Tuple[Asset, Asset]:
if len(pair) == 6:
tx_asset = bitcoinde_asset(pair[:3])
native_asset = bitcoinde_asset(pair[3:])
elif len(pair) in (7, 8):
tx_asset = bitcoinde_asset(pair[:4])
native_asset = bitcoinde_asset(pair[4:])
else:
raise DeserializationError(f'Could not parse pair: {pair}')
return tx_asset, native_asset
def deserialize(value: str) -> 'XpubType':
if value == 'p2pkh':
return XpubType.P2PKH
if value == 'p2sh_p2wpkh':
return XpubType.P2SH_P2WPKH
if value == 'wpkh':
return XpubType.WPKH
# else
raise DeserializationError(
f'Unknown xpub type {value} found at deserialization')
def _get_trade_pair_data_from_transaction(
raw_result: Dict[str, Any]) -> TradePairData:
"""Given a user transaction that contains the base and quote assets'
symbol as keys, return the Bitstamp trade pair data (raw pair str,
base/quote assets raw symbols, and TradePair).
NB: any custom pair conversion (e.g. from Bitstamp asset symbol to world)
should happen here.
Can raise DeserializationError.
"""
try:
pair = [
key for key in raw_result.keys()
if '_' in key and key != 'order_id'
][0]
except IndexError as e:
raise DeserializationError(
'Could not deserialize Bitstamp trade pair from user transaction. '
f'Trade pair not found in: {raw_result}.', ) from e
# NB: `pair_get_assets()` is not used for simplifying the calls and
# storing the raw pair strings.
base_asset_symbol, quote_asset_symbol = pair.split('_')
try:
base_asset = asset_from_bitstamp(base_asset_symbol)
quote_asset = asset_from_bitstamp(quote_asset_symbol)
except (UnknownAsset, UnsupportedAsset) as e:
log.error(str(e))
asset_tag = 'Unknown' if isinstance(
e, UnknownAsset) else 'Unsupported'
raise DeserializationError(
f'{asset_tag} {e.asset_name} found while processing trade pair.',
) from e
return TradePairData(
pair=pair,
base_asset_symbol=base_asset_symbol,
quote_asset_symbol=quote_asset_symbol,
base_asset=base_asset,
quote_asset=quote_asset,
)
def deserialize(cls: Type['AssetAcquisitionEvent'], data: Dict[str, Any]) -> 'AssetAcquisitionEvent': # noqa: E501
"""May raise DeserializationError"""
try:
return cls(
amount=data['full_amount'],
timestamp=data['timestamp'],
rate=data['rate'],
index=data['index'],
)
except KeyError as e:
raise DeserializationError(f'Missing key {str(e)}') from e
