def deserialize_fee(fee: Optional[str]) -> Fee:
"""Deserializes a fee from a json entry. Fee in the JSON entry can also be null
in which case a ZERO fee is returned.
Can throw DeserializationError if the fee is not as expected
"""
if not fee:
return Fee(ZERO)
try:
result = Fee(FVal(fee))
except ValueError as e:
raise DeserializationError(
f'Failed to deserialize a fee entry due to: {str(e)}')
return result
def __post_init__(self, form_with_incomplete_data: bool = False) -> None:
object.__setattr__(self, 'identifier',
ETHEREUM_DIRECTIVE + self.identifier)
super().__post_init__(form_with_incomplete_data)
# TODO: figure out a way to move this out. Moved in here due to cyclic imports
from rotkehlchen.assets.resolver import AssetResolver # isort:skip # noqa: E501 # pylint: disable=import-outside-toplevel
data = AssetResolver().get_asset_data(self.identifier) # pylint: disable=no-member
if not data.ethereum_address:
raise DeserializationError(
'Tried to initialize a non Ethereum asset as Ethereum Token', )
object.__setattr__(self, 'ethereum_address', data.ethereum_address)
object.__setattr__(self, 'decimals', data.decimals)
object.__setattr__(self, 'protocol', data.protocol)
def asset_from_binance(binance_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(binance_name, str):
raise DeserializationError(f'Got non-string type {type(binance_name)} for binance asset')
if binance_name in UNSUPPORTED_BINANCE_ASSETS:
raise UnsupportedAsset(binance_name)
if binance_name in RENAMED_BINANCE_ASSETS:
return Asset(RENAMED_BINANCE_ASSETS[binance_name])
name = BINANCE_TO_WORLD.get(binance_name, binance_name)
return Asset(name)
def bittrex_pair_to_world(given_pair: str) -> Tuple[Asset, Asset]:
"""
Turns a pair written in bittrex to way to rotki base/quote asset
Throws:
- UnsupportedAsset due to asset_from_bittrex()
- UnprocessableTradePair if the pair can't be split into its parts
"""
if not isinstance(given_pair, str):
raise DeserializationError(
f'Could not deserialize bittrex trade pair. Expected a string '
f'but found {type(given_pair)}',
)
pair = TradePair(given_pair.replace('-', '_'))
base_asset = asset_from_bittrex(get_pair_position_str(pair, 'first'))
quote_asset = asset_from_bittrex(get_pair_position_str(pair, 'second'))
return base_asset, quote_asset
def bittrex_pair_to_world(given_pair: str) -> TradePair:
"""
Turns a pair written in the bittrex way to Rotkehlchen way
Throws:
- UnsupportedAsset due to asset_from_bittrex()
- UnprocessableTradePair if the pair can't be split into its parts
"""
if not isinstance(given_pair, str):
raise DeserializationError(
f'Could not deserialize bittrex trade pair. Expected a string '
f'but found {type(given_pair)}', )
pair = TradePair(given_pair.replace('-', '_'))
# Check that there is no unsupported asset in the trade
_ = asset_from_bittrex(get_pair_position_str(pair, 'first'))
_ = asset_from_bittrex(get_pair_position_str(pair, 'second'))
return pair
def asset_from_binance(binance_name: str) -> Asset:
if not isinstance(binance_name, str):
raise DeserializationError(f'Got non-string type {type(binance_name)} for binance asset')
if len(binance_name) >= 5 and binance_name.startswith('LD'):
# this is a lending/savings coin. For now since we don't show lending/savings coins
# consider it as the normal version. In the future we may perhaps
# differentiate between them in the balances
binance_name = binance_name[2:]
if binance_name in UNSUPPORTED_BINANCE_ASSETS:
raise UnsupportedAsset(binance_name)
if binance_name in RENAMED_BINANCE_ASSETS:
return Asset(RENAMED_BINANCE_ASSETS[binance_name])
name = BINANCE_TO_WORLD.get(binance_name, binance_name)
return Asset(name)
def asset_from_ftx(ftx_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(ftx_name, str):
raise DeserializationError(
f'Got non-string type {type(ftx_name)} for ftx asset')
if ftx_name in UNSUPPORTED_FTX_ASSETS:
raise UnsupportedAsset(ftx_name)
if ftx_name == 'SRM_LOCKED':
name = strethaddress_to_identifier(
'0x476c5E26a75bd202a9683ffD34359C0CC15be0fF') # SRM
else:
name = FTX_TO_WORLD.get(ftx_name, ftx_name)
return symbol_to_asset_or_token(name)
def _deserialize_trade(
self,
raw_trade: Dict[str, Any],
) -> Trade:
"""Process a trade user transaction from Bitstamp and deserialize it.
Can raise DeserializationError.
"""
timestamp = deserialize_timestamp_from_bitstamp_date(raw_trade['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_trade)
base_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_trade[trade_pair_data.quote_asset_symbol],
)
rate = deserialize_price(raw_trade[trade_pair_data.pair])
fee_currency = trade_pair_data.quote_asset
if base_asset_amount >= ZERO:
trade_type = TradeType.BUY
else:
if quote_asset_amount < 0:
raise DeserializationError(
f'Unexpected bitstamp trade format. Both base and quote '
f'amounts are negative: {raw_trade}',
)
trade_type = TradeType.SELL
trade = Trade(
timestamp=timestamp,
location=Location.BITSTAMP,
base_asset=trade_pair_data.base_asset,
quote_asset=trade_pair_data.quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(base_asset_amount)),
rate=rate,
fee=deserialize_fee(raw_trade['fee']),
fee_currency=fee_currency,
link=str(raw_trade['id']),
notes='',
)
return trade
def _deserialize_unbond_request(
self,
raw_event: Dict[str, Any],
identity_address_map: Dict[ChecksumAddress, ChecksumAddress],
) -> UnbondRequest:
"""Deserialize an unbond request event.
It may raise KeyError.
"""
try:
adex_event = self._deserialize_adex_staking_event(
raw_event=raw_event,
identity_address_map=identity_address_map,
case='unbond_request',
)
bond_id = raw_event['bondId']
unlock_at = deserialize_timestamp(raw_event['willUnlock'])
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key in event: {msg}.'
log.error(
'Failed to deserialize an AdEx unbond request event',
error=msg,
raw_event=raw_event,
identity_address_map=identity_address_map,
)
raise DeserializationError(
'Failed to deserialize an AdEx unbond request event. Check logs for more details',
) from e
return UnbondRequest(
tx_hash=adex_event.tx_hash,
address=adex_event.address,
identity_address=adex_event.identity_address,
timestamp=adex_event.timestamp,
bond_id=bond_id,
value=Balance(),
unlock_at=unlock_at,
)
def bittrex_pair_to_world(given_pair: str) -> TradePair:
"""
Turns a pair written in the bittrex way to Rotkehlchen way
Throws:
- UnsupportedAsset due to asset_from_bittrex()
- UnprocessableTradePair if the pair can't be split into its parts
"""
if not isinstance(given_pair, str):
raise DeserializationError(
f'Could not deserialize bittrex trade pair. Expected a string '
f'but found {type(given_pair)}', )
pair = TradePair(given_pair.replace('-', '_'))
base_currency = asset_from_bittrex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_bittrex(get_pair_position_str(pair, 'second'))
# Since in Bittrex the base currency is the cost currency, iow in Bittrex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = trade_pair_from_assets(quote_currency, base_currency)
return pair
def calculate_trade_from_swaps(
swaps: List[AMMSwap],
trade_index: int = 0,
) -> AMMTrade:
"""Given a list of 1 or more AMMSwap (swap) return an AMMTrade (trade).
The trade is calculated using the first swap token (QUOTE) and last swap
token (BASE). Be aware that any token data in between will be ignored for
calculating the trade.
Examples:
[USDC -> AMPL] BASE_QUOTE pair is AMPL_USDC.
[USDC -> AMPL, AMPL -> WETH] BASE_QUOTE pair is WETH_USDC.
[USDC -> AMPL, AMPL -> WETH, WETH -> USDC] BASE_QUOTE pair is USDC_USDC.
May raise DeserializationError
"""
assert len(swaps) != 0, "Swaps can't be an empty list here"
if swaps[0].amount0_in == ZERO:
# Prevent a division by zero error when creating the trade.
# Swaps with `tokenIn` amount (<AMMSwap>.amount0_in) equals to zero are
# not expected nor supported. The function `deserialize_swap` will raise
# a DeserializationError, preventing to store them in the DB. In case
# of having a zero amount it means the db data was corrupted.
log.error(
'Failed to deserialize swap from db. First swap amount0_in is zero',
swaps=swaps,
)
raise DeserializationError('First swap amount0_in is zero.')
amm_trade = AMMTrade(
trade_type=TradeType.BUY, # AMMTrade is always a buy
base_asset=swaps[-1].token1,
quote_asset=swaps[0].token0,
amount=swaps[-1].amount1_out,
rate=Price(swaps[-1].amount1_out / swaps[0].amount0_in),
swaps=swaps,
trade_index=trade_index,
)
return amm_trade
def __post_init__(self, form_with_incomplete_data: bool = False) -> None:
"""
Asset post initialization
The only thing that is given to initialize an asset is a string.
If a non string is given then it's probably a deserialization error or
invalid data were given to us by the server if an API was queried.
If `form_with_incomplete_data` is given and is True then we allow the generation
of an asset object even if the corresponding underlying object is missing
important data such as name, symbol, token decimals etc. In most case this
is not wanted except for some exception like passing in some functions for
icon generation.
May raise UnknownAsset if the asset identifier can't be matched to anything
"""
if not isinstance(self.identifier, str):
raise DeserializationError(
'Tried to initialize an asset out of a non-string identifier',
)
# TODO: figure out a way to move this out. Moved in here due to cyclic imports
from rotkehlchen.assets.resolver import AssetResolver # isort:skip # noqa: E501 # pylint: disable=import-outside-toplevel
data = AssetResolver().get_asset_data(self.identifier,
form_with_incomplete_data)
# make sure same case of identifier as in DB is saved in the structure
object.__setattr__(self, 'identifier', data.identifier)
# Ugly hack to set attributes of a frozen data class as post init
# https://docs.python.org/3/library/dataclasses.html#frozen-instances
object.__setattr__(self, 'name', data.name)
object.__setattr__(self, 'symbol', data.symbol)
object.__setattr__(self, 'active', data.active)
object.__setattr__(self, 'asset_type', data.asset_type)
object.__setattr__(self, 'started', data.started)
object.__setattr__(self, 'ended', data.ended)
object.__setattr__(self, 'forked', data.forked)
object.__setattr__(self, 'swapped_for', data.swapped_for)
object.__setattr__(self, 'cryptocompare', data.cryptocompare)
object.__setattr__(self, 'coingecko', data.coingecko)
def asset_from_kraken(kraken_name: str) -> Asset:
if not isinstance(kraken_name, str):
raise DeserializationError(f'Got non-string type {type(kraken_name)} for kraken asset')
if kraken_name.endswith('.S'):
# this is a staked coin. For now since we don't show staked coins
# consider it as the normal version. In the future we may perhaps
# differentiate between them in the balances
kraken_name = kraken_name[:-2]
# Some names are not in the map since kraken can have multiple representations
# depending on the pair for the same asset. For example XXBT and XBT, XETH and ETH,
# ZUSD and USD
if kraken_name == 'XBT':
name = 'BTC'
elif kraken_name == 'XDG':
name = 'DOGE'
elif kraken_name in ('ETH', 'EUR', 'USD', 'GBP', 'CAD', 'JPY', 'KRW', 'CHF'):
name = kraken_name
else:
name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name)
return Asset(name)
def deserialize_invest_event(
raw_event: Dict[str, Any],
event_type: Literal[BalancerInvestEventType.ADD_LIQUIDITY,
BalancerInvestEventType.REMOVE_LIQUIDITY, ],
) -> BalancerInvestEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
timestamp = deserialize_timestamp(raw_event['timestamp'])
raw_user_address = raw_event['userAddress']['id']
raw_pool_address = raw_event['poolAddress']['id']
if event_type == BalancerInvestEventType.ADD_LIQUIDITY:
raw_token_address = raw_event['tokenIn']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountIn'])
elif event_type == BalancerInvestEventType.REMOVE_LIQUIDITY:
raw_token_address = raw_event['tokenOut']['address']
amount = deserialize_asset_amount(raw_event['tokenAmountOut'])
else:
raise AssertionError(f'Unexpected event type: {event_type}.')
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
token_address = deserialize_ethereum_address(raw_token_address)
invest_event = BalancerInvestEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
timestamp=timestamp,
event_type=event_type,
pool_address=pool_address,
token_address=token_address,
amount=amount,
)
return invest_event
def deserialize_unknown_ethereum_token_from_db(
ethereum_address: str,
symbol: str,
name: Optional[str],
decimals: Optional[int],
) -> UnknownEthereumToken:
"""Takes at least an ethereum address and a symbol, and returns an
<UnknownEthereumToken>
"""
try:
unknown_ethereum_token = UnknownEthereumToken(
ethereum_address=deserialize_ethereum_address(ethereum_address),
symbol=symbol,
name=name,
decimals=decimals,
)
except Exception as e:
raise DeserializationError(
f'Failed deserializing an unknown ethereum token with '
f'address {ethereum_address}, symbol {symbol}, name {name}, '
f'decimals {decimals}.', ) from e
return unknown_ethereum_token
def deserialize(
cls,
event_type: SchemaEventType,
data: Dict[str, Any],
) -> 'NamedJson':
"""Turns an event type and a data dict to a NamedJson object
May raise:
- a DeserializationError if something is wrong with given data or json validation fails.
"""
schema = event_type.get_schema()
try:
jsonschema.validate(data, schema)
except (jsonschema.exceptions.ValidationError,
jsonschema.exceptions.SchemaError) as e:
raise DeserializationError(
f'Failed jsonschema validation of {str(event_type)} data {data}. '
f'Error was {str(e)}', ) from e
return NamedJson(
event_type=event_type,
data=data,
)
def deserialize_from_db(
cls,
json_tuple: NamedJsonDBTuple,
) -> 'NamedJson':
"""Turns a tuple read from the database into an appropriate JsonSchema.
May raise:
- a DeserializationError if something is wrong with the DB data or json validation fails.
Event_tuple index - Schema columns
----------------------------------
0 - event_type
1 - data
"""
event_type = SchemaEventType.deserialize_from_db(json_tuple[0])
try:
data = json.loads(json_tuple[1])
except json.decoder.JSONDecodeError as e:
raise DeserializationError(
f'Could not decode json for {json_tuple} at NamedJson deserialization: {str(e)}',
) from e
return cls.deserialize(event_type=event_type, data=data)
def asset_from_bitfinex(
bitfinex_name: str,
currency_map: Dict[str, str],
is_currency_map_updated: bool = True,
) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
Currency map coming from `<Bitfinex>._query_currency_map()` is already
updated with BITFINEX_TO_WORLD (prevent updating it on each call)
"""
if not isinstance(bitfinex_name, str):
raise DeserializationError(f'Got non-string type {type(bitfinex_name)} for bitfinex asset')
if bitfinex_name in UNSUPPORTED_BITFINEX_ASSETS:
raise UnsupportedAsset(bitfinex_name)
if is_currency_map_updated is False:
currency_map.update(BITFINEX_TO_WORLD)
name = currency_map.get(bitfinex_name, bitfinex_name)
return Asset(name)
def deserialize_transaction_from_etherscan(
data: Dict[str, Any],
internal: bool,
) -> EthereumTransaction:
"""Reads dict data of a transaction from etherscan and deserializes it
Can raise DeserializationError if something is wrong
"""
try:
# internal tx list contains no gasprice
gas_price = -1 if internal else read_integer(data, 'gasPrice')
tx_hash = read_hash(data, 'hash')
input_data = read_hash(data, 'input')
timestamp = deserialize_timestamp(data['timeStamp'])
block_number = read_integer(data, 'blockNumber')
nonce = -1 if internal else read_integer(data, 'nonce')
return EthereumTransaction(
timestamp=timestamp,
block_number=block_number,
tx_hash=tx_hash,
from_address=to_checksum_address(data['from']),
to_address=to_checksum_address(data['to'])
if data['to'] != '' else None,
value=read_integer(data, 'value'),
gas=read_integer(data, 'gas'),
gas_price=gas_price,
gas_used=read_integer(data, 'gasUsed'),
input_data=input_data,
nonce=nonce,
)
except KeyError as e:
raise DeserializationError(
f'Etherscan ethereum transaction missing expected key {str(e)}',
) from e
def _parse_full_insert(self, insert_text: str) -> AssetData:
"""Parses the full insert line for an asset to give information for the conflict to the user
Note: In the future this needs to be different for each version
May raise:
- DeserializationError if the appropriate data is not found or if it can't
be properly parsed.
"""
asset_data = self._parse_asset_data(insert_text)
forked = address = decimals = protocol = None
if asset_data.asset_type == AssetType.ETHEREUM_TOKEN:
address, decimals, protocol = self._parse_ethereum_token_data(
insert_text)
else:
match = self.common_asset_details_re.match(insert_text)
if match is None:
raise DeserializationError(
f'At asset DB update could not parse common asset '
f'details data out of {insert_text}', )
forked = self._parse_optional_str(match.group(2), 'forked',
insert_text)
return AssetData( # types are not really proper here (except for asset_type)
identifier=asset_data.identifier,
name=asset_data.name,
symbol=asset_data.symbol,
asset_type=asset_data.asset_type,
started=asset_data.started,
forked=forked,
swapped_for=asset_data.swapped_for,
ethereum_address=address,
decimals=decimals,
cryptocompare=asset_data.cryptocompare,
coingecko=asset_data.coingecko,
protocol=protocol,
)
def iso8601ts_to_timestamp(datestr: str) -> Timestamp:
try:
return create_timestamp(datestr, formatstr='%Y-%m-%dT%H:%M:%S.%fZ')
except ValueError:
raise DeserializationError(f'Couldnt read {datestr} as iso8601ts timestamp')
def trade_from_binance(
binance_trade: Dict,
binance_symbols_to_pair: Dict[str, BinancePair],
) -> Trade:
"""Turn a binance trade returned from trade history to our common trade
history format
From the official binance api docs (01/09/18):
https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology
base asset refers to the asset that is the quantity of a symbol.
quote asset refers to the asset that is the price of a symbol.
Throws:
- UnsupportedAsset due to asset_from_binance
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(binance_trade['qty'])
rate = deserialize_price(binance_trade['price'])
if binance_trade['symbol'] not in binance_symbols_to_pair:
raise DeserializationError(
f'Error reading a binance trade. Could not find '
f'{binance_trade["symbol"]} in binance_symbols_to_pair',
)
binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
timestamp = deserialize_timestamp_from_binance(binance_trade['time'])
base_asset = asset_from_binance(binance_pair.binance_base_asset)
quote_asset = asset_from_binance(binance_pair.binance_quote_asset)
if binance_trade['isBuyer']:
order_type = TradeType.BUY
# e.g. in RDNETH we buy RDN by paying ETH
else:
order_type = TradeType.SELL
fee_currency = asset_from_binance(binance_trade['commissionAsset'])
fee = deserialize_fee(binance_trade['commission'])
log.debug(
'Processing binance Trade',
sensitive_log=True,
amount=amount,
rate=rate,
timestamp=timestamp,
pair=binance_trade['symbol'],
base_asset=base_asset,
quote=quote_asset,
order_type=order_type,
commision_asset=binance_trade['commissionAsset'],
fee=fee,
)
return Trade(
timestamp=timestamp,
location=Location.BINANCE,
pair=trade_pair_from_assets(base_asset, quote_asset),
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(binance_trade['id']),
)
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'crypto_withdrawal' or row_type == 'crypto_deposit':
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def hexstring_to_bytes(hexstr: str) -> bytes:
"""May raise DeserializationError if it can't convert"""
try:
return bytes.fromhex(hexstr.replace("0x", ""))
except ValueError as e:
raise DeserializationError(f'Failed to turn {hexstr} to bytes') from e
def deserialize(cls, name: str) -> 'CurrentPriceOracle':
if name == 'coingecko':
return cls.COINGECKO
if name == 'cryptocompare':
return cls.CRYPTOCOMPARE
raise DeserializationError(f'Failed to deserialize current price oracle: {name}')
def asset_from_kraken(kraken_name: str) -> Asset:
if not isinstance(kraken_name, str):
raise DeserializationError(f'Got non-string type {type(kraken_name)} for kraken asset')
name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name)
return Asset(name)
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def _deserialize_asset_movement(
self,
raw_movement: Dict[str, Any],
) -> AssetMovement:
"""Process a deposit/withdrawal user transaction from Bitstamp and
deserialize it.
Can raise DeserializationError.
From Bitstamp documentation, deposits/withdrawals can have a fee
(the amount is expected to be in the currency involved)
https://www.bitstamp.net/fee-schedule/
Bitstamp support confirmed the following withdrawal JSON:
{
"fee": "0.00050000",
"btc_usd": "0.00",
"datetime": "2020-12-04 09:30:00.000000",
"usd": "0.0",
"btc": "-0.50000000",
"type": "1",
"id": 123456789,
"eur": "0.0"
}
NB: any asset key not related with the pair is discarded (e.g. 'eur').
"""
type_ = raw_movement['type']
category: AssetMovementCategory
if type_ == 0:
category = AssetMovementCategory.DEPOSIT
elif type_ == 1:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.')
timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement)
base_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.quote_asset_symbol],
)
amount: FVal
fee_asset: Asset
if base_asset_amount != ZERO and quote_asset_amount == ZERO:
amount = base_asset_amount
fee_asset = trade_pair_data.base_asset
elif base_asset_amount == ZERO and quote_asset_amount != ZERO:
amount = quote_asset_amount
fee_asset = trade_pair_data.quote_asset
else:
raise DeserializationError(
'Could not deserialize Bitstamp asset movement from user transaction. '
f'Unexpected asset amount combination found in: {raw_movement}.',
)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.BITSTAMP,
category=category,
address=None, # requires query "crypto_transactions" endpoint
transaction_id=None, # requires query "crypto_transactions" endpoint
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=deserialize_fee(raw_movement['fee']),
link=str(raw_movement['id']),
)
return asset_movement
def deserialize_from_db(
cls,
event_tuple: LiquidityPoolEventDBTuple,
) -> 'LiquidityPoolEvent':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
May raise a DeserializationError if something is wrong with the DB data
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address
6 - is_token0_unknown
7 - token0_address
8 - token0_symbol
9 - token0_name
10 - token0_decimals
11 - is_token1_unknown
12 - token1_address
13 - token1_symbol
14 - token1_name
15 - token1_decimals
16 - amount0
17 - amount1
18 - usd_price
19 - lp_amount
"""
db_event_type = event_tuple[4]
if db_event_type not in {str(event_type) for event_type in EventType}:
raise DeserializationError(
f'Failed to deserialize event type. Unknown event: {db_event_type}.',
)
if db_event_type == str(EventType.MINT):
event_type = EventType.MINT
elif db_event_type == str(EventType.BURN):
event_type = EventType.BURN
else:
raise ValueError(f'Unexpected event type case: {db_event_type}.')
is_token0_unknown = event_tuple[6]
is_token1_unknown = event_tuple[11]
token0: Union[EthereumToken, UnknownEthereumToken]
token1: Union[EthereumToken, UnknownEthereumToken]
if is_token0_unknown:
token0 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[7],
symbol=event_tuple[8],
name=event_tuple[9],
decimals=event_tuple[10],
)
else:
token0 = deserialize_ethereum_token_from_db(
identifier=event_tuple[8])
if is_token1_unknown:
token1 = deserialize_unknown_ethereum_token_from_db(
ethereum_address=event_tuple[12],
symbol=event_tuple[13],
name=event_tuple[14],
decimals=event_tuple[15],
)
else:
token1 = deserialize_ethereum_token_from_db(
identifier=event_tuple[13])
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address=string_to_ethereum_address(event_tuple[5]),
token0=token0,
token1=token1,
amount0=deserialize_asset_amount(event_tuple[16]),
amount1=deserialize_asset_amount(event_tuple[17]),
usd_price=deserialize_price(event_tuple[18]),
lp_amount=deserialize_asset_amount(event_tuple[19]),
)
