def deserialize_asset(val: str) -> Asset:
match = ASSET_RE.match(val)
if match is None:
raise ValueError(f'Could not parse asset: {val}')
symbol, _identifier_outer, identifier = match.groups()
if identifier:
asset = symbol_to_asset_or_token(identifier)
else:
asset = symbol_to_asset_or_token(symbol)
if asset is None:
raise ValueError(f'Symbol not found or ambigous: {val}')
return asset
def asset_from_bitfinex(
bitfinex_name: str,
currency_map: Dict[str, str],
is_currency_map_updated: bool = True,
) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
Currency map coming from `<Bitfinex>._query_currency_map()` is already
updated with BITFINEX_TO_WORLD (prevent updating it on each call)
"""
if not isinstance(bitfinex_name, str):
raise DeserializationError(
f'Got non-string type {type(bitfinex_name)} for bitfinex asset')
if bitfinex_name in UNSUPPORTED_BITFINEX_ASSETS:
raise UnsupportedAsset(bitfinex_name)
if is_currency_map_updated is False:
currency_map.update(BITFINEX_TO_WORLD)
symbol = currency_map.get(bitfinex_name, bitfinex_name)
return symbol_to_asset_or_token(symbol)
def asset_from_kraken(kraken_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnknownAsset
"""
if not isinstance(kraken_name, str):
raise DeserializationError(
f'Got non-string type {type(kraken_name)} for kraken asset')
if kraken_name.endswith('.S') or kraken_name.endswith('.M'):
# this is a staked coin. For now since we don't show staked coins
# consider it as the normal version. In the future we may perhaps
# differentiate between them in the balances https://github.com/rotki/rotki/issues/569
kraken_name = kraken_name[:-2]
if kraken_name.endswith('.HOLD'):
kraken_name = kraken_name[:-5]
# Some names are not in the map since kraken can have multiple representations
# depending on the pair for the same asset. For example XXBT and XBT, XETH and ETH,
# ZUSD and USD
if kraken_name == 'SETH':
name = 'ETH2'
elif kraken_name == 'XBT':
name = 'BTC'
elif kraken_name == 'XDG':
name = 'DOGE'
elif kraken_name in ('ETH', 'EUR', 'USD', 'GBP', 'CAD', 'JPY', 'KRW',
'CHF', 'AUD'):
name = kraken_name
else:
name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name)
return symbol_to_asset_or_token(name)
def asset_from_coinbase(cb_name: str,
time: Optional[Timestamp] = None) -> Asset:
"""May raise:
- DeserializationError
- UnknownAsset
"""
# During the transition from DAI(SAI) to MCDAI(DAI) coinbase introduced an MCDAI
# wallet for the new DAI during the transition period. We should be able to handle this
# https://support.coinbase.com/customer/portal/articles/2982947
if cb_name == 'MCDAI':
return A_DAI
if cb_name == 'DAI':
# If it's dai and it's queried from the exchange before the end of the upgrade
if not time:
time = ts_now()
if time < COINBASE_DAI_UPGRADE_END_TS:
# Then it should be the single collateral version
return A_SAI
return A_DAI
if not isinstance(cb_name, str):
raise DeserializationError(
f'Got non-string type {type(cb_name)} for coinbase asset')
name = COINBASE_TO_WORLD.get(cb_name, cb_name)
return symbol_to_asset_or_token(name)
def _decode_redeem(
self,
tx_log: EthereumTxReceiptLog,
decoded_events: List[HistoryBaseEntry],
compound_token: EthereumToken,
) -> Tuple[Optional[HistoryBaseEntry], Optional[ActionItem]]:
redeemer = hex_or_bytes_to_address(tx_log.data[0:32])
if not self.base.is_tracked(redeemer):
return None, None
redeem_amount_raw = hex_or_bytes_to_int(tx_log.data[32:64])
redeem_tokens_raw = hex_or_bytes_to_int(tx_log.data[64:96])
underlying_token = symbol_to_asset_or_token(compound_token.symbol[1:])
redeem_amount = asset_normalized_value(redeem_amount_raw, underlying_token)
redeem_tokens = token_normalized_value(redeem_tokens_raw, compound_token)
out_event = in_event = None
for event in decoded_events:
# Find the transfer event which should have come before the redeeming
if event.event_type == HistoryEventType.RECEIVE and event.asset == underlying_token and event.balance.amount == redeem_amount: # noqa: E501
event.event_type = HistoryEventType.WITHDRAWAL
event.event_subtype = HistoryEventSubType.REMOVE_ASSET
event.counterparty = CPT_COMPOUND
event.notes = f'Withdraw {redeem_amount} {underlying_token.symbol} from compound'
in_event = event
if event.event_type == HistoryEventType.SPEND and event.asset == compound_token and event.balance.amount == redeem_tokens: # noqa: E501
event.event_type = HistoryEventType.SPEND
event.event_subtype = HistoryEventSubType.RETURN_WRAPPED
event.counterparty = CPT_COMPOUND
event.notes = f'Return {redeem_tokens} {compound_token.symbol} to compound'
out_event = event
maybe_reshuffle_events(out_event=out_event, in_event=in_event, events_list=decoded_events)
return None, None
def serialize_asset(asset: Asset) -> str:
try:
if asset == symbol_to_asset_or_token(asset.symbol):
return asset.symbol
except UnknownAsset:
pass
return f'{asset.symbol}[{asset.identifier}]'
def asset_from_gemini(symbol: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(symbol, str):
raise DeserializationError(f'Got non-string type {type(symbol)} for gemini asset')
return symbol_to_asset_or_token(symbol)
def asset_from_bitstamp(bitstamp_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(bitstamp_name, str):
raise DeserializationError(f'Got non-string type {type(bitstamp_name)} for bitstamp asset')
return symbol_to_asset_or_token(bitstamp_name)
def asset_from_uphold(symbol: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(symbol, str):
raise DeserializationError(
f'Got non-string type {type(symbol)} for uphold asset')
name = UPHOLD_TO_WORLD.get(symbol, symbol)
return symbol_to_asset_or_token(name)
def asset_from_nexo(nexo_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(nexo_name, str):
raise DeserializationError(
f'Got non-string type {type(nexo_name)} for nexo asset')
our_name = NEXO_TO_WORLD.get(nexo_name, nexo_name)
return symbol_to_asset_or_token(our_name)
def asset_from_bitpanda(bitpanda_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(bitpanda_name, str):
raise DeserializationError(
f'Got non-string type {type(bitpanda_name)} for bitpanda asset')
our_name = BITPANDA_TO_WORLD.get(bitpanda_name, bitpanda_name)
return symbol_to_asset_or_token(our_name)
def asset_from_coinbasepro(coinbase_pro_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(coinbase_pro_name, str):
raise DeserializationError(
f'Got non-string type {type(coinbase_pro_name)} for '
f'coinbasepro asset', )
name = COINBASE_PRO_TO_WORLD.get(coinbase_pro_name, coinbase_pro_name)
return symbol_to_asset_or_token(name)
def asset_from_cryptocom(cryptocom_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(cryptocom_name, str):
raise DeserializationError(
f'Got non-string type {type(cryptocom_name)} for cryptocom asset',
)
symbol = CRYPTOCOM_TO_WORLD.get(cryptocom_name, cryptocom_name)
return symbol_to_asset_or_token(symbol)
def assert_blockfi_trades_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing trades data from blockfi"""
trades = rotki.data.db.get_trades()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_trades = [Trade(
timestamp=Timestamp(1612051199),
location=Location.BLOCKFI,
base_asset=symbol_to_asset_or_token('USDC'),
quote_asset=symbol_to_asset_or_token('LTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('6404.6')),
rate=Price(FVal('151.6283999982779809352223797')),
fee=None,
fee_currency=None,
link='',
notes='One Time',
)]
assert trades == expected_trades
def asset_from_iconomi(symbol: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(symbol, str):
raise DeserializationError(f'Got non-string type {type(symbol)} for iconomi asset')
symbol = symbol.upper()
if symbol in UNSUPPORTED_ICONOMI_ASSETS:
raise UnsupportedAsset(symbol)
name = ICONOMI_TO_WORLD.get(symbol, symbol)
return symbol_to_asset_or_token(name)
def asset_from_poloniex(poloniex_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(poloniex_name, str):
raise DeserializationError(f'Got non-string type {type(poloniex_name)} for poloniex asset')
if poloniex_name in UNSUPPORTED_POLONIEX_ASSETS:
raise UnsupportedAsset(poloniex_name)
our_name = POLONIEX_TO_WORLD.get(poloniex_name, poloniex_name)
return symbol_to_asset_or_token(our_name)
def asset_from_ftx(ftx_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(ftx_name, str):
raise DeserializationError(f'Got non-string type {type(ftx_name)} for ftx asset')
if ftx_name in UNSUPPORTED_FTX_ASSETS:
raise UnsupportedAsset(ftx_name)
name = FTX_TO_WORLD.get(ftx_name, ftx_name)
return symbol_to_asset_or_token(name)
def asset_from_kucoin(kucoin_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(kucoin_name, str):
raise DeserializationError(f'Got non-string type {type(kucoin_name)} for kucoin asset')
if kucoin_name in UNSUPPORTED_KUCOIN_ASSETS:
raise UnsupportedAsset(kucoin_name)
name = KUCOIN_TO_WORLD.get(kucoin_name, kucoin_name)
return symbol_to_asset_or_token(name)
def _consume_blockfi_trade(self, csv_row: Dict[str, Any]) -> None:
"""
Consume the file containing only trades from BlockFi. As per my investigations
(@yabirgb) this file can only contain confirmed trades.
- UnknownAsset
- DeserializationError
"""
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
buy_asset = symbol_to_asset_or_token(csv_row['Buy Currency'])
buy_amount = deserialize_asset_amount(csv_row['Buy Quantity'])
sold_asset = symbol_to_asset_or_token(csv_row['Sold Currency'])
sold_amount = deserialize_asset_amount(csv_row['Sold Quantity'])
if sold_amount == ZERO:
log.debug(
f'Ignoring BlockFi trade with sold_amount equal to zero. {csv_row}'
)
return
rate = Price(buy_amount / sold_amount)
trade = Trade(
timestamp=timestamp,
location=Location.BLOCKFI,
base_asset=buy_asset,
quote_asset=sold_asset,
trade_type=TradeType.BUY,
amount=buy_amount,
rate=rate,
fee=None, # BlockFI doesn't provide this information
fee_currency=None,
link='',
notes=csv_row['Type'],
)
self.db.add_trades([trade])
def asset_from_gemini(symbol: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(symbol, str):
raise DeserializationError(
f'Got non-string type {type(symbol)} for gemini asset')
if symbol in UNSUPPORTED_GEMINI_ASSETS:
raise UnsupportedAsset(symbol)
name = GEMINI_TO_WORLD.get(symbol, symbol)
return symbol_to_asset_or_token(name)
def asset_from_binance(binance_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(binance_name, str):
raise DeserializationError(f'Got non-string type {type(binance_name)} for binance asset')
if binance_name in UNSUPPORTED_BINANCE_ASSETS:
raise UnsupportedAsset(binance_name)
if binance_name in RENAMED_BINANCE_ASSETS:
return Asset(RENAMED_BINANCE_ASSETS[binance_name])
name = BINANCE_TO_WORLD.get(binance_name, binance_name)
return symbol_to_asset_or_token(name)
def _decode_mint(
self,
transaction: EthereumTransaction,
tx_log: EthereumTxReceiptLog,
decoded_events: List[HistoryBaseEntry],
compound_token: EthereumToken,
) -> Tuple[Optional[HistoryBaseEntry], Optional[ActionItem]]:
minter = hex_or_bytes_to_address(tx_log.data[0:32])
if not self.base.is_tracked(minter):
return None, None
mint_amount_raw = hex_or_bytes_to_int(tx_log.data[32:64])
minted_amount_raw = hex_or_bytes_to_int(tx_log.data[64:96])
underlying_asset = symbol_to_asset_or_token(compound_token.symbol[1:])
mint_amount = asset_normalized_value(mint_amount_raw, underlying_asset)
minted_amount = token_normalized_value(minted_amount_raw, compound_token)
out_event = None
for event in decoded_events:
# Find the transfer event which should have come before the minting
if event.event_type == HistoryEventType.SPEND and event.asset == underlying_asset and event.balance.amount == mint_amount: # noqa: E501
event.event_type = HistoryEventType.DEPOSIT
event.event_subtype = HistoryEventSubType.DEPOSIT_ASSET
event.counterparty = CPT_COMPOUND
event.notes = f'Deposit {mint_amount} {underlying_asset.symbol} to compound'
out_event = event
break
if out_event is None:
log.debug(f'At compound mint decoding of tx {transaction.tx_hash.hex()} the out event was not found') # noqa: E501
return None, None
# also create an action item for the receive of the cTokens
action_item = ActionItem(
action='transform',
sequence_index=tx_log.log_index,
from_event_type=HistoryEventType.RECEIVE,
from_event_subtype=HistoryEventSubType.NONE,
asset=compound_token,
amount=minted_amount,
to_event_subtype=HistoryEventSubType.RECEIVE_WRAPPED,
to_notes=f'Receive {minted_amount} {compound_token.symbol} from compound',
to_counterparty=CPT_COMPOUND,
paired_event_data=(out_event, True),
)
return None, action_item
def asset_from_ftx(ftx_name: str) -> Asset:
"""May raise:
- DeserializationError
- UnsupportedAsset
- UnknownAsset
"""
if not isinstance(ftx_name, str):
raise DeserializationError(
f'Got non-string type {type(ftx_name)} for ftx asset')
if ftx_name in UNSUPPORTED_FTX_ASSETS:
raise UnsupportedAsset(ftx_name)
if ftx_name == 'SRM_LOCKED':
name = strethaddress_to_identifier(
'0x476c5E26a75bd202a9683ffD34359C0CC15be0fF') # SRM
else:
name = FTX_TO_WORLD.get(ftx_name, ftx_name)
return symbol_to_asset_or_token(name)
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = symbol_to_asset_or_token(to_currency)
quote_asset = symbol_to_asset_or_token(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = symbol_to_asset_or_token(currency)
quote_asset = symbol_to_asset_or_token(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = symbol_to_asset_or_token(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in ('airdrop_to_exchange_transfer', 'mco_stake_reward'):
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
self.db_ledger.add_ledger_action(action)
elif row_type == 'invest_deposit':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type == 'invest_withdrawal':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_associated_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
'interest_swap_credited',
'interest_swap_debited',
# The user has received an aidrop but can't claim it yet
'airdrop_locked',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCSVEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = symbol_to_asset_or_token(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = symbol_to_asset_or_token(csv_row['Cur.Buy'])
quote_asset = None if csv_row[
'Cur.Sell'] == '' else symbol_to_asset_or_token(
csv_row['Cur.Sell']) # noqa: E501
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('Deposit', 'Withdrawal'):
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = symbol_to_asset_or_token(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = symbol_to_asset_or_token(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCSVEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def bitcoinde_asset(symbol: str) -> Asset:
return symbol_to_asset_or_token(symbol.upper())
def test_global_db_restore(globaldb, database):
"""
Check that the user can recreate assets information from the packaged
database with rotki (hard reset). The test adds a new asset, restores
the database and checks that the added token is not in there and that
the amount of assets is the expected
"""
# Add a custom eth token
address_to_delete = make_ethereum_address()
token_to_delete = EthereumToken.initialize(
address=address_to_delete,
decimals=18,
name='willdell',
symbol='DELME',
)
globaldb.add_asset(
asset_id='DELMEID1',
asset_type=AssetType.ETHEREUM_TOKEN,
data=token_to_delete,
)
# Add a token with underlying token
with_underlying_address = make_ethereum_address()
with_underlying = EthereumToken.initialize(
address=with_underlying_address,
decimals=18,
name="Not a scam",
symbol="NSCM",
started=0,
underlying_tokens=[
UnderlyingToken(
address=address_to_delete,
weight=1,
)
],
)
globaldb.add_asset(
asset_id='xDELMEID1',
asset_type=AssetType.ETHEREUM_TOKEN,
data=with_underlying,
)
# Add asset that is not a token
globaldb.add_asset(
asset_id='1',
asset_type=AssetType.OWN_CHAIN,
data={
'name': 'Lolcoin',
'symbol': 'LOLZ',
'started': 0,
},
)
# Add asset that is not a token
globaldb.add_asset(
asset_id='2',
asset_type=AssetType.OWN_CHAIN,
data={
'name': 'Lolcoin2',
'symbol': 'LOLZ2',
'started': 0,
},
)
database.add_asset_identifiers('1')
database.add_asset_identifiers('2')
# Try to reset DB it if we have a trade that uses a custom asset
buy_asset = symbol_to_asset_or_token('LOLZ2')
buy_amount = deserialize_asset_amount(1)
sold_asset = symbol_to_asset_or_token('LOLZ')
sold_amount = deserialize_asset_amount(2)
rate = Price(buy_amount / sold_amount)
trade = Trade(
timestamp=Timestamp(12312312),
location=Location.BLOCKFI,
base_asset=buy_asset,
quote_asset=sold_asset,
trade_type=TradeType.BUY,
amount=buy_amount,
rate=rate,
fee=None,
fee_currency=None,
link='',
notes="",
)
database.add_trades([trade])
status, _ = GlobalDBHandler().hard_reset_assets_list(database)
assert status is False
# Now do it without the trade
database.delete_trade(trade.identifier)
status, msg = GlobalDBHandler().hard_reset_assets_list(database, True)
assert status, msg
cursor = globaldb._conn.cursor()
query = f'SELECT COUNT(*) FROM ethereum_tokens where address == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (0, ), 'Ethereum token should have been deleted'
query = f'SELECT COUNT(*) FROM assets where details_reference == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (
0, ), 'Ethereum token should have been deleted from assets'
query = f'SELECT COUNT(*) FROM ethereum_tokens where address == "{with_underlying_address}";'
r = cursor.execute(query)
assert r.fetchone() == (
0,
), 'Token with underlying token should have been deleted from assets'
query = f'SELECT COUNT(*) FROM assets where details_reference == "{with_underlying_address}";'
r = cursor.execute(query)
assert r.fetchone() == (0, )
query = f'SELECT COUNT(*) FROM underlying_tokens_list where address == "{address_to_delete}";'
r = cursor.execute(query)
assert r.fetchone() == (0, )
query = 'SELECT COUNT(*) FROM assets where identifier == "1";'
r = cursor.execute(query)
assert r.fetchone() == (0, ), 'Non ethereum token should be deleted'
# Check that the user database is correctly updated
query = 'SELECT identifier from assets'
r = cursor.execute(query)
user_db_cursor = database.conn.cursor()
user_db_cursor.execute(query)
assert r.fetchall() == user_db_cursor.fetchall()
# Check that the number of assets is the expected
root_dir = Path(__file__).resolve().parent.parent.parent
builtin_database = root_dir / 'data' / 'global.db'
conn = sqlite3.connect(builtin_database)
cursor_clean_db = conn.cursor()
tokens_expected = cursor_clean_db.execute('SELECT COUNT(*) FROM assets;')
tokens_local = cursor.execute('SELECT COUNT(*) FROM assets;')
assert tokens_expected.fetchone() == tokens_local.fetchone()
cursor.execute('SELECT asset_id FROM user_owned_assets')
msg = 'asset id in trade should not be in the owned table'
assert "'2'" not in [entry[0] for entry in cursor.fetchall()], msg
conn.close()
