def test_all_action_types_writtable_in_db(database, function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
for entry in LedgerActionType:
db.add_ledger_action(
timestamp=1,
action_type=entry,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
link='',
notes='',
)
assert len(db.get_ledger_actions(None, None, None)) == len(LedgerActionType)
def test_ledger_action_can_be_removed(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT COUNT(*) FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
# Add the entry that we want to delete
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Delete ledger action
assert db.remove_ledger_action(identifier) is None
# Check that the change has been committed
cursor.execute(query, (identifier, ))
assert cursor.fetchone() == (0, )
def test_all_action_types_writtable_in_db(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT COUNT(*) FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
for entry in LedgerActionType:
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=entry,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Check that changes have been committed to db
cursor.execute(query, (identifier, ))
assert cursor.fetchone() == (1, )
assert len(db.get_ledger_actions(None, None,
None)) == len(LedgerActionType)
def test_ledger_action_can_be_edited(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT * FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
# Add the entry that we want to edit
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Data for the new entry
new_entry = LedgerAction(
identifier=identifier,
timestamp=2,
action_type=LedgerActionType.GIFT,
location=Location.EXTERNAL,
amount=FVal(3),
asset=A_ETH,
rate=FVal(100),
rate_asset=A_USD,
link='foo',
notes='updated',
)
assert db.edit_ledger_action(new_entry) is None
# Check that changes have been committed
cursor.execute(query, (identifier, ))
updated_entry = LedgerAction.deserialize_from_db(cursor.fetchone())
new_entry.identifier = identifier
assert updated_entry == new_entry
# now try to see if the optional assets can also be set to None
new_entry.rate = new_entry.rate_asset = new_entry.link = new_entry.notes = None
assert db.edit_ledger_action(new_entry) is None
cursor.execute(query, (identifier, ))
updated_entry = LedgerAction.deserialize_from_db(cursor.fetchone())
assert updated_entry.rate is None
assert updated_entry.rate_asset is None
assert updated_entry.link is None
assert updated_entry.notes is None
def process_entry(
self,
db: DBHandler,
db_ledger: DBLedgerActions,
timestamp: Timestamp,
data: BinanceCsvRow,
) -> None:
asset = data['Coin']
amount = data['Change']
ledger_action = LedgerAction(
identifier=0,
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BINANCE,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'Imported from binance CSV file. Binance operation: {data["Operation"]}',
)
db_ledger.add_ledger_action(ledger_action)
def test_ledger_action_can_be_edited(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT * FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
# Add the entry that we want to edit
identifier = db.add_ledger_action(
timestamp=1,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
link='',
notes='',
)
# Data for the new entry
new_entry = LedgerAction(
identifier=identifier,
timestamp=2,
action_type=LedgerActionType.GIFT,
location=Location.EXTERNAL,
amount=FVal(3),
asset=A_ETH,
link='',
notes='updated',
)
assert db.edit_ledger_action(new_entry) is None
# Check that changes have been committed
cursor.execute(query, (identifier, ))
updated_entry = LedgerAction(*cursor.fetchone())
assert updated_entry.timestamp == new_entry.timestamp
assert str(updated_entry.amount) == str(new_entry.amount)
assert updated_entry.action_type == new_entry.action_type.serialize_for_db(
)
class DataImporter():
def __init__(self, db: DBHandler) -> None:
self.db = db
self.db_ledger = DBLedgerActions(self.db, self.db.msg_aggregator)
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = symbol_to_asset_or_token(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = symbol_to_asset_or_token(csv_row['Cur.Buy'])
quote_asset = None if csv_row[
'Cur.Sell'] == '' else symbol_to_asset_or_token(
csv_row['Cur.Sell']) # noqa: E501
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('Deposit', 'Withdrawal'):
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = symbol_to_asset_or_token(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(
csv_row['Sell'])
asset = symbol_to_asset_or_token(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCSVEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def import_cointracking_csv(self, filepath: Path) -> Tuple[bool, str]:
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.reader(csvfile, delimiter=',', quotechar='"')
header = remap_header(next(data))
for row in data:
try:
self._consume_cointracking_entry(dict(zip(header, row)))
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During cointracking CSV import found action with unknown '
f'asset {e.asset_name}. Ignoring entry', )
continue
except IndexError:
self.db.msg_aggregator.add_warning(
'During cointracking CSV import found entry with '
'unexpected number of columns', )
continue
except DeserializationError as e:
self.db.msg_aggregator.add_warning(
f'Error during cointracking CSV import deserialization. '
f'Error was {str(e)}. Ignoring entry', )
continue
except UnsupportedCSVEntry as e:
self.db.msg_aggregator.add_warning(str(e))
continue
except KeyError as e:
return False, str(e)
return True, ''
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = symbol_to_asset_or_token(to_currency)
quote_asset = symbol_to_asset_or_token(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = symbol_to_asset_or_token(currency)
quote_asset = symbol_to_asset_or_token(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = symbol_to_asset_or_token(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in ('airdrop_to_exchange_transfer', 'mco_stake_reward'):
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
self.db_ledger.add_ledger_action(action)
elif row_type == 'invest_deposit':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type == 'invest_withdrawal':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_associated_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
'interest_swap_credited',
'interest_swap_debited',
# The user has received an aidrop but can't claim it yet
'airdrop_locked',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCSVEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
def import_cryptocom_csv(self, filepath: Path) -> Tuple[bool, str]:
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.DictReader(csvfile)
try:
# Notice: Crypto.com csv export gathers all swapping entries (`lockup_swap_*`,
# `crypto_wallet_swap_*`, ...) into one entry named `dynamic_coin_swap_*`.
self._import_cryptocom_associated_entries(
data, 'dynamic_coin_swap')
# reset the iterator
csvfile.seek(0)
# pass the header since seek(0) make the first row to be the header
next(data)
self._import_cryptocom_associated_entries(
data, 'dust_conversion')
csvfile.seek(0)
next(data)
self._import_cryptocom_associated_entries(
data, 'interest_swap')
csvfile.seek(0)
next(data)
self._import_cryptocom_associated_entries(data, 'invest')
csvfile.seek(0)
next(data)
except KeyError as e:
return False, f'Crypto.com csv missing entry for {str(e)}'
except UnknownAsset as e:
return False, f'Encountered unknown asset {str(e)} at crypto.com csv import'
for row in data:
try:
self._consume_cryptocom_entry(row)
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During cryptocom CSV import found action with unknown '
f'asset {e.asset_name}. Ignoring entry', )
continue
except DeserializationError as e:
self.db.msg_aggregator.add_warning(
f'Error during cryptocom CSV import deserialization. '
f'Error was {str(e)}. Ignoring entry', )
continue
except UnsupportedCSVEntry as e:
self.db.msg_aggregator.add_warning(str(e))
continue
except KeyError as e:
return False, str(e)
return True, ''
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def import_blockfi_transactions_csv(self,
filepath: Path) -> Tuple[bool, str]:
"""
Information for the values that the columns can have has been obtained from
https://github.com/BittyTax/BittyTax/blob/06794f51223398759852d6853bc7112ffb96129a/bittytax/conv/parsers/blockfi.py#L67
"""
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.DictReader(csvfile)
for row in data:
try:
self._consume_blockfi_entry(row)
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During BlockFi CSV import found action with unknown '
f'asset {e.asset_name}. Ignoring entry', )
continue
except DeserializationError as e:
self.db.msg_aggregator.add_warning(
f'Deserialization error during BlockFi CSV import. '
f'{str(e)}. Ignoring entry', )
continue
except UnsupportedCSVEntry as e:
self.db.msg_aggregator.add_warning(str(e))
continue
except KeyError as e:
return False, str(e)
return True, ''
def _consume_blockfi_trade(self, csv_row: Dict[str, Any]) -> None:
"""
Consume the file containing only trades from BlockFi. As per my investigations
(@yabirgb) this file can only contain confirmed trades.
- UnknownAsset
- DeserializationError
"""
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
buy_asset = symbol_to_asset_or_token(csv_row['Buy Currency'])
buy_amount = deserialize_asset_amount(csv_row['Buy Quantity'])
sold_asset = symbol_to_asset_or_token(csv_row['Sold Currency'])
sold_amount = deserialize_asset_amount(csv_row['Sold Quantity'])
if sold_amount == ZERO:
log.debug(
f'Ignoring BlockFi trade with sold_amount equal to zero. {csv_row}'
)
return
rate = Price(buy_amount / sold_amount)
trade = Trade(
timestamp=timestamp,
location=Location.BLOCKFI,
base_asset=buy_asset,
quote_asset=sold_asset,
trade_type=TradeType.BUY,
amount=buy_amount,
rate=rate,
fee=None, # BlockFI doesn't provide this information
fee_currency=None,
link='',
notes=csv_row['Type'],
)
self.db.add_trades([trade])
def import_blockfi_trades_csv(self, filepath: Path) -> Tuple[bool, str]:
"""
Information for the values that the columns can have has been obtained from
the issue in github #1674
"""
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.DictReader(csvfile)
for row in data:
try:
self._consume_blockfi_trade(row)
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During BlockFi CSV import found action with unknown '
f'asset {e.asset_name}. Ignoring entry', )
continue
except DeserializationError as e:
self.db.msg_aggregator.add_warning(
f'Deserialization error during BlockFi CSV import. '
f'{str(e)}. Ignoring entry', )
continue
except UnsupportedCSVEntry as e:
self.db.msg_aggregator.add_warning(str(e))
continue
except KeyError as e:
return False, str(e)
return True, ''
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def import_nexo_csv(self, filepath: Path) -> Tuple[bool, str]:
"""
Information for the values that the columns can have has been obtained from
https://github.com/BittyTax/BittyTax/blob/06794f51223398759852d6853bc7112ffb96129a/bittytax/conv/parsers/nexo.py
"""
with open(filepath, 'r', encoding='utf-8-sig') as csvfile:
data = csv.DictReader(csvfile)
for row in data:
try:
self._consume_nexo(row)
except UnknownAsset as e:
self.db.msg_aggregator.add_warning(
f'During Nexo CSV import found action with unknown '
f'asset {e.asset_name}. Ignoring entry', )
continue
except DeserializationError as e:
self.db.msg_aggregator.add_warning(
f'Deserialization error during Nexo CSV import. '
f'{str(e)}. Ignoring entry', )
continue
except UnsupportedCSVEntry as e:
self.db.msg_aggregator.add_warning(str(e))
continue
except KeyError as e:
return False, str(e)
return True, ''
def test_query_ledger_actions(events_historian, function_scope_messages_aggregator):
"""
Create actions and query the events historian to check that the history
has events previous to the selected from_ts. This allows us to verify that
actions before one period are counted in the PnL report to calculate cost basis.
https://github.com/rotki/rotki/issues/2541
"""
selected_timestamp = 10
db = DBLedgerActions(events_historian.db, function_scope_messages_aggregator)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp - 2,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp + 3,
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
amount=FVal(0.5),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp + 5,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(10),
asset=A_USDC,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
actions, length = events_historian.query_ledger_actions(
has_premium=True,
from_ts=None,
to_ts=Timestamp(selected_timestamp + 4),
)
assert any((action.timestamp < selected_timestamp for action in actions))
assert length == 2
