def test_taxable_ledger_action_setting(accountant, expected, google_service):
"""Test that ledger actions respect the taxable setting"""
history = [
LedgerAction(
identifier=1,
timestamp=1476979735,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1), # 578.505 EUR/BTC from mocked prices
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=2,
timestamp=1491062063,
action_type=LedgerActionType.AIRDROP,
location=Location.EXTERNAL,
amount=FVal(10), # 47.865 EUR/ETH from mocked prices
asset=A_ETH,
rate=None,
rate_asset=None,
link='foo',
notes='boo',
), LedgerAction(
identifier=3,
timestamp=1501062063,
action_type=LedgerActionType.LOSS,
location=Location.BLOCKCHAIN,
amount=FVal(2), # 175.44 EUR/ETH from mocked prices
asset=A_ETH,
rate=FVal(400), # but should use the given rate of 400 EUR
rate_asset=A_EUR,
link='goo',
notes='hoo',
), LedgerAction( # include a non taxed ledger action too
identifier=4,
timestamp=1501062064,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKCHAIN,
amount=FVal(1),
asset=A_ETH,
rate=FVal(400),
rate_asset=A_EUR,
link='goo2',
notes='hoo2',
),
]
accounting_history_process(
accountant=accountant,
start_ts=1436979735,
end_ts=1519693374,
history_list=history,
)
expected_pnls = PnlTotals()
if expected != 0:
expected_pnls[AccountingEventType.LEDGER_ACTION] = PNL(taxable=FVal(expected), free=ZERO)
check_pnls_and_csv(accountant, expected_pnls, google_service)
def test_kfee_price_in_accounting(accountant):
"""
Test that KFEEs are correctly handled during accounting
"""
history = [
{
'timestamp': 1609537953,
'base_asset': 'ETH',
'quote_asset': A_USDT.identifier,
'trade_type': 'sell',
'rate': 1000,
'fee': '30',
'fee_currency': A_KFEE.identifier,
'amount': 0.02,
'location': 'kraken',
},
]
ledger_actions_list = [
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238),
action_type=LedgerActionType.INCOME,
location=Location.KRAKEN,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='',
),
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238),
action_type=LedgerActionType.INCOME,
location=Location.KRAKEN,
amount=FVal(1000),
asset=A_KFEE,
rate=None,
rate_asset=None,
link=None,
notes='',
),
]
report, _ = accounting_history_process(
accountant,
start_ts=1539713238,
end_ts=1624395187,
history_list=history,
ledger_actions_list=ledger_actions_list,
)
warnings = accountant.msg_aggregator.consume_warnings()
assert len(warnings) == 0
errors = accountant.msg_aggregator.consume_errors()
assert len(errors) == 0
# The ledger actions income doesn't count for the income as the 1
# year rule is applied. Only the sell is computed.
# The expected PnL without the fee is 14.2277000
# counting the fee is 14.2277000 - 30 * 0.01 * 0.82411
assert FVal(report['total_profit_loss']) == FVal(13.980467)
def test_coinbase_query_income_loss_expense(function_scope_coinbase):
"""Test that coinbase deposit/withdrawals history query works fine for the happy path"""
coinbase = function_scope_coinbase
with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query):
ledger_actions = coinbase.query_online_income_loss_expense(
start_ts=0,
end_ts=1611426233,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(ledger_actions) == 2
expected_ledger_actions = [LedgerAction(
identifier=ledger_actions[0].identifier,
location=Location.COINBASE,
action_type=LedgerActionType.INCOME,
timestamp=1609877514,
asset=asset_from_coinbase('NMR'),
amount=FVal('0.02762431'),
rate=FVal('36.56199919563601769600761069'),
rate_asset=A_USD,
link='id4',
notes=('Received Numeraire '
'From Coinbase Earn '
'Received 0.02762431 NMR ($1.01)'),
), LedgerAction(
identifier=ledger_actions[1].identifier,
location=Location.COINBASE,
action_type=LedgerActionType.INCOME,
timestamp=1611426233,
asset=asset_from_coinbase('ALGO'),
amount=FVal('0.000076'),
rate=ZERO,
rate_asset=A_USD,
link='id5',
notes=('Algorand reward '
'From Coinbase '
'Received 0.000076 ALGO ($0.00)'),
)]
assert expected_ledger_actions == ledger_actions
# and now try to query within a specific range
with patch.object(coinbase.session, 'get', side_effect=mock_normal_coinbase_query):
ledger_actions = coinbase.query_online_income_loss_expense(
start_ts=0,
end_ts=1609877514,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(ledger_actions) == 1
assert ledger_actions[0].action_type == LedgerActionType.INCOME
assert ledger_actions[0].timestamp == 1609877514
def test_kfee_price_in_accounting(accountant, google_service):
"""
Test that KFEEs are correctly handled during accounting
KFEE price is fixed at $0.01
"""
history = [
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238), # 178.615 EUR/ETH
action_type=LedgerActionType.INCOME,
location=Location.KRAKEN,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='',
), LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238), # 0.8612 USD/EUR. 1 KFEE = $0.01 so 8.612 EUR
action_type=LedgerActionType.INCOME,
location=Location.KRAKEN,
amount=FVal(1000),
asset=A_KFEE,
rate=None,
rate_asset=None,
link=None,
notes='',
), Trade(
timestamp=1609537953,
location=Location.KRAKEN, # 0.89 USDT/EUR -> PNL: 20 * 0.89 - 0.02*178.615 -> 14.2277
base_asset=A_ETH,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=FVal('0.02'),
rate=FVal(1000),
fee=FVal(30), # KFEE should not be taken into account
fee_currency=A_KFEE,
link=None,
),
]
accounting_history_process(
accountant,
start_ts=1539713238,
end_ts=1624395187,
history_list=history,
)
no_message_errors(accountant.msg_aggregator)
expected_pnls = PnlTotals({
AccountingEventType.TRADE: PNL(taxable=ZERO, free=FVal('14.2277')),
AccountingEventType.LEDGER_ACTION: PNL(taxable=FVal('187.227'), free=ZERO),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
def test_csv_import(database, price_historian): # pylint: disable=unused-argument
imp = GitcoinDataImporter(database)
csv_path = Path(__file__).resolve().parent.parent / 'data' / 'gitcoin.csv'
imp.import_gitcoin_csv(csv_path)
actions = imp.db_ledger.get_ledger_actions(
filter_query=LedgerActionsFilterQuery.make(location=Location.GITCOIN),
has_premium=True,
)
assert len(actions) == 10
expected_actions = [
LedgerAction(
identifier=1,
timestamp=Timestamp(1624798800),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.0004789924016679019628604417823'),
asset=A_ETH,
rate=FVal('1983.33'),
rate_asset=A_USD,
link=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ETHEREUM,
),
),
LedgerAction(
identifier=2,
timestamp=Timestamp(1624798800),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.0005092445533521905078832065264'),
asset=A_ETH,
rate=FVal('1983.33'),
rate_asset=A_USD,
link=
'sync-tx:5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ZKSYNC,
),
)
]
assert expected_actions == actions[:2]
def test_ignore_ledger_actions_in_accountant(rotkehlchen_api_server):
"""Test that ignored ledger actions are correctly ignored by the accountant"""
accountant = rotkehlchen_api_server.rest_api.rotkehlchen.accountant
ledger_actions_list = [
LedgerAction(
identifier=1,
timestamp=1467279735,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKCHAIN,
amount=FVal(1000),
asset=A_ETH,
rate=FVal(100),
rate_asset=A_USD,
link=None,
notes=None,
),
LedgerAction(
identifier=2,
timestamp=1467279735,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKCHAIN,
amount=FVal(5),
asset=A_DAI,
rate=None,
rate_asset=None,
link='foo',
notes='boo',
),
]
# Set the server to ignore second ledger action
response = requests.put(
api_url_for(
rotkehlchen_api_server,
'ignoredactionsresource',
),
json={
'action_type': 'ledger action',
'action_ids': ['2']
},
)
result = assert_proper_response_with_result(response)
assert result == {'ledger action': ['2']}
# Retrieve ignored actions mapping. Should contain 2
ignored_actions = accountant.db.get_ignored_action_ids(action_type=None)
ignored = []
# Call the should_ignore method used in the accountant
for action in ledger_actions_list:
should_ignore = action.should_ignore(ignored_actions)
ignored.append(should_ignore)
assert ignored == [False, True]
def test_ledger_action_can_be_edited(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT * FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
# Add the entry that we want to edit
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Data for the new entry
new_entry = LedgerAction(
identifier=identifier,
timestamp=2,
action_type=LedgerActionType.GIFT,
location=Location.EXTERNAL,
amount=FVal(3),
asset=A_ETH,
rate=FVal(100),
rate_asset=A_USD,
link='foo',
notes='updated',
)
assert db.edit_ledger_action(new_entry) is None
# Check that changes have been committed
cursor.execute(query, (identifier, ))
updated_entry = LedgerAction.deserialize_from_db(cursor.fetchone())
new_entry.identifier = identifier
assert updated_entry == new_entry
# now try to see if the optional assets can also be set to None
new_entry.rate = new_entry.rate_asset = new_entry.link = new_entry.notes = None
assert db.edit_ledger_action(new_entry) is None
cursor.execute(query, (identifier, ))
updated_entry = LedgerAction.deserialize_from_db(cursor.fetchone())
assert updated_entry.rate is None
assert updated_entry.rate_asset is None
assert updated_entry.link is None
assert updated_entry.notes is None
def test_taxable_ledger_action_setting(accountant, expected_pnl):
"""Test that ledger actions respect the taxable setting"""
ledger_actions_list = [
LedgerAction(
identifier=1,
timestamp=1476979735,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1), # 578.505 EUR total from mocked prices
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes=None,
),
LedgerAction(
identifier=2,
timestamp=1491062063,
action_type=LedgerActionType.AIRDROP,
location=Location.EXTERNAL,
amount=FVal(10), # 478.65 EUR total from mocked prices
asset=A_ETH,
rate=None,
rate_asset=None,
link='foo',
notes='boo',
),
LedgerAction(
identifier=3,
timestamp=1501062063,
action_type=LedgerActionType.LOSS,
location=Location.BLOCKCHAIN,
amount=FVal(2), # 350.88 EUR total from mocked prices
asset=A_ETH,
rate=FVal(400), # but should use the given rate of 400 EUR
rate_asset=A_EUR,
link='goo',
notes='hoo',
),
]
result = accounting_history_process(
accountant,
1436979735,
1519693374,
history_list=[],
ledger_actions_list=ledger_actions_list,
)
assert FVal(
result['overview']['total_taxable_profit_loss']).is_close(expected_pnl)
def put(
self,
timestamp: Timestamp,
action_type: LedgerActionType,
location: Location,
amount: AssetAmount,
asset: Asset,
rate: Optional[Price],
rate_asset: Optional[Asset],
link: Optional[str],
notes: Optional[str],
) -> Response:
action = LedgerAction(
identifier=0, # whatever -- is not used at insertion
timestamp=timestamp,
action_type=action_type,
location=location,
amount=amount,
asset=asset,
rate=rate,
rate_asset=rate_asset,
link=link,
notes=notes,
)
return self.rest_api.add_ledger_action(action)
def test_all_action_types_writtable_in_db(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT COUNT(*) FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
for entry in LedgerActionType:
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=entry,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Check that changes have been committed to db
cursor.execute(query, (identifier, ))
assert cursor.fetchone() == (1, )
assert len(db.get_ledger_actions(None, None,
None)) == len(LedgerActionType)
def test_ledger_action_can_be_removed(database,
function_scope_messages_aggregator):
db = DBLedgerActions(database, function_scope_messages_aggregator)
query = 'SELECT COUNT(*) FROM ledger_actions WHERE identifier=?'
cursor = database.conn.cursor()
# Add the entry that we want to delete
action = LedgerAction(
identifier=0, # whatever
timestamp=1,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
identifier = db.add_ledger_action(action)
# Delete ledger action
assert db.remove_ledger_action(identifier) is None
# Check that the change has been committed
cursor.execute(query, (identifier, ))
assert cursor.fetchone() == (0, )
def test_fees_in_received_asset(accountant, google_service):
"""
Test the sell trade where the fee is nominated in the asset received. We had a bug
where the PnL report said that there was no documented acquisition.
"""
history = [
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238), # 178.615 EUR/ETH
action_type=LedgerActionType.INCOME,
location=Location.BINANCE,
amount=ONE,
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='',
),
Trade(
# Sell 0.02 ETH for USDT with rate 1000 USDT/ETH and 0.10 USDT fee
# So acquired 20 USDT for 0.02 ETH + 0.10 USDT
# So acquired 20 USDT for 0.02 * 598.26 + 0.10 * 0.89 -> 12.0542 EUR
# So paid 12.0542/20 -> 0.60271 EUR/USDT
timestamp=1609537953, # 0.89 EUR/USDT
location=Location.BINANCE,
base_asset=A_ETH, # 598.26 EUR/ETH
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=FVal('0.02'),
rate=FVal(1000),
fee=FVal('0.10'),
fee_currency=A_USDT,
link=None,
),
]
accounting_history_process(
accountant,
start_ts=1539713238,
end_ts=1624395187,
history_list=history,
)
no_message_errors(accountant.msg_aggregator)
assert accountant.pots[0].cost_basis.get_calculated_asset_amount(A_USDT.identifier).is_close('19.90') # noqa: E501
expected_pnls = PnlTotals({
AccountingEventType.TRADE: PNL(taxable=ZERO, free=FVal('14.2277')),
AccountingEventType.FEE: PNL(taxable=FVal('-0.060271'), free=ZERO),
AccountingEventType.LEDGER_ACTION: PNL(taxable=FVal('178.615'), free=ZERO),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
def _deserialize_transaction(grant_id: int, rawtx: Dict[str,
Any]) -> LedgerAction:
"""May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
timestamp = deserialize_timestamp_from_date(
date=rawtx['timestamp'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin API',
skip_milliseconds=True,
)
asset = get_gitcoin_asset(symbol=rawtx['asset'],
token_address=rawtx['token_address'])
raw_amount = deserialize_int_from_str(symbol=rawtx['amount'],
location='gitcoin api')
amount = asset_normalized_value(raw_amount, asset)
if amount == ZERO:
raise ZeroGitcoinAmount()
# let's use gitcoin's calculated rate for now since they include it in the response
usd_value = Price(
ZERO) if rawtx['usd_value'] is None else deserialize_price(
rawtx['usd_value']) # noqa: E501
rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount)
raw_txid = rawtx['tx_hash']
tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx)
# until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501
clr_round = _calculate_clr_round(timestamp, rawtx)
notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501
return LedgerAction(
identifier=1, # whatever -- does not end up in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=AssetAmount(amount),
asset=asset,
rate=rate,
rate_asset=A_USD,
link=raw_txid,
notes=notes,
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=clr_round,
tx_type=tx_type,
),
)
def test_fees_in_received_asset(accountant):
"""
Test the sell trade where the fee is nominated in the asset received. We had a bug
where the PnL report said that there was no documented adquisition.
"""
history = [
{
'timestamp': 1609537953,
'base_asset': 'ETH',
'quote_asset': A_USDT.identifier,
'trade_type': 'sell',
'rate': 1000,
'fee': '0.10',
'fee_currency': A_USDT.identifier,
'amount': 0.02,
'location': 'binance',
},
]
ledger_actions_list = [
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238),
action_type=LedgerActionType.INCOME,
location=Location.BINANCE,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='',
),
]
report, _ = accounting_history_process(
accountant,
start_ts=1539713238,
end_ts=1624395187,
history_list=history,
ledger_actions_list=ledger_actions_list,
)
warnings = accountant.msg_aggregator.consume_warnings()
assert len(warnings) == 0
errors = accountant.msg_aggregator.consume_errors()
assert len(errors) == 0
assert accountant.events.cost_basis.get_calculated_asset_amount(
A_USDT.identifier).is_close('19.90') # noqa: E501
# The ethereum income doesn't count for the income as the 1
# year rule is applied. Only the sell is computed.
assert FVal(report['total_profit_loss']) == FVal(14.13870)
def test_gas_fees_after_year(accountant, google_service):
"""
Test that for an expense like gas fees after year the "selling" part is tax free
PnL, and the expense part is taxable pnl.
"""
tx_hash = '0x5cc0e6e62753551313412492296d5e57bea0a9d1ce507cc96aa4aa076c5bde7a'
history = [
LedgerAction(
identifier=0,
timestamp=Timestamp(1539713238), # 178.615 EUR/ETH
action_type=LedgerActionType.
GIFT, # gift so not counting as income
location=Location.KRAKEN,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='',
),
HistoryBaseEntry(
event_identifier=tx_hash,
sequence_index=0,
timestamp=1640493374000, # 4072.51 EUR/ETH
location=Location.BLOCKCHAIN,
location_label=make_ethereum_address(),
asset=A_ETH,
balance=Balance(amount=FVal('0.01')),
notes='Burned 0.01 ETH in gas',
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
counterparty=CPT_GAS,
)
]
accounting_history_process(
accountant,
start_ts=0,
end_ts=1640493376,
history_list=history,
)
no_message_errors(accountant.msg_aggregator)
expected_pnls = PnlTotals({
AccountingEventType.TRANSACTION_EVENT:
PNL(taxable=FVal('-40.7251'), free=FVal('38.93895')),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
def process_entry(
self,
db: DBHandler,
db_ledger: DBLedgerActions,
timestamp: Timestamp,
data: BinanceCsvRow,
) -> None:
asset = data['Coin']
amount = data['Change']
ledger_action = LedgerAction(
identifier=0,
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BINANCE,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'Imported from binance CSV file. Binance operation: {data["Operation"]}',
)
db_ledger.add_ledger_action(ledger_action)
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = symbol_to_asset_or_token(to_currency)
quote_asset = symbol_to_asset_or_token(currency)
if quote_asset is None:
raise DeserializationError(
'Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = symbol_to_asset_or_token(currency)
quote_asset = symbol_to_asset_or_token(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(
csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = symbol_to_asset_or_token(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in ('airdrop_to_exchange_transfer', 'mco_stake_reward'):
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
self.db_ledger.add_ledger_action(action)
elif row_type == 'invest_deposit':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type == 'invest_withdrawal':
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount(csv_row['Amount'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_associated_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
'interest_swap_credited',
'interest_swap_debited',
# The user has received an aidrop but can't claim it yet
'airdrop_locked',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCSVEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry', )
def test_store_same_tx_hash_in_db(database):
"""Test that if somehow during addition a duplicate is added,
it's ignored and only 1 ends up in the db"""
action1 = LedgerAction(
identifier=1,
timestamp=Timestamp(1624791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.0004789924016679019628604417823'),
asset=A_ETH,
rate=FVal('1983.33'),
rate_asset=A_USD,
link=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ETHEREUM,
),
)
action2 = LedgerAction(
identifier=2,
timestamp=Timestamp(1634791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('0.789924016679019628604417823'),
asset=A_ETH,
rate=FVal('1913.33'),
rate_asset=A_USD,
link=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'0x00298f72ad40167051e111e6dc2924de08cce7cf0ad00d04ad5a9e58426536a1',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ETHEREUM,
),
)
action3 = LedgerAction(
identifier=2,
timestamp=Timestamp(1654791600),
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=FVal('2445533521905078832065264'),
asset=A_ETH,
rate=FVal('1973.33'),
rate_asset=A_USD,
link=
'sync-tx:5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
notes='Gitcoin grant 149 event',
extra_data=GitcoinEventData(
tx_id=
'5612f84bc20cda25b911af39b792c973bdd5916b3b6868db2420b5dafd705a90',
grant_id=149,
clr_round=None,
tx_type=GitcoinEventTxType.ZKSYNC,
),
)
dbledger = DBLedgerActions(database, database.msg_aggregator)
dbledger.add_ledger_actions([action1, action2, action3])
stored_actions = dbledger.get_ledger_actions(
filter_query=LedgerActionsFilterQuery.make(location=Location.GITCOIN),
has_premium=True,
)
assert stored_actions == [action1, action3]
errors = database.msg_aggregator.consume_errors()
warnings = database.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
assert 'Did not add ledger action to DB' in warnings[0]
def test_ledger_actions_accounting(accountant):
"""Test for accounting for ledger actions
Makes sure that Ledger actions are processed in accounting, range is respected
and that they contribute to the "bought" amount per asset and that also if
a rate is given then that is used instead of the queried price
"""
ledger_actions_history = [
LedgerAction( # before range - read only for amount not profit
identifier=1,
timestamp=1435979735, # 0.1 EUR per ETH
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal(1)),
rate=None,
rate_asset=None,
link=None,
notes=None,
),
LedgerAction(
identifier=2,
timestamp=1437279735, # 250 EUR per BTC
action_type=LedgerActionType.INCOME,
location=Location.BLOCKCHAIN,
asset=A_BTC,
amount=AssetAmount(FVal(1)),
rate=FVal('400'),
rate_asset=A_EUR,
link='foo',
notes='we give a rate here',
),
LedgerAction(
identifier=3,
timestamp=1447279735, # 0.4 EUR per XMR
action_type=LedgerActionType.DIVIDENDS_INCOME,
location=Location.KRAKEN,
asset=A_XMR,
amount=AssetAmount(FVal(10)),
rate=None,
rate_asset=None,
link=None,
notes=None,
),
LedgerAction(
identifier=4,
timestamp=1457279735, # 1 EUR per ETH
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal('0.1')),
rate=None,
rate_asset=None,
link=None,
notes=None,
),
LedgerAction(
identifier=5,
timestamp=1467279735, # 420 EUR per BTC
action_type=LedgerActionType.LOSS,
location=Location.EXTERNAL,
asset=A_BTC,
amount=AssetAmount(FVal('0.1')),
rate=FVal(500),
rate_asset=A_USD,
link='foo2',
notes='we give a rate here',
),
LedgerAction( # after range and should be completely ignored
identifier=6,
timestamp=1529693374,
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal('0.5')),
rate=FVal(400),
rate_asset=A_EUR,
link='foo3',
notes='we give a rate here too but doesnt matter',
)
]
result = accounting_history_process(
accountant=accountant,
start_ts=1436979735,
end_ts=1519693374,
history_list=[],
ledger_actions_list=ledger_actions_history,
)
assert accountant.events.cost_basis.get_calculated_asset_amount(
A_BTC).is_close('0.9')
assert accountant.events.cost_basis.get_calculated_asset_amount(
A_ETH).is_close('0.9')
assert accountant.events.cost_basis.get_calculated_asset_amount(
A_XMR).is_close('10')
# 400 * 1 + 0.4 * 10 - 1 * 0.1 - 500 * 0.9004 * 0.1 = 358.88
expected_pnl = '358.88'
assert FVal(result['overview']['ledger_actions_profit_loss']).is_close(
expected_pnl)
assert FVal(result['overview']['total_profit_loss']).is_close(expected_pnl)
assert FVal(
result['overview']['total_taxable_profit_loss']).is_close(expected_pnl)
def test_ledger_actions_accounting(accountant, google_service):
"""Test for accounting for ledger actions
Makes sure that Ledger actions are processed in accounting, range is respected
and that they contribute to the "bought" amount per asset and that also if
a rate is given then that is used instead of the queried price
"""
history = [LedgerAction( # before range - read only for amount not profit
identifier=1,
timestamp=1435979735, # 0.1 EUR per ETH
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal(1)),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=2,
timestamp=1437279735, # 250 EUR per BTC
action_type=LedgerActionType.INCOME,
location=Location.BLOCKCHAIN,
asset=A_BTC,
amount=AssetAmount(FVal(1)),
rate=FVal('400'),
rate_asset=A_EUR,
link='foo',
notes='we give a rate here',
), LedgerAction(
identifier=3,
timestamp=1447279735, # 0.4 EUR per XMR
action_type=LedgerActionType.DIVIDENDS_INCOME,
location=Location.KRAKEN,
asset=A_XMR,
amount=AssetAmount(FVal(10)),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=4,
timestamp=1457279735, # 1 EUR per ETH
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal('0.1')),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=5,
timestamp=1467279735, # 420 EUR per BTC
action_type=LedgerActionType.LOSS,
location=Location.EXTERNAL,
asset=A_BTC,
amount=AssetAmount(FVal('0.1')),
rate=FVal(500),
rate_asset=A_USD,
link='foo2',
notes='we give a rate here',
), LedgerAction( # after range and should be completely ignored
identifier=6,
timestamp=1529693374,
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
asset=A_ETH,
amount=AssetAmount(FVal('0.5')),
rate=FVal(400),
rate_asset=A_EUR,
link='foo3',
notes='we give a rate here too but doesnt matter',
)]
accounting_history_process(
accountant=accountant,
start_ts=1436979735,
end_ts=1519693374,
history_list=history,
)
assert accountant.pots[0].cost_basis.get_calculated_asset_amount(A_BTC).is_close('0.9')
assert accountant.pots[0].cost_basis.get_calculated_asset_amount(A_ETH).is_close('0.9')
assert accountant.pots[0].cost_basis.get_calculated_asset_amount(A_XMR).is_close('10')
expected_pnls = PnlTotals({
# 400 + 0.4*10 - 1*0.1 + 1*0.1 - 1*0.01 - 0.1*500*0.9004 + 0.1*500*0.9004 - 0.1* 400
AccountingEventType.LEDGER_ACTION: PNL(taxable=FVal('363.99'), free=ZERO),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
def _deserialize_ledger_action(
self, raw_data: Dict[str, Any]) -> Optional[LedgerAction]:
"""Processes a single transaction from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data.get('status', '') != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
get_key_if_has_val(raw_data, 'created_at'),
'iso8601',
'coinbase',
)
if 'type' in raw_data:
# The parent method filtered with 'from' attribute, so it is from another user.
# https://developers.coinbase.com/api/v2?python#transaction-resource
action_type = LedgerActionType.INCOME
if raw_data.get('type',
'') not in ('send', 'inflation_reward'):
msg = ('Non "send" or "inflation_reward" type '
'found in coinbase transactions processing')
raise DeserializationError(msg)
amount_data = raw_data.get('amount', {})
amount = deserialize_asset_amount(amount_data['amount'])
asset = asset_from_coinbase(amount_data['currency'],
time=timestamp)
native_amount_data = raw_data.get('native_amount', {})
native_amount = deserialize_asset_amount(
native_amount_data['amount'])
native_asset = asset_from_coinbase(
native_amount_data['currency'])
rate = ZERO
if amount_data and native_amount_data and native_amount and amount != ZERO:
rate = native_amount / amount
if 'details' in raw_data and 'title' in raw_data['details'] \
and 'subtitle' in raw_data['details'] and 'header' in raw_data['details']:
details = raw_data.get('details', {})
notes = (f"{details.get('title', '')} "
f"{details.get('subtitle', '')} "
f"{details.get('header', '')}")
else:
notes = ''
return LedgerAction(identifier=0,
location=Location.COINBASE,
action_type=action_type,
timestamp=timestamp,
asset=asset,
amount=amount,
rate=Price(rate),
rate_asset=native_asset,
link=str(raw_data['id']),
notes=notes)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase transaction with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'ledger action. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'ledger action {raw_data}. Error was: {msg}', )
return None
def _consume_grant_entry(self, entry: Dict[str,
Any]) -> Optional[LedgerAction]:
"""
Consumes a grant entry from the CSV and turns it into a LedgerAction
May raise:
- DeserializationError
- KeyError
- UnknownAsset
"""
if entry['Type'] != 'grant':
return None
timestamp = deserialize_timestamp_from_date(
date=entry['date'],
formatstr='%Y-%m-%dT%H:%M:%S',
location='Gitcoin CSV',
skip_milliseconds=True,
)
usd_value = deserialize_asset_amount(entry['Value In USD'])
asset = get_asset_by_symbol(entry['token_name'])
if asset is None:
raise UnknownAsset(entry['token_name'])
token_amount = deserialize_asset_amount(entry['token_value'])
if token_amount == ZERO: # try to make up for https://github.com/gitcoinco/web/issues/9213
price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=f'Gitcoin CSV entry {entry["txid"]}',
msg_aggregator=self.db.msg_aggregator,
)
if price == ZERO:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to amount being zero and inability to find price. Skipping.',
)
return None
# calculate the amount from price and value
token_amount = usd_value / price # type: ignore
match = self.grantid_re.search(entry['url'])
if match is None:
self.db.msg_aggregator.add_warning(
f'Could not process gitcoin grant entry at {entry["date"]} for {asset.symbol} '
f'due to inability to read grant id. Skipping.', )
return None
grant_id = int(match.group(1))
rate = Price(usd_value / token_amount)
raw_txid = entry['txid']
tx_type, tx_id = process_gitcoin_txid(key='txid', entry=entry)
return LedgerAction(
identifier=1, # whatever does not go in the DB
timestamp=timestamp,
action_type=LedgerActionType.DONATION_RECEIVED,
location=Location.GITCOIN,
amount=token_amount,
asset=asset,
rate=rate,
rate_asset=A_USD, # let's use the rate gitcoin calculated
link=raw_txid,
notes=f'Gitcoin grant {grant_id} event',
extra_data=GitcoinEventData(
tx_id=tx_id,
grant_id=grant_id,
clr_round=None, # can't get round from CSV
tx_type=tx_type,
),
)
def assert_cryptocom_special_events_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from crypto.com"""
trades = rotki.data.db.get_trades()
ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator)
ledger_actions = ledger_db.get_ledger_actions(None, None, None)
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_actions = [LedgerAction(
identifier=5,
timestamp=Timestamp(1609884000),
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(FVal('1')),
asset=symbol_to_asset_or_token('CRO'),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=4,
timestamp=Timestamp(1609884000),
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(FVal('0.5')),
asset=symbol_to_asset_or_token('MCO'),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=3,
timestamp=Timestamp(1609884000),
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(FVal('1')),
asset=symbol_to_asset_or_token('CRO'),
rate=None,
rate_asset=None,
link=None,
notes=None,
), LedgerAction(
identifier=2,
timestamp=Timestamp(1609797600),
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(FVal('0.02005')),
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes='Stake profit for asset BTC',
), LedgerAction(
identifier=1,
timestamp=Timestamp(1609624800),
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(FVal('0.00005')),
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes='Stake profit for asset BTC',
)]
assert list(reversed(expected_actions)) == ledger_actions
expected_trades = [Trade(
timestamp=Timestamp(1609884000),
location=Location.CRYPTOCOM,
base_asset=symbol_to_asset_or_token('CRO'),
quote_asset=symbol_to_asset_or_token('MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('10')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='MCO Earnings/Rewards Swap\nSource: crypto.com (CSV import)',
)]
assert trades == expected_trades
def test_query_ledger_actions(events_historian, function_scope_messages_aggregator):
"""
Create actions and query the events historian to check that the history
has events previous to the selected from_ts. This allows us to verify that
actions before one period are counted in the PnL report to calculate cost basis.
https://github.com/rotki/rotki/issues/2541
"""
selected_timestamp = 10
db = DBLedgerActions(events_historian.db, function_scope_messages_aggregator)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp - 2,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(1),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp + 3,
action_type=LedgerActionType.EXPENSE,
location=Location.EXTERNAL,
amount=FVal(0.5),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
action = LedgerAction(
identifier=0, # whatever
timestamp=selected_timestamp + 5,
action_type=LedgerActionType.INCOME,
location=Location.EXTERNAL,
amount=FVal(10),
asset=A_USDC,
rate=None,
rate_asset=None,
link=None,
notes=None,
)
db.add_ledger_action(action)
actions, length = events_historian.query_ledger_actions(
has_premium=True,
from_ts=None,
to_ts=Timestamp(selected_timestamp + 4),
)
assert any((action.timestamp < selected_timestamp for action in actions))
assert length == 2
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _import_cryptocom_associated_entries(self, data: Any,
tx_kind: str) -> None:
"""Look for events that have associated entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known kind: 'dynamic_coin_swap' or 'dust_conversion'
May raise:
- UnknownAsset if an unknown asset is encountered in the imported files
- KeyError if a row contains unexpected data entries
"""
multiple_rows: Dict[Any, Dict[str, Any]] = {}
investments_deposits: Dict[str, List[Any]] = defaultdict(list)
investments_withdrawals: Dict[str, List[Any]] = defaultdict(list)
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{tx_kind}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
if 'debited' not in multiple_rows[timestamp]:
multiple_rows[timestamp]['debited'] = []
multiple_rows[timestamp]['debited'].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_credited':
# They only is one credited row
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if timestamp not in multiple_rows:
multiple_rows[timestamp] = {}
multiple_rows[timestamp]['credited'] = row
elif row['Transaction Kind'] == f'{tx_kind}_deposit':
asset = row['Currency']
investments_deposits[asset].append(row)
elif row['Transaction Kind'] == f'{tx_kind}_withdrawal':
asset = row['Currency']
investments_withdrawals[asset].append(row)
for timestamp in multiple_rows:
# When we convert multiple assets dust to CRO
# in one time, it will create multiple debited rows with
# the same timestamp
debited_rows = multiple_rows[timestamp]['debited']
credited_row = multiple_rows[timestamp]['credited']
total_debited_usd = functools.reduce(
lambda acc, row: acc + deserialize_asset_amount(row[
'Native Amount (in USD)']),
debited_rows,
ZERO,
)
# If the value of the transaction is too small (< 0,01$),
# crypto.com will display 0 as native amount
# if we have multiple debited rows, we can't import them
# since we can't compute their dedicated rates, so we skip them
if len(debited_rows) > 1 and total_debited_usd == 0:
return
if credited_row is not None and len(debited_rows) != 0:
for debited_row in debited_rows:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = symbol_to_asset_or_token(
credited_row['Currency'])
quote_asset = symbol_to_asset_or_token(
debited_row['Currency'])
part_of_total = (
FVal(1) if len(debited_rows) == 1 else
deserialize_asset_amount(
debited_row["Native Amount (in USD)"], ) /
total_debited_usd)
quote_amount_sold = deserialize_asset_amount(
debited_row['Amount'], ) * part_of_total
base_amount_bought = deserialize_asset_amount(
credited_row['Amount'], ) * part_of_total
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=TradeType.BUY,
amount=AssetAmount(base_amount_bought),
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
# Compute investments profit
if len(investments_withdrawals) != 0:
for asset in investments_withdrawals:
asset_object = symbol_to_asset_or_token(asset)
if asset not in investments_deposits:
log.error(
f'Investment withdrawal without deposit at crypto.com. Ignoring '
f'staking info for asset {asset_object}', )
continue
# Sort by date in ascending order
withdrawals_rows = sorted(
investments_withdrawals[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
investments_rows = sorted(
investments_deposits[asset],
key=lambda x: deserialize_timestamp_from_date(
date=x['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
),
)
last_date = Timestamp(0)
for withdrawal in withdrawals_rows:
withdrawal_date = deserialize_timestamp_from_date(
date=withdrawal['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
amount_deposited = ZERO
for deposit in investments_rows:
deposit_date = deserialize_timestamp_from_date(
date=deposit['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='cryptocom',
)
if last_date < deposit_date <= withdrawal_date:
# Amount is negative
amount_deposited += deserialize_asset_amount(
deposit['Amount'])
amount_withdrawal = deserialize_asset_amount(
withdrawal['Amount'])
# Compute profit
profit = amount_withdrawal + amount_deposited
if profit >= ZERO:
last_date = withdrawal_date
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=withdrawal_date,
action_type=LedgerActionType.INCOME,
location=Location.CRYPTOCOM,
amount=AssetAmount(profit),
asset=asset_object,
rate=None,
rate_asset=None,
link=None,
notes=f'Stake profit for asset {asset}',
)
self.db_ledger.add_ledger_action(action)
def assert_blockfi_transactions_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from blockfi"""
ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator)
ledger_actions = ledger_db.get_ledger_actions(None, None, None)
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_actions = [LedgerAction(
identifier=3,
timestamp=Timestamp(1600293599),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.48385358')),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='Bonus Payment from BlockFi',
), LedgerAction(
identifier=2,
timestamp=Timestamp(1606953599),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.00052383')),
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes='Referral Bonus from BlockFi',
), LedgerAction(
identifier=1,
timestamp=Timestamp(1612051199),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.56469042')),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='Interest Payment from BlockFi',
)]
assert expected_actions == ledger_actions
expected_movements = [AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1595247055),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('1.11415058')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1605977971),
asset=A_ETH,
amount=AssetAmount(FVal('3')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
)]
assert expected_movements == asset_movements
def assert_nexo_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from nexo"""
ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator)
ledger_actions = ledger_db.get_ledger_actions(None, None, None)
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_actions = [LedgerAction(
identifier=3,
timestamp=Timestamp(1565888464),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('22.5653042')),
asset=symbol_to_asset_or_token('NEXO'),
rate=None,
rate_asset=None,
link='NXT0000000009',
notes='Dividend from Nexo',
), LedgerAction(
identifier=2,
timestamp=Timestamp(1597492915),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('10.3585507')),
asset=symbol_to_asset_or_token('NEXO'),
rate=None,
rate_asset=None,
link='NXT0000000007',
notes='Dividend from Nexo',
), LedgerAction(
identifier=1,
timestamp=Timestamp(1614993620),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('1')),
asset=symbol_to_asset_or_token('USDC'),
rate=None,
rate_asset=None,
link='NXT0000000002',
notes='Interest from Nexo',
)]
expected_movements = [AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1556116964),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('1')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000013',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1556122699),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('0.9995')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000012',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1558720210),
address=None,
transaction_id=None,
asset=symbol_to_asset_or_token('NEXO'),
amount=AssetAmount(FVal('1.00001')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000011',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1565912821),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('10000')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000010',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1608131364),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('2000.79')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000005',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1614366540),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('10')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000003',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1615024314),
address=None,
transaction_id=None,
asset=symbol_to_asset_or_token('USDC'),
amount=AssetAmount(FVal('1')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000001',
)]
assert ledger_actions == expected_actions
assert asset_movements == expected_movements