def _handle_pooltogether(normalized_balance: FVal,
token_name: str) -> Optional[DefiBalance]:
"""Special handling for pooltogether
https://github.com/rotki/rotki/issues/1429
"""
if 'DAI' in token_name:
dai_price = Inquirer.find_usd_price(A_DAI)
return DefiBalance(
token_address=string_to_ethereum_address(
'0x49d716DFe60b37379010A75329ae09428f17118d'),
token_name='Pool Together DAI token',
token_symbol='plDAI',
balance=Balance(
amount=normalized_balance,
usd_value=normalized_balance * dai_price,
),
)
if 'USDC' in token_name:
usdc_price = Inquirer.find_usd_price(A_USDC)
return DefiBalance(
token_address=string_to_ethereum_address(
'0xBD87447F48ad729C5c4b8bcb503e1395F62e8B98'),
token_name='Pool Together USDC token',
token_symbol='plUSDC',
balance=Balance(
amount=normalized_balance,
usd_value=normalized_balance * usdc_price,
),
)
# else
return None
def _process_vault_events(self, events: List[YearnVaultEvent]) -> Balance:
"""Process the events for a single vault and returns total profit/loss after all events"""
total = Balance()
profit_so_far = Balance()
if len(events) < 2:
return total
for event in events:
if event.event_type == 'deposit':
total -= event.from_value
else: # withdraws
profit_amount = total.amount + event.to_value.amount - profit_so_far.amount
profit: Optional[Balance]
if profit_amount >= 0:
usd_price = get_usd_price_zero_if_error(
asset=event.to_asset,
time=event.timestamp,
location=f'yearn vault event {event.tx_hash} processing',
msg_aggregator=self.msg_aggregator,
)
profit = Balance(profit_amount, profit_amount * usd_price)
profit_so_far += profit
else:
profit = None
event.realized_pnl = profit
total += event.to_value
return total
def test_query_balances_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument
assets_balance, msg = sandbox_kuckoin.query_balances()
assert assets_balance == {
A_BTC:
Balance(
amount=FVal('2.61018067'),
usd_value=FVal('3.915271005'),
),
A_ETH:
Balance(
amount=FVal('47.43934995'),
usd_value=FVal('71.159024925'),
),
A_KCS:
Balance(
amount=FVal('0.2'),
usd_value=FVal('0.30'),
),
A_USDT:
Balance(
amount=FVal('45097.26244755'),
usd_value=FVal('67645.893671325'),
),
}
assert msg == ''
def test_get_vault_balance(
inquirer, # pylint: disable=unused-argument
mocked_current_prices,
):
debt_value = FVal('2000')
owner = make_ethereum_address()
vault = MakerdaoVault(
identifier=1,
collateral_type='ETH-A',
collateral_asset=A_ETH,
owner=owner,
collateral=Balance(FVal('100'), FVal('20000')),
debt=Balance(debt_value, debt_value * mocked_current_prices['DAI']),
collateralization_ratio='990%',
liquidation_ratio=FVal(1.5),
liquidation_price=FVal('50'), # not calculated to be correct
urn=make_ethereum_address(),
stability_fee=ZERO,
)
expected_result = BalanceSheet(
assets=defaultdict(Balance,
{A_ETH: Balance(FVal('100'), FVal('20000'))}),
liabilities=defaultdict(Balance,
{A_DAI: Balance(FVal('2000'), FVal('2020'))}),
)
assert vault.get_balance() == expected_result
def test_balance_sheet_to_dict():
a = BalanceSheet(
assets={
A_USD: Balance(amount=FVal('2'), usd_value=FVal('2')),
A_ETH: Balance(amount=FVal('3'), usd_value=FVal('900')),
},
liabilities={
A_DAI: Balance(amount=FVal('5'), usd_value=FVal('5.1')),
A_ETH: Balance(amount=FVal('0.5'), usd_value=FVal('150')),
},
)
assert a.to_dict() == {
'assets': {
'USD': {
'amount': FVal('2'),
'usd_value': FVal('2')
},
'ETH': {
'amount': FVal('3'),
'usd_value': FVal('900')
},
},
'liabilities': {
ethaddress_to_identifier('0x6B175474E89094C44Da98b954EedeAC495271d0F'):
{
'amount': FVal('5'),
'usd_value': FVal('5.1')
}, # noqa: E501
'ETH': {
'amount': FVal('0.5'),
'usd_value': FVal('150')
},
},
}
def _get_staking_history(
staking_balances: Dict[ChecksumAddress, List[Dict[str, Any]]],
staking_events: ADXStakingEvents,
) -> Dict[ChecksumAddress, ADXStakingHistory]:
"""Given the following params:
- staking_balances: the balances of the addresses per pool.
- staking_events: all the events of the addresses mixed but grouped by
type.
Return a map between an address and its <ADXStakingDetail>, which contains
all the events that belong to the address, and the performance details
per staking pool.
"""
staking_history = {}
address_staking_events = defaultdict(list)
all_events = staking_events.get_all()
# Map addresses with their events
for event in all_events:
address_staking_events[event.address].append(event)
# Sort staking events per address by timestamp (older first) and event type
for address in address_staking_events.keys():
address_staking_events[address].sort(
key=lambda event: (event.timestamp, EVENT_TYPE_ORDER_IN_ASC[type(event)]),
)
for address, adx_staking_balances in staking_balances.items():
adx_staking_details = []
for adx_staking_balance in adx_staking_balances:
adx_total_profit = Balance()
adx_balance = Balance(
amount=adx_staking_balance['adx_balance']['amount'],
usd_value=adx_staking_balance['adx_balance']['usd_value'],
)
# Add claimed amounts and their historical usd value
for event in address_staking_events[address]:
if isinstance(event, ChannelWithdraw):
if event.token == A_ADX:
adx_total_profit += event.value
pool_staking_detail = ADXStakingDetail(
contract_address=adx_staking_balance['contract_address'],
pool_id=adx_staking_balance['pool_id'],
pool_name=adx_staking_balance['pool_name'],
total_staked_amount=ZERO, # unable to calculate for now
apr=ZERO, # unable to calculate for now
adx_balance=adx_balance,
adx_unclaimed_balance=Balance(), # unable to calculate for now
dai_unclaimed_balance=Balance(), # unable to calculate for now
adx_profit_loss=adx_total_profit,
dai_profit_loss=Balance(), # unable to calculate for now
)
adx_staking_details.append(pool_staking_detail)
staking_history[address] = ADXStakingHistory(
events=address_staking_events[address],
staking_details=adx_staking_details,
)
return staking_history
def test_compound_ether_withdraw(database, ethereum_manager,
function_scope_messages_aggregator):
"""Data taken from:
https://etherscan.io/tx/0x024bd402420c3ba2f95b875f55ce2a762338d2a14dac4887b78174254c9ab807
"""
# TODO: For faster tests hard-code the transaction and the logs here so no remote query needed
tx_hash = deserialize_evm_tx_hash(
'0x024bd402420c3ba2f95b875f55ce2a762338d2a14dac4887b78174254c9ab807'
) # noqa: E501
events = get_decoded_events_of_transaction(
ethereum_manager=ethereum_manager,
database=database,
msg_aggregator=function_scope_messages_aggregator,
tx_hash=tx_hash,
)
expected_events = [
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=0,
timestamp=1598813490000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
asset=A_ETH,
balance=Balance(amount=FVal('0.02858544'), usd_value=ZERO),
location_label=ADDY,
notes=f'Burned 0.02858544 ETH in gas from {ADDY}',
counterparty=CPT_GAS,
),
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=1,
timestamp=1598813490000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.RETURN_WRAPPED,
asset=A_CETH,
balance=Balance(amount=FVal('24.97649991'), usd_value=ZERO),
location_label=ADDY,
notes='Return 24.97649991 cETH to compound',
counterparty=CPT_COMPOUND,
),
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=2,
timestamp=1598813490000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.WITHDRAWAL,
event_subtype=HistoryEventSubType.REMOVE_ASSET,
asset=A_ETH,
balance=Balance(amount=FVal('0.500003923413507454'),
usd_value=ZERO),
location_label=ADDY,
notes='Withdraw 0.500003923413507454 ETH from compound',
counterparty=CPT_COMPOUND,
)
]
assert events == expected_events
def _query_vault_data(
self,
identifier: int,
owner: ChecksumEthAddress,
urn: ChecksumEthAddress,
ilk: bytes,
) -> Optional[MakerdaoVault]:
collateral_type = ilk.split(b'\0', 1)[0].decode()
asset = COLLATERAL_TYPE_MAPPING.get(collateral_type, None)
if asset is None:
self.msg_aggregator.add_warning(
f'Detected vault with collateral_type {collateral_type}. That '
f'is not yet supported by rotki. Skipping...', )
return None
result = MAKERDAO_VAT.call(self.ethereum, 'urns', arguments=[ilk, urn])
# also known as ink in their contract
collateral_amount = FVal(result[0] / WAD)
normalized_debt = result[1] # known as art in their contract
result = MAKERDAO_VAT.call(self.ethereum, 'ilks', arguments=[ilk])
rate = result[1] # Accumulated Rates
spot = FVal(result[2]) # Price with Safety Margin
# How many DAI owner needs to pay back to the vault
debt_value = FVal(((normalized_debt / WAD) * rate) / RAY)
result = MAKERDAO_SPOT.call(self.ethereum, 'ilks', arguments=[ilk])
mat = result[1]
liquidation_ratio = FVal(mat / RAY)
price = FVal((spot / RAY) * liquidation_ratio)
self.usd_price[asset.identifier] = price
collateral_value = FVal(price * collateral_amount)
if debt_value == 0:
collateralization_ratio = None
else:
collateralization_ratio = FVal(collateral_value /
debt_value).to_percentage(2)
collateral_usd_value = price * collateral_amount
if collateral_amount == 0:
liquidation_price = None
else:
liquidation_price = (debt_value *
liquidation_ratio) / collateral_amount
dai_usd_price = Inquirer().find_usd_price(A_DAI)
return MakerdaoVault(
identifier=identifier,
owner=owner,
collateral_type=collateral_type,
collateral_asset=asset,
collateral=Balance(collateral_amount, collateral_usd_value),
debt=Balance(debt_value, dai_usd_price * debt_value),
liquidation_ratio=liquidation_ratio,
collateralization_ratio=collateralization_ratio,
liquidation_price=liquidation_price,
urn=urn,
stability_fee=self.get_stability_fee(ilk),
)
def test_compound_ether_deposit(database, ethereum_manager,
function_scope_messages_aggregator):
"""Data taken from:
https://etherscan.io/tx/0x06a8b9f758b0471886186c2a48dea189b3044916c7f94ee7f559026fefd91c39
"""
# TODO: For faster tests hard-code the transaction and the logs here so no remote query needed
tx_hash = deserialize_evm_tx_hash(
'0x06a8b9f758b0471886186c2a48dea189b3044916c7f94ee7f559026fefd91c39'
) # noqa: E501
events = get_decoded_events_of_transaction(
ethereum_manager=ethereum_manager,
database=database,
msg_aggregator=function_scope_messages_aggregator,
tx_hash=tx_hash,
)
expected_events = [
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=0,
timestamp=1598639099000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
asset=A_ETH,
balance=Balance(amount=FVal('0.014122318'), usd_value=ZERO),
location_label=ADDY,
notes=f'Burned 0.014122318 ETH in gas from {ADDY}',
counterparty=CPT_GAS,
),
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=1,
timestamp=1598639099000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.DEPOSIT,
event_subtype=HistoryEventSubType.DEPOSIT_ASSET,
asset=A_ETH,
balance=Balance(amount=FVal('0.5'), usd_value=ZERO),
location_label=ADDY,
notes='Deposit 0.5 ETH to compound',
counterparty=CPT_COMPOUND,
),
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=33,
timestamp=1598639099000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.RECEIVE,
event_subtype=HistoryEventSubType.RECEIVE_WRAPPED,
asset=A_CETH,
balance=Balance(amount=FVal('24.97649991'), usd_value=ZERO),
location_label=ADDY,
notes='Receive 24.97649991 cETH from compound',
counterparty=CPT_COMPOUND,
)
]
assert events == expected_events
def test_tx_decode(evm_transaction_decoder, database):
dbethtx = DBEthTx(database)
addr1 = '0x2B888954421b424C5D3D9Ce9bB67c9bD47537d12'
approve_tx_hash = deserialize_evm_tx_hash('0x5cc0e6e62753551313412492296d5e57bea0a9d1ce507cc96aa4aa076c5bde7a') # noqa: E501
transactions = dbethtx.get_ethereum_transactions(
filter_=ETHTransactionsFilterQuery.make(
addresses=[addr1],
tx_hash=approve_tx_hash,
),
has_premium=True,
)
decoder = evm_transaction_decoder
with patch.object(decoder, 'decode_transaction', wraps=decoder.decode_transaction) as decode_mock: # noqa: E501
for tx in transactions:
receipt = dbethtx.get_receipt(tx.tx_hash)
assert receipt is not None, 'all receipts should be queried in the test DB'
events = decoder.get_or_decode_transaction_events(tx, receipt, ignore_cache=False)
if tx.tx_hash == approve_tx_hash:
assert len(events) == 2
assert_events_equal(events[0], HistoryBaseEntry(
# The no-member is due to https://github.com/PyCQA/pylint/issues/3162
event_identifier=approve_tx_hash.hex(), # pylint: disable=no-member
sequence_index=0,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
location_label=addr1,
asset=A_ETH,
balance=Balance(amount=FVal('0.000030921')),
# The no-member is due to https://github.com/PyCQA/pylint/issues/3162
notes=f'Burned 0.000030921 ETH in gas from {addr1}',
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
counterparty=CPT_GAS,
))
assert_events_equal(events[1], HistoryBaseEntry(
# The no-member is due to https://github.com/PyCQA/pylint/issues/3162
event_identifier=approve_tx_hash.hex(), # pylint: disable=no-member
sequence_index=163,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
location_label=addr1,
asset=A_SAI,
balance=Balance(amount=1),
notes=f'Approve 1 SAI of {addr1} for spending by 0xdf869FAD6dB91f437B59F1EdEFab319493D4C4cE', # noqa: E501
event_type=HistoryEventType.INFORMATIONAL,
event_subtype=HistoryEventSubType.APPROVE,
counterparty='0xdf869FAD6dB91f437B59F1EdEFab319493D4C4cE',
))
assert decode_mock.call_count == len(transactions)
# now go again, and see that no more decoding happens as it's all pulled from the DB
for tx in transactions:
receipt = dbethtx.get_receipt(tx.tx_hash)
assert receipt is not None, 'all receipts should be queried in the test DB'
events = decoder.get_or_decode_transaction_events(tx, receipt, ignore_cache=False)
assert decode_mock.call_count == len(transactions)
def test_balance_raddition():
a = Balance(amount=FVal('1.5'), usd_value=FVal('1.6'))
b = Balance(amount=FVal('2.5'), usd_value=FVal('2.7'))
c = Balance(amount=FVal('3'), usd_value=FVal('3.21'))
result = sum([a, b, c])
assert isinstance(result, Balance)
assert result.amount == FVal('7')
assert result.usd_value == FVal('7.51')
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> List[DefiEvent]:
"""Gets the history events from DSR for accounting
This is a premium only call. Check happens only in the API level.
"""
history = self.get_historical_dsr()
events = []
for _, report in history.items():
total_balance = Balance()
counted_profit = Balance()
for movement in report.movements:
if movement.timestamp < from_timestamp:
continue
if movement.timestamp > to_timestamp:
break
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
balance = Balance(
amount=_dsrdai_to_dai(movement.amount),
usd_value=movement.amount_usd_value,
)
if movement.movement_type == 'deposit':
spent_asset = A_DAI
spent_balance = balance
total_balance -= balance
else:
got_asset = A_DAI
got_balance = balance
total_balance += balance
if total_balance.amount - counted_profit.amount > ZERO:
pnl_balance = total_balance - counted_profit
counted_profit += pnl_balance
pnl = [AssetBalance(asset=A_DAI, balance=pnl_balance)]
events.append(
DefiEvent(
timestamp=movement.timestamp,
wrapped_event=movement,
event_type=DefiEventType.DSR_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Depositing and withdrawing from DSR is not counted in
# cost basis. DAI were always yours, you did not rebuy them
count_spent_got_cost_basis=False,
tx_hash=movement.tx_hash,
))
return events
def test_kraken_staking_events(accountant, google_service):
"""
Test that staking events from kraken are correctly processed
"""
history = [
HistoryBaseEntry(
event_identifier='XXX',
sequence_index=0,
timestamp=1640493374000,
location=Location.KRAKEN,
location_label='Kraken 1',
asset=A_ETH2,
balance=Balance(
amount=FVal(0.0000541090),
usd_value=FVal(0.212353475950),
),
notes=None,
event_type=HistoryEventType.STAKING,
event_subtype=HistoryEventSubType.REWARD,
),
HistoryBaseEntry(
event_identifier='YYY',
sequence_index=0,
timestamp=1636638550000,
location=Location.KRAKEN,
location_label='Kraken 1',
asset=A_ETH2,
balance=Balance(
amount=FVal(0.0000541090),
usd_value=FVal(0.212353475950),
),
notes=None,
event_type=HistoryEventType.STAKING,
event_subtype=HistoryEventSubType.REWARD,
)
]
_, events = accounting_history_process(
accountant,
start_ts=1636638549,
end_ts=1640493376,
history_list=history,
)
no_message_errors(accountant.msg_aggregator)
expected_pnls = PnlTotals({
AccountingEventType.STAKING:
PNL(taxable=FVal('0.471505826'), free=ZERO),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
assert len(events) == 2
expected_pnls = [FVal('0.25114638241'), FVal('0.22035944359')]
for idx, event in enumerate(events):
assert event.pnl.taxable == expected_pnls[idx]
assert event.type == AccountingEventType.STAKING
def _parse_repays(
self,
repays: List[Dict[str, Any]],
from_ts: Timestamp,
to_ts: Timestamp,
) -> List[AaveRepayEvent]:
events = []
for entry in repays:
common = _parse_common_event_data(entry, from_ts, to_ts)
if common is None:
continue # either timestamp out of range or error (logged in the function above)
timestamp, tx_hash, index = common
result = _get_reserve_asset_and_decimals(entry,
reserve_key='reserve')
if result is None:
continue # problem parsing, error already logged
asset, decimals = result
if 'amountAfterFee' in entry:
amount_after_fee = token_normalized_value_decimals(
int(entry['amountAfterFee']),
token_decimals=decimals,
)
fee = token_normalized_value_decimals(int(entry['fee']),
token_decimals=decimals)
else:
# In the V2 subgraph the amountAfterFee and Fee keys are replaced by amount
amount_after_fee = token_normalized_value_decimals(
int(entry['amount']),
token_decimals=decimals,
)
fee = ZERO
usd_price = query_usd_price_zero_if_error(
asset=asset,
time=timestamp,
location=f'aave repay event {tx_hash} from graph query',
msg_aggregator=self.msg_aggregator,
)
events.append(
AaveRepayEvent(
event_type='repay',
asset=asset,
value=Balance(amount=amount_after_fee,
usd_value=amount_after_fee * usd_price),
fee=Balance(amount=fee, usd_value=fee * usd_price),
block_number=0, # can't get from graph query
timestamp=timestamp,
tx_hash=tx_hash,
log_index=
index, # not really the log index, but should also be unique
))
return events
def test_gitcoin_old_donation(database, ethereum_manager, function_scope_messages_aggregator):
"""Data taken from
https://etherscan.io/tx/0x811ba23a10c76111289133ec6f90d3c33a604baa50053739210e870687a456d9
"""
# TODO: For faster tests hard-code the transaction and the logs here so no remote query needed
tx_hash = deserialize_evm_tx_hash('0x811ba23a10c76111289133ec6f90d3c33a604baa50053739210e870687a456d9') # noqa: E501
events = get_decoded_events_of_transaction(
ethereum_manager=ethereum_manager,
database=database,
msg_aggregator=function_scope_messages_aggregator,
tx_hash=tx_hash,
)
expected_events = [
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=0,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
asset=A_ETH,
balance=Balance(amount=FVal('0.000055118'), usd_value=ZERO),
location_label=ADDY,
notes=f'Burned 0.000055118 ETH in gas from {ADDY}',
counterparty=CPT_GAS,
), HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=164,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.DONATE,
asset=A_SAI,
balance=Balance(amount=FVal('0.95'), usd_value=ZERO),
location_label=ADDY,
notes='Donate 0.95 SAI to 0xEbDb626C95a25f4e304336b1adcAd0521a1Bdca1 via gitcoin', # noqa: E501
counterparty=CPT_GITCOIN,
), HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=165,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.DONATE,
asset=A_SAI,
balance=Balance(amount=FVal('0.05'), usd_value=ZERO),
location_label=ADDY,
notes='Donate 0.05 SAI to 0x00De4B13153673BCAE2616b67bf822500d325Fc3 via gitcoin', # noqa: E501
counterparty=CPT_GITCOIN,
),
]
assert events == expected_events
def test_query_some_balances(
function_scope_independentreserve,
inquirer, # pylint: disable=unused-argument
):
"""Just like test_query_balances but make sure 0 balances are skipped"""
exchange = function_scope_independentreserve
def mock_api_return(method, url, **kwargs): # pylint: disable=unused-argument
assert method == 'post'
response = """[{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 1.2, "CurrencyCode": "Aud", "TotalBalance": 2.5},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Usd", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Nzd", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Sgd", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Xbt", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Eth", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Xrp", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Ada", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Dot", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Uni", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Link", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Usdt", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Usdc", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Bch", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Ltc", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Mkr", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Dai", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Comp", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Snx", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Grt", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Eos", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Xlm", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Etc", "TotalBalance": 100.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Bat", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Pmgt", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Yfi", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Aave", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Zrx", "TotalBalance": 0.0},
{"AccountGuid": "foo", "AccountStatus": "Active", "AvailableBalance": 0.0, "CurrencyCode": "Omg", "TotalBalance": 0.0}]""" # noqa: E501
return MockResponse(200, response)
with patch.object(exchange.session, 'request',
side_effect=mock_api_return):
balances, msg = exchange.query_balances()
assert msg == ''
assert balances == {
A_AUD: Balance(amount=FVal(2.5), usd_value=FVal(3.75)),
A_ETC: Balance(amount=FVal(100), usd_value=FVal(150)),
}
def _update_events_value(
self,
staking_events: ADXStakingEvents,
) -> None:
# Update amounts for unbonds and unbond requests
bond_id_bond_map: Dict[str, Optional[Bond]] = {
bond.bond_id: bond for bond in staking_events.bonds
}
for event in (
staking_events.unbonds +
staking_events.unbond_requests # type: ignore # mypy bug concatenating lists
):
has_bond = True
bond = bond_id_bond_map.get(event.bond_id, None)
if bond:
event.value = Balance(amount=bond.value.amount)
event.pool_id = bond.pool_id
elif event.bond_id not in bond_id_bond_map:
bond_id_bond_map[event.bond_id] = None
db_bonds = cast(List[Bond], self.database.get_adex_events(
bond_id=event.bond_id,
event_type=AdexEventType.BOND,
))
if db_bonds:
db_bond = db_bonds[0]
bond_id_bond_map[event.bond_id] = db_bond
event.value = Balance(amount=db_bond.value.amount)
event.pool_id = db_bond.pool_id
else:
has_bond = False
else:
has_bond = False
if has_bond is False:
log.warning(
'Failed to update an AdEx event data. Unable to find its related bond event',
event=event,
)
# Update usd_value for all events
for event in staking_events.get_all(): # type: ignore # event can have all types
token = event.token if isinstance(event, ChannelWithdraw) else A_ADX
usd_price = PriceHistorian().query_historical_price(
from_asset=token,
to_asset=A_USD,
timestamp=event.timestamp,
)
event.value.usd_value = event.value.amount * usd_price
def decode_proxy_creation(
self,
tx_log: EthereumTxReceiptLog,
transaction: EthereumTransaction,
decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument
all_logs: List[EthereumTxReceiptLog], # pylint: disable=unused-argument
action_items: List[ActionItem], # pylint: disable=unused-argument
) -> Tuple[Optional[HistoryBaseEntry], Optional[ActionItem]]:
if tx_log.topics[
0] == b'%\x9b0\xca9\x88\\m\x80\x1a\x0b]\xbc\x98\x86@\xf3\xc2^/7S\x1f\xe18\xc5\xc5\xaf\x89U\xd4\x1b': # noqa: E501
owner_address = hex_or_bytes_to_address(tx_log.topics[2])
if not self.base.is_tracked(owner_address):
return None, None
proxy_address = hex_or_bytes_to_address(tx_log.data[0:32])
notes = f'Create DSR proxy {proxy_address} with owner {owner_address}'
event = HistoryBaseEntry(
event_identifier=transaction.tx_hash.hex(),
sequence_index=self.base.get_sequence_index(tx_log),
timestamp=ts_sec_to_ms(transaction.timestamp),
location=Location.BLOCKCHAIN,
location_label=owner_address,
# TODO: This should be null for proposals and other informational events
asset=A_ETH,
balance=Balance(),
notes=notes,
event_type=HistoryEventType.INFORMATIONAL,
event_subtype=HistoryEventSubType.DEPLOY,
counterparty=proxy_address,
)
return event, None
return None, None
def liquity_staking_balances(
self,
addresses: List[ChecksumEthAddress],
) -> Dict[ChecksumEthAddress, StakePosition]:
staked = self._get_raw_history(addresses, 'stake')
lqty_price = Inquirer().find_usd_price(A_LQTY)
data = {}
for stake in staked['lqtyStakes']:
try:
owner = to_checksum_address(stake['id'])
amount = deserialize_optional_to_fval(
value=stake['amount'],
name='amount',
location='liquity',
)
position = AssetBalance(
asset=A_LQTY,
balance=Balance(
amount=amount,
usd_value=lqty_price * amount,
),
)
data[owner] = StakePosition(position)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_warning(
f'Ignoring Liquity staking information. '
f'Failed to decode remote response. {msg}.', )
continue
return data
def _maybe_decode_erc20_approve(
self,
token: Optional[EthereumToken],
tx_log: EthereumTxReceiptLog,
transaction: EthereumTransaction,
decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument
action_items: List[ActionItem], # pylint: disable=unused-argument
) -> Optional[HistoryBaseEntry]:
if tx_log.topics[0] != ERC20_APPROVE or token is None:
return None
owner_address = hex_or_bytes_to_address(tx_log.topics[1])
spender_address = hex_or_bytes_to_address(tx_log.topics[2])
if not any(
self.base.is_tracked(x)
for x in (owner_address, spender_address)):
return None
amount_raw = hex_or_bytes_to_int(tx_log.data)
amount = token_normalized_value(token_amount=amount_raw, token=token)
notes = f'Approve {amount} {token.symbol} of {owner_address} for spending by {spender_address}' # noqa: E501
return HistoryBaseEntry(
event_identifier=transaction.tx_hash.hex(),
sequence_index=self.base.get_sequence_index(tx_log),
timestamp=ts_sec_to_ms(transaction.timestamp),
location=Location.BLOCKCHAIN,
location_label=owner_address,
asset=token,
balance=Balance(amount=amount),
notes=notes,
event_type=HistoryEventType.INFORMATIONAL,
event_subtype=HistoryEventSubType.APPROVE,
counterparty=spender_address,
)
def test_receiving_value_from_tx(accountant, google_service):
"""
Test that receiving a transaction that provides value works fine
"""
addr2 = make_ethereum_address()
tx_hash = '0x5cc0e6e62753551313412492296d5e57bea0a9d1ce507cc96aa4aa076c5bde7a'
history = [
HistoryBaseEntry(
event_identifier=tx_hash,
sequence_index=0,
timestamp=1569924574000,
location=Location.BLOCKCHAIN,
location_label=make_ethereum_address(),
asset=A_ETH,
balance=Balance(amount=FVal('1.5')),
notes=f'Received 1.5 ETH from {addr2}',
event_type=HistoryEventType.RECEIVE,
event_subtype=HistoryEventSubType.NONE,
counterparty=addr2,
)
]
accounting_history_process(
accountant,
start_ts=0,
end_ts=1640493376,
history_list=history,
)
no_message_errors(accountant.msg_aggregator)
expected_pnls = PnlTotals({
AccountingEventType.TRANSACTION_EVENT:
PNL(taxable=FVal('242.385'), free=ZERO),
})
check_pnls_and_csv(accountant, expected_pnls, google_service)
def _decode_vault_creation(
self,
tx_log: EthereumTxReceiptLog,
transaction: EthereumTransaction,
decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument
all_logs: List[EthereumTxReceiptLog], # pylint: disable=unused-argument
) -> Tuple[Optional[HistoryBaseEntry], Optional[ActionItem]]:
owner_address = self._get_address_or_proxy(
hex_or_bytes_to_address(tx_log.topics[2]))
if owner_address is None:
return None, None
if not self.base.is_tracked(owner_address):
return None, None
cdp_id = hex_or_bytes_to_int(tx_log.topics[3])
notes = f'Create MakerDAO vault with id {cdp_id} and owner {owner_address}'
event = HistoryBaseEntry(
event_identifier=transaction.tx_hash.hex(),
sequence_index=self.base.get_sequence_index(tx_log),
timestamp=ts_sec_to_ms(transaction.timestamp),
location=Location.BLOCKCHAIN,
location_label=owner_address,
# TODO: This should be null for proposals and other informational events
asset=A_ETH,
balance=Balance(),
notes=notes,
event_type=HistoryEventType.INFORMATIONAL,
event_subtype=HistoryEventSubType.DEPLOY,
counterparty='makerdao vault',
)
return event, None
def get_manually_tracked_balances(
db: 'DBHandler',
balance_type: Optional[BalanceType] = None,
) -> List[ManuallyTrackedBalanceWithValue]:
"""Gets the manually tracked balances"""
balances = db.get_manually_tracked_balances(balance_type=balance_type)
balances_with_value = []
for entry in balances:
try:
price = Inquirer().find_usd_price(entry.asset)
except RemoteError as e:
db.msg_aggregator.add_warning(
f'Could not find price for {entry.asset.identifier} during '
f'manually tracked balance querying due to {str(e)}', )
price = Price(ZERO)
value = Balance(amount=entry.amount, usd_value=price * entry.amount)
balances_with_value.append(
ManuallyTrackedBalanceWithValue(
id=entry.id,
asset=entry.asset,
label=entry.label,
value=value,
location=entry.location,
tags=entry.tags,
balance_type=entry.balance_type,
))
return balances_with_value
def deserialize_from_db(
cls,
deposit_tuple: Eth2DepositDBTuple,
) -> 'Eth2Deposit':
"""Turns a tuple read from DB into an appropriate LiquidityPoolEvent.
Deposit_tuple index - Schema columns
------------------------------------
0 - tx_hash
1 - tx_index
2 - from_address
3 - timestamp
4 - pubkey
5 - withdrawal_credentials
6 - amount
7 - usd_value
"""
return cls(
tx_hash=make_evm_tx_hash(deposit_tuple[0]),
tx_index=int(deposit_tuple[1]),
from_address=string_to_ethereum_address(deposit_tuple[2]),
timestamp=Timestamp(int(deposit_tuple[3])),
pubkey=deposit_tuple[4],
withdrawal_credentials=deposit_tuple[5],
value=Balance(amount=FVal(deposit_tuple[6]), usd_value=FVal(deposit_tuple[7])),
)
def query_balances(self) -> ExchangeQueryBalances:
returned_balances: Dict[Asset, Balance] = {}
try:
resp = self._api_query_dict('get', 'user/wallet',
{'currency': 'XBt'})
# Bitmex shows only BTC balance
usd_price = Inquirer().find_usd_price(A_BTC)
except RemoteError as e:
msg = f'Bitmex API request failed due to: {str(e)}'
log.error(msg)
return None, msg
# result is in satoshis
try:
amount = satoshis_to_btc(deserialize_asset_amount(resp['amount']))
except DeserializationError as e:
msg = f'Bitmex API request failed. Failed to deserialized amount due to {str(e)}'
log.error(msg)
return None, msg
usd_value = amount * usd_price
returned_balances[A_BTC] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Bitmex balance query result',
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def query_balances(self) -> ExchangeQueryBalances:
try:
balances = self._private_api_query('balances')
balances.extend(self._private_api_query('balances/earn'))
except (GeminiPermissionError, RemoteError) as e:
msg = f'Gemini API request failed. {str(e)}'
log.error(msg)
return None, msg
returned_balances: DefaultDict[Asset, Balance] = defaultdict(Balance)
for entry in balances:
try:
balance_type = entry['type']
if balance_type == 'exchange':
amount = deserialize_asset_amount(entry['amount'])
else: # should be 'Earn'
amount = deserialize_asset_amount(entry['balance'])
# ignore empty balances
if amount == ZERO:
continue
asset = asset_from_gemini(entry['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing gemini {balance_type} balance result due to '
f'inability to query USD price: {str(e)}. Skipping balance entry',
)
continue
returned_balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini {balance_type} balance result with unsupported '
f'asset {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a gemini {balance_type} balance. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a gemini {balance_type} balance',
error=msg,
)
continue
return returned_balances, ''
def deserialize_from_db(
cls,
event_tuple: BalancerEventDBTuple,
) -> 'BalancerEvent':
"""May raise DeserializationError
Event_tuple index - Schema columns
----------------------------------
0 - tx_hash
1 - log_index
2 - address
3 - timestamp
4 - type
5 - pool_address_token
6 - lp_amount
7 - usd_value
8 - amount0
9 - amount1
10 - amount2
11 - amount3
12 - amount4
13 - amount5
14 - amount6
15 - amount7
"""
event_tuple_type = event_tuple[4]
try:
event_type = getattr(BalancerBPTEventType, event_tuple_type.upper())
except AttributeError as e:
raise DeserializationError(f'Unexpected event type: {event_tuple_type}.') from e
pool_address_token = EthereumToken.from_identifier(
event_tuple[5],
form_with_incomplete_data=True, # since some may not have decimals input correctly
)
if pool_address_token is None:
raise DeserializationError(
f'Balancer event pool token: {event_tuple[5]} not found in the DB.',
)
amounts: List[AssetAmount] = [
deserialize_asset_amount(item)
for item in event_tuple[8:16]
if item is not None
]
return cls(
tx_hash=event_tuple[0],
log_index=event_tuple[1],
address=string_to_ethereum_address(event_tuple[2]),
timestamp=deserialize_timestamp(event_tuple[3]),
event_type=event_type,
pool_address_token=pool_address_token,
lp_balance=Balance(
amount=deserialize_asset_amount(event_tuple[6]),
usd_value=deserialize_price(event_tuple[7]),
),
amounts=amounts,
)
def test_hop_l2_deposit(database, ethereum_manager,
function_scope_messages_aggregator):
"""Data taken from
https://etherscan.io/tx/0xd46640417a686b399b2f2a920b0c58a35095759365cbe7b795bddec34b8c5eee
"""
# TODO: For faster tests hard-code the transaction and the logs here so no remote query needed
tx_hash = deserialize_evm_tx_hash(
'0xd46640417a686b399b2f2a920b0c58a35095759365cbe7b795bddec34b8c5eee'
) # noqa: E501
events = get_decoded_events_of_transaction(
ethereum_manager=ethereum_manager,
database=database,
msg_aggregator=function_scope_messages_aggregator,
tx_hash=tx_hash,
)
expected_events = [
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=0,
timestamp=1653219722000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.SPEND,
event_subtype=HistoryEventSubType.FEE,
asset=A_ETH,
balance=Balance(amount=FVal('0.001964214783875487')),
location_label=ADDY,
notes=f'Burned 0.001964214783875487 ETH in gas from {ADDY}',
counterparty=CPT_GAS,
),
HistoryBaseEntry(
event_identifier=tx_hash.hex(), # pylint: disable=no-member
sequence_index=1,
timestamp=1653219722000,
location=Location.BLOCKCHAIN,
event_type=HistoryEventType.TRANSFER,
event_subtype=HistoryEventSubType.BRIDGE,
asset=A_ETH,
balance=Balance(amount=FVal('0.2')),
location_label=ADDY,
notes=
'Bridge 0.2 ETH to Optimism at the same address via Hop protocol',
counterparty=CPT_HOP,
)
]
assert expected_events == events
def query_balances(self) -> ExchangeQueryBalances:
try:
wallets, _, _ = self._api_query('wallets')
# asset_wallets = self._api_query('asset-wallets')
fiat_wallets, _, _ = self._api_query('fiatwallets')
except RemoteError as e:
msg = f'Failed to query Bitpanda balances. {str(e)}'
return None, msg
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
wallets_len = len(wallets)
for idx, entry in enumerate(wallets + fiat_wallets):
if idx < wallets_len:
symbol_key = 'cryptocoin_symbol'
else:
symbol_key = 'fiat_symbol'
try:
amount = deserialize_asset_amount(
entry['attributes']['balance'])
asset = asset_from_bitpanda(entry['attributes'][symbol_key])
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported/unknown Bitpanda asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing Bitpanda balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing bitpanda balance',
entry=entry,
error=msg,
)
continue
if amount == ZERO:
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing Bitpanda balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return dict(assets_balance), ''
def get_validator_deposits(
self,
indices_or_pubkeys: Union[List[int], List[Eth2PubKey]],
) -> List[Eth2Deposit]:
"""Get the deposits of all the validators given from the list of indices or pubkeys
Queries in chunks of 100 due to api limitations
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
chunks = _calculate_query_chunks(indices_or_pubkeys)
data = []
for chunk in chunks:
result = self._query(
module='validator',
endpoint='deposits',
encoded_args=','.join(str(x) for x in chunk),
)
if isinstance(result, list):
data.extend(result)
else:
data.append(result)
deposits = []
for entry in data:
try:
amount = from_gwei(FVal(entry['amount']))
timestamp = entry['block_ts']
usd_price = query_usd_price_zero_if_error(
asset=A_ETH,
time=timestamp,
location=f'Eth2 staking deposit at time {timestamp}',
msg_aggregator=self.msg_aggregator,
)
deposits.append(Eth2Deposit(
from_address=deserialize_ethereum_address(entry['from_address']),
pubkey=entry['publickey'],
withdrawal_credentials=entry['withdrawal_credentials'],
value=Balance(
amount=amount,
usd_value=amount * usd_price,
),
tx_hash=deserialize_evm_tx_hash(entry['tx_hash']),
tx_index=entry['tx_index'],
timestamp=entry['block_ts'],
))
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
raise RemoteError(
f'Beaconchai.in deposits response processing error. {msg}',
) from e
return deposits