def add_defi_event(self, event: DefiEvent,
profit_loss_in_profit_currency: FVal) -> None:
if not self.create_csv:
return
self.defi_events_csv.append({
'time':
self.timestamp_to_date(event.timestamp),
'type':
str(event.event_type),
'asset':
str(event.asset),
'amount':
str(event.amount),
f'profit_loss_in_{self.profit_currency.identifier}':
profit_loss_in_profit_currency,
'tx_hashes':
event.serialize_tx_hashes(),
'notes':
event.notes,
})
paid_asset: Union[EmptyStr, Asset]
received_asset: Union[EmptyStr, Asset]
if event.is_profitable():
paid_in_profit_currency = ZERO
paid_in_asset = ZERO
paid_asset = S_EMPTYSTR
received_asset = event.asset
received_in_asset = event.amount
received_in_profit_currency = profit_loss_in_profit_currency
else:
paid_in_profit_currency = -1 * profit_loss_in_profit_currency
paid_in_asset = event.amount
paid_asset = event.asset
received_asset = S_EMPTYSTR
received_in_asset = ZERO
received_in_profit_currency = ZERO
self.add_to_allevents(
event_type=EV_DEFI,
location=Location.BLOCKCHAIN,
paid_in_profit_currency=paid_in_profit_currency,
paid_asset=paid_asset,
paid_in_asset=paid_in_asset,
received_asset=received_asset,
received_in_asset=received_in_asset,
taxable_received_in_profit_currency=received_in_profit_currency,
timestamp=event.timestamp,
)
def test_defi_event_zero_amount(accountant):
"""Test that if a Defi Event with a zero amount obtained
comes in we don't raise an error
Regression test for a division by zero error a user reported
"""
defi_events = [
DefiEvent(
timestamp=1467279735,
wrapped_event=AaveInterestEvent(
event_type='interest',
asset=A_WBTC,
value=Balance(amount=FVal(0), usd_value=FVal(0)),
block_number=4,
timestamp=Timestamp(1467279735),
tx_hash=
'0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=4,
),
event_type=DefiEventType.AAVE_EVENT,
got_asset=A_WBTC,
got_balance=Balance(amount=FVal(0), usd_value=FVal(0)),
spent_asset=A_WBTC,
spent_balance=Balance(amount=FVal(0), usd_value=FVal(0)),
pnl=[
AssetBalance(asset=A_WBTC,
balance=Balance(amount=FVal(0),
usd_value=FVal(0)))
],
count_spent_got_cost_basis=True,
tx_hash=
'0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
)
]
result = accounting_history_process(
accountant=accountant,
start_ts=1466979735,
end_ts=1519693374,
history_list=[],
defi_events_list=defi_events,
)
assert result['all_events'][0] == {
'cost_basis': None,
'is_virtual': False,
'location': 'blockchain',
'net_profit_or_loss': ZERO,
'paid_asset': 'WBTC(0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599)',
'paid_in_asset': ZERO,
'paid_in_profit_currency': ZERO,
'received_asset': '',
'received_in_asset': ZERO,
'taxable_amount': ZERO,
'taxable_bought_cost_in_profit_currency': ZERO,
'taxable_received_in_profit_currency': ZERO,
'time': 1467279735,
'type': 'defi_event',
}
def add_defi_event(
self,
event: DefiEvent,
profit_loss_in_profit_currency_list: List[FVal],
) -> None:
if not self.create_csv:
return
profit_loss_sum = FVal(sum(profit_loss_in_profit_currency_list))
self.defi_events_csv.append({
'time': self.timestamp_to_date(event.timestamp),
'type': str(event.event_type),
'got_asset': str(event.got_asset) if event.got_asset else '',
'got_amount': str(event.got_balance.amount) if event.got_balance else '',
'spent_asset': str(event.spent_asset) if event.spent_asset else '',
'spent_amount': str(event.spent_balance.amount) if event.spent_balance else '',
f'profit_loss_in_{self.profit_currency.symbol}': profit_loss_sum,
'tx_hash': event.tx_hash if event.tx_hash else '',
'description': event.to_string(timestamp_converter=self.timestamp_to_date),
})
paid_asset: Union[EmptyStr, Asset]
received_asset: Union[EmptyStr, Asset]
if event.pnl is None:
return # don't pollute all events csv with entries that are not useful
for idx, entry in enumerate(event.pnl):
if entry.balance.amount > ZERO:
paid_in_profit_currency = ZERO
paid_in_asset = ZERO
paid_asset = S_EMPTYSTR
received_asset = entry.asset
received_in_asset = entry.balance.amount
# The index should be the same as the precalculated profit_currency list amounts
received_in_profit_currency = profit_loss_in_profit_currency_list[idx]
else: # pnl is a loss
# The index should be the same as the precalculated profit_currency list amounts
paid_in_profit_currency = profit_loss_in_profit_currency_list[idx]
paid_in_asset = entry.balance.amount
paid_asset = entry.asset
received_asset = S_EMPTYSTR
received_in_asset = ZERO
received_in_profit_currency = ZERO
self.add_to_allevents(
event_type=EV_DEFI,
location=Location.BLOCKCHAIN,
paid_in_profit_currency=paid_in_profit_currency,
paid_asset=paid_asset,
paid_in_asset=paid_in_asset,
received_asset=received_asset,
received_in_asset=received_in_asset,
taxable_received_in_profit_currency=received_in_profit_currency,
total_received_in_profit_currency=received_in_profit_currency,
timestamp=event.timestamp,
)
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> List[DefiEvent]:
"""Gets the history events from DSR for accounting
This is a premium only call. Check happens only in the API level.
"""
history = self.get_historical_dsr()
events = []
for _, report in history.items():
total_balance = Balance()
counted_profit = Balance()
for movement in report.movements:
if movement.timestamp < from_timestamp:
continue
if movement.timestamp > to_timestamp:
break
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
balance = Balance(
amount=_dsrdai_to_dai(movement.amount),
usd_value=movement.amount_usd_value,
)
if movement.movement_type == 'deposit':
spent_asset = A_DAI
spent_balance = balance
total_balance -= balance
else:
got_asset = A_DAI
got_balance = balance
total_balance += balance
if total_balance.amount - counted_profit.amount > ZERO:
pnl_balance = total_balance - counted_profit
counted_profit += pnl_balance
pnl = [AssetBalance(asset=A_DAI, balance=pnl_balance)]
events.append(
DefiEvent(
timestamp=movement.timestamp,
wrapped_event=movement,
event_type=DefiEventType.DSR_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Depositing and withdrawing from DSR is not counted in
# cost basis. DAI were always yours, you did not rebuy them
count_spent_got_cost_basis=False,
tx_hash=movement.tx_hash,
))
return events
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
addresses: List[ChecksumEthAddress],
) -> List[DefiEvent]:
if len(addresses) == 0:
return []
mapping = self.get_history(
addresses=addresses,
reset_db_data=False,
from_timestamp=from_timestamp,
to_timestamp=to_timestamp,
)
events = []
for _, history in mapping.items():
for event in history.events:
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
if isinstance(event, Bond):
spent_asset = A_ADX
spent_balance = event.value
elif isinstance(event, Unbond):
got_asset = A_ADX
got_balance = event.value
elif isinstance(event, UnbondRequest):
continue # just ignore those for accounting purposes
elif isinstance(event, ChannelWithdraw):
got_asset = event.token
got_balance = event.value
pnl = [AssetBalance(asset=got_asset, balance=got_balance)]
else:
raise AssertionError(
f'Unexpected adex event type {type(event)}')
events.append(
DefiEvent(
timestamp=event.timestamp,
wrapped_event=event,
event_type=DefiEventType.ADEX_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Do not count staking deposit/withdrawals as cost basis events
# the ADX was always ours. PnL will ofc still be counted.
count_spent_got_cost_basis=False,
tx_hash=event.tx_hash,
))
return events
def add_defi_event(self, event: DefiEvent) -> None:
log.debug(
'Accounting for DeFi event',
sensitive_log=True,
event=event,
)
rate = self.get_rate_in_profit_currency(event.asset, event.timestamp)
profit_loss = event.amount * rate
if not event.is_profitable():
profit_loss *= -1
self.defi_profit_loss += profit_loss
self.csv_exporter.add_defi_event(
event=event, profit_loss_in_profit_currency=profit_loss)
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> List[DefiEvent]:
"""Gets the history events from maker vaults for accounting
This is a premium only call. Check happens only in the API level.
"""
vault_details = self.get_vault_details()
events = []
for detail in vault_details:
total_vault_dai_balance = Balance()
realized_vault_dai_loss = Balance()
for event in detail.events:
timestamp = event.timestamp
if timestamp < from_timestamp:
continue
if timestamp > to_timestamp:
break
got_asset: Optional[Asset]
spent_asset: Optional[Asset]
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
count_spent_got_cost_basis = False
if event.event_type == VaultEventType.GENERATE_DEBT:
count_spent_got_cost_basis = True
got_asset = A_DAI
got_balance = event.value
total_vault_dai_balance += event.value
elif event.event_type == VaultEventType.PAYBACK_DEBT:
count_spent_got_cost_basis = True
spent_asset = A_DAI
spent_balance = event.value
total_vault_dai_balance -= event.value
if total_vault_dai_balance.amount + realized_vault_dai_loss.amount < ZERO:
pnl_balance = total_vault_dai_balance + realized_vault_dai_loss
realized_vault_dai_loss += -pnl_balance
pnl = [AssetBalance(asset=A_DAI, balance=pnl_balance)]
elif event.event_type == VaultEventType.DEPOSIT_COLLATERAL:
spent_asset = detail.collateral_asset
spent_balance = event.value
elif event.event_type == VaultEventType.WITHDRAW_COLLATERAL:
got_asset = detail.collateral_asset
got_balance = event.value
elif event.event_type == VaultEventType.LIQUIDATION:
count_spent_got_cost_basis = True
# TODO: Don't you also get the dai here -- but how to calculate it?
spent_asset = detail.collateral_asset
spent_balance = event.value
pnl = [
AssetBalance(asset=detail.collateral_asset,
balance=-spent_balance)
]
else:
raise AssertionError(
f'Invalid Makerdao vault event type {event.event_type}'
)
events.append(
DefiEvent(
timestamp=timestamp,
wrapped_event=event,
event_type=DefiEventType.MAKERDAO_VAULT_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Depositing and withdrawing from a vault is not counted in
# cost basis. Assets were always yours, you did not rebuy them.
# Other actions are counted though to track debt and liquidations
count_spent_got_cost_basis=count_spent_got_cost_basis,
tx_hash=event.tx_hash,
))
return events
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
addresses: List[ChecksumEthAddress],
) -> List[DefiEvent]:
history = self.get_history(
given_defi_balances={},
addresses=addresses,
reset_db_data=False,
from_timestamp=from_timestamp,
to_timestamp=to_timestamp,
)
events = []
for event in history['events']:
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
if event.event_type == 'mint':
spent_asset = event.asset
spent_balance = event.value
got_asset = event.to_asset
got_balance = event.to_value
elif event.event_type == 'redeem':
spent_asset = event.asset
spent_balance = event.value
got_asset = event.to_asset
got_balance = event.to_value
if event.realized_pnl is not None:
pnl = [
AssetBalance(asset=got_asset,
balance=event.realized_pnl)
]
elif event.event_type == 'borrow':
got_asset = event.asset
got_balance = event.value
elif event.event_type == 'repay':
spent_asset = event.asset
spent_balance = event.value
if event.realized_pnl is not None:
pnl = [
AssetBalance(asset=spent_asset,
balance=-event.realized_pnl)
]
elif event.event_type == 'liquidation':
spent_asset = event.to_asset
spent_balance = event.to_value
got_asset = event.asset
got_balance = event.value
pnl = [
# collateral lost
AssetBalance(asset=spent_asset, balance=-spent_balance),
# borrowed asset gained since you can keep it
AssetBalance(asset=got_asset, balance=got_balance),
]
elif event.event_type == 'comp':
got_asset = event.asset
got_balance = event.value
if event.realized_pnl is not None:
pnl = [
AssetBalance(asset=got_asset,
balance=event.realized_pnl)
]
else:
raise AssertionError(
f'Unexpected compound event {event.event_type}')
events.append(
DefiEvent(
timestamp=event.timestamp,
wrapped_event=event,
event_type=DefiEventType.COMPOUND_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Count all compound events in cost basis since there is a swap
# from normal to cToken and back involved. Also to track debt.
count_spent_got_cost_basis=True,
tx_hash=event.tx_hash,
))
return events
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
addresses: List[ChecksumEthAddress],
) -> List[DefiEvent]:
if len(addresses) == 0:
return []
mapping = self.get_history(
addresses=addresses,
reset_db_data=False,
from_timestamp=from_timestamp,
to_timestamp=to_timestamp,
given_defi_balances={},
)
events = []
for _, history in mapping.items():
total_borrow: Dict[Asset, Balance] = defaultdict(Balance)
realized_borrow_loss: Dict[Asset, Balance] = defaultdict(Balance)
for event in history.events:
got_asset: Optional[Asset]
spent_asset: Optional[Asset]
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
if event.event_type == 'deposit':
event = cast(AaveDepositWithdrawalEvent, event)
spent_asset = event.asset
spent_balance = event.value
# this will need editing for v2
got_asset = event.atoken
got_balance = event.value
elif event.event_type == 'withdrawal':
event = cast(AaveDepositWithdrawalEvent, event)
got_asset = event.asset
got_balance = event.value
# this will need editing for v2
spent_asset = event.atoken
spent_balance = got_balance
elif event.event_type == 'interest':
event = cast(AaveInterestEvent, event)
pnl = [
AssetBalance(asset=event.asset, balance=event.value)
]
elif event.event_type == 'borrow':
event = cast(AaveBorrowEvent, event)
got_asset = event.asset
got_balance = event.value
total_borrow[got_asset] += got_balance
elif event.event_type == 'repay':
event = cast(AaveRepayEvent, event)
spent_asset = event.asset
spent_balance = event.value
if total_borrow[spent_asset].amount + realized_borrow_loss[
spent_asset].amount < ZERO: # noqa: E501
pnl_balance = total_borrow[
spent_asset] + realized_borrow_loss[spent_asset]
realized_borrow_loss[spent_asset] += -pnl_balance
pnl = [
AssetBalance(asset=spent_asset,
balance=pnl_balance)
]
elif event.event_type == 'liquidation':
event = cast(AaveLiquidationEvent, event)
got_asset = event.principal_asset
got_balance = event.principal_balance
spent_asset = event.collateral_asset
spent_balance = event.collateral_balance
pnl = [
AssetBalance(asset=spent_asset,
balance=-spent_balance),
AssetBalance(asset=got_asset, balance=got_balance),
]
# The principal needs to also be removed from the total_borrow
total_borrow[got_asset] -= got_balance
else:
raise AssertionError(
f'Unexpected aave event {event.event_type}')
events.append(
DefiEvent(
timestamp=event.timestamp,
wrapped_event=event,
event_type=DefiEventType.AAVE_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Count all aave events in cost basis since there is a swap
# involved from normal to aTokens and then back again. Also
# borrowing/repaying for debt tracking.
count_spent_got_cost_basis=True,
tx_hash=event.tx_hash,
))
return events
def get_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
has_premium: bool,
) -> HistoryResult:
"""Creates trades and loans history from start_ts to end_ts"""
log.info(
'Get/create trade history',
start_ts=start_ts,
end_ts=end_ts,
)
now = ts_now()
# start creating the all trades history list
history: List[Union[Trade, MarginPosition]] = []
asset_movements = []
loans = []
empty_or_error = ''
def populate_history_cb(
trades_history: List[Trade],
margin_history: List[MarginPosition],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(trades_history)
history.extend(margin_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
loans.extend(process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
# We need to have full history of loans available
start_ts=Timestamp(0),
end_ts=now,
))
def fail_history_cb(error_msg: str) -> None:
"""This callback will run for failure in exchange history query"""
nonlocal empty_or_error
empty_or_error += '\n' + error_msg
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange.query_history_with_callbacks(
# We need to have full history of exchanges available
start_ts=Timestamp(0),
end_ts=now,
success_callback=populate_history_cb,
fail_callback=fail_history_cb,
)
try:
eth_transactions = self.chain_manager.ethereum.transactions.query(
address=None, # all addresses
# We need to have full history of transactions available
from_ts=Timestamp(0),
to_ts=now,
with_limit=False, # at the moment ignore the limit for historical processing,
recent_first=False, # for history processing we need oldest first
)
except RemoteError as e:
eth_transactions = []
msg = str(e)
self.msg_aggregator.add_error(
f'There was an error when querying etherscan for ethereum transactions: {msg}'
f'The final history result will not include ethereum transactions',
)
empty_or_error += '\n' + msg
# Include the external trades in the history
external_trades = self.db.get_trades(
# We need to have full history of trades available
from_ts=Timestamp(0),
to_ts=now,
location=Location.EXTERNAL,
)
history.extend(external_trades)
# Include makerdao DSR gains
defi_events = []
if self.chain_manager.makerdao_dsr and has_premium:
dsr_gains = self.chain_manager.makerdao_dsr.get_dsr_gains_in_period(
from_ts=start_ts,
to_ts=end_ts,
)
for gain, timestamp in dsr_gains:
if gain > ZERO:
defi_events.append(DefiEvent(
timestamp=timestamp,
event_type=DefiEventType.DSR_LOAN_GAIN,
asset=A_DAI,
amount=gain,
))
# Include makerdao vault events
if self.chain_manager.makerdao_vaults and has_premium:
vault_details = self.chain_manager.makerdao_vaults.get_vault_details()
# We count the loss on a vault in the period if the last event is within
# the given period. It's not a very accurate approach but it's good enough
# for now. A more detailed approach would need archive node or log querying
# to find owed debt at any given timestamp
for detail in vault_details:
last_event_ts = detail.events[-1].timestamp
if last_event_ts >= start_ts and last_event_ts <= end_ts:
defi_events.append(DefiEvent(
timestamp=last_event_ts,
event_type=DefiEventType.MAKERDAO_VAULT_LOSS,
asset=A_USD,
amount=detail.total_liquidated.usd_value + detail.total_interest_owed,
))
# include yearn vault events
if self.chain_manager.yearn_vaults and has_premium:
yearn_vaults_history = self.chain_manager.yearn_vaults.get_history(
given_defi_balances=self.chain_manager.defi_balances,
addresses=self.chain_manager.queried_addresses_for_module('yearn_vaults'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
for _, vault_mappings in yearn_vaults_history.items():
for _, vault_history in vault_mappings.items():
# For the vaults since we can't get historical values of vault tokens
# yet, for the purposes of the tax report count everything as USD
defi_events.append(DefiEvent(
timestamp=Timestamp(end_ts - 1),
event_type=DefiEventType.YEARN_VAULTS_PNL,
asset=A_USD,
amount=vault_history.profit_loss.usd_value,
))
# include compound events
if self.chain_manager.compound and has_premium:
compound_history = self.chain_manager.compound.get_history(
given_defi_balances=self.chain_manager.defi_balances,
addresses=self.chain_manager.queried_addresses_for_module('compound'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
for event in compound_history['events']:
skip_event = (
event.event_type != 'liquidation' and
(event.realized_pnl is None or event.realized_pnl.amount == ZERO)
)
if skip_event:
continue # skip events with no realized profit/loss
if event.event_type == 'redeem':
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_LOAN_INTEREST,
asset=event.to_asset,
amount=event.realized_pnl.amount,
))
elif event.event_type == 'repay':
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_DEBT_REPAY,
asset=event.asset,
amount=event.realized_pnl.amount,
))
elif event.event_type == 'liquidation':
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_LIQUIDATION_DEBT_REPAID,
asset=event.asset,
amount=event.value.amount,
))
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_LIQUIDATION_COLLATERAL_LOST,
asset=event.to_asset,
amount=event.to_value.amount,
))
elif event.event_type == 'comp':
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_REWARDS,
asset=event.asset,
amount=event.realized_pnl.amount,
))
# include aave lending events
aave = self.chain_manager.aave
if aave is not None and has_premium:
mapping = aave.get_history(
addresses=self.chain_manager.queried_addresses_for_module('aave'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
now = ts_now()
for _, aave_history in mapping.items():
total_amount_per_token: Dict[Asset, FVal] = defaultdict(FVal)
for event in aave_history.events:
if event.timestamp < start_ts:
continue
if event.timestamp > end_ts:
break
if event.event_type == 'interest':
defi_events.append(DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.AAVE_LOAN_INTEREST,
asset=event.asset,
amount=event.value.amount,
))
total_amount_per_token[event.asset] += event.value.amount
for token, balance in aave_history.total_earned.items():
# Αdd an extra event per token per address for the remaining not paid amount
if token in total_amount_per_token:
defi_events.append(DefiEvent(
timestamp=now,
event_type=DefiEventType.AAVE_LOAN_INTEREST,
asset=event.asset,
amount=balance.amount - total_amount_per_token[token],
))
history.sort(key=lambda trade: action_get_timestamp(trade))
return (
empty_or_error,
history,
loans,
asset_movements,
eth_transactions,
defi_events,
)
def _process_trove_events(
self,
changes: List[Dict[str, Any]],
from_timestamp: Timestamp,
to_timestamp: Timestamp,
) -> List[DefiEvent]:
events = []
total_lusd_trove_balance = Balance()
realized_trove_lusd_loss = Balance()
for change in changes:
try:
operation = TroveOperation.deserialize(
change['troveOperation'])
collateral_change = deserialize_asset_amount(
change['collateralChange'])
debt_change = deserialize_asset_amount(change['debtChange'])
timestamp = change['transaction']['timestamp']
if timestamp < from_timestamp:
continue
if timestamp > to_timestamp:
break
got_asset: Optional[Asset]
spent_asset: Optional[Asset]
pnl = got_asset = got_balance = spent_asset = spent_balance = None
count_spent_got_cost_basis = False
# In one transaction it is possible to generate debt and change the collateral
if debt_change != AssetAmount(ZERO):
if debt_change > ZERO:
# Generate debt
count_spent_got_cost_basis = True
got_asset = A_LUSD
got_balance = Balance(
amount=debt_change,
usd_value=query_usd_price_or_use_default(
asset=A_LUSD,
time=timestamp,
default_value=ZERO,
location='Liquity',
),
)
total_lusd_trove_balance += got_balance
else: # payback debt
count_spent_got_cost_basis = True
spent_asset = A_LUSD
spent_balance = Balance(
amount=abs(debt_change),
usd_value=query_usd_price_or_use_default(
asset=A_LUSD,
time=timestamp,
default_value=ZERO,
location='Liquity',
),
)
total_lusd_trove_balance -= spent_balance
balance = total_lusd_trove_balance.amount + realized_trove_lusd_loss.amount
if balance < ZERO:
pnl_balance = total_lusd_trove_balance + realized_trove_lusd_loss
realized_trove_lusd_loss += -pnl_balance
pnl = [
AssetBalance(asset=A_LUSD, balance=pnl_balance)
]
if collateral_change != AssetAmount(ZERO):
if collateral_change < ZERO:
# Withdraw collateral
got_asset = A_ETH
got_balance = Balance(
amount=abs(collateral_change),
usd_value=query_usd_price_or_use_default(
asset=A_ETH,
time=timestamp,
default_value=ZERO,
location='Liquity',
),
)
else: # Deposit collateral
spent_asset = A_ETH
spent_balance = Balance(
amount=collateral_change,
usd_value=query_usd_price_or_use_default(
asset=A_ETH,
time=timestamp,
default_value=ZERO,
location='Liquity',
),
)
if operation in (
TroveOperation.LIQUIDATEINNORMALMODE,
TroveOperation.LIQUIDATEINRECOVERYMODE,
):
count_spent_got_cost_basis = True
spent_asset = A_ETH
spent_balance = Balance(
amount=abs(collateral_change),
usd_value=query_usd_price_or_use_default(
asset=A_ETH,
time=timestamp,
default_value=ZERO,
location='Liquity',
),
)
pnl = [AssetBalance(asset=A_ETH, balance=-spent_balance)]
event = DefiEvent(
timestamp=Timestamp(change['transaction']['timestamp']),
wrapped_event=change,
event_type=DefiEventType.LIQUITY,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
count_spent_got_cost_basis=count_spent_got_cost_basis,
tx_hash=change['transaction']['id'],
)
events.append(event)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
log.debug(
f'Failed to extract defievent in Liquity from {change}')
self.msg_aggregator.add_warning(
f'Ignoring Liquity Trove event in Liquity. '
f'Failed to decode remote information. {msg}.', )
continue
return events
def get_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
has_premium: bool,
) -> HistoryResult:
"""Creates trades and loans history from start_ts to end_ts"""
log.info(
'Get/create trade history',
start_ts=start_ts,
end_ts=end_ts,
)
now = ts_now()
# start creating the all trades history list
history: List[Union[Trade, MarginPosition]] = []
asset_movements = []
loans = []
empty_or_error = ''
def populate_history_cb(
trades_history: List[Trade],
margin_history: List[MarginPosition],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(trades_history)
history.extend(margin_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
loans.extend(process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
# We need to have full history of loans available
start_ts=Timestamp(0),
end_ts=now,
))
def fail_history_cb(error_msg: str) -> None:
"""This callback will run for failure in exchange history query"""
nonlocal empty_or_error
empty_or_error += '\n' + error_msg
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange.query_history_with_callbacks(
# We need to have full history of exchanges available
start_ts=Timestamp(0),
end_ts=now,
success_callback=populate_history_cb,
fail_callback=fail_history_cb,
)
try:
eth_transactions = query_ethereum_transactions(
database=self.db,
etherscan=self.chain_manager.ethereum.etherscan,
# We need to have full history of transactions available
from_ts=Timestamp(0),
to_ts=now,
)
except RemoteError as e:
eth_transactions = []
msg = str(e)
self.msg_aggregator.add_error(
f'There was an error when querying etherscan for ethereum transactions: {msg}'
f'The final history result will not include ethereum transactions',
)
empty_or_error += '\n' + msg
# Include the external trades in the history
external_trades = self.db.get_trades(
# We need to have full history of trades available
from_ts=Timestamp(0),
to_ts=now,
location=Location.EXTERNAL,
)
history.extend(external_trades)
# Include makerdao DSR gains and vault events
defi_events = []
if self.chain_manager.makerdao and has_premium:
dsr_gains = self.chain_manager.makerdao.get_dsr_gains_in_period(
from_ts=start_ts,
to_ts=end_ts,
)
log.debug('DSR GAINS: {dsr_gains}')
for gain, timestamp in dsr_gains:
if gain > ZERO:
defi_events.append(DefiEvent(
timestamp=timestamp,
event_type=DefiEventType.DSR_LOAN_GAIN,
asset=A_DAI,
amount=gain,
))
vault_details = self.chain_manager.makerdao.get_vault_details()
# We count the loss on a vault in the period if the last event is within
# the given period. It's not a very accurate approach but it's good enough
# for now. A more detailed approach would need archive node or log querying
# to find owed debt at any given timestamp
for detail in vault_details:
last_event_ts = detail.events[-1].timestamp
if last_event_ts >= start_ts and last_event_ts <= end_ts:
defi_events.append(DefiEvent(
timestamp=last_event_ts,
event_type=DefiEventType.MAKERDAO_VAULT_LOSS,
asset=A_USD,
amount=detail.total_liquidated_usd + detail.total_interest_owed,
))
history.sort(key=lambda trade: action_get_timestamp(trade))
return (
empty_or_error,
history,
loans,
asset_movements,
eth_transactions,
defi_events,
)
def get_history_events(
self,
from_timestamp: Timestamp,
to_timestamp: Timestamp,
addresses: List[ChecksumEthAddress],
) -> List[DefiEvent]:
"""Gets the history events from maker vaults for accounting
This is a premium only call. Check happens only in the API level.
"""
if len(addresses) == 0:
return []
from_block = self.ethereum.get_blocknumber_by_time(from_timestamp)
to_block = self.ethereum.get_blocknumber_by_time(to_timestamp)
events = []
for address in addresses:
for _, vault in YEARN_VAULTS.items():
vault_history = self.get_vault_history(
defi_balances=[],
vault=vault,
address=address,
from_block=from_block,
to_block=to_block,
)
if vault_history is None:
continue
if len(vault_history.events) != 0:
# process the vault's events to populate realized_pnl
self._process_vault_events(vault_history.events)
for event in vault_history.events:
pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501
if event.event_type == 'deposit':
spent_asset = event.from_asset
spent_balance = event.from_value
got_asset = event.to_asset
got_balance = event.to_value
else: # withdraw
spent_asset = event.from_asset
spent_balance = event.from_value
got_asset = event.to_asset
got_balance = event.to_value
if event.realized_pnl is not None:
pnl = [
AssetBalance(asset=got_asset,
balance=event.realized_pnl)
]
events.append(
DefiEvent(
timestamp=event.timestamp,
wrapped_event=event,
event_type=DefiEventType.YEARN_VAULTS_EVENT,
got_asset=got_asset,
got_balance=got_balance,
spent_asset=spent_asset,
spent_balance=spent_balance,
pnl=pnl,
# Depositing and withdrawing from a vault is not counted in
# cost basis. Assets were always yours, you did not rebuy them
count_spent_got_cost_basis=False,
tx_hash=event.tx_hash,
))
return events
def get_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
has_premium: bool,
) -> HistoryResult:
"""Creates trades and loans history from start_ts to end_ts"""
self._reset_variables()
step = 0
total_steps = len(
self.exchange_manager.connected_exchanges) + HISTORY_QUERY_STEPS
log.info(
'Get/create trade history',
start_ts=start_ts,
end_ts=end_ts,
)
# start creating the all trades history list
history: List[Union[Trade, MarginPosition, AMMTrade]] = []
asset_movements = []
loans = []
empty_or_error = ''
def populate_history_cb(
trades_history: List[Trade],
margin_history: List[MarginPosition],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(trades_history)
history.extend(margin_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
loans.extend(
process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
# We need to have history of loans since before the range
start_ts=Timestamp(0),
end_ts=end_ts,
))
def fail_history_cb(error_msg: str) -> None:
"""This callback will run for failure in exchange history query"""
nonlocal empty_or_error
empty_or_error += '\n' + error_msg
for name, exchange in self.exchange_manager.connected_exchanges.items(
):
self.processing_state_name = f'Querying {name} exchange history'
exchange.query_history_with_callbacks(
# We need to have history of exchanges since before the range
start_ts=Timestamp(0),
end_ts=end_ts,
success_callback=populate_history_cb,
fail_callback=fail_history_cb,
)
step = self._increase_progress(step, total_steps)
try:
self.processing_state_name = 'Querying ethereum transactions history'
eth_transactions = self.chain_manager.ethereum.transactions.query(
addresses=None, # all addresses
# We need to have history of transactions since before the range
from_ts=Timestamp(0),
to_ts=end_ts,
with_limit=
False, # at the moment ignore the limit for historical processing,
recent_first=
False, # for history processing we need oldest first
)
except RemoteError as e:
eth_transactions = []
msg = str(e)
self.msg_aggregator.add_error(
f'There was an error when querying etherscan for ethereum transactions: {msg}'
f'The final history result will not include ethereum transactions',
)
empty_or_error += '\n' + msg
step = self._increase_progress(step, total_steps)
# Include the external trades in the history
self.processing_state_name = 'Querying external trades history'
external_trades = self.db.get_trades(
# We need to have history of trades since before the range
from_ts=Timestamp(0),
to_ts=end_ts,
location=Location.EXTERNAL,
)
history.extend(external_trades)
step = self._increase_progress(step, total_steps)
# include the ledger actions
self.processing_state_name = 'Querying ledger actions history'
ledger_actions, _ = self.query_ledger_actions(has_premium,
from_ts=start_ts,
to_ts=end_ts)
step = self._increase_progress(step, total_steps)
# include uniswap trades
if has_premium and self.chain_manager.uniswap:
self.processing_state_name = 'Querying uniswap history'
uniswap_trades = self.chain_manager.uniswap.get_trades(
addresses=self.chain_manager.queried_addresses_for_module(
'uniswap'),
from_timestamp=Timestamp(0),
to_timestamp=end_ts,
)
history.extend(uniswap_trades)
step = self._increase_progress(step, total_steps)
# Include makerdao DSR gains
defi_events = []
if self.chain_manager.makerdao_dsr and has_premium:
self.processing_state_name = 'Querying makerDAO DSR history'
dsr_gains = self.chain_manager.makerdao_dsr.get_dsr_gains_in_period(
from_ts=start_ts,
to_ts=end_ts,
)
for gain in dsr_gains:
if gain.amount <= ZERO:
continue
notes = (
f'MakerDAO DSR Gains from {self.timestamp_to_date(gain.from_timestamp)}'
f' to {self.timestamp_to_date(gain.to_timestamp)}')
defi_events.append(
DefiEvent(
timestamp=gain.to_timestamp,
event_type=DefiEventType.DSR_LOAN_GAIN,
asset=A_DAI,
amount=gain.amount,
tx_hashes=gain.tx_hashes,
notes=notes,
))
step = self._increase_progress(step, total_steps)
# Include makerdao vault events
if self.chain_manager.makerdao_vaults and has_premium:
self.processing_state_name = 'Querying makerDAO vaults history'
vault_details = self.chain_manager.makerdao_vaults.get_vault_details(
)
# We count the loss on a vault in the period if the last event is within
# the given period. It's not a very accurate approach but it's good enough
# for now. A more detailed approach would need archive node or log querying
# to find owed debt at any given timestamp
for detail in vault_details:
last_event_ts = detail.events[-1].timestamp
if start_ts <= last_event_ts <= end_ts:
notes = (
f'USD value of DAI lost for MakerDAO vault {detail.identifier} '
f'due to accrued debt or liquidations. IMPORTANT: At the moment rotki '
f'can\'t figure debt until a given time, so this is debt until '
f'now. If you are looking at a past range this may be bigger '
f'than it should be. We are actively working on improving this'
)
defi_events.append(
DefiEvent(
timestamp=last_event_ts,
event_type=DefiEventType.MAKERDAO_VAULT_LOSS,
asset=A_USD,
amount=detail.total_liquidated.usd_value +
detail.total_interest_owed,
tx_hashes=[x.tx_hash for x in detail.events],
notes=notes,
))
step = self._increase_progress(step, total_steps)
# include yearn vault events
if self.chain_manager.yearn_vaults and has_premium:
self.processing_state_name = 'Querying yearn vaults history'
yearn_vaults_history = self.chain_manager.yearn_vaults.get_history(
given_defi_balances=self.chain_manager.defi_balances,
addresses=self.chain_manager.queried_addresses_for_module(
'yearn_vaults'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
for address, vault_mappings in yearn_vaults_history.items():
for vault_name, vault_history in vault_mappings.items():
# For the vaults since we can't get historical values of vault tokens
# yet, for the purposes of the tax report count everything as USD
for yearn_event in vault_history.events:
if start_ts <= yearn_event.timestamp <= end_ts and yearn_event.realized_pnl is not None: # noqa: E501
defi_events.append(
DefiEvent(
timestamp=yearn_event.timestamp,
event_type=DefiEventType.YEARN_VAULTS_PNL,
asset=A_USD,
amount=yearn_event.realized_pnl.usd_value,
tx_hashes=[yearn_event.tx_hash],
notes=
(f'USD equivalent PnL for {address} and yearn '
f'{vault_name} at event'),
))
step = self._increase_progress(step, total_steps)
# include compound events
if self.chain_manager.compound and has_premium:
self.processing_state_name = 'Querying compound history'
compound_history = self.chain_manager.compound.get_history(
given_defi_balances=self.chain_manager.defi_balances,
addresses=self.chain_manager.queried_addresses_for_module(
'compound'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
for event in compound_history['events']:
skip_event = (event.event_type != 'liquidation'
and (event.realized_pnl is None
or event.realized_pnl.amount == ZERO))
if skip_event:
continue # skip events with no realized profit/loss
if event.event_type == 'redeem':
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_LOAN_INTEREST,
asset=event.to_asset,
amount=event.realized_pnl.amount,
tx_hashes=[event.tx_hash],
notes=(
f'Interest earned in compound for '
f'{event.to_asset.identifier} until this event'
),
))
elif event.event_type == 'repay':
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_DEBT_REPAY,
asset=event.asset,
amount=event.realized_pnl.amount,
tx_hashes=[event.tx_hash],
notes=(
f'Amount of {event.asset.identifier} lost in '
f'compound due to debt repayment'),
))
elif event.event_type == 'liquidation':
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.
COMPOUND_LIQUIDATION_DEBT_REPAID,
asset=event.asset,
amount=event.value.amount,
tx_hashes=[event.tx_hash],
notes=(
f'Amount of {event.asset.identifier} gained in '
f'compound due to liquidation debt repayment'),
))
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.
COMPOUND_LIQUIDATION_COLLATERAL_LOST,
asset=event.to_asset,
amount=event.to_value.amount,
tx_hashes=[event.tx_hash],
notes=
(f'Amount of {event.to_asset.identifier} collateral lost '
f'in compound due to liquidation'),
))
elif event.event_type == 'comp':
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.COMPOUND_REWARDS,
asset=event.asset,
amount=event.realized_pnl.amount,
tx_hashes=[event.tx_hash],
))
step = self._increase_progress(step, total_steps)
# include adex staking profit
adex = self.chain_manager.adex
if adex is not None and has_premium:
self.processing_state_name = 'Querying adex staking history'
adx_mapping = adex.get_events_history(
addresses=self.chain_manager.queried_addresses_for_module(
'adex'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
for _, adex_history in adx_mapping.items():
# The transaction hashes here are not accurate. Need to figure out
# a way to have accurate transaction hashes for events in a time period
adex_tx_hashes = [x.tx_hash for x in adex_history.events]
for adx_detail in adex_history.staking_details:
defi_events.append(
DefiEvent(
timestamp=end_ts,
event_type=DefiEventType.ADEX_STAKE_PROFIT,
asset=A_ADX,
amount=adx_detail.adx_profit_loss.amount,
tx_hashes=adex_tx_hashes, # type: ignore
))
defi_events.append(
DefiEvent(
timestamp=end_ts,
event_type=DefiEventType.ADEX_STAKE_PROFIT,
asset=A_DAI,
amount=adx_detail.dai_profit_loss.amount,
tx_hashes=adex_tx_hashes, # type: ignore
))
step = self._increase_progress(step, total_steps)
# include aave lending events
aave = self.chain_manager.aave
if aave is not None and has_premium:
self.processing_state_name = 'Querying aave history'
mapping = aave.get_history(
given_defi_balances=self.chain_manager.defi_balances,
addresses=self.chain_manager.queried_addresses_for_module(
'aave'),
reset_db_data=False,
from_timestamp=start_ts,
to_timestamp=end_ts,
)
now = ts_now()
for _, aave_history in mapping.items():
total_amount_per_token: Dict[Asset, FVal] = defaultdict(FVal)
for event in aave_history.events:
if event.timestamp < start_ts:
continue
if event.timestamp > end_ts:
break
if event.event_type == 'interest':
defi_events.append(
DefiEvent(
timestamp=event.timestamp,
event_type=DefiEventType.AAVE_LOAN_INTEREST,
asset=event.asset,
amount=event.value.amount,
tx_hashes=[event.tx_hash],
))
total_amount_per_token[
event.asset] += event.value.amount
# TODO: Here we should also calculate any unclaimed interest payments
# within the time range. IT's quite complicated to do that though and
# would most probably require an archive node
# Add all losses from aave borrowing/liquidations
for asset, balance in aave_history.total_lost.items():
aave_tx_hashes = []
for event in aave_history.events:
if event not in ('borrow', 'repay', 'liquidation'):
continue
if event in ('borrow',
'repay') and event.asset == asset:
aave_tx_hashes.append(event.tx_hash)
continue
relevant_liquidation = (
event.event_type == 'liquidation' and asset
in (event.collateral_asset, event.principal_asset))
if relevant_liquidation:
aave_tx_hashes.append(event.tx_hash)
defi_events.append(
DefiEvent(
timestamp=now,
event_type=DefiEventType.AAVE_LOSS,
asset=asset,
amount=balance.amount,
tx_hashes=aave_tx_hashes,
notes=
(f'All {asset.identifier} lost in Aave due to borrowing '
f'debt or liquidations in the PnL period.'),
))
# Add earned assets from aave liquidations
for asset, balance in aave_history.total_earned_liquidations.items(
):
aave_tx_hashes = []
for event in aave_history.events:
relevant_liquidation = (
event.event_type == 'liquidation' and asset
in (event.collateral_asset, event.principal_asset))
if relevant_liquidation:
aave_tx_hashes.append(event.tx_hash)
defi_events.append(
DefiEvent(
timestamp=now,
event_type=DefiEventType.AAVE_LOAN_INTEREST,
asset=asset,
amount=balance.amount,
tx_hashes=aave_tx_hashes,
notes=
(f'All {asset.identifier} gained in Aave due to liquidation leftovers'
),
))
self._increase_progress(step, total_steps)
history.sort(key=action_get_timestamp)
return (
empty_or_error,
history,
loans,
asset_movements,
eth_transactions,
defi_events,
ledger_actions,
)
