def add_defi_event(self, event: DefiEvent, profit_loss_in_profit_currency: FVal) -> None: if not self.create_csv: return self.defi_events_csv.append({ 'time': self.timestamp_to_date(event.timestamp), 'type': str(event.event_type), 'asset': str(event.asset), 'amount': str(event.amount), f'profit_loss_in_{self.profit_currency.identifier}': profit_loss_in_profit_currency, 'tx_hashes': event.serialize_tx_hashes(), 'notes': event.notes, }) paid_asset: Union[EmptyStr, Asset] received_asset: Union[EmptyStr, Asset] if event.is_profitable(): paid_in_profit_currency = ZERO paid_in_asset = ZERO paid_asset = S_EMPTYSTR received_asset = event.asset received_in_asset = event.amount received_in_profit_currency = profit_loss_in_profit_currency else: paid_in_profit_currency = -1 * profit_loss_in_profit_currency paid_in_asset = event.amount paid_asset = event.asset received_asset = S_EMPTYSTR received_in_asset = ZERO received_in_profit_currency = ZERO self.add_to_allevents( event_type=EV_DEFI, location=Location.BLOCKCHAIN, paid_in_profit_currency=paid_in_profit_currency, paid_asset=paid_asset, paid_in_asset=paid_in_asset, received_asset=received_asset, received_in_asset=received_in_asset, taxable_received_in_profit_currency=received_in_profit_currency, timestamp=event.timestamp, )
def test_defi_event_zero_amount(accountant): """Test that if a Defi Event with a zero amount obtained comes in we don't raise an error Regression test for a division by zero error a user reported """ defi_events = [ DefiEvent( timestamp=1467279735, wrapped_event=AaveInterestEvent( event_type='interest', asset=A_WBTC, value=Balance(amount=FVal(0), usd_value=FVal(0)), block_number=4, timestamp=Timestamp(1467279735), tx_hash= '0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', log_index=4, ), event_type=DefiEventType.AAVE_EVENT, got_asset=A_WBTC, got_balance=Balance(amount=FVal(0), usd_value=FVal(0)), spent_asset=A_WBTC, spent_balance=Balance(amount=FVal(0), usd_value=FVal(0)), pnl=[ AssetBalance(asset=A_WBTC, balance=Balance(amount=FVal(0), usd_value=FVal(0))) ], count_spent_got_cost_basis=True, tx_hash= '0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de', ) ] result = accounting_history_process( accountant=accountant, start_ts=1466979735, end_ts=1519693374, history_list=[], defi_events_list=defi_events, ) assert result['all_events'][0] == { 'cost_basis': None, 'is_virtual': False, 'location': 'blockchain', 'net_profit_or_loss': ZERO, 'paid_asset': 'WBTC(0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599)', 'paid_in_asset': ZERO, 'paid_in_profit_currency': ZERO, 'received_asset': '', 'received_in_asset': ZERO, 'taxable_amount': ZERO, 'taxable_bought_cost_in_profit_currency': ZERO, 'taxable_received_in_profit_currency': ZERO, 'time': 1467279735, 'type': 'defi_event', }
def add_defi_event( self, event: DefiEvent, profit_loss_in_profit_currency_list: List[FVal], ) -> None: if not self.create_csv: return profit_loss_sum = FVal(sum(profit_loss_in_profit_currency_list)) self.defi_events_csv.append({ 'time': self.timestamp_to_date(event.timestamp), 'type': str(event.event_type), 'got_asset': str(event.got_asset) if event.got_asset else '', 'got_amount': str(event.got_balance.amount) if event.got_balance else '', 'spent_asset': str(event.spent_asset) if event.spent_asset else '', 'spent_amount': str(event.spent_balance.amount) if event.spent_balance else '', f'profit_loss_in_{self.profit_currency.symbol}': profit_loss_sum, 'tx_hash': event.tx_hash if event.tx_hash else '', 'description': event.to_string(timestamp_converter=self.timestamp_to_date), }) paid_asset: Union[EmptyStr, Asset] received_asset: Union[EmptyStr, Asset] if event.pnl is None: return # don't pollute all events csv with entries that are not useful for idx, entry in enumerate(event.pnl): if entry.balance.amount > ZERO: paid_in_profit_currency = ZERO paid_in_asset = ZERO paid_asset = S_EMPTYSTR received_asset = entry.asset received_in_asset = entry.balance.amount # The index should be the same as the precalculated profit_currency list amounts received_in_profit_currency = profit_loss_in_profit_currency_list[idx] else: # pnl is a loss # The index should be the same as the precalculated profit_currency list amounts paid_in_profit_currency = profit_loss_in_profit_currency_list[idx] paid_in_asset = entry.balance.amount paid_asset = entry.asset received_asset = S_EMPTYSTR received_in_asset = ZERO received_in_profit_currency = ZERO self.add_to_allevents( event_type=EV_DEFI, location=Location.BLOCKCHAIN, paid_in_profit_currency=paid_in_profit_currency, paid_asset=paid_asset, paid_in_asset=paid_in_asset, received_asset=received_asset, received_in_asset=received_in_asset, taxable_received_in_profit_currency=received_in_profit_currency, total_received_in_profit_currency=received_in_profit_currency, timestamp=event.timestamp, )
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, ) -> List[DefiEvent]: """Gets the history events from DSR for accounting This is a premium only call. Check happens only in the API level. """ history = self.get_historical_dsr() events = [] for _, report in history.items(): total_balance = Balance() counted_profit = Balance() for movement in report.movements: if movement.timestamp < from_timestamp: continue if movement.timestamp > to_timestamp: break pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 balance = Balance( amount=_dsrdai_to_dai(movement.amount), usd_value=movement.amount_usd_value, ) if movement.movement_type == 'deposit': spent_asset = A_DAI spent_balance = balance total_balance -= balance else: got_asset = A_DAI got_balance = balance total_balance += balance if total_balance.amount - counted_profit.amount > ZERO: pnl_balance = total_balance - counted_profit counted_profit += pnl_balance pnl = [AssetBalance(asset=A_DAI, balance=pnl_balance)] events.append( DefiEvent( timestamp=movement.timestamp, wrapped_event=movement, event_type=DefiEventType.DSR_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Depositing and withdrawing from DSR is not counted in # cost basis. DAI were always yours, you did not rebuy them count_spent_got_cost_basis=False, tx_hash=movement.tx_hash, )) return events
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, addresses: List[ChecksumEthAddress], ) -> List[DefiEvent]: if len(addresses) == 0: return [] mapping = self.get_history( addresses=addresses, reset_db_data=False, from_timestamp=from_timestamp, to_timestamp=to_timestamp, ) events = [] for _, history in mapping.items(): for event in history.events: pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 if isinstance(event, Bond): spent_asset = A_ADX spent_balance = event.value elif isinstance(event, Unbond): got_asset = A_ADX got_balance = event.value elif isinstance(event, UnbondRequest): continue # just ignore those for accounting purposes elif isinstance(event, ChannelWithdraw): got_asset = event.token got_balance = event.value pnl = [AssetBalance(asset=got_asset, balance=got_balance)] else: raise AssertionError( f'Unexpected adex event type {type(event)}') events.append( DefiEvent( timestamp=event.timestamp, wrapped_event=event, event_type=DefiEventType.ADEX_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Do not count staking deposit/withdrawals as cost basis events # the ADX was always ours. PnL will ofc still be counted. count_spent_got_cost_basis=False, tx_hash=event.tx_hash, )) return events
def add_defi_event(self, event: DefiEvent) -> None: log.debug( 'Accounting for DeFi event', sensitive_log=True, event=event, ) rate = self.get_rate_in_profit_currency(event.asset, event.timestamp) profit_loss = event.amount * rate if not event.is_profitable(): profit_loss *= -1 self.defi_profit_loss += profit_loss self.csv_exporter.add_defi_event( event=event, profit_loss_in_profit_currency=profit_loss)
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, ) -> List[DefiEvent]: """Gets the history events from maker vaults for accounting This is a premium only call. Check happens only in the API level. """ vault_details = self.get_vault_details() events = [] for detail in vault_details: total_vault_dai_balance = Balance() realized_vault_dai_loss = Balance() for event in detail.events: timestamp = event.timestamp if timestamp < from_timestamp: continue if timestamp > to_timestamp: break got_asset: Optional[Asset] spent_asset: Optional[Asset] pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 count_spent_got_cost_basis = False if event.event_type == VaultEventType.GENERATE_DEBT: count_spent_got_cost_basis = True got_asset = A_DAI got_balance = event.value total_vault_dai_balance += event.value elif event.event_type == VaultEventType.PAYBACK_DEBT: count_spent_got_cost_basis = True spent_asset = A_DAI spent_balance = event.value total_vault_dai_balance -= event.value if total_vault_dai_balance.amount + realized_vault_dai_loss.amount < ZERO: pnl_balance = total_vault_dai_balance + realized_vault_dai_loss realized_vault_dai_loss += -pnl_balance pnl = [AssetBalance(asset=A_DAI, balance=pnl_balance)] elif event.event_type == VaultEventType.DEPOSIT_COLLATERAL: spent_asset = detail.collateral_asset spent_balance = event.value elif event.event_type == VaultEventType.WITHDRAW_COLLATERAL: got_asset = detail.collateral_asset got_balance = event.value elif event.event_type == VaultEventType.LIQUIDATION: count_spent_got_cost_basis = True # TODO: Don't you also get the dai here -- but how to calculate it? spent_asset = detail.collateral_asset spent_balance = event.value pnl = [ AssetBalance(asset=detail.collateral_asset, balance=-spent_balance) ] else: raise AssertionError( f'Invalid Makerdao vault event type {event.event_type}' ) events.append( DefiEvent( timestamp=timestamp, wrapped_event=event, event_type=DefiEventType.MAKERDAO_VAULT_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Depositing and withdrawing from a vault is not counted in # cost basis. Assets were always yours, you did not rebuy them. # Other actions are counted though to track debt and liquidations count_spent_got_cost_basis=count_spent_got_cost_basis, tx_hash=event.tx_hash, )) return events
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, addresses: List[ChecksumEthAddress], ) -> List[DefiEvent]: history = self.get_history( given_defi_balances={}, addresses=addresses, reset_db_data=False, from_timestamp=from_timestamp, to_timestamp=to_timestamp, ) events = [] for event in history['events']: pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 if event.event_type == 'mint': spent_asset = event.asset spent_balance = event.value got_asset = event.to_asset got_balance = event.to_value elif event.event_type == 'redeem': spent_asset = event.asset spent_balance = event.value got_asset = event.to_asset got_balance = event.to_value if event.realized_pnl is not None: pnl = [ AssetBalance(asset=got_asset, balance=event.realized_pnl) ] elif event.event_type == 'borrow': got_asset = event.asset got_balance = event.value elif event.event_type == 'repay': spent_asset = event.asset spent_balance = event.value if event.realized_pnl is not None: pnl = [ AssetBalance(asset=spent_asset, balance=-event.realized_pnl) ] elif event.event_type == 'liquidation': spent_asset = event.to_asset spent_balance = event.to_value got_asset = event.asset got_balance = event.value pnl = [ # collateral lost AssetBalance(asset=spent_asset, balance=-spent_balance), # borrowed asset gained since you can keep it AssetBalance(asset=got_asset, balance=got_balance), ] elif event.event_type == 'comp': got_asset = event.asset got_balance = event.value if event.realized_pnl is not None: pnl = [ AssetBalance(asset=got_asset, balance=event.realized_pnl) ] else: raise AssertionError( f'Unexpected compound event {event.event_type}') events.append( DefiEvent( timestamp=event.timestamp, wrapped_event=event, event_type=DefiEventType.COMPOUND_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Count all compound events in cost basis since there is a swap # from normal to cToken and back involved. Also to track debt. count_spent_got_cost_basis=True, tx_hash=event.tx_hash, )) return events
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, addresses: List[ChecksumEthAddress], ) -> List[DefiEvent]: if len(addresses) == 0: return [] mapping = self.get_history( addresses=addresses, reset_db_data=False, from_timestamp=from_timestamp, to_timestamp=to_timestamp, given_defi_balances={}, ) events = [] for _, history in mapping.items(): total_borrow: Dict[Asset, Balance] = defaultdict(Balance) realized_borrow_loss: Dict[Asset, Balance] = defaultdict(Balance) for event in history.events: got_asset: Optional[Asset] spent_asset: Optional[Asset] pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 if event.event_type == 'deposit': event = cast(AaveDepositWithdrawalEvent, event) spent_asset = event.asset spent_balance = event.value # this will need editing for v2 got_asset = event.atoken got_balance = event.value elif event.event_type == 'withdrawal': event = cast(AaveDepositWithdrawalEvent, event) got_asset = event.asset got_balance = event.value # this will need editing for v2 spent_asset = event.atoken spent_balance = got_balance elif event.event_type == 'interest': event = cast(AaveInterestEvent, event) pnl = [ AssetBalance(asset=event.asset, balance=event.value) ] elif event.event_type == 'borrow': event = cast(AaveBorrowEvent, event) got_asset = event.asset got_balance = event.value total_borrow[got_asset] += got_balance elif event.event_type == 'repay': event = cast(AaveRepayEvent, event) spent_asset = event.asset spent_balance = event.value if total_borrow[spent_asset].amount + realized_borrow_loss[ spent_asset].amount < ZERO: # noqa: E501 pnl_balance = total_borrow[ spent_asset] + realized_borrow_loss[spent_asset] realized_borrow_loss[spent_asset] += -pnl_balance pnl = [ AssetBalance(asset=spent_asset, balance=pnl_balance) ] elif event.event_type == 'liquidation': event = cast(AaveLiquidationEvent, event) got_asset = event.principal_asset got_balance = event.principal_balance spent_asset = event.collateral_asset spent_balance = event.collateral_balance pnl = [ AssetBalance(asset=spent_asset, balance=-spent_balance), AssetBalance(asset=got_asset, balance=got_balance), ] # The principal needs to also be removed from the total_borrow total_borrow[got_asset] -= got_balance else: raise AssertionError( f'Unexpected aave event {event.event_type}') events.append( DefiEvent( timestamp=event.timestamp, wrapped_event=event, event_type=DefiEventType.AAVE_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Count all aave events in cost basis since there is a swap # involved from normal to aTokens and then back again. Also # borrowing/repaying for debt tracking. count_spent_got_cost_basis=True, tx_hash=event.tx_hash, )) return events
def get_history( self, start_ts: Timestamp, end_ts: Timestamp, has_premium: bool, ) -> HistoryResult: """Creates trades and loans history from start_ts to end_ts""" log.info( 'Get/create trade history', start_ts=start_ts, end_ts=end_ts, ) now = ts_now() # start creating the all trades history list history: List[Union[Trade, MarginPosition]] = [] asset_movements = [] loans = [] empty_or_error = '' def populate_history_cb( trades_history: List[Trade], margin_history: List[MarginPosition], result_asset_movements: List[AssetMovement], exchange_specific_data: Any, ) -> None: """This callback will run for succesfull exchange history query""" history.extend(trades_history) history.extend(margin_history) asset_movements.extend(result_asset_movements) if exchange_specific_data: # This can only be poloniex at the moment polo_loans_data = exchange_specific_data loans.extend(process_polo_loans( msg_aggregator=self.msg_aggregator, data=polo_loans_data, # We need to have full history of loans available start_ts=Timestamp(0), end_ts=now, )) def fail_history_cb(error_msg: str) -> None: """This callback will run for failure in exchange history query""" nonlocal empty_or_error empty_or_error += '\n' + error_msg for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange.query_history_with_callbacks( # We need to have full history of exchanges available start_ts=Timestamp(0), end_ts=now, success_callback=populate_history_cb, fail_callback=fail_history_cb, ) try: eth_transactions = self.chain_manager.ethereum.transactions.query( address=None, # all addresses # We need to have full history of transactions available from_ts=Timestamp(0), to_ts=now, with_limit=False, # at the moment ignore the limit for historical processing, recent_first=False, # for history processing we need oldest first ) except RemoteError as e: eth_transactions = [] msg = str(e) self.msg_aggregator.add_error( f'There was an error when querying etherscan for ethereum transactions: {msg}' f'The final history result will not include ethereum transactions', ) empty_or_error += '\n' + msg # Include the external trades in the history external_trades = self.db.get_trades( # We need to have full history of trades available from_ts=Timestamp(0), to_ts=now, location=Location.EXTERNAL, ) history.extend(external_trades) # Include makerdao DSR gains defi_events = [] if self.chain_manager.makerdao_dsr and has_premium: dsr_gains = self.chain_manager.makerdao_dsr.get_dsr_gains_in_period( from_ts=start_ts, to_ts=end_ts, ) for gain, timestamp in dsr_gains: if gain > ZERO: defi_events.append(DefiEvent( timestamp=timestamp, event_type=DefiEventType.DSR_LOAN_GAIN, asset=A_DAI, amount=gain, )) # Include makerdao vault events if self.chain_manager.makerdao_vaults and has_premium: vault_details = self.chain_manager.makerdao_vaults.get_vault_details() # We count the loss on a vault in the period if the last event is within # the given period. It's not a very accurate approach but it's good enough # for now. A more detailed approach would need archive node or log querying # to find owed debt at any given timestamp for detail in vault_details: last_event_ts = detail.events[-1].timestamp if last_event_ts >= start_ts and last_event_ts <= end_ts: defi_events.append(DefiEvent( timestamp=last_event_ts, event_type=DefiEventType.MAKERDAO_VAULT_LOSS, asset=A_USD, amount=detail.total_liquidated.usd_value + detail.total_interest_owed, )) # include yearn vault events if self.chain_manager.yearn_vaults and has_premium: yearn_vaults_history = self.chain_manager.yearn_vaults.get_history( given_defi_balances=self.chain_manager.defi_balances, addresses=self.chain_manager.queried_addresses_for_module('yearn_vaults'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) for _, vault_mappings in yearn_vaults_history.items(): for _, vault_history in vault_mappings.items(): # For the vaults since we can't get historical values of vault tokens # yet, for the purposes of the tax report count everything as USD defi_events.append(DefiEvent( timestamp=Timestamp(end_ts - 1), event_type=DefiEventType.YEARN_VAULTS_PNL, asset=A_USD, amount=vault_history.profit_loss.usd_value, )) # include compound events if self.chain_manager.compound and has_premium: compound_history = self.chain_manager.compound.get_history( given_defi_balances=self.chain_manager.defi_balances, addresses=self.chain_manager.queried_addresses_for_module('compound'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) for event in compound_history['events']: skip_event = ( event.event_type != 'liquidation' and (event.realized_pnl is None or event.realized_pnl.amount == ZERO) ) if skip_event: continue # skip events with no realized profit/loss if event.event_type == 'redeem': defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_LOAN_INTEREST, asset=event.to_asset, amount=event.realized_pnl.amount, )) elif event.event_type == 'repay': defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_DEBT_REPAY, asset=event.asset, amount=event.realized_pnl.amount, )) elif event.event_type == 'liquidation': defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_LIQUIDATION_DEBT_REPAID, asset=event.asset, amount=event.value.amount, )) defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_LIQUIDATION_COLLATERAL_LOST, asset=event.to_asset, amount=event.to_value.amount, )) elif event.event_type == 'comp': defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_REWARDS, asset=event.asset, amount=event.realized_pnl.amount, )) # include aave lending events aave = self.chain_manager.aave if aave is not None and has_premium: mapping = aave.get_history( addresses=self.chain_manager.queried_addresses_for_module('aave'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) now = ts_now() for _, aave_history in mapping.items(): total_amount_per_token: Dict[Asset, FVal] = defaultdict(FVal) for event in aave_history.events: if event.timestamp < start_ts: continue if event.timestamp > end_ts: break if event.event_type == 'interest': defi_events.append(DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.AAVE_LOAN_INTEREST, asset=event.asset, amount=event.value.amount, )) total_amount_per_token[event.asset] += event.value.amount for token, balance in aave_history.total_earned.items(): # Αdd an extra event per token per address for the remaining not paid amount if token in total_amount_per_token: defi_events.append(DefiEvent( timestamp=now, event_type=DefiEventType.AAVE_LOAN_INTEREST, asset=event.asset, amount=balance.amount - total_amount_per_token[token], )) history.sort(key=lambda trade: action_get_timestamp(trade)) return ( empty_or_error, history, loans, asset_movements, eth_transactions, defi_events, )
def _process_trove_events( self, changes: List[Dict[str, Any]], from_timestamp: Timestamp, to_timestamp: Timestamp, ) -> List[DefiEvent]: events = [] total_lusd_trove_balance = Balance() realized_trove_lusd_loss = Balance() for change in changes: try: operation = TroveOperation.deserialize( change['troveOperation']) collateral_change = deserialize_asset_amount( change['collateralChange']) debt_change = deserialize_asset_amount(change['debtChange']) timestamp = change['transaction']['timestamp'] if timestamp < from_timestamp: continue if timestamp > to_timestamp: break got_asset: Optional[Asset] spent_asset: Optional[Asset] pnl = got_asset = got_balance = spent_asset = spent_balance = None count_spent_got_cost_basis = False # In one transaction it is possible to generate debt and change the collateral if debt_change != AssetAmount(ZERO): if debt_change > ZERO: # Generate debt count_spent_got_cost_basis = True got_asset = A_LUSD got_balance = Balance( amount=debt_change, usd_value=query_usd_price_or_use_default( asset=A_LUSD, time=timestamp, default_value=ZERO, location='Liquity', ), ) total_lusd_trove_balance += got_balance else: # payback debt count_spent_got_cost_basis = True spent_asset = A_LUSD spent_balance = Balance( amount=abs(debt_change), usd_value=query_usd_price_or_use_default( asset=A_LUSD, time=timestamp, default_value=ZERO, location='Liquity', ), ) total_lusd_trove_balance -= spent_balance balance = total_lusd_trove_balance.amount + realized_trove_lusd_loss.amount if balance < ZERO: pnl_balance = total_lusd_trove_balance + realized_trove_lusd_loss realized_trove_lusd_loss += -pnl_balance pnl = [ AssetBalance(asset=A_LUSD, balance=pnl_balance) ] if collateral_change != AssetAmount(ZERO): if collateral_change < ZERO: # Withdraw collateral got_asset = A_ETH got_balance = Balance( amount=abs(collateral_change), usd_value=query_usd_price_or_use_default( asset=A_ETH, time=timestamp, default_value=ZERO, location='Liquity', ), ) else: # Deposit collateral spent_asset = A_ETH spent_balance = Balance( amount=collateral_change, usd_value=query_usd_price_or_use_default( asset=A_ETH, time=timestamp, default_value=ZERO, location='Liquity', ), ) if operation in ( TroveOperation.LIQUIDATEINNORMALMODE, TroveOperation.LIQUIDATEINRECOVERYMODE, ): count_spent_got_cost_basis = True spent_asset = A_ETH spent_balance = Balance( amount=abs(collateral_change), usd_value=query_usd_price_or_use_default( asset=A_ETH, time=timestamp, default_value=ZERO, location='Liquity', ), ) pnl = [AssetBalance(asset=A_ETH, balance=-spent_balance)] event = DefiEvent( timestamp=Timestamp(change['transaction']['timestamp']), wrapped_event=change, event_type=DefiEventType.LIQUITY, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, count_spent_got_cost_basis=count_spent_got_cost_basis, tx_hash=change['transaction']['id'], ) events.append(event) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' log.debug( f'Failed to extract defievent in Liquity from {change}') self.msg_aggregator.add_warning( f'Ignoring Liquity Trove event in Liquity. ' f'Failed to decode remote information. {msg}.', ) continue return events
def get_history( self, start_ts: Timestamp, end_ts: Timestamp, has_premium: bool, ) -> HistoryResult: """Creates trades and loans history from start_ts to end_ts""" log.info( 'Get/create trade history', start_ts=start_ts, end_ts=end_ts, ) now = ts_now() # start creating the all trades history list history: List[Union[Trade, MarginPosition]] = [] asset_movements = [] loans = [] empty_or_error = '' def populate_history_cb( trades_history: List[Trade], margin_history: List[MarginPosition], result_asset_movements: List[AssetMovement], exchange_specific_data: Any, ) -> None: """This callback will run for succesfull exchange history query""" history.extend(trades_history) history.extend(margin_history) asset_movements.extend(result_asset_movements) if exchange_specific_data: # This can only be poloniex at the moment polo_loans_data = exchange_specific_data loans.extend(process_polo_loans( msg_aggregator=self.msg_aggregator, data=polo_loans_data, # We need to have full history of loans available start_ts=Timestamp(0), end_ts=now, )) def fail_history_cb(error_msg: str) -> None: """This callback will run for failure in exchange history query""" nonlocal empty_or_error empty_or_error += '\n' + error_msg for _, exchange in self.exchange_manager.connected_exchanges.items(): exchange.query_history_with_callbacks( # We need to have full history of exchanges available start_ts=Timestamp(0), end_ts=now, success_callback=populate_history_cb, fail_callback=fail_history_cb, ) try: eth_transactions = query_ethereum_transactions( database=self.db, etherscan=self.chain_manager.ethereum.etherscan, # We need to have full history of transactions available from_ts=Timestamp(0), to_ts=now, ) except RemoteError as e: eth_transactions = [] msg = str(e) self.msg_aggregator.add_error( f'There was an error when querying etherscan for ethereum transactions: {msg}' f'The final history result will not include ethereum transactions', ) empty_or_error += '\n' + msg # Include the external trades in the history external_trades = self.db.get_trades( # We need to have full history of trades available from_ts=Timestamp(0), to_ts=now, location=Location.EXTERNAL, ) history.extend(external_trades) # Include makerdao DSR gains and vault events defi_events = [] if self.chain_manager.makerdao and has_premium: dsr_gains = self.chain_manager.makerdao.get_dsr_gains_in_period( from_ts=start_ts, to_ts=end_ts, ) log.debug('DSR GAINS: {dsr_gains}') for gain, timestamp in dsr_gains: if gain > ZERO: defi_events.append(DefiEvent( timestamp=timestamp, event_type=DefiEventType.DSR_LOAN_GAIN, asset=A_DAI, amount=gain, )) vault_details = self.chain_manager.makerdao.get_vault_details() # We count the loss on a vault in the period if the last event is within # the given period. It's not a very accurate approach but it's good enough # for now. A more detailed approach would need archive node or log querying # to find owed debt at any given timestamp for detail in vault_details: last_event_ts = detail.events[-1].timestamp if last_event_ts >= start_ts and last_event_ts <= end_ts: defi_events.append(DefiEvent( timestamp=last_event_ts, event_type=DefiEventType.MAKERDAO_VAULT_LOSS, asset=A_USD, amount=detail.total_liquidated_usd + detail.total_interest_owed, )) history.sort(key=lambda trade: action_get_timestamp(trade)) return ( empty_or_error, history, loans, asset_movements, eth_transactions, defi_events, )
def get_history_events( self, from_timestamp: Timestamp, to_timestamp: Timestamp, addresses: List[ChecksumEthAddress], ) -> List[DefiEvent]: """Gets the history events from maker vaults for accounting This is a premium only call. Check happens only in the API level. """ if len(addresses) == 0: return [] from_block = self.ethereum.get_blocknumber_by_time(from_timestamp) to_block = self.ethereum.get_blocknumber_by_time(to_timestamp) events = [] for address in addresses: for _, vault in YEARN_VAULTS.items(): vault_history = self.get_vault_history( defi_balances=[], vault=vault, address=address, from_block=from_block, to_block=to_block, ) if vault_history is None: continue if len(vault_history.events) != 0: # process the vault's events to populate realized_pnl self._process_vault_events(vault_history.events) for event in vault_history.events: pnl = got_asset = got_balance = spent_asset = spent_balance = None # noqa: E501 if event.event_type == 'deposit': spent_asset = event.from_asset spent_balance = event.from_value got_asset = event.to_asset got_balance = event.to_value else: # withdraw spent_asset = event.from_asset spent_balance = event.from_value got_asset = event.to_asset got_balance = event.to_value if event.realized_pnl is not None: pnl = [ AssetBalance(asset=got_asset, balance=event.realized_pnl) ] events.append( DefiEvent( timestamp=event.timestamp, wrapped_event=event, event_type=DefiEventType.YEARN_VAULTS_EVENT, got_asset=got_asset, got_balance=got_balance, spent_asset=spent_asset, spent_balance=spent_balance, pnl=pnl, # Depositing and withdrawing from a vault is not counted in # cost basis. Assets were always yours, you did not rebuy them count_spent_got_cost_basis=False, tx_hash=event.tx_hash, )) return events
def get_history( self, start_ts: Timestamp, end_ts: Timestamp, has_premium: bool, ) -> HistoryResult: """Creates trades and loans history from start_ts to end_ts""" self._reset_variables() step = 0 total_steps = len( self.exchange_manager.connected_exchanges) + HISTORY_QUERY_STEPS log.info( 'Get/create trade history', start_ts=start_ts, end_ts=end_ts, ) # start creating the all trades history list history: List[Union[Trade, MarginPosition, AMMTrade]] = [] asset_movements = [] loans = [] empty_or_error = '' def populate_history_cb( trades_history: List[Trade], margin_history: List[MarginPosition], result_asset_movements: List[AssetMovement], exchange_specific_data: Any, ) -> None: """This callback will run for succesfull exchange history query""" history.extend(trades_history) history.extend(margin_history) asset_movements.extend(result_asset_movements) if exchange_specific_data: # This can only be poloniex at the moment polo_loans_data = exchange_specific_data loans.extend( process_polo_loans( msg_aggregator=self.msg_aggregator, data=polo_loans_data, # We need to have history of loans since before the range start_ts=Timestamp(0), end_ts=end_ts, )) def fail_history_cb(error_msg: str) -> None: """This callback will run for failure in exchange history query""" nonlocal empty_or_error empty_or_error += '\n' + error_msg for name, exchange in self.exchange_manager.connected_exchanges.items( ): self.processing_state_name = f'Querying {name} exchange history' exchange.query_history_with_callbacks( # We need to have history of exchanges since before the range start_ts=Timestamp(0), end_ts=end_ts, success_callback=populate_history_cb, fail_callback=fail_history_cb, ) step = self._increase_progress(step, total_steps) try: self.processing_state_name = 'Querying ethereum transactions history' eth_transactions = self.chain_manager.ethereum.transactions.query( addresses=None, # all addresses # We need to have history of transactions since before the range from_ts=Timestamp(0), to_ts=end_ts, with_limit= False, # at the moment ignore the limit for historical processing, recent_first= False, # for history processing we need oldest first ) except RemoteError as e: eth_transactions = [] msg = str(e) self.msg_aggregator.add_error( f'There was an error when querying etherscan for ethereum transactions: {msg}' f'The final history result will not include ethereum transactions', ) empty_or_error += '\n' + msg step = self._increase_progress(step, total_steps) # Include the external trades in the history self.processing_state_name = 'Querying external trades history' external_trades = self.db.get_trades( # We need to have history of trades since before the range from_ts=Timestamp(0), to_ts=end_ts, location=Location.EXTERNAL, ) history.extend(external_trades) step = self._increase_progress(step, total_steps) # include the ledger actions self.processing_state_name = 'Querying ledger actions history' ledger_actions, _ = self.query_ledger_actions(has_premium, from_ts=start_ts, to_ts=end_ts) step = self._increase_progress(step, total_steps) # include uniswap trades if has_premium and self.chain_manager.uniswap: self.processing_state_name = 'Querying uniswap history' uniswap_trades = self.chain_manager.uniswap.get_trades( addresses=self.chain_manager.queried_addresses_for_module( 'uniswap'), from_timestamp=Timestamp(0), to_timestamp=end_ts, ) history.extend(uniswap_trades) step = self._increase_progress(step, total_steps) # Include makerdao DSR gains defi_events = [] if self.chain_manager.makerdao_dsr and has_premium: self.processing_state_name = 'Querying makerDAO DSR history' dsr_gains = self.chain_manager.makerdao_dsr.get_dsr_gains_in_period( from_ts=start_ts, to_ts=end_ts, ) for gain in dsr_gains: if gain.amount <= ZERO: continue notes = ( f'MakerDAO DSR Gains from {self.timestamp_to_date(gain.from_timestamp)}' f' to {self.timestamp_to_date(gain.to_timestamp)}') defi_events.append( DefiEvent( timestamp=gain.to_timestamp, event_type=DefiEventType.DSR_LOAN_GAIN, asset=A_DAI, amount=gain.amount, tx_hashes=gain.tx_hashes, notes=notes, )) step = self._increase_progress(step, total_steps) # Include makerdao vault events if self.chain_manager.makerdao_vaults and has_premium: self.processing_state_name = 'Querying makerDAO vaults history' vault_details = self.chain_manager.makerdao_vaults.get_vault_details( ) # We count the loss on a vault in the period if the last event is within # the given period. It's not a very accurate approach but it's good enough # for now. A more detailed approach would need archive node or log querying # to find owed debt at any given timestamp for detail in vault_details: last_event_ts = detail.events[-1].timestamp if start_ts <= last_event_ts <= end_ts: notes = ( f'USD value of DAI lost for MakerDAO vault {detail.identifier} ' f'due to accrued debt or liquidations. IMPORTANT: At the moment rotki ' f'can\'t figure debt until a given time, so this is debt until ' f'now. If you are looking at a past range this may be bigger ' f'than it should be. We are actively working on improving this' ) defi_events.append( DefiEvent( timestamp=last_event_ts, event_type=DefiEventType.MAKERDAO_VAULT_LOSS, asset=A_USD, amount=detail.total_liquidated.usd_value + detail.total_interest_owed, tx_hashes=[x.tx_hash for x in detail.events], notes=notes, )) step = self._increase_progress(step, total_steps) # include yearn vault events if self.chain_manager.yearn_vaults and has_premium: self.processing_state_name = 'Querying yearn vaults history' yearn_vaults_history = self.chain_manager.yearn_vaults.get_history( given_defi_balances=self.chain_manager.defi_balances, addresses=self.chain_manager.queried_addresses_for_module( 'yearn_vaults'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) for address, vault_mappings in yearn_vaults_history.items(): for vault_name, vault_history in vault_mappings.items(): # For the vaults since we can't get historical values of vault tokens # yet, for the purposes of the tax report count everything as USD for yearn_event in vault_history.events: if start_ts <= yearn_event.timestamp <= end_ts and yearn_event.realized_pnl is not None: # noqa: E501 defi_events.append( DefiEvent( timestamp=yearn_event.timestamp, event_type=DefiEventType.YEARN_VAULTS_PNL, asset=A_USD, amount=yearn_event.realized_pnl.usd_value, tx_hashes=[yearn_event.tx_hash], notes= (f'USD equivalent PnL for {address} and yearn ' f'{vault_name} at event'), )) step = self._increase_progress(step, total_steps) # include compound events if self.chain_manager.compound and has_premium: self.processing_state_name = 'Querying compound history' compound_history = self.chain_manager.compound.get_history( given_defi_balances=self.chain_manager.defi_balances, addresses=self.chain_manager.queried_addresses_for_module( 'compound'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) for event in compound_history['events']: skip_event = (event.event_type != 'liquidation' and (event.realized_pnl is None or event.realized_pnl.amount == ZERO)) if skip_event: continue # skip events with no realized profit/loss if event.event_type == 'redeem': defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_LOAN_INTEREST, asset=event.to_asset, amount=event.realized_pnl.amount, tx_hashes=[event.tx_hash], notes=( f'Interest earned in compound for ' f'{event.to_asset.identifier} until this event' ), )) elif event.event_type == 'repay': defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_DEBT_REPAY, asset=event.asset, amount=event.realized_pnl.amount, tx_hashes=[event.tx_hash], notes=( f'Amount of {event.asset.identifier} lost in ' f'compound due to debt repayment'), )) elif event.event_type == 'liquidation': defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType. COMPOUND_LIQUIDATION_DEBT_REPAID, asset=event.asset, amount=event.value.amount, tx_hashes=[event.tx_hash], notes=( f'Amount of {event.asset.identifier} gained in ' f'compound due to liquidation debt repayment'), )) defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType. COMPOUND_LIQUIDATION_COLLATERAL_LOST, asset=event.to_asset, amount=event.to_value.amount, tx_hashes=[event.tx_hash], notes= (f'Amount of {event.to_asset.identifier} collateral lost ' f'in compound due to liquidation'), )) elif event.event_type == 'comp': defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.COMPOUND_REWARDS, asset=event.asset, amount=event.realized_pnl.amount, tx_hashes=[event.tx_hash], )) step = self._increase_progress(step, total_steps) # include adex staking profit adex = self.chain_manager.adex if adex is not None and has_premium: self.processing_state_name = 'Querying adex staking history' adx_mapping = adex.get_events_history( addresses=self.chain_manager.queried_addresses_for_module( 'adex'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) for _, adex_history in adx_mapping.items(): # The transaction hashes here are not accurate. Need to figure out # a way to have accurate transaction hashes for events in a time period adex_tx_hashes = [x.tx_hash for x in adex_history.events] for adx_detail in adex_history.staking_details: defi_events.append( DefiEvent( timestamp=end_ts, event_type=DefiEventType.ADEX_STAKE_PROFIT, asset=A_ADX, amount=adx_detail.adx_profit_loss.amount, tx_hashes=adex_tx_hashes, # type: ignore )) defi_events.append( DefiEvent( timestamp=end_ts, event_type=DefiEventType.ADEX_STAKE_PROFIT, asset=A_DAI, amount=adx_detail.dai_profit_loss.amount, tx_hashes=adex_tx_hashes, # type: ignore )) step = self._increase_progress(step, total_steps) # include aave lending events aave = self.chain_manager.aave if aave is not None and has_premium: self.processing_state_name = 'Querying aave history' mapping = aave.get_history( given_defi_balances=self.chain_manager.defi_balances, addresses=self.chain_manager.queried_addresses_for_module( 'aave'), reset_db_data=False, from_timestamp=start_ts, to_timestamp=end_ts, ) now = ts_now() for _, aave_history in mapping.items(): total_amount_per_token: Dict[Asset, FVal] = defaultdict(FVal) for event in aave_history.events: if event.timestamp < start_ts: continue if event.timestamp > end_ts: break if event.event_type == 'interest': defi_events.append( DefiEvent( timestamp=event.timestamp, event_type=DefiEventType.AAVE_LOAN_INTEREST, asset=event.asset, amount=event.value.amount, tx_hashes=[event.tx_hash], )) total_amount_per_token[ event.asset] += event.value.amount # TODO: Here we should also calculate any unclaimed interest payments # within the time range. IT's quite complicated to do that though and # would most probably require an archive node # Add all losses from aave borrowing/liquidations for asset, balance in aave_history.total_lost.items(): aave_tx_hashes = [] for event in aave_history.events: if event not in ('borrow', 'repay', 'liquidation'): continue if event in ('borrow', 'repay') and event.asset == asset: aave_tx_hashes.append(event.tx_hash) continue relevant_liquidation = ( event.event_type == 'liquidation' and asset in (event.collateral_asset, event.principal_asset)) if relevant_liquidation: aave_tx_hashes.append(event.tx_hash) defi_events.append( DefiEvent( timestamp=now, event_type=DefiEventType.AAVE_LOSS, asset=asset, amount=balance.amount, tx_hashes=aave_tx_hashes, notes= (f'All {asset.identifier} lost in Aave due to borrowing ' f'debt or liquidations in the PnL period.'), )) # Add earned assets from aave liquidations for asset, balance in aave_history.total_earned_liquidations.items( ): aave_tx_hashes = [] for event in aave_history.events: relevant_liquidation = ( event.event_type == 'liquidation' and asset in (event.collateral_asset, event.principal_asset)) if relevant_liquidation: aave_tx_hashes.append(event.tx_hash) defi_events.append( DefiEvent( timestamp=now, event_type=DefiEventType.AAVE_LOAN_INTEREST, asset=asset, amount=balance.amount, tx_hashes=aave_tx_hashes, notes= (f'All {asset.identifier} gained in Aave due to liquidation leftovers' ), )) self._increase_progress(step, total_steps) history.sort(key=action_get_timestamp) return ( empty_or_error, history, loans, asset_movements, eth_transactions, defi_events, ledger_actions, )