コード例 #1
0
ファイル: accountant.py プロジェクト: jbrit/rotki
class Accountant():

    def __init__(
            self,
            db: DBHandler,
            user_directory: FilePath,
            msg_aggregator: MessagesAggregator,
            create_csv: bool,
    ) -> None:
        self.db = db
        profit_currency = db.get_main_currency()
        self.msg_aggregator = msg_aggregator
        self.csvexporter = CSVExporter(profit_currency, user_directory, create_csv)
        self.events = TaxableEvents(self.csvexporter, profit_currency)

        self.asset_movement_fees = FVal(0)
        self.last_gas_price = FVal(0)

        self.started_processing_timestamp = Timestamp(-1)
        self.currently_processing_timestamp = Timestamp(-1)

    def __del__(self) -> None:
        del self.events
        del self.csvexporter

    @property
    def general_trade_pl(self) -> FVal:
        return self.events.general_trade_profit_loss

    @property
    def taxable_trade_pl(self) -> FVal:
        return self.events.taxable_trade_profit_loss

    def _customize(self, settings: DBSettings) -> None:
        """Customize parameters after pulling DBSettings"""
        if settings.include_crypto2crypto is not None:
            self.events.include_crypto2crypto = settings.include_crypto2crypto

        if settings.taxfree_after_period is not None:
            given_taxfree_after_period: Optional[int] = settings.taxfree_after_period
            if given_taxfree_after_period == -1:
                # That means user requested to disable taxfree_after_period
                given_taxfree_after_period = None

            self.events.taxfree_after_period = given_taxfree_after_period

        self.profit_currency = settings.main_currency
        self.events.profit_currency = settings.main_currency
        self.csvexporter.profit_currency = settings.main_currency

    def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
        """Get the profit_currency rate of the fee of the given trade

        May raise:
        - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from the price oracle
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        fee_rate = PriceHistorian().query_historical_price(
            from_asset=trade.fee_currency,
            to_asset=self.profit_currency,
            timestamp=trade.timestamp,
        )
        return Fee(fee_rate * trade.fee)

    def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
        """
        Adds the given asset movement to the processed events

        May raise:
        - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        timestamp = movement.timestamp
        if timestamp < self.start_ts:
            return

        if movement.asset.identifier == 'KFEE':
            # There is no reason to process deposits of KFEE for kraken as it has only value
            # internal to kraken and KFEE has no value and will error at cryptocompare price query
            return

        fee_rate = self.events.get_rate_in_profit_currency(movement.fee_asset, timestamp)
        cost = movement.fee * fee_rate
        self.asset_movement_fees += cost
        log.debug(
            'Accounting for asset movement',
            sensitive_log=True,
            category=movement.category,
            asset=movement.asset,
            cost_in_profit_currency=cost,
            timestamp=timestamp,
            exchange_name=movement.location,
        )

        self.csvexporter.add_asset_movement(
            exchange=movement.location,
            category=movement.category,
            asset=movement.asset,
            fee=movement.fee,
            rate=fee_rate,
            timestamp=timestamp,
        )

    def account_for_gas_costs(
            self,
            transaction: EthereumTransaction,
            include_gas_costs: bool,
    ) -> None:
        """
        Accounts for the gas costs of the given ethereum transaction

        May raise:
        - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        if not include_gas_costs:
            return
        if transaction.timestamp < self.start_ts:
            return

        if transaction.gas_price == -1:
            gas_price = self.last_gas_price
        else:
            gas_price = transaction.gas_price
            self.last_gas_price = transaction.gas_price

        rate = self.events.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
        eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10 ** 18)
        cost = eth_burned_as_gas * rate
        self.eth_transactions_gas_costs += cost

        log.debug(
            'Accounting for ethereum transaction gas cost',
            sensitive_log=True,
            gas_used=transaction.gas_used,
            gas_price=gas_price,
            timestamp=transaction.timestamp,
        )

        self.csvexporter.add_tx_gas_cost(
            transaction_hash=transaction.tx_hash,
            eth_burned_as_gas=eth_burned_as_gas,
            rate=rate,
            timestamp=transaction.timestamp,
        )

    def trade_add_to_sell_events(self, trade: Trade, loan_settlement: bool) -> None:
        """
        Adds the given trade to the sell events

        May raise:
        - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        selling_asset = trade.base_asset
        receiving_asset = trade.quote_asset
        receiving_asset_rate = self.events.get_rate_in_profit_currency(
            receiving_asset,
            trade.timestamp,
        )
        selling_rate = receiving_asset_rate * trade.rate
        fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
        gain_in_profit_currency = selling_rate * trade.amount

        if not loan_settlement:
            self.events.add_sell_and_corresponding_buy(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=receiving_asset,
                receiving_amount=trade.amount * trade.rate,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
            )
        else:
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
                is_virtual=False,
            )

    def process_history(
            self,
            start_ts: Timestamp,
            end_ts: Timestamp,
            trade_history: List[Union[Trade, MarginPosition]],
            loan_history: List[Loan],
            asset_movements: List[AssetMovement],
            eth_transactions: List[EthereumTransaction],
    ) -> Dict[str, Any]:
        """Processes the entire history of cryptoworld actions in order to determine
        the price and time at which every asset was obtained and also
        the general and taxable profit/loss.

        start_ts here is the timestamp at which to start taking trades and other
        taxable events into account. Not where processing starts from. Processing
        always starts from the very first event we find in the history.
        """
        log.info(
            'Start of history processing',
            start_ts=start_ts,
            end_ts=end_ts,
        )
        self.events.reset(start_ts, end_ts)
        self.last_gas_price = FVal("2000000000")
        self.start_ts = start_ts
        self.eth_transactions_gas_costs = FVal(0)
        self.asset_movement_fees = FVal(0)
        self.csvexporter.reset_csv_lists()
        # Used only in the "avoid zerorpc remote lost after 10ms problem"
        self.last_sleep_ts = 0

        # Ask the DB for the settings once at the start of processing so we got the
        # same settings through the entire task
        db_settings = self.db.get_settings()
        self._customize(db_settings)

        actions: List[TaxableAction] = list(trade_history)
        # If we got loans, we need to interleave them with the full history and re-sort
        if len(loan_history) != 0:
            actions.extend(loan_history)

        if len(asset_movements) != 0:
            actions.extend(asset_movements)

        if len(eth_transactions) != 0:
            actions.extend(eth_transactions)

        actions.sort(
            key=lambda action: action_get_timestamp(action),
        )
        # The first ts is the ts of the first action we have in history or 0 for empty history
        first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(actions[0])
        self.currently_processing_timestamp = first_ts
        self.started_processing_timestamp = first_ts

        prev_time = Timestamp(0)
        count = 0
        for action in actions:
            try:
                (
                    should_continue,
                    prev_time,
                    count,
                ) = self.process_action(action, end_ts, prev_time, count, db_settings)
            except PriceQueryUnknownFromAsset as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
                    f'during history processing due to an asset unknown to '
                    f'cryptocompare being involved. Check logs for details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'cryptocompare not supporting an involved asset: {str(e)}',
                )
                continue
            except NoPriceForGivenTimestamp as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
                    f'during history processing due to inability to find a price '
                    f'at that point in time: {str(e)}. Check the logs for more details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'inability to query a price at that time: {str(e)}',
                )
                continue
            except RemoteError as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
                    f'during history processing due to inability to reach an external '
                    f'service at that point in time: {str(e)}. Check the logs for more details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'inability to reach an external service at that time: {str(e)}',
                )
                continue

            if not should_continue:
                break

        self.events.calculate_asset_details()
        Inquirer().save_historical_forex_data()

        sum_other_actions = (
            self.events.margin_positions_profit_loss +
            self.events.loan_profit -
            self.events.settlement_losses -
            self.asset_movement_fees -
            self.eth_transactions_gas_costs
        )
        total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
        return {
            'overview': {
                'loan_profit': str(self.events.loan_profit),
                'margin_positions_profit_loss': str(self.events.margin_positions_profit_loss),
                'settlement_losses': str(self.events.settlement_losses),
                'ethereum_transaction_gas_costs': str(self.eth_transactions_gas_costs),
                'asset_movement_fees': str(self.asset_movement_fees),
                'general_trade_profit_loss': str(self.events.general_trade_profit_loss),
                'taxable_trade_profit_loss': str(self.events.taxable_trade_profit_loss),
                'total_taxable_profit_loss': str(total_taxable_pl),
                'total_profit_loss': str(
                    self.events.general_trade_profit_loss +
                    sum_other_actions,
                ),
            },
            'all_events': self.csvexporter.all_events,
        }

    def process_action(
            self,
            action: TaxableAction,
            end_ts: Timestamp,
            prev_time: Timestamp,
            count: int,
            db_settings: DBSettings,
    ) -> Tuple[bool, Timestamp, int]:
        """Processes each individual action and returns whether we should continue
        looping through the rest of the actions or not

        May raise:
        - PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        ignored_assets = self.db.get_ignored_assets()

        # Assert we are sorted in ascending time order.
        timestamp = action_get_timestamp(action)
        assert timestamp >= prev_time, (
            "During history processing the trades/loans are not in ascending order"
        )
        prev_time = timestamp

        if timestamp > end_ts:
            return False, prev_time, count

        self.currently_processing_timestamp = timestamp

        action_type = action_get_type(action)

        try:
            asset1, asset2 = action_get_assets(action)
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unknown asset {e.asset_name}. '
                f'Ignoring the trade.',
            )
            return True, prev_time, count
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unsupported asset {e.asset_name}. '
                f'Ignoring the trade.',
            )
            return True, prev_time, count
        except DeserializationError:
            self.msg_aggregator.add_error(
                f'At history processing found trade with non string asset type. '
                f'Ignoring the trade.',
            )
            return True, prev_time, count

        if asset1 in ignored_assets or asset2 in ignored_assets:
            log.debug(
                'Ignoring action with ignored asset',
                action_type=action_type,
                asset1=asset1,
                asset2=asset2,
            )

            return True, prev_time, count

        if action_type == 'loan':
            action = cast(Loan, action)
            self.events.add_loan_gain(
                gained_asset=action.currency,
                lent_amount=action.amount_lent,
                gained_amount=action.earned,
                fee_in_asset=action.fee,
                open_time=action.open_time,
                close_time=timestamp,
            )
            return True, prev_time, count
        elif action_type == 'asset_movement':
            action = cast(AssetMovement, action)
            self.add_asset_movement_to_events(action)
            return True, prev_time, count
        elif action_type == 'margin_position':
            action = cast(MarginPosition, action)
            self.events.add_margin_position(margin=action)
            return True, prev_time, count
        elif action_type == 'ethereum_transaction':
            action = cast(EthereumTransaction, action)
            self.account_for_gas_costs(action, db_settings.include_gas_costs)
            return True, prev_time, count

        # if we get here it's a trade
        trade = cast(Trade, action)

        # When you buy, you buy with the cost_currency and receive the other one
        # When you sell, you sell the amount in non-cost_currency and receive
        # costs in cost_currency
        if trade.trade_type == TradeType.BUY:
            self.events.add_buy_and_corresponding_sell(
                bought_asset=trade.base_asset,
                bought_amount=trade.amount,
                paid_with_asset=trade.quote_asset,
                trade_rate=trade.rate,
                fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
                fee_currency=trade.fee_currency,
                timestamp=trade.timestamp,
            )
        elif trade.trade_type == TradeType.SELL:
            self.trade_add_to_sell_events(trade, False)
        elif trade.trade_type == TradeType.SETTLEMENT_SELL:
            # in poloniex settlements sell some asset to get BTC to repay a loan
            self.trade_add_to_sell_events(trade, True)
        elif trade.trade_type == TradeType.SETTLEMENT_BUY:
            # in poloniex settlements you buy some asset with BTC to repay a loan
            # so in essense you sell BTC to repay the loan
            selling_asset = A_BTC
            selling_asset_rate = self.events.get_rate_in_profit_currency(
                selling_asset,
                trade.timestamp,
            )
            selling_rate = selling_asset_rate * trade.rate
            fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
            gain_in_profit_currency = selling_rate * trade.amount
            # Since the original trade is a buy of some asset with BTC, then the
            # when we invert the sell, the sold amount of BTC should be the cost
            # (amount*rate) of the original buy
            selling_amount = trade.rate * trade.amount
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=selling_amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_asset_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
            )
        else:
            # Should never happen
            raise AssertionError(f'Unknown trade type "{trade.trade_type}" encountered')

        return True, prev_time, count

    def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
        """Get the amount of asset accounting has calculated we should have after
        the history has been processed
        """
        if asset not in self.events.events:
            return None

        amount = FVal(0)
        for buy_event in self.events.events[asset].buys:
            amount += buy_event.amount
        return amount
コード例 #2
0
ファイル: accountant.py プロジェクト: yakkomajuri/rotki
class Accountant():
    def __init__(
        self,
        db: DBHandler,
        user_directory: Path,
        msg_aggregator: MessagesAggregator,
        create_csv: bool,
    ) -> None:
        self.db = db
        profit_currency = db.get_main_currency()
        self.msg_aggregator = msg_aggregator
        self.csvexporter = CSVExporter(
            database=db,
            user_directory=user_directory,
            create_csv=create_csv,
        )
        self.events = TaxableEvents(self.csvexporter, profit_currency)

        self.asset_movement_fees = FVal(0)
        self.last_gas_price = 0
        self.currently_processing_timestamp = -1
        self.first_processed_timestamp = -1

    def __del__(self) -> None:
        del self.events
        del self.csvexporter

    @property
    def general_trade_pl(self) -> FVal:
        return self.events.general_trade_profit_loss

    @property
    def taxable_trade_pl(self) -> FVal:
        return self.events.taxable_trade_profit_loss

    def _customize(self, settings: DBSettings) -> None:
        """Customize parameters after pulling DBSettings"""
        if settings.include_crypto2crypto is not None:
            self.events.include_crypto2crypto = settings.include_crypto2crypto

        if settings.taxfree_after_period is not None:
            given_taxfree_after_period: Optional[
                int] = settings.taxfree_after_period
            if given_taxfree_after_period == -1:
                # That means user requested to disable taxfree_after_period
                given_taxfree_after_period = None

            self.events.taxfree_after_period = given_taxfree_after_period

        self.profit_currency = settings.main_currency
        self.events.profit_currency = settings.main_currency
        self.csvexporter.profit_currency = settings.main_currency

        if settings.account_for_assets_movements is not None:
            self.events.account_for_assets_movements = settings.account_for_assets_movements

    def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
        """Get the profit_currency rate of the fee of the given trade

        May raise:
        - PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from the price oracle
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        fee_rate = PriceHistorian().query_historical_price(
            from_asset=trade.fee_currency,
            to_asset=self.profit_currency,
            timestamp=trade.timestamp,
        )
        return Fee(fee_rate * trade.fee)

    def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
        """
        Adds the given asset movement to the processed events

        May raise:
        - PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        timestamp = movement.timestamp
        if timestamp < self.start_ts:
            return

        if movement.asset.identifier == 'KFEE' or not self.events.account_for_assets_movements:
            # There is no reason to process deposits of KFEE for kraken as it has only value
            # internal to kraken and KFEE has no value and will error at cryptocompare price query
            return

        fee_rate = self.events.get_rate_in_profit_currency(
            movement.fee_asset, timestamp)
        cost = movement.fee * fee_rate
        self.asset_movement_fees += cost
        log.debug(
            'Accounting for asset movement',
            sensitive_log=True,
            category=movement.category,
            asset=movement.asset,
            cost_in_profit_currency=cost,
            timestamp=timestamp,
            exchange_name=movement.location,
        )

        self.csvexporter.add_asset_movement(
            exchange=movement.location,
            category=movement.category,
            asset=movement.asset,
            fee=movement.fee,
            rate=fee_rate,
            timestamp=timestamp,
        )

    def account_for_gas_costs(
        self,
        transaction: EthereumTransaction,
        include_gas_costs: bool,
    ) -> None:
        """
        Accounts for the gas costs of the given ethereum transaction

        May raise:
        - PriceQueryUnsupportedAsset if from/to assets are missing from all price oracles
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        if not include_gas_costs:
            return
        if transaction.timestamp < self.start_ts:
            return

        if transaction.gas_price == -1:
            gas_price = self.last_gas_price
        else:
            gas_price = transaction.gas_price
            self.last_gas_price = transaction.gas_price

        rate = self.events.get_rate_in_profit_currency(A_ETH,
                                                       transaction.timestamp)
        eth_burned_as_gas = FVal(transaction.gas_used * gas_price) / FVal(10**
                                                                          18)
        cost = eth_burned_as_gas * rate
        self.eth_transactions_gas_costs += cost

        log.debug(
            'Accounting for ethereum transaction gas cost',
            sensitive_log=True,
            gas_used=transaction.gas_used,
            gas_price=gas_price,
            timestamp=transaction.timestamp,
        )

        self.csvexporter.add_tx_gas_cost(
            transaction_hash=transaction.tx_hash,
            eth_burned_as_gas=eth_burned_as_gas,
            rate=rate,
            timestamp=transaction.timestamp,
        )

    def trade_add_to_sell_events(self, trade: Trade,
                                 loan_settlement: bool) -> None:
        """
        Adds the given trade to the sell events

        May raise:
        - PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from cryptocompare
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        selling_asset = trade.base_asset
        receiving_asset = trade.quote_asset
        receiving_asset_rate = self.events.get_rate_in_profit_currency(
            receiving_asset,
            trade.timestamp,
        )
        selling_rate = receiving_asset_rate * trade.rate
        fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
        gain_in_profit_currency = selling_rate * trade.amount

        if not loan_settlement:
            self.events.add_sell_and_corresponding_buy(
                location=trade.location,
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=receiving_asset,
                receiving_amount=trade.amount * trade.rate,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
            )
        else:
            self.events.add_sell(
                location=trade.location,
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
                is_virtual=False,
            )

    def process_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
        trade_history: List[Union[Trade, MarginPosition, AMMTrade]],
        loan_history: List[Loan],
        asset_movements: List[AssetMovement],
        eth_transactions: List[EthereumTransaction],
        defi_events: List[DefiEvent],
        ledger_actions: List[LedgerAction],
    ) -> Dict[str, Any]:
        """Processes the entire history of cryptoworld actions in order to determine
        the price and time at which every asset was obtained and also
        the general and taxable profit/loss.

        start_ts here is the timestamp at which to start taking trades and other
        taxable events into account. Not where processing starts from. Processing
        always starts from the very first event we find in the history.
        """
        log.info(
            'Start of history processing',
            start_ts=start_ts,
            end_ts=end_ts,
        )
        self.events.reset(start_ts, end_ts)
        self.last_gas_price = 2000000000
        self.start_ts = start_ts
        self.eth_transactions_gas_costs = FVal(0)
        self.asset_movement_fees = FVal(0)
        self.csvexporter.reset()

        # Ask the DB for the settings once at the start of processing so we got the
        # same settings through the entire task
        db_settings = self.db.get_settings()
        self._customize(db_settings)

        actions: List[TaxableAction] = list(trade_history)
        # If we got loans, we need to interleave them with the full history and re-sort
        if len(loan_history) != 0:
            actions.extend(loan_history)

        if len(asset_movements) != 0:
            actions.extend(asset_movements)

        if len(eth_transactions) != 0:
            actions.extend(eth_transactions)

        if len(defi_events) != 0:
            actions.extend(defi_events)

        if len(ledger_actions) != 0:
            actions.extend(ledger_actions)

        actions.sort(key=action_get_timestamp)
        # The first ts is the ts of the first action we have in history or 0 for empty history
        first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(
            actions[0])
        self.currently_processing_timestamp = first_ts
        self.first_processed_timestamp = first_ts

        prev_time = Timestamp(0)
        count = 0
        ignored_actionids_mapping = self.db.get_ignored_action_ids(
            action_type=None)
        for action in actions:
            try:
                (
                    should_continue,
                    prev_time,
                ) = self._process_action(
                    action=action,
                    start_ts=start_ts,
                    end_ts=end_ts,
                    prev_time=prev_time,
                    db_settings=db_settings,
                    ignored_actionids_mapping=ignored_actionids_mapping,
                )
            except PriceQueryUnsupportedAsset as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f'{self.csvexporter.timestamp_to_date(ts)} '
                    f'during history processing due to an asset unknown to '
                    f'cryptocompare being involved. Check logs for details', )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'cryptocompare not supporting an involved asset: {str(e)}',
                )
                continue
            except NoPriceForGivenTimestamp as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f'{self.csvexporter.timestamp_to_date(ts)} '
                    f'during history processing due to inability to find a price '
                    f'at that point in time: {str(e)}. Check the logs for more details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'inability to query a price at that time: {str(e)}', )
                continue
            except RemoteError as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f'{self.csvexporter.timestamp_to_date(ts)} '
                    f'during history processing due to inability to reach an external '
                    f'service at that point in time: {str(e)}. Check the logs for more details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'inability to reach an external service at that time: {str(e)}',
                )
                continue

            if not should_continue:
                break

            if count % 500 == 0:
                # This loop can take a very long time depending on the amount of actions
                # to process. We need to yield to other greenlets or else calls to the
                # API may time out
                gevent.sleep(0.5)
            count += 1

        self.events.calculate_asset_details()
        Inquirer().save_historical_forex_data()

        sum_other_actions = (self.events.margin_positions_profit_loss +
                             self.events.defi_profit_loss +
                             self.events.ledger_actions_profit_loss +
                             self.events.loan_profit -
                             self.events.settlement_losses -
                             self.asset_movement_fees -
                             self.eth_transactions_gas_costs)
        total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
        return {
            'overview': {
                'ledger_actions_profit_loss':
                str(self.events.ledger_actions_profit_loss),
                'defi_profit_loss':
                str(self.events.defi_profit_loss),
                'loan_profit':
                str(self.events.loan_profit),
                'margin_positions_profit_loss':
                str(self.events.margin_positions_profit_loss),
                'settlement_losses':
                str(self.events.settlement_losses),
                'ethereum_transaction_gas_costs':
                str(self.eth_transactions_gas_costs),
                'asset_movement_fees':
                str(self.asset_movement_fees),
                'general_trade_profit_loss':
                str(self.events.general_trade_profit_loss),
                'taxable_trade_profit_loss':
                str(self.events.taxable_trade_profit_loss),
                'total_taxable_profit_loss':
                str(total_taxable_pl),
                'total_profit_loss':
                str(self.events.general_trade_profit_loss +
                    sum_other_actions, ),
            },
            'all_events': self.csvexporter.all_events,
        }

    @staticmethod
    def _should_ignore_action(
        action: TaxableAction,
        action_type: str,
        ignored_actionids_mapping: Dict[ActionType, List[str]],
    ) -> Tuple[bool, Any]:
        # TODO: These ifs/mappings of action type str to the enum
        # are only due to mix of new and old code. They should be removed and only
        # the enum should be used everywhere eventually
        should_ignore = False
        identifier: Optional[Any] = None
        if action_type == 'trade':
            trade = cast(Trade, action)
            identifier = trade.identifier
            should_ignore = identifier in ignored_actionids_mapping.get(
                ActionType.TRADE, [])

        elif action_type == 'asset_movement':
            movement = cast(AssetMovement, action)
            identifier = movement.identifier
            should_ignore = identifier in ignored_actionids_mapping.get(
                ActionType.ASSET_MOVEMENT,
                [],
            )

        elif action_type == 'ethereum_transaction':
            tx = cast(EthereumTransaction, action)
            identifier = tx.identifier
            should_ignore = tx.identifier in ignored_actionids_mapping.get(
                ActionType.ETHEREUM_TX,
                [],
            )

        elif action_type == 'ledger_action':
            ledger_action = cast(LedgerAction, action)
            identifier = ledger_action.identifier
            should_ignore = identifier in ignored_actionids_mapping.get(
                ActionType.LEDGER_ACTION,
                [],
            )

        return should_ignore, identifier

    def _process_action(
        self,
        action: TaxableAction,
        start_ts: Timestamp,
        end_ts: Timestamp,
        prev_time: Timestamp,
        db_settings: DBSettings,
        ignored_actionids_mapping: Dict[ActionType, List[str]],
    ) -> Tuple[bool, Timestamp]:
        """Processes each individual action and returns whether we should continue
        looping through the rest of the actions or not

        May raise:
        - PriceQueryUnsupportedAsset if from/to asset is missing from price oracles
        - NoPriceForGivenTimestamp if we can't find a price for the asset in the given
        timestamp from the price oracle
        - RemoteError if there is a problem reaching the price oracle server
        or with reading the response returned by the server
        """
        ignored_assets = self.db.get_ignored_assets()

        # Assert we are sorted in ascending time order.
        timestamp = action_get_timestamp(action)
        assert timestamp >= prev_time, (
            "During history processing the trades/loans are not in ascending order"
        )
        prev_time = timestamp

        if not db_settings.calculate_past_cost_basis and timestamp < start_ts:
            # ignore older actions than start_ts if we don't want past cost basis
            return True, prev_time

        if timestamp > end_ts:
            # reached the end of the time period for the report
            return False, prev_time

        self.currently_processing_timestamp = timestamp
        action_type = action_get_type(action)

        try:
            asset1, asset2 = action_get_assets(action)
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unknown asset {e.asset_name}. '
                f'Ignoring the trade.', )
            return True, prev_time
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unsupported asset {e.asset_name}. '
                f'Ignoring the trade.', )
            return True, prev_time
        except DeserializationError:
            self.msg_aggregator.add_error(
                'At history processing found trade with non string asset type. '
                'Ignoring the trade.', )
            return True, prev_time

        if isinstance(asset1, UnknownEthereumToken) or isinstance(
                asset2, UnknownEthereumToken):  # type: ignore  # noqa: E501
            # TODO: Typing needs fixing here  # type: ignore
            log.debug(  # type: ignore
                'Ignoring action with unknown token',
                action_type=action_type,
                asset1=asset1,
                asset2=asset2,
            )
            return True, prev_time

        if asset1 in ignored_assets or asset2 in ignored_assets:
            log.debug(
                'Ignoring action with ignored asset',
                action_type=action_type,
                asset1=asset1,
                asset2=asset2,
            )
            return True, prev_time

        should_ignore, identifier = self._should_ignore_action(
            action=action,
            action_type=action_type,
            ignored_actionids_mapping=ignored_actionids_mapping,
        )
        if should_ignore:
            log.info(
                f'Ignoring {action_type} action with identifier {identifier} '
                f'at {timestamp} since the user asked to ignore it', )
            return True, prev_time

        if action_type == 'loan':
            action = cast(Loan, action)
            self.events.add_loan_gain(
                location=action.location,
                gained_asset=action.currency,
                lent_amount=action.amount_lent,
                gained_amount=action.earned,
                fee_in_asset=action.fee,
                open_time=action.open_time,
                close_time=timestamp,
            )
            return True, prev_time
        if action_type == 'asset_movement':
            action = cast(AssetMovement, action)
            self.add_asset_movement_to_events(action)
            return True, prev_time
        if action_type == 'margin_position':
            action = cast(MarginPosition, action)
            self.events.add_margin_position(margin=action)
            return True, prev_time
        if action_type == 'ethereum_transaction':
            action = cast(EthereumTransaction, action)
            self.account_for_gas_costs(action, db_settings.include_gas_costs)
            return True, prev_time
        if action_type == 'defi_event':
            action = cast(DefiEvent, action)
            self.events.add_defi_event(action)
            return True, prev_time
        if action_type == 'ledger_action':
            action = cast(LedgerAction, action)
            self.events.add_ledger_action(action)
            return True, prev_time

        # else if we get here it's a trade
        trade = cast(Trade, action)
        # When you buy, you buy with the cost_currency and receive the other one
        # When you sell, you sell the amount in non-cost_currency and receive
        # costs in cost_currency
        if trade.trade_type == TradeType.BUY:
            self.events.add_buy_and_corresponding_sell(
                location=trade.location,
                bought_asset=trade.base_asset,
                bought_amount=trade.amount,
                paid_with_asset=trade.quote_asset,
                trade_rate=trade.rate,
                fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
                fee_currency=trade.fee_currency,
                timestamp=trade.timestamp,
            )
        elif trade.trade_type == TradeType.SELL:
            self.trade_add_to_sell_events(trade, False)
        elif trade.trade_type == TradeType.SETTLEMENT_SELL:
            # in poloniex settlements sell some asset to get BTC to repay a loan
            self.trade_add_to_sell_events(trade, True)
        elif trade.trade_type == TradeType.SETTLEMENT_BUY:
            # in poloniex settlements you buy some asset with BTC to repay a loan
            # so in essense you sell BTC to repay the loan
            selling_asset = A_BTC
            selling_asset_rate = self.events.get_rate_in_profit_currency(
                selling_asset,
                trade.timestamp,
            )
            selling_rate = selling_asset_rate * trade.rate
            fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
            gain_in_profit_currency = selling_rate * trade.amount
            # Since the original trade is a buy of some asset with BTC, then the
            # when we invert the sell, the sold amount of BTC should be the cost
            # (amount*rate) of the original buy
            selling_amount = trade.rate * trade.amount
            self.events.add_sell(
                location=trade.location,
                selling_asset=selling_asset,
                selling_amount=selling_amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_asset_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
            )
        else:
            # Should never happen
            raise AssertionError(
                f'Unknown trade type "{trade.trade_type}" encountered')

        return True, prev_time

    def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
        """Get the amount of asset accounting has calculated we should have after
        the history has been processed
        """
        if asset not in self.events.events:
            return None

        amount = FVal(0)
        for buy_event in self.events.events[asset].buys:
            amount += buy_event.amount
        return amount
コード例 #3
0
ファイル: accountant.py プロジェクト: esaulkov/rotkehlchen
class Accountant():
    def __init__(
        self,
        profit_currency: Asset,
        user_directory: FilePath,
        msg_aggregator: MessagesAggregator,
        create_csv: bool,
        ignored_assets: List[Asset],
        include_crypto2crypto: bool,
        taxfree_after_period: int,
        include_gas_costs: bool,
    ):
        self.msg_aggregator = msg_aggregator
        self.csvexporter = CSVExporter(profit_currency, user_directory,
                                       create_csv)
        self.events = TaxableEvents(self.csvexporter, profit_currency)
        self.set_main_currency(profit_currency.identifier)

        self.asset_movement_fees = FVal(0)
        self.last_gas_price = FVal(0)

        self.started_processing_timestamp = -1
        self.currently_processing_timestamp = -1

        # Customizable Options
        self.ignored_assets = ignored_assets
        self.include_gas_costs = include_gas_costs
        self.events.include_crypto2crypto = include_crypto2crypto
        self.events.taxfree_after_period = taxfree_after_period

    def __del__(self):
        del self.events
        del self.csvexporter

    @property
    def general_trade_pl(self) -> FVal:
        return self.events.general_trade_profit_loss

    @property
    def taxable_trade_pl(self) -> FVal:
        return self.events.taxable_trade_profit_loss

    def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
        if 'include_crypto2crypto' in settings:
            given_include_c2c = settings['include_crypto2crypto']
            if not isinstance(given_include_c2c, bool):
                return False, 'Value for include_crypto2crypto must be boolean'

            self.events.include_crypto2crypto = given_include_c2c

        if 'taxfree_after_period' in settings:
            given_taxfree_after_period = settings['taxfree_after_period']
            if given_taxfree_after_period is not None:
                if not isinstance(given_taxfree_after_period, int):
                    return False, 'Value for taxfree_after_period must be an integer'

                if given_taxfree_after_period == 0:
                    return False, 'Value for taxfree_after_period can not be 0 days'

                # turn to seconds
                given_taxfree_after_period = given_taxfree_after_period * 86400
                settings['taxfree_after_period'] = given_taxfree_after_period

            self.events.taxfree_after_period = given_taxfree_after_period

        return True, ''

    def set_main_currency(self, given_currency: str) -> None:
        currency = Asset(given_currency)
        msg = 'main currency checks should have happened at rotkehlchen.set_settings()'
        assert currency.is_fiat(), msg

        self.profit_currency = currency
        self.events.profit_currency = currency

    @staticmethod
    def query_historical_price(
        from_asset: Asset,
        to_asset: Asset,
        timestamp: Timestamp,
    ) -> FVal:
        price = PriceHistorian().query_historical_price(
            from_asset, to_asset, timestamp)
        return price

    def get_rate_in_profit_currency(self, asset: Asset,
                                    timestamp: Timestamp) -> FVal:
        # TODO: Moved this to events.py too. Is it still needed here?
        if asset == self.profit_currency:
            rate = FVal(1)
        else:
            rate = self.query_historical_price(
                asset,
                self.profit_currency,
                timestamp,
            )
        assert isinstance(rate,
                          (FVal, int))  # TODO Remove. Is temporary assert
        return rate

    def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
        fee_rate = self.query_historical_price(
            from_asset=trade.fee_currency,
            to_asset=self.profit_currency,
            timestamp=trade.timestamp,
        )
        return fee_rate * trade.fee

    def add_asset_movement_to_events(
        self,
        category: str,
        asset: Asset,
        timestamp: Timestamp,
        exchange: Exchange,
        fee: Fee,
    ) -> None:
        if timestamp < self.start_ts:
            return

        rate = self.get_rate_in_profit_currency(asset, timestamp)
        cost = fee * rate
        self.asset_movement_fees += cost
        log.debug(
            'Accounting for asset movement',
            sensitive_log=True,
            category=category,
            asset=asset,
            cost_in_profit_currency=cost,
            timestamp=timestamp,
            exchange_name=exchange,
        )

        self.csvexporter.add_asset_movement(
            exchange=exchange,
            category=category,
            asset=asset,
            fee=fee,
            rate=rate,
            timestamp=timestamp,
        )

    def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
        if not self.include_gas_costs:
            return
        if transaction.timestamp < self.start_ts:
            return

        if transaction.gas_price == -1:
            gas_price = self.last_gas_price
        else:
            gas_price = transaction.gas_price
            self.last_gas_price = transaction.gas_price

        rate = self.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
        eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
        cost = eth_burned_as_gas * rate
        self.eth_transactions_gas_costs += cost

        log.debug(
            'Accounting for ethereum transaction gas cost',
            sensitive_log=True,
            gas_used=transaction.gas_used,
            gas_price=gas_price,
            timestamp=transaction.timestamp,
        )

        self.csvexporter.add_tx_gas_cost(
            transaction_hash=transaction.hash,
            eth_burned_as_gas=eth_burned_as_gas,
            rate=rate,
            timestamp=transaction.timestamp,
        )

    def trade_add_to_sell_events(self, trade: Trade,
                                 loan_settlement: bool) -> None:
        selling_asset = trade.base_asset
        receiving_asset = trade.quote_asset
        receiving_asset_rate = self.get_rate_in_profit_currency(
            receiving_asset,
            trade.timestamp,
        )
        selling_rate = receiving_asset_rate * trade.rate
        fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
        gain_in_profit_currency = selling_rate * trade.amount

        if not loan_settlement:
            self.events.add_sell_and_corresponding_buy(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=receiving_asset,
                receiving_amount=trade.amount * trade.rate,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
            )
        else:
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
                is_virtual=False,
            )

    def process_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
        trade_history: List[Trade],
        loan_history: List[Loan],
        asset_movements: List[AssetMovement],
        eth_transactions: List[EthereumTransaction],
    ) -> Dict[str, Any]:
        """Processes the entire history of cryptoworld actions in order to determine
        the price and time at which every asset was obtained and also
        the general and taxable profit/loss.

        start_ts here is the timestamp at which to start taking trades and other
        taxable events into account. Not where processing starts from. Processing
        always starts from the very first event we find in the history.
        """
        log.info(
            'Start of history processing',
            start_ts=start_ts,
            end_ts=end_ts,
        )
        self.events.reset(start_ts, end_ts)
        self.last_gas_price = FVal("2000000000")
        self.start_ts = start_ts
        self.eth_transactions_gas_costs = FVal(0)
        self.asset_movement_fees = FVal(0)
        self.csvexporter.reset_csv_lists()
        # Used only in the "avoid zerorpc remote lost after 10ms problem"
        self.last_sleep_ts = 0

        actions: List[TaxableAction] = list(trade_history)
        # If we got loans, we need to interleave them with the full history and re-sort
        if len(loan_history) != 0:
            actions.extend(loan_history)

        if len(asset_movements) != 0:
            actions.extend(asset_movements)

        if len(eth_transactions) != 0:
            actions.extend(eth_transactions)

        actions.sort(key=lambda action: action_get_timestamp(action), )
        # The first timestamp is the timestamp of the first action we have in history
        first_ts = action_get_timestamp(actions[0])
        self.currently_processing_timestamp = first_ts
        self.started_processing_timestamp = first_ts

        prev_time = Timestamp(0)
        count = 0
        for action in actions:
            try:
                (
                    should_continue,
                    prev_time,
                    count,
                ) = self.process_action(action, end_ts, prev_time, count)
            except PriceQueryUnknownFromAsset as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
                    f'during history processing due to an asset unknown to '
                    f'cryptocompare being involved. Check logs for details', )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'cryptocompare not supporting an involved asset: {str(e)}',
                )
                continue
            except NoPriceForGivenTimestamp as e:
                ts = action_get_timestamp(action)
                self.msg_aggregator.add_error(
                    f'Skipping action at '
                    f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
                    f'during history processing due to inability to find a price '
                    f'at that point in time: {str(e)}. Check the logs for more details',
                )
                log.error(
                    f'Skipping action {str(action)} during history processing due to '
                    f'inability to query a price at that time: {str(e)}', )
                continue

            if not should_continue:
                break

        self.events.calculate_asset_details()
        Inquirer().save_historical_forex_data()

        sum_other_actions = (self.events.margin_positions_profit_loss +
                             self.events.loan_profit -
                             self.events.settlement_losses -
                             self.asset_movement_fees -
                             self.eth_transactions_gas_costs)
        total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
        return {
            'overview': {
                'loan_profit':
                str(self.events.loan_profit),
                'margin_positions_profit_loss':
                str(self.events.margin_positions_profit_loss),
                'settlement_losses':
                str(self.events.settlement_losses),
                'ethereum_transaction_gas_costs':
                str(self.eth_transactions_gas_costs),
                'asset_movement_fees':
                str(self.asset_movement_fees),
                'general_trade_profit_loss':
                str(self.events.general_trade_profit_loss),
                'taxable_trade_profit_loss':
                str(self.events.taxable_trade_profit_loss),
                'total_taxable_profit_loss':
                str(total_taxable_pl),
                'total_profit_loss':
                str(self.events.general_trade_profit_loss +
                    sum_other_actions, ),
            },
            'all_events': self.csvexporter.all_events,
        }

    def process_action(
        self,
        action: TaxableAction,
        end_ts: Timestamp,
        prev_time: Timestamp,
        count: int,
    ) -> Tuple[bool, Timestamp, int]:
        """Processes each individual action and returns whether we should continue
        looping through the rest of the actions or not"""

        # Hack to periodically yield back to the gevent IO loop to avoid getting
        # the losing remote after hearbeat error for the zerorpc client. (after 10s)
        # https://github.com/0rpc/zerorpc-python/issues/37
        # TODO: Find better way to do this. Perhaps enforce this only if method
        # is a synced call, and if async don't do this yielding. In any case
        # this calculation should definitely be async
        now = ts_now()
        if now - self.last_sleep_ts >= 7:  # choose 7 seconds to be safe
            self.last_sleep_ts = now
            gevent.sleep(0.01)  # context switch

        # Assert we are sorted in ascending time order.
        timestamp = action_get_timestamp(action)
        assert timestamp >= prev_time, (
            "During history processing the trades/loans are not in ascending order"
        )
        prev_time = timestamp

        if timestamp > end_ts:
            return False, prev_time, count

        self.currently_processing_timestamp = timestamp

        action_type = action_get_type(action)

        try:
            asset1, asset2 = action_get_assets(action)
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unknown asset {e.asset_name}. '
                f'Ignoring the trade.', )
            return True, prev_time, count
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'At history processing found trade with unsupported asset {e.asset_name}. '
                f'Ignoring the trade.', )
            return True, prev_time, count
        except DeserializationError:
            self.msg_aggregator.add_error(
                f'At history processing found trade with non string asset type. '
                f'Ignoring the trade.', )
            return True, prev_time, count

        if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
            log.debug(
                'Ignoring action with ignored asset',
                action_type=action_type,
                asset1=asset1,
                asset2=asset2,
            )

            return True, prev_time, count

        if action_type == 'loan':
            action = cast(Loan, action)
            self.events.add_loan_gain(
                gained_asset=action.currency,
                lent_amount=action.amount_lent,
                gained_amount=action.earned,
                fee_in_asset=action.fee,
                open_time=action.open_time,
                close_time=timestamp,
            )
            return True, prev_time, count
        elif action_type == 'asset_movement':
            action = cast(AssetMovement, action)
            self.add_asset_movement_to_events(
                category=action.category,
                asset=action.asset,
                timestamp=action.timestamp,
                exchange=action.exchange,
                fee=action.fee,
            )
            return True, prev_time, count
        elif action_type == 'margin_position':
            action = cast(MarginPosition, action)
            self.events.add_margin_position(
                gain_loss_asset=action.pl_currency,
                gain_loss_amount=action.profit_loss,
                fee_in_asset=Fee(ZERO),
                margin_notes=action.notes,
                timestamp=action.close_time,
            )
            return True, prev_time, count
        elif action_type == 'ethereum_transaction':
            action = cast(EthereumTransaction, action)
            self.account_for_gas_costs(action)
            return True, prev_time, count

        # if we get here it's a trade
        trade = cast(Trade, action)

        # When you buy, you buy with the cost_currency and receive the other one
        # When you sell, you sell the amount in non-cost_currency and receive
        # costs in cost_currency
        if trade.trade_type == TradeType.BUY:
            self.events.add_buy_and_corresponding_sell(
                bought_asset=trade.base_asset,
                bought_amount=trade.amount,
                paid_with_asset=trade.quote_asset,
                trade_rate=trade.rate,
                fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
                fee_currency=trade.fee_currency,
                timestamp=trade.timestamp,
            )
        elif trade.trade_type == TradeType.SELL:
            self.trade_add_to_sell_events(trade, False)
        elif trade.trade_type == TradeType.SETTLEMENT_SELL:
            # in poloniex settlements sell some asset to get BTC to repay a loan
            self.trade_add_to_sell_events(trade, True)
        elif trade.trade_type == TradeType.SETTLEMENT_BUY:
            # in poloniex settlements you buy some asset with BTC to repay a loan
            # so in essense you sell BTC to repay the loan
            selling_asset = A_BTC
            selling_asset_rate = self.get_rate_in_profit_currency(
                selling_asset,
                trade.timestamp,
            )
            selling_rate = selling_asset_rate * trade.rate
            fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
            gain_in_profit_currency = selling_rate * trade.amount
            # Since the original trade is a buy of some asset with BTC, then the
            # when we invert the sell, the sold amount of BTC should be the cost
            # (amount*rate) of the original buy
            selling_amount = trade.rate * trade.amount
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=selling_amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_asset_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
            )
        else:
            raise ValueError(
                f'Unknown trade type "{trade.trade_type}" encountered')

        return True, prev_time, count

    def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
        """Get the amount of asset accounting has calculated we should have after
        the history has been processed
        """
        if asset not in self.events.events:
            return None

        amount = FVal(0)
        for buy_event in self.events.events[asset].buys:
            amount += buy_event.amount
        return amount
コード例 #4
0
ファイル: accountant.py プロジェクト: ewingrj/rotkehlchen
class Accountant(object):
    def __init__(
        self,
        price_historian: PriceHistorian,
        profit_currency: FiatAsset,
        user_directory: FilePath,
        create_csv: bool,
        ignored_assets: List[Asset],
        include_crypto2crypto: bool,
        taxfree_after_period: int,
        include_gas_costs: bool,
    ):
        self.price_historian = price_historian
        self.csvexporter = CSVExporter(profit_currency, user_directory,
                                       create_csv)
        self.events = TaxableEvents(price_historian, self.csvexporter,
                                    profit_currency)
        self.set_main_currency(profit_currency)

        self.asset_movement_fees = FVal(0)
        self.last_gas_price = FVal(0)

        # Customizable Options
        self.ignored_assets = ignored_assets
        self.include_gas_costs = include_gas_costs
        self.events.include_crypto2crypto = include_crypto2crypto
        self.events.taxfree_after_period = taxfree_after_period

    def __del__(self):
        del self.events
        del self.csvexporter
        del self.price_historian

    @property
    def general_trade_pl(self) -> FVal:
        return self.events.general_trade_profit_loss

    @property
    def taxable_trade_pl(self) -> FVal:
        return self.events.taxable_trade_profit_loss

    def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
        include_c2c = self.events.include_crypto2crypto
        taxfree_after_period = self.events.taxfree_after_period

        if 'include_crypto2crypto' in settings:
            include_c2c = settings['include_crypto2crypto']
            if not isinstance(include_c2c, bool):
                return False, 'Value for include_crypto2crypto must be boolean'

        if 'taxfree_after_period' in settings:
            taxfree_after_period = settings['taxfree_after_period']
            if taxfree_after_period is not None:
                if not isinstance(taxfree_after_period, int):
                    return False, 'Value for taxfree_after_period must be an integer'

                if taxfree_after_period == 0:
                    return False, 'Value for taxfree_after_period can not be 0 days'

                # turn to seconds
                taxfree_after_period = taxfree_after_period * 86400
                settings['taxfree_after_period'] = taxfree_after_period

        self.events.include_crypto2crypto = include_c2c
        self.events.taxfree_after_period = taxfree_after_period

        return True, ''

    def set_main_currency(self, currency: FiatAsset) -> None:
        if currency not in FIAT_CURRENCIES:
            raise ValueError(
                'Attempted to set unsupported "{}" as main currency.'.format(
                    currency), )

        self.profit_currency = currency
        self.events.profit_currency = currency

    def query_historical_price(
        self,
        from_asset: Asset,
        to_asset: Asset,
        timestamp: Timestamp,
    ) -> FVal:
        price = self.price_historian.query_historical_price(
            from_asset, to_asset, timestamp)
        return price

    def get_rate_in_profit_currency(self, asset: Asset,
                                    timestamp: Timestamp) -> FVal:
        # TODO: Moved this to events.py too. Is it still needed here?
        if asset == self.profit_currency:
            rate = FVal(1)
        else:
            rate = self.query_historical_price(
                asset,
                self.profit_currency,
                timestamp,
            )
        assert isinstance(rate,
                          (FVal, int))  # TODO Remove. Is temporary assert
        return rate

    def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
        fee_rate = self.query_historical_price(
            trade.fee_currency,
            self.profit_currency,
            trade.timestamp,
        )
        return fee_rate * trade.fee

    def add_asset_movement_to_events(
        self,
        category: str,
        asset: Asset,
        amount: FVal,
        timestamp: Timestamp,
        exchange: str,
        fee: Fee,
    ) -> None:
        if timestamp < self.start_ts:
            return

        rate = self.get_rate_in_profit_currency(asset, timestamp)
        cost = fee * rate
        self.asset_movement_fees += cost
        log.debug(
            'Accounting for asset movement',
            sensitive_log=True,
            category=category,
            asset=asset,
            cost_in_profit_currency=cost,
            timestamp=timestamp,
            exchange_name=exchange,
        )
        if category == 'withdrawal':
            assert fee != 0, 'So far all exchanges charge you for withdrawing'

        self.csvexporter.add_asset_movement(
            exchange=exchange,
            category=category,
            asset=asset,
            fee=fee,
            rate=rate,
            timestamp=timestamp,
        )

    def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
        if not self.include_gas_costs:
            return
        if transaction.timestamp < self.start_ts:
            return

        if transaction.gas_price == -1:
            gas_price = self.last_gas_price
        else:
            gas_price = transaction.gas_price
            self.last_gas_price = transaction.gas_price

        rate = self.get_rate_in_profit_currency(S_ETH, transaction.timestamp)
        eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
        cost = eth_burned_as_gas * rate
        self.eth_transactions_gas_costs += cost

        log.debug(
            'Accounting for ethereum transaction gas cost',
            sensitive_log=True,
            gas_used=transaction.gas_used,
            gas_price=gas_price,
            timestamp=transaction.timestamp,
        )

        self.csvexporter.add_tx_gas_cost(
            transaction_hash=transaction.hash,
            eth_burned_as_gas=eth_burned_as_gas,
            rate=rate,
            timestamp=transaction.timestamp,
        )

    def trade_add_to_sell_events(self, trade: Trade,
                                 loan_settlement: bool) -> None:
        selling_asset = trade_get_other_pair(trade, trade.cost_currency)
        selling_asset_rate = self.get_rate_in_profit_currency(
            trade.cost_currency,
            trade.timestamp,
        )
        selling_rate = selling_asset_rate * trade.rate
        fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
        gain_in_profit_currency = selling_rate * trade.amount

        if not loan_settlement:
            self.events.add_sell_and_corresponding_buy(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=trade.cost_currency,
                receiving_amount=trade.cost,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
            )
        else:
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
                is_virtual=False,
            )

    def process_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
        trade_history: List[Trade],
        margin_history: List[Trade],
        loan_history: Dict,
        asset_movements: List[AssetMovement],
        eth_transactions: List[EthereumTransaction],
    ) -> Dict[str, Any]:
        """Processes the entire history of cryptoworld actions in order to determine
        the price and time at which every asset was obtained and also
        the general and taxable profit/loss.
        """
        log.info(
            'Start of history processing',
            start_ts=start_ts,
            end_ts=end_ts,
        )
        self.events.reset(start_ts, end_ts)
        self.last_gas_price = FVal("2000000000")
        self.start_ts = start_ts
        self.eth_transactions_gas_costs = FVal(0)
        self.asset_movement_fees = FVal(0)
        self.csvexporter.reset_csv_lists()

        actions: List[TaxableAction] = list(trade_history)
        # If we got loans, we need to interleave them with the full history and re-sort
        if len(loan_history) != 0:
            actions.extend(loan_history)

        if len(asset_movements) != 0:
            actions.extend(asset_movements)

        if len(margin_history) != 0:
            actions.extend(margin_history)

        if len(eth_transactions) != 0:
            actions.extend(eth_transactions)

        actions.sort(key=lambda action: action_get_timestamp(action), )

        prev_time = Timestamp(0)
        count = 0
        for action in actions:
            try:
                (
                    should_continue,
                    prev_time,
                    count,
                ) = self.process_action(action, end_ts, prev_time, count)
            except PriceQueryUnknownFromAsset as e:
                log.error(
                    f'Skipping trade during history processing: {str(e)}')
                continue

            if not should_continue:
                break

        self.events.calculate_asset_details()
        self.price_historian.inquirer.save_historical_forex_data()

        sum_other_actions = (self.events.margin_positions_profit_loss +
                             self.events.loan_profit -
                             self.events.settlement_losses -
                             self.asset_movement_fees -
                             self.eth_transactions_gas_costs)
        total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
        return {
            'overview': {
                'loan_profit':
                str(self.events.loan_profit),
                'margin_positions_profit_loss':
                str(self.events.margin_positions_profit_loss),
                'settlement_losses':
                str(self.events.settlement_losses),
                'ethereum_transaction_gas_costs':
                str(self.eth_transactions_gas_costs),
                'asset_movement_fees':
                str(self.asset_movement_fees),
                'general_trade_profit_loss':
                str(self.events.general_trade_profit_loss),
                'taxable_trade_profit_loss':
                str(self.events.taxable_trade_profit_loss),
                'total_taxable_profit_loss':
                str(total_taxable_pl),
                'total_profit_loss':
                str(self.events.general_trade_profit_loss +
                    sum_other_actions, ),
            },
            'all_events': self.csvexporter.all_events,
        }

    def process_action(
        self,
        action: TaxableAction,
        end_ts: Timestamp,
        prev_time: Timestamp,
        count: int,
    ) -> Tuple[bool, Timestamp, int]:
        """Processes each individual action and returns whether we should continue
        looping through the rest of the actions or not"""

        # Hack to periodically yield back to the gevent IO loop to avoid getting
        # the losing remote after hearbeat error for the zerorpc client.
        # https://github.com/0rpc/zerorpc-python/issues/37
        # TODO: Find better way to do this. Perhaps enforce this only if method
        # is a synced call, and if async don't do this yielding. In any case
        # this calculation should definitely by async
        count += 1
        if count % 500 == 0:
            gevent.sleep(0.01)  # context switch

        # Assert we are sorted in ascending time order.
        timestamp = action_get_timestamp(action)
        assert timestamp >= prev_time, (
            "During history processing the trades/loans are not in ascending order"
        )
        prev_time = timestamp

        if timestamp > end_ts:
            return False, prev_time, count

        action_type = action_get_type(action)

        asset1, asset2 = action_get_assets(action)
        if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
            log.debug(
                'Ignoring action with ignored asset',
                action_type=action_type,
                asset1=asset1,
                asset2=asset2,
            )

            return True, prev_time, count

        if action_type == 'loan':
            action = cast(Dict, action)
            self.events.add_loan_gain(
                gained_asset=action['currency'],
                lent_amount=action['amount_lent'],
                gained_amount=action['earned'],
                fee_in_asset=action['fee'],
                open_time=action['open_time'],
                close_time=timestamp,
            )
            return True, prev_time, count
        elif action_type == 'asset_movement':
            action = cast(AssetMovement, action)
            self.add_asset_movement_to_events(
                category=action.category,
                asset=action.asset,
                amount=action.amount,
                timestamp=action.timestamp,
                exchange=action.exchange,
                fee=action.fee,
            )
            return True, prev_time, count
        elif action_type == 'margin_position':
            action = cast(MarginPosition, action)
            self.events.add_margin_position(
                gain_loss_asset=action.pl_currency,
                gain_loss_amount=action.profit_loss,
                fee_in_asset=Fee(ZERO),
                margin_notes=action.notes,
                timestamp=action.close_time,
            )
            return True, prev_time, count
        elif action_type == 'ethereum_transaction':
            action = cast(EthereumTransaction, action)
            self.account_for_gas_costs(action)
            return True, prev_time, count

        # if we get here it's a trade
        trade = cast(Trade, action)

        # if the cost is not equal to rate * amount then the data is somehow corrupt
        if not trade.cost.is_close(trade.amount * trade.rate, max_diff="1e-4"):
            raise CorruptData(
                "Trade found with cost {} which is not equal to trade.amount"
                "({}) * trade.rate({})".format(trade.cost, trade.amount,
                                               trade.rate), )

        # When you buy, you buy with the cost_currency and receive the other one
        # When you sell, you sell the amount in non-cost_currency and receive
        # costs in cost_currency
        if trade.type == 'buy':
            self.events.add_buy_and_corresponding_sell(
                bought_asset=trade_get_other_pair(trade, trade.cost_currency),
                bought_amount=trade.amount,
                paid_with_asset=trade.cost_currency,
                trade_rate=trade.rate,
                fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
                fee_currency=trade.fee_currency,
                timestamp=trade.timestamp,
            )
        elif trade.type == 'sell':
            self.trade_add_to_sell_events(trade, False)
        elif trade.type == 'settlement_sell':
            # in poloniex settlements sell some asset to get BTC to repay a loan
            self.trade_add_to_sell_events(trade, True)
        elif trade.type == 'settlement_buy':
            # in poloniex settlements you buy some asset with BTC to repay a loan
            # so in essense you sell BTC to repay the loan
            selling_asset = S_BTC
            selling_asset_rate = self.get_rate_in_profit_currency(
                selling_asset,
                trade.timestamp,
            )
            selling_rate = selling_asset_rate * trade.rate
            fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
            gain_in_profit_currency = selling_rate * trade.amount
            selling_amount = trade.cost
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=selling_amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=fee_in_profit_currency,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_asset_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
            )
        else:
            raise ValueError('Unknown trade type "{}" encountered'.format(
                trade.type))

        return True, prev_time, count

    def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
        """Get the amount of asset accounting has calculated we should have after
        the history has been processed
        """
        if asset not in self.events.events:
            return None

        amount = FVal(0)
        for buy_event in self.events.events[asset].buys:
            amount += buy_event.amount
        return amount
コード例 #5
0
ファイル: accountant.py プロジェクト: premachb/rotkehlchen
class Accountant(object):
    def __init__(
        self,
        price_historian,
        profit_currency,
        user_directory,
        create_csv,
        ignored_assets,
        include_crypto2crypto,
        taxfree_after_period,
    ):

        self.price_historian = price_historian
        self.csvexporter = CSVExporter(profit_currency, user_directory,
                                       create_csv)
        self.events = TaxableEvents(price_historian, self.csvexporter,
                                    profit_currency)
        self.set_main_currency(profit_currency)

        # Customizable Options
        self.ignored_assets = ignored_assets
        self.events.include_crypto2crypto = include_crypto2crypto
        self.events.taxfree_after_period = taxfree_after_period

    @property
    def general_trade_pl(self):
        return self.events.general_trade_profit_loss

    @property
    def taxable_trade_pl(self):
        return self.events.taxable_trade_profit_loss

    def customize(self, settings):
        include_c2c = self.events.include_crypto2crypto
        taxfree_after_period = self.events.taxfree_after_period

        if 'include_crypto2crypto' in settings:
            include_c2c = settings['include_crypto2crypto']
            if not isinstance(include_c2c, bool):
                return False, 'Value for include_crypto2crypto must be boolean'

        if 'taxfree_after_period' in settings:
            taxfree_after_period = settings['taxfree_after_period']
            if taxfree_after_period is not None:
                if not isinstance(taxfree_after_period, int):
                    return False, 'Value for taxfree_after_period must be an integer'

                if taxfree_after_period == 0:
                    return False, 'Value for taxfree_after_period can not be 0 days'

                # turn to seconds
                taxfree_after_period = taxfree_after_period * 86400
                settings['taxfree_after_period'] = taxfree_after_period

        self.events.include_crypto2crypto = include_c2c
        self.events.taxfree_after_period = taxfree_after_period

        return True, ''

    def set_main_currency(self, currency):
        if currency not in FIAT_CURRENCIES:
            raise ValueError(
                'Attempted to set unsupported "{}" as main currency.'.format(
                    currency))

        self.profit_currency = currency
        self.events.profit_currency = currency

    def query_historical_price(self, from_asset, to_asset, timestamp):
        price = self.price_historian.query_historical_price(
            from_asset, to_asset, timestamp)
        return price

    def get_rate_in_profit_currency(self, asset, timestamp):
        # TODO: Moved this to events.py too. Is it still needed here?
        if asset == self.profit_currency:
            rate = 1
        else:
            rate = self.query_historical_price(asset, self.profit_currency,
                                               timestamp)
        assert isinstance(rate,
                          (FVal, int))  # TODO Remove. Is temporary assert
        return rate

    def add_asset_movement_to_events(self, category, asset, amount, timestamp,
                                     exchange, fee):
        rate = self.get_rate_in_profit_currency(asset, timestamp)
        self.asset_movement_fees += fee * rate
        if category == 'withdrawal':
            assert fee != 0, "So far all exchanges charge you for withdrawing"

        self.csvexporter.add_asset_movement(
            exchange=exchange,
            category=category,
            asset=asset,
            fee=fee,
            rate=rate,
            timestamp=timestamp,
        )

    def account_for_gas_costs(self, transaction):

        if transaction.gas_price == -1:
            gas_price = self.last_gas_price
        else:
            gas_price = transaction.gas_price
            self.last_gas_price = transaction.gas_price

        rate = self.get_rate_in_profit_currency('ETH', transaction.timestamp)
        eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
        self.eth_transactions_gas_costs += eth_burned_as_gas * rate
        self.csvexporter.add_tx_gas_cost(
            transaction_hash=transaction.hash,
            eth_burned_as_gas=eth_burned_as_gas,
            rate=rate,
            timestamp=transaction.timestamp,
        )

    def trade_add_to_sell_events(self, trade, loan_settlement):
        selling_asset = trade_get_other_pair(trade, trade.cost_currency)
        selling_asset_rate = self.get_rate_in_profit_currency(
            trade.cost_currency, trade.timestamp)
        selling_rate = selling_asset_rate * trade.rate
        fee_rate = self.query_historical_price(trade.fee_currency,
                                               self.profit_currency,
                                               trade.timestamp)
        total_sell_fee_cost = fee_rate * trade.fee
        gain_in_profit_currency = selling_rate * trade.amount

        if not loan_settlement:
            self.events.add_sell_and_corresponding_buy(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=trade.cost_currency,
                receiving_amount=trade.cost,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=total_sell_fee_cost,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
            )
        else:
            self.events.add_sell(
                selling_asset=selling_asset,
                selling_amount=trade.amount,
                receiving_asset=None,
                receiving_amount=None,
                gain_in_profit_currency=gain_in_profit_currency,
                total_fee_in_profit_currency=total_sell_fee_cost,
                trade_rate=trade.rate,
                rate_in_profit_currency=selling_rate,
                timestamp=trade.timestamp,
                loan_settlement=True,
                is_virtual=False)

    def process_history(self, start_ts, end_ts, trade_history, margin_history,
                        loan_history, asset_movements, eth_transactions):
        """Processes the entire history of cryptoworld actions in order to determine
        the price and time at which every asset was obtained and also
        the general and taxable profit/loss.
        """
        self.events.reset(start_ts, end_ts)
        self.last_gas_price = FVal("2000000000")
        self.eth_transactions_gas_costs = FVal(0)
        self.asset_movement_fees = FVal(0)
        self.csvexporter.reset_csv_lists()

        actions = list(trade_history)
        # If we got loans, we need to interleave them with the full history and re-sort
        if len(loan_history) != 0:
            actions.extend(loan_history)

        if len(asset_movements) != 0:
            actions.extend(asset_movements)

        if len(margin_history) != 0:
            actions.extend(margin_history)

        if len(eth_transactions) != 0:
            actions.extend(eth_transactions)

        actions.sort(key=lambda action: action_get_timestamp(action))

        prev_time = 0
        count = 0
        for action in actions:

            # Hack to periodically yield back to the gevent IO loop to avoid getting
            # the losing remote after hearbeat error for the zerorpc client.
            # https://github.com/0rpc/zerorpc-python/issues/37
            # TODO: Find better way to do this. Perhaps enforce this only if method
            # is a synced call, and if async don't do this yielding. In any case
            # this calculation should definitely by async
            count += 1
            if count % 500 == 0:
                gevent.sleep(0.01)  # context switch

            # Assert we are sorted in ascending time order.
            timestamp = action_get_timestamp(action)
            assert timestamp >= prev_time, (
                "During history processing the trades/loans are not in ascending order"
            )
            prev_time = timestamp

            if timestamp > end_ts:
                break

            action_type = action_get_type(action)

            asset1, asset2 = action_get_assets(action)
            if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
                if logger.isEnabledFor(logging.DEBUG):
                    logger.debug("Ignoring {} with {} {}".format(
                        action_type, asset1, asset2))
                    continue

            if action_type == 'loan':
                self.events.add_loan_gain(
                    gained_asset=action['currency'],
                    lent_amount=action['amount_lent'],
                    gained_amount=action['earned'],
                    fee_in_asset=action['fee'],
                    open_time=action['open_time'],
                    close_time=timestamp,
                )
                continue
            elif action_type == 'asset_movement':
                self.add_asset_movement_to_events(category=action.category,
                                                  asset=action.asset,
                                                  amount=action.amount,
                                                  timestamp=action.timestamp,
                                                  exchange=action.exchange,
                                                  fee=action.fee)
                continue
            elif action_type == 'margin_position':
                self.events.add_margin_position(
                    gained_asset='BTC',
                    gained_amount=action['btc_profit_loss'],
                    fee_in_asset=0,
                    margin_notes=action['notes'],
                    timestamp=timestamp)
                continue
            elif action_type == 'ethereum_transaction':
                self.account_for_gas_costs(action)
                continue

            # if we get here it's a trade
            trade = action

            # if the cost is not equal to rate * amount then the data is somehow corrupt
            if not trade.cost.is_close(trade.amount * trade.rate,
                                       max_diff="1e-5"):
                raise CorruptData(
                    "Trade found with cost {} which is not equal to trade.amount"
                    "({}) * trade.rate({})".format(trade.cost, trade.amount,
                                                   trade.rate))

            # When you buy, you buy with the cost_currency and receive the other one
            # When you sell, you sell the amount in non-cost_currency and receive
            # costs in cost_currency
            if trade.type == 'buy':
                self.events.add_buy_and_corresponding_sell(
                    bought_asset=trade_get_other_pair(trade,
                                                      trade.cost_currency),
                    bought_amount=trade.amount,
                    paid_with_asset=trade.cost_currency,
                    trade_rate=trade.rate,
                    trade_fee=trade.fee,
                    fee_currency=trade.fee_currency,
                    timestamp=trade.timestamp)
            elif trade.type == 'sell':
                self.trade_add_to_sell_events(trade, False)
            elif trade.type == 'settlement_sell':
                # in poloniex settlements sell some asset to get BTC to repay a loan
                self.trade_add_to_sell_events(trade, True)
            elif trade.type == 'settlement_buy':
                # in poloniex settlements you buy some asset with BTC to repay a loan
                # so in essense you sell BTC to repay the loan
                selling_asset = 'BTC'
                selling_asset_rate = self.get_rate_in_profit_currency(
                    selling_asset, trade.timestamp)
                selling_rate = selling_asset_rate * trade.rate
                fee_rate = self.query_historical_price(trade.fee_currency,
                                                       self.profit_currency,
                                                       trade.timestamp)
                total_sell_fee_cost = fee_rate * trade.fee
                gain_in_profit_currency = selling_rate * trade.amount
                self.events.add_sell(
                    selling_asset=selling_asset,
                    selling_amount=trade.cost,
                    receiving_asset=None,
                    receiving_amount=None,
                    gain_in_profit_currency=gain_in_profit_currency,
                    total_fee_in_profit_currency=total_sell_fee_cost,
                    trade_rate=trade.rate,
                    rate_in_profit_currency=selling_rate,
                    timestamp=trade.timestamp,
                    loan_settlement=True)
            else:
                raise ValueError('Unknown trade type "{}" encountered'.format(
                    trade.type))

        self.events.calculate_asset_details()

        sum_other_actions = (self.events.margin_positions_profit +
                             self.events.loan_profit -
                             self.events.settlement_losses -
                             self.asset_movement_fees -
                             self.eth_transactions_gas_costs)
        total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
        return {
            'overview': {
                'loan_profit':
                str(self.events.loan_profit),
                'margin_positions_profit':
                str(self.events.margin_positions_profit),
                'settlement_losses':
                str(self.events.settlement_losses),
                'ethereum_transaction_gas_costs':
                str(self.eth_transactions_gas_costs),
                'asset_movement_fees':
                str(self.asset_movement_fees),
                'general_trade_profit_loss':
                str(self.events.general_trade_profit_loss),
                'taxable_trade_profit_loss':
                str(self.events.taxable_trade_profit_loss),
                'total_taxable_profit_loss':
                str(total_taxable_pl),
                'total_profit_loss':
                str(self.events.general_trade_profit_loss + sum_other_actions),
            },
            'all_events': self.csvexporter.all_events,
        }