Esempio n. 1
0
    def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
        """Consumes a cointracking entry row from the CSV and adds it into the database
        Can raise:
            - DeserializationError if something is wrong with the format of the expected values
            - UnsupportedCointrackingEntry if importing of this entry is not supported.
            - IndexError if the CSV file is corrupt
            - KeyError if the an expected CSV key is missing
            - UnknownAsset if one of the assets founds in the entry are not supported
        """
        row_type = csv_row['Type']
        timestamp = deserialize_timestamp_from_date(
            date=csv_row['Date'],
            formatstr='%d.%m.%Y %H:%M:%S',
            location='cointracking.info',
        )
        notes = csv_row['Comment']
        location = exchange_row_to_location(csv_row['Exchange'])

        fee = Fee(ZERO)
        fee_currency = A_USD  # whatever (used only if there is no fee)
        if csv_row['Fee'] != '':
            fee = deserialize_fee(csv_row['Fee'])
            fee_currency = Asset(csv_row['Cur.Fee'])

        if row_type in ('Gift/Tip', 'Trade', 'Income'):
            base_asset = Asset(csv_row['Cur.Buy'])
            quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(
                csv_row['Cur.Sell'])
            if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
                raise DeserializationError(
                    'Got a trade entry with an empty quote asset')

            if quote_asset is None:
                # Really makes no difference as this is just a gift and the amount is zero
                quote_asset = A_USD
            pair = TradePair(
                f'{base_asset.identifier}_{quote_asset.identifier}')
            base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
            if csv_row['Sell'] != '-':
                quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
            else:
                quote_amount_sold = AssetAmount(ZERO)
            rate = Price(quote_amount_sold / base_amount_bought)

            trade = Trade(
                timestamp=timestamp,
                location=location,
                pair=pair,
                trade_type=TradeType.
                BUY,  # It's always a buy during cointracking import
                amount=base_amount_bought,
                rate=rate,
                fee=fee,
                fee_currency=fee_currency,
                link='',
                notes=notes,
            )
            self.db.add_trades([trade])
        elif row_type == 'Deposit' or row_type == 'Withdrawal':
            category = deserialize_asset_movement_category(row_type.lower())
            if category == AssetMovementCategory.DEPOSIT:
                amount = deserialize_asset_amount(csv_row['Buy'])
                asset = Asset(csv_row['Cur.Buy'])
            else:
                amount = deserialize_asset_amount_force_positive(
                    csv_row['Sell'])
                asset = Asset(csv_row['Cur.Sell'])

            asset_movement = AssetMovement(
                location=location,
                category=category,
                address=None,
                transaction_id=None,
                timestamp=timestamp,
                asset=asset,
                amount=amount,
                fee=fee,
                fee_asset=fee_currency,
                link='',
            )
            self.db.add_asset_movements([asset_movement])
        else:
            raise UnsupportedCointrackingEntry(
                f'Unknown entrype type "{row_type}" encountered during cointracking '
                f'data import. Ignoring entry', )
Esempio n. 2
0
    def _deserialize_asset_movement(
            self,
            movement_type: AssetMovementCategory,
            movement_data: Dict[str, Any],
    ) -> Optional[AssetMovement]:
        """Processes a single deposit/withdrawal from polo and deserializes it

        Can log error/warning and return None if something went wrong at deserialization
        """
        try:
            if movement_type == AssetMovementCategory.DEPOSIT:
                fee = Fee(ZERO)
                uid_key = 'depositNumber'
                transaction_id = get_key_if_has_val(movement_data, 'txid')
            else:
                fee = deserialize_fee(movement_data['fee'])
                uid_key = 'withdrawalNumber'
                split = movement_data['status'].split(':')
                if len(split) != 2:
                    transaction_id = None
                else:
                    transaction_id = split[1].lstrip()
                    if transaction_id == '':
                        transaction_id = None

            asset = asset_from_poloniex(movement_data['currency'])
            return AssetMovement(
                location=Location.POLONIEX,
                category=movement_type,
                address=deserialize_asset_movement_address(movement_data, 'address', asset),
                transaction_id=transaction_id,
                timestamp=deserialize_timestamp(movement_data['timestamp']),
                asset=asset,
                amount=deserialize_asset_amount_force_positive(movement_data['amount']),
                fee_asset=asset,
                fee=fee,
                link=str(movement_data[uid_key]),
            )
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'Found {str(movement_type)} of unsupported poloniex asset '
                f'{e.asset_name}. Ignoring it.',
            )
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'Found {str(movement_type)} of unknown poloniex asset '
                f'{e.asset_name}. Ignoring it.',
            )
        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key entry for {msg}.'
            self.msg_aggregator.add_error(
                'Unexpected data encountered during deserialization of a poloniex '
                'asset movement. Check logs for details and open a bug report.',
            )
            log.error(
                f'Unexpected data encountered during deserialization of poloniex '
                f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
            )

        return None
Esempio n. 3
0
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
    """Turn a poloniex trade returned from poloniex trade history to our common trade
    history format

    Throws:
        - UnsupportedAsset due to asset_from_poloniex()
        - DeserializationError due to the data being in unexpected format
        - UnprocessableTradePair due to the pair data being in an unexpected format
    """

    try:
        trade_type = deserialize_trade_type(poloniex_trade['type'])
        amount = deserialize_asset_amount(poloniex_trade['amount'])
        rate = deserialize_price(poloniex_trade['rate'])
        perc_fee = deserialize_fee(poloniex_trade['fee'])
        base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
        quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
        timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
    except KeyError as e:
        raise DeserializationError(
            f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
        )

    cost = rate * amount
    if trade_type == TradeType.BUY:
        fee = Fee(amount * perc_fee)
        fee_currency = quote_currency
    elif trade_type == TradeType.SELL:
        fee = Fee(cost * perc_fee)
        fee_currency = base_currency
    else:
        raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')

    if poloniex_trade['category'] == 'settlement':
        if trade_type == TradeType.BUY:
            trade_type = TradeType.SETTLEMENT_BUY
        else:
            trade_type = TradeType.SETTLEMENT_SELL

    log.debug(
        'Processing poloniex Trade',
        sensitive_log=True,
        timestamp=timestamp,
        order_type=trade_type,
        pair=pair,
        base_currency=base_currency,
        quote_currency=quote_currency,
        amount=amount,
        fee=fee,
        rate=rate,
    )

    # Use the converted assets in our pair
    pair = trade_pair_from_assets(base_currency, quote_currency)
    # Since in Poloniex the base currency is the cost currency, iow in poloniex
    # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
    # into the Rotkehlchen way which is following the base/quote approach.
    pair = invert_pair(pair)
    return Trade(
        timestamp=timestamp,
        location=Location.POLONIEX,
        pair=pair,
        trade_type=trade_type,
        amount=amount,
        rate=rate,
        fee=fee,
        fee_currency=fee_currency,
        link=str(poloniex_trade['globalTradeID']),
    )
Esempio n. 4
0
    def query_online_trade_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> List[Trade]:
        """Queries gemini for trades
        """
        log.debug('Query gemini trade history',
                  start_ts=start_ts,
                  end_ts=end_ts)
        trades = []
        gemini_trades = []
        for symbol in self.symbols:
            gemini_trades = self._get_trades_for_symbol(
                symbol=symbol,
                start_ts=start_ts,
                end_ts=end_ts,
            )
            for entry in gemini_trades:
                try:
                    timestamp = deserialize_timestamp(entry['timestamp'])
                    if timestamp > end_ts:
                        break

                    trades.append(
                        Trade(
                            timestamp=timestamp,
                            location=Location.GEMINI,
                            pair=gemini_symbol_to_pair(symbol),
                            trade_type=deserialize_trade_type(entry['type']),
                            amount=deserialize_asset_amount(entry['amount']),
                            rate=deserialize_price(entry['price']),
                            fee=deserialize_fee(entry['fee_amount']),
                            fee_currency=Asset(entry['fee_currency']),
                            link=str(entry['tid']),
                            notes='',
                        ))
                except UnprocessableTradePair as e:
                    self.msg_aggregator.add_warning(
                        f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
                    )
                    continue
                except UnknownAsset as e:
                    self.msg_aggregator.add_warning(
                        f'Found unknown Gemini asset {e.asset_name}. '
                        f'Ignoring the trade.', )
                    continue
                except (DeserializationError, KeyError) as e:
                    msg = str(e)
                    if isinstance(e, KeyError):
                        msg = f'Missing key entry for {msg}.'
                    self.msg_aggregator.add_error(
                        'Failed to deserialize a gemini trade. '
                        'Check logs for details. Ignoring it.', )
                    log.error(
                        'Error processing a gemini trade.',
                        raw_trade=entry,
                        error=msg,
                    )
                    continue

        return trades
Esempio n. 5
0
def process_polo_loans(
        msg_aggregator: MessagesAggregator,
        data: List[Dict],
        start_ts: Timestamp,
        end_ts: Timestamp,
) -> List[Loan]:
    """Takes in the list of loans from poloniex as returned by the return_lending_history
    api call, processes it and returns it into our loan format
    """
    new_data = []

    for loan in reversed(data):
        log.debug('processing poloniex loan', **make_sensitive(loan))
        try:
            close_time = deserialize_timestamp_from_poloniex_date(loan['close'])
            open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
            if open_time < start_ts:
                continue
            if close_time > end_ts:
                continue

            our_loan = Loan(
                location=Location.POLONIEX,
                open_time=open_time,
                close_time=close_time,
                currency=asset_from_poloniex(loan['currency']),
                fee=deserialize_fee(loan['fee']),
                earned=deserialize_asset_amount(loan['earned']),
                amount_lent=deserialize_asset_amount(loan['amount']),
            )
        except UnsupportedAsset as e:
            msg_aggregator.add_warning(
                f'Found poloniex loan with unsupported asset'
                f' {e.asset_name}. Ignoring it.',
            )
            continue
        except UnknownAsset as e:
            msg_aggregator.add_warning(
                f'Found poloniex loan with unknown asset'
                f' {e.asset_name}. Ignoring it.',
            )
            continue
        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key entry for {msg}.'
            msg_aggregator.add_error(
                'Deserialization error while reading a poloniex loan. Check '
                'logs for more details. Ignoring it.',
            )
            log.error(
                'Deserialization error while reading a poloniex loan',
                loan=loan,
                error=msg,
            )
            continue

        new_data.append(our_loan)

    new_data.sort(key=lambda loan: loan.open_time)
    return new_data
Esempio n. 6
0
    def query_online_deposits_withdrawals(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> List[AssetMovement]:
        result = self.query_until_finished(
            endpoint='Ledgers',
            keyname='ledger',
            start_ts=start_ts,
            end_ts=end_ts,
            extra_dict={'type': 'deposit'},
        )
        result.extend(
            self.query_until_finished(
                endpoint='Ledgers',
                keyname='ledger',
                start_ts=start_ts,
                end_ts=end_ts,
                extra_dict={'type': 'withdrawal'},
            ))

        log.debug('Kraken deposit/withdrawals query result',
                  num_results=len(result))

        movements = []
        for movement in result:
            try:
                asset = asset_from_kraken(movement['asset'])
                movement_type = movement['type']
                if movement_type not in ('deposit', 'withdrawal'):
                    # Other known types: 'transfer'
                    continue  # Can be for moving funds from spot to stake etc.
                movements.append(
                    AssetMovement(
                        location=Location.KRAKEN,
                        category=deserialize_asset_movement_category(
                            movement_type),
                        timestamp=deserialize_timestamp_from_kraken(
                            movement['time']),
                        address=None,  # no data from kraken ledger endpoint
                        transaction_id=
                        None,  # no data from kraken ledger endpoint
                        asset=asset,
                        amount=deserialize_asset_amount_force_positive(
                            movement['amount']),
                        fee_asset=asset,
                        fee=deserialize_fee(movement['fee']),
                        link=str(movement['refid']),
                    ))
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found unknown kraken asset {e.asset_name}. '
                    f'Ignoring its deposit/withdrawals query.', )
                continue
            except (DeserializationError, KeyError) as e:
                msg = str(e)
                if isinstance(e, KeyError):
                    msg = f'Missing key entry for {msg}.'
                self.msg_aggregator.add_error(
                    'Failed to deserialize a kraken deposit/withdrawals. '
                    'Check logs for details. Ignoring it.', )
                log.error(
                    'Error processing a kraken deposit/withdrawal.',
                    raw_asset_movement=movement,
                    error=msg,
                )
                continue

        return movements
Esempio n. 7
0
    def _deserialize_wallettx(
        self,
        entry: Dict[str, Any],
        from_ts: Timestamp,
        to_ts: Timestamp,
    ) -> Optional[AssetMovement]:
        """Deserializes a bitpanda fiatwallets/transactions or wallets/transactions
        entry to a deposit/withdrawal

        Returns None and logs error is there is a problem or simpy None if
        it's not a type of entry we are interested in
        """
        try:
            transaction_type = entry['type']
            if (transaction_type
                    not in ('fiat_wallet_transaction', 'wallet_transaction')
                    or entry['attributes']['status'] != 'finished'):
                return None
            time = Timestamp(
                deserialize_int_from_str(
                    symbol=entry['attributes']['time']['unix'],
                    location='bitpanda wallet transaction',
                ))
            if time < from_ts or time > to_ts:
                # should we also stop querying from calling method?
                # Probably yes but docs don't mention anything about results
                # being ordered by time so let's be conservative
                return None

            try:
                movement_category = deserialize_asset_movement_category(
                    entry['attributes']['type'])  # noqa: E501
            except DeserializationError:
                return None  # not a deposit/withdrawal

            if transaction_type == 'fiat_wallet_transaction':
                asset_id = entry['attributes']['fiat_id']
                asset = self.fiat_map.get(asset_id)
            else:
                asset_id = entry['attributes']['cryptocoin_id']
                asset = self.cryptocoin_map.get(asset_id)
            if asset is None:
                self.msg_aggregator.add_error(
                    f'While deserializing Bitpanda fiat transaction, could not find '
                    f'bitpanda asset with id {asset_id} in the mapping', )
                return None
            amount = deserialize_asset_amount(entry['attributes']['amount'])
            fee = deserialize_fee(entry['attributes']['fee'])
            tx_id = entry['id']

            transaction_id = entry['attributes'].get('tx_id')
            address = entry['attributes'].get('recipient')
        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key {msg} for wallet transaction entry'

            self.msg_aggregator.add_error(
                f'Error processing bitpanda wallet transaction entry due to {msg}'
            )  # noqa: E501
            log.error(
                'Error processing bitpanda wallet transaction entry',
                error=msg,
                entry=entry,
            )
            return None

        return AssetMovement(
            location=Location.BITPANDA,
            category=movement_category,
            address=address,
            transaction_id=transaction_id,
            timestamp=time,
            asset=asset,
            amount=amount,
            fee_asset=asset,
            fee=fee,
            link=tx_id,
        )
Esempio n. 8
0
    def query_online_trade_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
        """Queries coinbase pro for trades"""
        log.debug('Query coinbasepro trade history',
                  start_ts=start_ts,
                  end_ts=end_ts)
        self.first_connection()

        trades = []
        # first get all orders, to see which product ids we need to query fills for
        orders = []
        for batch in self._paginated_query(
                endpoint='orders',
                query_options={'status': 'done'},
        ):
            orders.extend(batch)

        queried_product_ids = set()

        for order_entry in orders:
            product_id = order_entry.get('product_id', None)
            if product_id is None:
                msg = (
                    'Skipping coinbasepro trade since it lacks a product_id. '
                    'Check logs for details')
                self.msg_aggregator.add_error(msg)
                log.error(
                    'Error processing a coinbasepro order.',
                    raw_trade=order_entry,
                    error=msg,
                )
                continue

            if product_id in queried_product_ids or product_id not in self.available_products:
                continue  # already queried this product id or delisted product id

            # Now let's get all fills for this product id
            queried_product_ids.add(product_id)
            fills = []
            for batch in self._paginated_query(
                    endpoint='fills',
                    query_options={'product_id': product_id},
            ):
                fills.extend(batch)

            try:
                base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
            except UnprocessableTradePair as e:
                self.msg_aggregator.add_warning(
                    f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
                )
                continue
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found unknown Coinbasepro asset {e.asset_name}. '
                    f'Ignoring the trade.', )
                continue

            for fill_entry in fills:
                try:
                    timestamp = coinbasepro_deserialize_timestamp(
                        fill_entry, 'created_at')
                    if timestamp < start_ts or timestamp > end_ts:
                        continue

                    # Fee currency seems to always be quote asset
                    # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724  # noqa: E501
                    trades.append(
                        Trade(
                            timestamp=timestamp,
                            location=Location.COINBASEPRO,
                            base_asset=base_asset,
                            quote_asset=quote_asset,
                            trade_type=TradeType.deserialize(
                                fill_entry['side']),
                            amount=deserialize_asset_amount(
                                fill_entry['size']),
                            rate=deserialize_price(fill_entry['price']),
                            fee=deserialize_fee(fill_entry['fee']),
                            fee_currency=quote_asset,
                            link=str(fill_entry['trade_id']) + '_' +
                            fill_entry['order_id'],
                            notes='',
                        ))
                except UnprocessableTradePair as e:
                    self.msg_aggregator.add_warning(
                        f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
                    )
                    continue
                except UnknownAsset as e:
                    self.msg_aggregator.add_warning(
                        f'Found unknown Coinbasepro asset {e.asset_name}. '
                        f'Ignoring the trade.', )
                    continue
                except (DeserializationError, KeyError) as e:
                    msg = str(e)
                    if isinstance(e, KeyError):
                        msg = f'Missing key entry for {msg}.'
                    self.msg_aggregator.add_error(
                        'Failed to deserialize a coinbasepro trade. '
                        'Check logs for details. Ignoring it.', )
                    log.error(
                        'Error processing a coinbasepro fill.',
                        raw_trade=fill_entry,
                        error=msg,
                    )
                    continue

        return trades, (start_ts, end_ts)
Esempio n. 9
0
    def _deserialize_asset_movement(
            self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
        """Processes a single deposit/withdrawal from coinbase and deserializes it

        Can log error/warning and return None if something went wrong at deserialization
        """
        try:
            if raw_data['status'] != 'completed':
                return None

            payout_date = raw_data.get('payout_at', None)
            if payout_date:
                timestamp = deserialize_timestamp_from_date(
                    payout_date, 'iso8601', 'coinbase')
            else:
                timestamp = deserialize_timestamp_from_date(
                    raw_data['created_at'],
                    'iso8601',
                    'coinbase',
                )

            # Only get address/transaction id for "send" type of transactions
            address = None
            transaction_id = None
            # movement_category: Union[Literal['deposit'], Literal['withdrawal']]
            if 'type' in raw_data:
                # Then this should be a "send" which is the way Coinbase uses to send
                # crypto outside of the exchange
                # https://developers.coinbase.com/api/v2?python#transaction-resource
                msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
                assert raw_data['type'] == 'send', msg
                movement_category = AssetMovementCategory.WITHDRAWAL
                # Can't see the fee being charged from the "send" resource

                amount = deserialize_asset_amount_force_positive(
                    raw_data['amount']['amount'])
                asset = asset_from_coinbase(raw_data['amount']['currency'],
                                            time=timestamp)
                # Fees dont appear in the docs but from an experiment of sending ETH
                # to an address from coinbase there is the network fee in the response
                fee = Fee(ZERO)
                raw_network = raw_data.get('network', None)
                if raw_network:
                    raw_fee = raw_network.get('transaction_fee', None)

                if raw_fee:
                    # Since this is a withdrawal the fee should be the same as the moved asset
                    if asset != asset_from_coinbase(raw_fee['currency'],
                                                    time=timestamp):
                        # If not we set ZERO fee and ignore
                        log.error(
                            f'In a coinbase withdrawal of {asset.identifier} the fee'
                            f'is denoted in {raw_fee["currency"]}', )
                    else:
                        fee = deserialize_fee(raw_fee['amount'])

                if 'network' in raw_data:
                    transaction_id = get_key_if_has_val(
                        raw_data['network'], 'hash')
                if 'to' in raw_data:
                    address = deserialize_asset_movement_address(
                        raw_data['to'], 'address', asset)
            else:
                movement_category = deserialize_asset_movement_category(
                    raw_data['resource'])
                amount = deserialize_asset_amount_force_positive(
                    raw_data['amount']['amount'])
                fee = deserialize_fee(raw_data['fee']['amount'])
                asset = asset_from_coinbase(raw_data['amount']['currency'],
                                            time=timestamp)

            return AssetMovement(
                location=Location.COINBASE,
                category=movement_category,
                address=address,
                transaction_id=transaction_id,
                timestamp=timestamp,
                asset=asset,
                amount=amount,
                fee_asset=asset,
                fee=fee,
                link=str(raw_data['id']),
            )
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'Found coinbase deposit/withdrawal with unknown asset '
                f'{e.asset_name}. Ignoring it.', )
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'Found coinbase deposit/withdrawal with unsupported asset '
                f'{e.asset_name}. Ignoring it.', )
        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key entry for {msg}.'
            self.msg_aggregator.add_error(
                'Unexpected data encountered during deserialization of a coinbase '
                'asset movement. Check logs for details and open a bug report.',
            )
            log.error(
                f'Unexpected data encountered during deserialization of coinbase '
                f'asset_movement {raw_data}. Error was: {str(e)}', )

        return None
Esempio n. 10
0
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
    """
    Takes in the trade data dictionary, validates it and returns a trade instance

    If there is an error it returns an error message in the second part of the tuple
    """
    for field in otc_fields:
        if field not in data:
            return None, f'{field} was not provided'

        if data[field] in ('', None) and field not in otc_optional_fields:
            return None, f'{field} was empty'

        if field in otc_numerical_fields and not is_number(data[field]):
            return None, f'{field} should be a number'

    # Satisfy mypy typing
    assert isinstance(data['otc_pair'], str)
    assert isinstance(data['otc_fee_currency'], str)
    assert isinstance(data['otc_fee'], str)

    pair = TradePair(data['otc_pair'])
    try:
        first = get_pair_position_asset(pair, 'first')
        second = get_pair_position_asset(pair, 'second')
        fee_currency = Asset(data['otc_fee_currency'])
    except UnknownAsset as e:
        return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
    # Not catching DeserializationError here since we have asserts for the data
    # being strings right above

    try:
        trade_type = deserialize_trade_type(str(data['otc_type']))
        amount = deserialize_asset_amount(data['otc_amount'])
        rate = deserialize_price(data['otc_rate'])
        fee = deserialize_fee(data['otc_fee'])
    except DeserializationError as e:
        return None, f'Deserialization Error: {str(e)}'
    try:
        assert isinstance(data['otc_timestamp'], str)
        timestamp = create_timestamp(data['otc_timestamp'],
                                     formatstr='%d/%m/%Y %H:%M')
    except ValueError as e:
        return None, f'Could not process the given datetime: {e}'

    log.debug(
        'Creating OTC trade data',
        sensitive_log=True,
        pair=pair,
        trade_type=trade_type,
        amount=amount,
        rate=rate,
        fee=fee,
        fee_currency=fee_currency,
    )

    if data['otc_fee_currency'] not in (first, second):
        return None, 'Trade fee currency should be one of the two in the currency pair'

    if data['otc_type'] not in ('buy', 'sell'):
        return None, 'Trade type can only be buy or sell'

    trade = Trade(
        timestamp=timestamp,
        location=Location.EXTERNAL,
        pair=pair,
        trade_type=trade_type,
        amount=amount,
        rate=rate,
        fee=fee,
        fee_currency=fee_currency,
        link=str(data['otc_link']),
        notes=str(data['otc_notes']),
    )

    return trade, ''
Esempio n. 11
0
        """
        if case == KucoinCase.DEPOSITS:
            category = AssetMovementCategory.DEPOSIT
        elif case == KucoinCase.WITHDRAWALS:
            category = AssetMovementCategory.WITHDRAWAL
        else:
            raise AssertionError(f'Unexpected case: {case}')

        try:
            timestamp_ms = deserialize_timestamp(raw_result['createdAt'])
            timestamp = Timestamp(int(timestamp_ms / 1000))
            address = raw_result['address']
            # The transaction id can have an @ which we should just get rid of
            transaction_id = raw_result['walletTxId'].split('@')[0]
            amount = deserialize_asset_amount(raw_result['amount'])
            fee = deserialize_fee(raw_result['fee'])
            fee_currency_symbol = raw_result['currency']
            link_id = raw_result.get('id',
                                     '')  # NB: id only exists for withdrawals
        except KeyError as e:
            raise DeserializationError(f'Missing key: {str(e)}.') from e

        fee_asset = asset_from_kucoin(fee_currency_symbol)

        asset_movement = AssetMovement(
            timestamp=timestamp,
            location=Location.KUCOIN,
            category=category,
            address=address,
            transaction_id=transaction_id,
            asset=fee_asset,
Esempio n. 12
0
File: ftx.py Progetto: rotki/rotki
    def _deserialize_asset_movement(
        self,
        raw_data: Dict[str, Any],
        movement_type: AssetMovementCategory,
    ) -> Optional[AssetMovement]:
        """Processes a single deposit/withdrawal from FTX and deserializes it

        Can log error/warning and return None if something went wrong at deserialization
        """
        try:
            if raw_data['status'] not in ('complete', 'confirmed'):
                return None

            timestamp = deserialize_timestamp_from_date(
                raw_data['time'], 'iso8601', 'FTX')
            amount = deserialize_asset_amount_force_positive(raw_data['size'])
            asset = asset_from_ftx(raw_data['coin'])
            fee = Fee(ZERO)
            movement_category = movement_type
            if raw_data.get('fee', None) is not None:
                fee = deserialize_fee(raw_data['fee'])
            address = raw_data.get('address', None)
            if isinstance(address, dict):
                address = raw_data['address'].get('address', None)
            transaction_id = raw_data.get('txid', None)

            return AssetMovement(
                location=Location.FTX,
                category=movement_category,
                address=address,
                transaction_id=transaction_id,
                timestamp=timestamp,
                asset=asset,
                amount=amount,
                fee_asset=asset,
                fee=fee,
                link=str(raw_data['id']),
            )
        except UnknownAsset as e:
            self.msg_aggregator.add_warning(
                f'Found FTX deposit/withdrawal with unknown asset '
                f'{e.asset_name}. Ignoring it.', )
        except UnsupportedAsset as e:
            self.msg_aggregator.add_warning(
                f'Found FTX deposit/withdrawal with unsupported asset '
                f'{e.asset_name}. Ignoring it.', )
        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key entry for {msg}.'
            self.msg_aggregator.add_error(
                'Unexpected data encountered during deserialization of an FTX '
                'asset movement. Check logs for details and open a bug report.',
            )
            log.error(
                'Error processing FTX trade',
                trade=raw_data,
                error=msg,
            )

        return None
Esempio n. 13
0
    def query_online_trade_history(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> List[Trade]:
        """Queries coinbase pro for trades"""
        log.debug('Query coinbasepro trade history',
                  start_ts=start_ts,
                  end_ts=end_ts)

        trades = []
        raw_trades = []
        for batch in self._paginated_query(
                endpoint='orders',
                query_options={'status': 'done'},
        ):
            raw_trades.extend(batch)

        for entry in raw_trades:
            timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at')
            if timestamp < start_ts or timestamp > end_ts:
                continue

            try:
                pair = coinbasepro_to_worldpair(entry['product_id'])
                # Fee currency seems to always be quote asset
                # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724  # noqa: E501
                _, quote_asset = pair_get_assets(pair)
                trades.append(
                    Trade(
                        timestamp=timestamp,
                        location=Location.COINBASEPRO,
                        pair=coinbasepro_to_worldpair(entry['product_id']),
                        trade_type=deserialize_trade_type(entry['side']),
                        amount=deserialize_asset_amount(entry['size']),
                        rate=deserialize_price(entry['price']),
                        fee=deserialize_fee(entry['fill_fees']),
                        fee_currency=quote_asset,
                        link=entry['id'],
                        notes='',
                    ))
            except UnprocessableTradePair as e:
                self.msg_aggregator.add_warning(
                    f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
                )
                continue
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found unknown Coinbasepro asset {e.asset_name}. '
                    f'Ignoring the trade.', )
                continue
            except (DeserializationError, KeyError) as e:
                msg = str(e)
                if isinstance(e, KeyError):
                    msg = f'Missing key entry for {msg}.'
                self.msg_aggregator.add_error(
                    'Failed to deserialize a coinbasepro trade. '
                    'Check logs for details. Ignoring it.', )
                log.error(
                    'Error processing a coinbasepro trade.',
                    raw_trade=entry,
                    error=msg,
                )
                continue

        return trades
Esempio n. 14
0
    def _deserialize_trade(
        self,
        entry: Dict[str, Any],
        from_ts: Timestamp,
        to_ts: Timestamp,
    ) -> Optional[Trade]:
        """Deserializes a bitpanda trades result entry to a Trade

        Returns None and logs error is there is a problem or simpy None if
        it's not a type of trade we are interested in
        """
        try:
            if entry['type'] != 'trade' or entry['attributes'][
                    'status'] != 'finished':
                return None
            time = Timestamp(
                deserialize_int_from_str(
                    symbol=entry['attributes']['time']['unix'],
                    location='bitpanda trade',
                ))
            if time < from_ts or time > to_ts:
                # should we also stop querying from calling method?
                # Probably yes but docs don't mention anything about results
                # being ordered by time so let's be conservative
                return None

            cryptocoin_id = entry['attributes']['cryptocoin_id']
            crypto_asset = self.cryptocoin_map.get(cryptocoin_id)
            if crypto_asset is None:
                self.msg_aggregator.add_error(
                    f'While deserializing a trade, could not find bitpanda cryptocoin '
                    f'with id {cryptocoin_id} in the mapping. Skipping trade.',
                )
                return None

            fiat_id = entry['attributes']['fiat_id']
            fiat_asset = self.fiat_map.get(fiat_id)
            if fiat_asset is None:
                self.msg_aggregator.add_error(
                    f'While deserializing a trade, could not find bitpanda fiat '
                    f'with id {fiat_id} in the mapping. Skipping trade.', )
                return None

            trade_type = TradeType.deserialize(entry['attributes']['type'])
            if trade_type in (TradeType.BUY, TradeType.SELL):
                # you buy crypto with fiat and sell it for fiat
                base_asset = crypto_asset
                quote_asset = fiat_asset
                amount = deserialize_asset_amount(
                    entry['attributes']['amount_cryptocoin'])
                price = deserialize_price(entry['attributes']['price'])
            else:
                self.msg_aggregator.add_error(
                    'Found bitpanda trade with unknown trade type {trade_type}'
                )  # noqa: E501
                return None

            trade_id = entry['id']
            fee = Fee(ZERO)
            fee_asset = A_BEST
            if entry['attributes']['bfc_used'] is True:
                fee = deserialize_fee(
                    entry['attributes']['best_fee_collection']['attributes']
                    ['wallet_transaction']['attributes']['fee'],  # noqa: E501
                )

        except (DeserializationError, KeyError) as e:
            msg = str(e)
            if isinstance(e, KeyError):
                msg = f'Missing key {msg} for trade entry'

            self.msg_aggregator.add_error(
                f'Error processing bitpanda trade due to {msg}')
            log.error(
                'Error processing bitpanda trade entry',
                error=msg,
                entry=entry,
            )
            return None

        return Trade(
            timestamp=time,
            location=Location.BITPANDA,
            base_asset=base_asset,
            quote_asset=quote_asset,
            trade_type=trade_type,
            amount=amount,
            rate=price,
            fee=fee,
            fee_currency=fee_asset,
            link=trade_id,
        )
Esempio n. 15
0
def trade_from_binance(
        binance_trade: Dict,
        binance_symbols_to_pair: Dict[str, BinancePair],
) -> Trade:
    """Turn a binance trade returned from trade history to our common trade
    history format

    From the official binance api docs (01/09/18):
    https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology

    base asset refers to the asset that is the quantity of a symbol.
    quote asset refers to the asset that is the price of a symbol.

    Throws:
        - UnsupportedAsset due to asset_from_binance
        - DeserializationError due to unexpected format of dict entries
        - KeyError due to dict entries missing an expected entry
    """
    amount = deserialize_asset_amount(binance_trade['qty'])
    rate = deserialize_price(binance_trade['price'])
    if binance_trade['symbol'] not in binance_symbols_to_pair:
        raise DeserializationError(
            f'Error reading a binance trade. Could not find '
            f'{binance_trade["symbol"]} in binance_symbols_to_pair',
        )

    binance_pair = binance_symbols_to_pair[binance_trade['symbol']]
    timestamp = deserialize_timestamp_from_binance(binance_trade['time'])

    base_asset = asset_from_binance(binance_pair.binance_base_asset)
    quote_asset = asset_from_binance(binance_pair.binance_quote_asset)

    if binance_trade['isBuyer']:
        order_type = TradeType.BUY
        # e.g. in RDNETH we buy RDN by paying ETH
    else:
        order_type = TradeType.SELL

    fee_currency = asset_from_binance(binance_trade['commissionAsset'])
    fee = deserialize_fee(binance_trade['commission'])

    log.debug(
        'Processing binance Trade',
        sensitive_log=True,
        amount=amount,
        rate=rate,
        timestamp=timestamp,
        pair=binance_trade['symbol'],
        base_asset=base_asset,
        quote=quote_asset,
        order_type=order_type,
        commision_asset=binance_trade['commissionAsset'],
        fee=fee,
    )

    return Trade(
        timestamp=timestamp,
        location=Location.BINANCE,
        pair=trade_pair_from_assets(base_asset, quote_asset),
        trade_type=order_type,
        amount=amount,
        rate=rate,
        fee=fee,
        fee_currency=fee_currency,
        link=str(binance_trade['id']),
    )
Esempio n. 16
0
    def query_online_deposits_withdrawals(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> List:
        resp = self._api_query_list('get', 'user/walletHistory')

        log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))

        movements = list()
        for movement in resp:
            try:
                transaction_type = movement['transactType']
                if transaction_type == 'Deposit':
                    transaction_type = AssetMovementCategory.DEPOSIT
                elif transaction_type == 'Withdrawal':
                    transaction_type = AssetMovementCategory.WITHDRAWAL
                else:
                    continue

                timestamp = iso8601ts_to_timestamp(movement['timestamp'])
                if timestamp < start_ts:
                    continue
                if timestamp > end_ts:
                    continue

                asset = bitmex_to_world(movement['currency'])
                amount = deserialize_asset_amount(movement['amount'])
                fee = deserialize_fee(movement['fee'])
                # bitmex has negative numbers for withdrawals
                if amount < 0:
                    amount *= -1

                if asset == A_BTC:
                    # bitmex stores amounts in satoshis
                    amount = satoshis_to_btc(amount)
                    fee = satoshis_to_btc(fee)

                movements.append(
                    AssetMovement(
                        location=Location.BITMEX,
                        category=transaction_type,
                        timestamp=timestamp,
                        asset=asset,
                        amount=amount,
                        fee_asset=asset,
                        fee=fee,
                        link=str(movement['transactID']),
                    ))
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found bitmex deposit/withdrawal with unknown asset '
                    f'{e.asset_name}. Ignoring it.', )
                continue
            except (DeserializationError, KeyError) as e:
                msg = str(e)
                if isinstance(e, KeyError):
                    msg = f'Missing key entry for {msg}.'
                self.msg_aggregator.add_error(
                    f'Unexpected data encountered during deserialization of a bitmex '
                    f'asset movement. Check logs for details and open a bug report.',
                )
                log.error(
                    f'Unexpected data encountered during deserialization of bitmex '
                    f'asset_movement {movement}. Error was: {str(e)}', )
                continue
        return movements
Esempio n. 17
0
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
    """Turn a kraken trade returned from kraken trade history to our common trade
    history format

    - Can raise UnknownAsset due to kraken_to_world_pair
    - Can raise UnprocessableTradePair due to kraken_to_world_pair
    - Can raise DeserializationError due to dict entries not being as expected
    - Can raise KeyError due to dict entries missing an expected entry
    """
    currency_pair = kraken_to_world_pair(kraken_trade['pair'])
    quote_currency = get_pair_position_asset(currency_pair, 'second')

    timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
    amount = deserialize_asset_amount(kraken_trade['vol'])
    cost = deserialize_price(kraken_trade['cost'])
    fee = deserialize_fee(kraken_trade['fee'])
    order_type = deserialize_trade_type(kraken_trade['type'])
    rate = deserialize_price(kraken_trade['price'])

    # pylint does not seem to see that Price is essentially FVal
    if not cost.is_close(amount * rate):  # pylint: disable=no-member
        log.warning(
            f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
        )

    log.debug(
        'Processing kraken Trade',
        sensitive_log=True,
        timestamp=timestamp,
        order_type=order_type,
        kraken_pair=kraken_trade['pair'],
        pair=currency_pair,
        quote_currency=quote_currency,
        amount=amount,
        cost=cost,
        fee=fee,
        rate=rate,
    )

    # Kraken trades can have the same ordertxid and postxid for different trades ..
    # Also note postxid is optional and can be missing
    # The only thing that could differentiate them is timestamps in the milliseconds range
    # For example here are parts of two different kraken_trade:
    # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
    # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
    # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
    # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
    #
    # In order to counter this for the unique exchange trade link we are going
    # to use a concatenation of the above
    exchange_uuid = (
        str(kraken_trade['ordertxid']) +
        str(kraken_trade.get('postxid', '')) +  # postxid is optional
        str(kraken_trade['time']))

    return Trade(
        timestamp=timestamp,
        location=Location.KRAKEN,
        pair=currency_pair,
        trade_type=order_type,
        amount=amount,
        rate=rate,
        fee=fee,
        fee_currency=quote_currency,
        link=exchange_uuid,
    )
Esempio n. 18
0
    def _deserialize_asset_movement(
            self,
            raw_movement: Dict[str, Any],
    ) -> AssetMovement:
        """Process a deposit/withdrawal user transaction from Bitstamp and
        deserialize it.

        Can raise DeserializationError.

        From Bitstamp documentation, deposits/withdrawals can have a fee
        (the amount is expected to be in the currency involved)
        https://www.bitstamp.net/fee-schedule/

        Bitstamp support confirmed the following withdrawal JSON:
        {
            "fee": "0.00050000",
            "btc_usd": "0.00",
            "datetime": "2020-12-04 09:30:00.000000",
            "usd": "0.0",
            "btc": "-0.50000000",
            "type": "1",
            "id": 123456789,
            "eur": "0.0"
        }
        NB: any asset key not related with the pair is discarded (e.g. 'eur').
        """
        type_ = raw_movement['type']
        category: AssetMovementCategory
        if type_ == 0:
            category = AssetMovementCategory.DEPOSIT
        elif type_ == 1:
            category = AssetMovementCategory.WITHDRAWAL
        else:
            raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.')

        timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime'])
        trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement)
        base_asset_amount = deserialize_asset_amount(
            raw_movement[trade_pair_data.base_asset_symbol],
        )
        quote_asset_amount = deserialize_asset_amount(
            raw_movement[trade_pair_data.quote_asset_symbol],
        )
        amount: FVal
        fee_asset: Asset
        if base_asset_amount != ZERO and quote_asset_amount == ZERO:
            amount = base_asset_amount
            fee_asset = trade_pair_data.base_asset
        elif base_asset_amount == ZERO and quote_asset_amount != ZERO:
            amount = quote_asset_amount
            fee_asset = trade_pair_data.quote_asset
        else:
            raise DeserializationError(
                'Could not deserialize Bitstamp asset movement from user transaction. '
                f'Unexpected asset amount combination found in: {raw_movement}.',
            )

        asset_movement = AssetMovement(
            timestamp=timestamp,
            location=Location.BITSTAMP,
            category=category,
            address=None,  # requires query "crypto_transactions" endpoint
            transaction_id=None,  # requires query "crypto_transactions" endpoint
            asset=fee_asset,
            amount=abs(amount),
            fee_asset=fee_asset,
            fee=deserialize_fee(raw_movement['fee']),
            link=str(raw_movement['id']),
        )
        return asset_movement
Esempio n. 19
0
    def query_online_deposits_withdrawals(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
    ) -> List[AssetMovement]:
        """Queries coinbase pro for asset movements"""
        log.debug('Query coinbasepro asset movements',
                  start_ts=start_ts,
                  end_ts=end_ts)
        movements = []
        raw_movements = []
        for batch in self._paginated_query(
                endpoint='transfers',
                query_options={'type': 'withdraw'},
        ):
            raw_movements.extend(batch)
        for batch in self._paginated_query(
                endpoint='transfers',
                query_options={'type': 'deposit'},
        ):
            raw_movements.extend(batch)

        account_to_currency = self.create_or_return_account_to_currency_map()
        for entry in raw_movements:
            try:
                # Check if the transaction has not been completed. If so it should be skipped
                if entry.get('completed_at', None) is None:
                    log.warning(
                        f'Skipping coinbase pro deposit/withdrawal '
                        f'due not having been completed: {entry}', )
                    continue

                timestamp = coinbasepro_deserialize_timestamp(
                    entry, 'completed_at')
                if timestamp < start_ts or timestamp > end_ts:
                    continue

                category = deserialize_asset_movement_category(entry['type'])
                asset = account_to_currency.get(entry['account_id'], None)
                if asset is None:
                    log.warning(
                        f'Skipping coinbase pro asset_movement {entry} due to inability to '
                        f'match account id to an asset', )
                    continue

                address = None
                transaction_id = None
                fee = Fee(ZERO)
                if category == AssetMovementCategory.DEPOSIT:
                    try:
                        address = entry['details']['crypto_address']
                        transaction_id = entry['details'][
                            'crypto_transaction_hash']
                    except KeyError:
                        pass
                else:  # withdrawal
                    try:
                        address = entry['details']['sent_to_address']
                        transaction_id = entry['details'][
                            'crypto_transaction_hash']
                        fee = deserialize_fee(entry['details']['fee'])
                    except KeyError:
                        pass

                if transaction_id and (
                        asset == A_ETH or asset.asset_type
                        == AssetType.ETHEREUM_TOKEN):  # noqa: E501
                    transaction_id = '0x' + transaction_id

                movements.append(
                    AssetMovement(
                        location=Location.COINBASEPRO,
                        category=category,
                        address=address,
                        transaction_id=transaction_id,
                        timestamp=timestamp,
                        asset=asset,
                        amount=deserialize_asset_amount_force_positive(
                            entry['amount']),
                        fee_asset=asset,
                        fee=fee,
                        link=str(entry['id']),
                    ))
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found unknown Coinbasepro asset {e.asset_name}. '
                    f'Ignoring its deposit/withdrawal.', )
                continue
            except (DeserializationError, KeyError) as e:
                msg = str(e)
                if isinstance(e, KeyError):
                    msg = f'Missing key entry for {msg}.'
                self.msg_aggregator.add_error(
                    'Failed to deserialize a Coinbasepro deposit/withdrawal. '
                    'Check logs for details. Ignoring it.', )
                log.error(
                    'Error processing a coinbasepro  deposit/withdrawal.',
                    raw_asset_movement=entry,
                    error=msg,
                )
                continue

        return movements
Esempio n. 20
0
    def query_deposits_withdrawals(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
        end_at_least_ts: Timestamp,
    ) -> List[AssetMovement]:
        with self.lock:
            cache = self.check_trades_cache_list(
                start_ts,
                end_at_least_ts,
                special_name='deposits_withdrawals',
            )
        result: List[Any]

        if cache is not None:
            result = cache
        else:
            result = self.query_until_finished(
                endpoint='Ledgers',
                keyname='ledger',
                start_ts=start_ts,
                end_ts=end_ts,
                extra_dict=dict(type='deposit'),
            )
            result.extend(
                self.query_until_finished(
                    endpoint='Ledgers',
                    keyname='ledger',
                    start_ts=start_ts,
                    end_ts=end_ts,
                    extra_dict=dict(type='withdrawal'),
                ))

            with self.lock:
                self.update_trades_cache(
                    result,
                    start_ts,
                    end_ts,
                    special_name='deposits_withdrawals',
                )

        log.debug('Kraken deposit/withdrawals query result',
                  num_results=len(result))

        movements = list()
        for movement in result:
            try:
                movements.append(
                    AssetMovement(
                        exchange=Exchange.KRAKEN,
                        category=deserialize_asset_movement_category(
                            movement['type']),
                        timestamp=deserialize_timestamp_from_kraken(
                            movement['time']),
                        asset=asset_from_kraken(movement['asset']),
                        amount=deserialize_asset_amount(movement['amount']),
                        fee=deserialize_fee(movement['fee']),
                    ))
            except UnknownAsset as e:
                self.msg_aggregator.add_warning(
                    f'Found unknown kraken asset {e.asset_name}. '
                    f'Ignoring its deposit/withdrawals query.', )
                continue
            except (DeserializationError, KeyError) as e:
                msg = str(e)
                if isinstance(e, KeyError):
                    msg = f'Missing key entry for {msg}.'
                self.msg_aggregator.add_error(
                    'Failed to deserialize a kraken deposit/withdrawals. '
                    'Check logs for details. Ignoring it.', )
                log.error(
                    'Error processing a kraken deposit/withdrawal.',
                    raw_asset_movement=movement,
                    error=msg,
                )
                continue

        return movements