def test_add_margin_positions(data_dir, username, caplog):
"""Test that adding and retrieving margin positions from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
margin1 = MarginPosition(
location=Location.BITMEX,
open_time=1451606400,
close_time=1451616500,
profit_loss=FVal('1.0'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin2 = MarginPosition(
location=Location.BITMEX,
open_time=1451626500,
close_time=1451636500,
profit_loss=FVal('0.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin3 = MarginPosition(
location=Location.POLONIEX,
open_time=1452636501,
close_time=1459836501,
profit_loss=FVal('2.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_margin_positions([margin1, margin2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2]
# Add the last 2 margins. Since margin2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_margin_positions([margin2, margin3])
assert (
'Did not add "Margin position with id 0a57acc1f4c09da0f194c59c4cd240e6'
'8e2d36e56c05b3f7115def9b8ee3943f') in caplog.text
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2, margin3]
def test_query_deposits_withdrawals(mock_ftx: Ftx):
"""Test happy path of deposits/withdrawls"""
with patch.object(mock_ftx.session,
'get',
side_effect=mock_normal_ftx_query):
movements = mock_ftx.query_online_deposits_withdrawals(
start_ts=Timestamp(0),
end_ts=TEST_END_TS,
)
warnings = mock_ftx.msg_aggregator.consume_warnings()
errors = mock_ftx.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
expected_movements = [
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.DEPOSIT,
address='0x541163adf0a2e830d9f940763e912807d1a359f5',
transaction_id=
'0xf787fa6b62edf1c97fb3f73f80a5eb7550bbf3dcf4269b9bfb9e8c1c0a3bc1a9',
timestamp=Timestamp(1612159566),
asset=A_ETH,
amount=FVal('20'),
fee_asset=A_ETH,
fee=Fee(ZERO),
link='3',
),
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612159566),
address='0x903d12bf2c57a29f32365917c706ce0e1a84cce3',
transaction_id=
'0xbb27f24c2a348526fc23767d3d8bb303099e90f253ef9fdbb28ce38c1635d116',
asset=A_ETH,
amount=FVal('11.0'),
fee_asset=A_ETH,
fee=Fee(ZERO),
link='1',
),
AssetMovement(
location=Location.FTX,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1612159566),
asset=A_USD,
amount=FVal('21.0'),
fee_asset=A_USD,
fee=Fee(ZERO),
link='2',
)
]
assert len(movements) == 3
assert movements == expected_movements
def test_deserialize_asset_movement_withdrawal(mock_bitstamp):
raw_movement = {
'id': 5,
'type': '1',
'datetime': '2020-12-02 09:30:00',
'btc': '0.00000000',
'usd': '-10000.00000000',
'btc_usd': '0.00',
'fee': '50.00000000',
'order_id': 2,
'eur': '0.00',
}
asset = A_USD
movement = AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('10000'),
fee_asset=asset,
fee=Fee(FVal('50')),
link='5',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
raw_movement = {
'id': 5,
'type': '1',
'datetime': '2018-03-21 06:46:06.559877',
'btc': '0',
'usd': '0',
'btc_usd': '0.00',
'fee': '0.1',
'order_id': 2,
'eur': '500',
}
asset = A_EUR
movement = AssetMovement(
timestamp=1521614766,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset,
amount=FVal('500'),
fee_asset=asset,
fee=Fee(FVal('0.1')),
link='5',
)
expected_movement = mock_bitstamp._deserialize_asset_movement(raw_movement)
assert movement == expected_movement
def test_query_online_deposits_withdrawals(mock_bitstamp, start_ts, since_id):
"""Test `since_id` value will change depending on `start_ts` value.
Also tests `db_asset_movements` are sorted by `link` (as int) in ascending
mode.
"""
asset_btc = A_BTC
asset_usd = A_USD
movements = [
AssetMovement(
timestamp=1606901400,
location=Location.BITSTAMP,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=asset_usd,
amount=FVal('10000'),
fee_asset=asset_usd,
fee=Fee(FVal('50')),
link='5',
),
AssetMovement(
timestamp=1606801400,
location=Location.BITSTAMP,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
asset=asset_btc,
amount=FVal('0.5'),
fee_asset=asset_btc,
fee=Fee(FVal('0.0005')),
link='2',
),
]
mock_bitstamp.db.add_asset_movements(movements)
end_ts = Timestamp(1606901401)
expected_call = call(
start_ts=start_ts,
end_ts=end_ts,
options={
'since_id': since_id,
'limit': 1000,
'sort': 'asc',
'offset': 0,
},
case='asset_movements',
)
with patch.object(mock_bitstamp,
'_api_query_paginated') as mock_api_query_paginated:
mock_bitstamp.query_online_deposits_withdrawals(
start_ts=Timestamp(start_ts),
end_ts=end_ts,
)
assert mock_api_query_paginated.call_args == expected_call
def test_deserialize_trade_sell(mock_bitfinex):
mock_bitfinex.currency_map = {'UST': 'USDt'}
mock_bitfinex.pair_bfx_symbols_map = {'ETHUST': ('ETH', 'UST')}
raw_result = [
399251013,
'tETHUST',
1573485493000,
33963608932,
-0.26334268,
187.37,
'LIMIT',
None,
-1,
-0.09868591,
'USD',
]
expected_trade = Trade(
timestamp=Timestamp(1573485493),
location=Location.BITFINEX,
base_asset=A_ETH,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.26334268')),
rate=Price(FVal('187.37')),
fee=Fee(FVal('0.09868591')),
fee_currency=A_USD,
link='399251013',
notes='',
)
trade = mock_bitfinex._deserialize_trade(raw_result=raw_result)
assert trade == expected_trade
def test_deserialize_asset_movement_withdrawal(mock_kucoin):
raw_result = {
'id': '5c2dc64e03aa675aa263f1ac',
'address': '0x5bedb060b8eb8d823e2414d82acce78d38be7fe9',
'memo': '',
'currency': 'ETH',
'amount': 1,
'fee': 0.01,
'walletTxId': '3e2414d82acce78d38be7fe9',
'isInner': False,
'status': 'SUCCESS',
'remark': 'test',
'createdAt': 1612556794259,
'updatedAt': 1612556795000,
}
expected_asset_movement = AssetMovement(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
address='0x5bedb060b8eb8d823e2414d82acce78d38be7fe9',
transaction_id='3e2414d82acce78d38be7fe9',
asset=A_ETH,
amount=AssetAmount(FVal('1')),
fee_asset=A_ETH,
fee=Fee(FVal('0.01')),
link='5c2dc64e03aa675aa263f1ac',
)
asset_movement = mock_kucoin._deserialize_asset_movement(
raw_result=raw_result,
case=KucoinCase.WITHDRAWALS,
)
assert asset_movement == expected_asset_movement
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self._get_paginated_query(
endpoint='transfers',
start_ts=start_ts,
end_ts=end_ts,
)
movements = []
for entry in result:
try:
timestamp = deserialize_timestamp(entry['timestampms'])
timestamp = Timestamp(int(timestamp / 1000))
asset = asset_from_gemini(entry['currency'])
movement = AssetMovement(
location=Location.GEMINI,
category=deserialize_asset_movement_category(
entry['type']),
address=deserialize_asset_movement_address(
entry, 'destination', asset),
transaction_id=get_key_if_has_val(entry, 'txHash'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
# Gemini does not include withdrawal fees neither in the API nor in their UI
fee=Fee(ZERO),
link=str(entry['eid']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini deposit/withdrawal. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a gemini deposit_withdrawal',
asset_movement=entry,
error=msg,
)
continue
movements.append(movement)
return movements
def _trade_from_independentreserve(raw_trade: Dict) -> Trade:
"""Convert IndependentReserve raw data to a trade
https://www.independentreserve.com/products/api#GetClosedFilledOrders
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
log.debug(f'Processing raw IndependentReserve trade: {raw_trade}')
trade_type = TradeType.BUY if 'Bid' in raw_trade[
'OrderType'] else TradeType.SELL
base_asset = independentreserve_asset(raw_trade['PrimaryCurrencyCode'])
quote_asset = independentreserve_asset(raw_trade['SecondaryCurrencyCode'])
amount = FVal(raw_trade['Volume']) - FVal(raw_trade['Outstanding'])
timestamp = deserialize_timestamp_from_date(
date=raw_trade['CreatedTimestampUtc'],
formatstr='iso8601',
location='IndependentReserve',
)
rate = Price(FVal(raw_trade['AvgPrice']))
fee_amount = FVal(raw_trade['FeePercent']) * amount
fee_asset = base_asset
return Trade(
timestamp=timestamp,
location=Location.INDEPENDENTRESERVE,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(amount),
rate=rate,
fee=Fee(fee_amount),
fee_currency=fee_asset,
link=str(raw_trade['OrderGuid']),
)
def mock_exchange_data_in_db(exchange_locations, rotki) -> None:
db = rotki.data.db
for exchange_location in exchange_locations:
db.add_trades([
Trade(
timestamp=Timestamp(1),
location=exchange_location,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_ETH,
link='foo',
notes='boo',
)
])
db.update_used_query_range(
name=f'{str(exchange_location)}_trades_{str(exchange_location)}',
start_ts=0,
end_ts=9999) # noqa: E501
db.update_used_query_range(
name=f'{str(exchange_location)}_margins_{str(exchange_location)}',
start_ts=0,
end_ts=9999) # noqa: E501
db.update_used_query_range(
name=
f'{str(exchange_location)}_asset_movements_{str(exchange_location)}',
start_ts=0,
end_ts=9999) # noqa: E501
def deserialize_fee(fee: Optional[str]) -> Fee:
"""Deserializes a fee from a json entry. Fee in the JSON entry can also be null
in which case a ZERO fee is returned.
Can throw DeserializationError if the fee is not as expected
"""
if fee is None:
return Fee(ZERO)
try:
result = Fee(FVal(fee))
except ValueError as e:
raise DeserializationError(
f'Failed to deserialize a fee entry due to: {str(e)}') from e
return result
def test_deserialize_v1_trade(mock_kucoin):
raw_result = {
'id': 'xxxx',
'symbol': 'NANO-ETH',
'dealPrice': '0.015743',
'dealValue': '0.00003441',
'amount': '0.002186',
'fee': '0.00000003',
'side': 'sell',
'createdAt': 1520471876,
}
expected_trade = Trade(
timestamp=Timestamp(1520471876),
location=Location.KUCOIN,
base_asset=A_NANO,
quote_asset=A_ETH,
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.002186')),
rate=Price(FVal('0.015743')),
fee=Fee(FVal('0.00000003')),
fee_currency=A_ETH,
link='xxxx',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.OLD_TRADES,
)
assert trade == expected_trade
def process_entry(
self,
db: DBHandler,
db_ledger: DBLedgerActions,
timestamp: Timestamp,
data: BinanceCsvRow,
) -> None:
amount = data['Change']
asset = data['Coin']
category = AssetMovementCategory.DEPOSIT if data['Operation'] == 'Deposit' else AssetMovementCategory.WITHDRAWAL # noqa: E501
if category == AssetMovementCategory.WITHDRAWAL:
amount = -amount
asset_movement = AssetMovement(
location=Location.BINANCE,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=AssetAmount(amount),
fee=Fee(ZERO),
fee_asset=A_USD,
link=f'Imported from binance CSV file. Binance operation: {data["Operation"]}',
)
db.add_asset_movements([asset_movement])
def test_int_overflow_at_tuple_insertion(database, caplog):
"""Test that if somehow an int that will overflow makes it there we handle it
Related: https://github.com/rotki/rotki/issues/2175
"""
caplog.set_level(logging.INFO)
database.add_asset_movements([
AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.DEPOSIT,
timestamp=177778,
address='0xfoo',
transaction_id=99999999999999999999999999999999999999999,
asset=A_BTC,
amount=FVal(1),
fee_asset=A_BTC,
fee=Fee(FVal('0.0001')),
link='a link',
)
])
errors = database.msg_aggregator.consume_errors()
assert len(errors) == 1
assert 'Failed to add "asset_movement" to the DB with overflow error' in errors[
0]
assert 'Overflow error while trying to add "asset_movement" tuples to the DB. Tuples:' in caplog.text # noqa: E501
def test_get_associated_locations(
rotkehlchen_api_server_with_exchanges,
added_exchanges,
ethereum_accounts, # pylint: disable=unused-argument
start_with_valid_premium, # pylint: disable=unused-argument
):
rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
mock_exchange_data_in_db(added_exchanges, rotki)
db = rotki.data.db
db.add_trades([
Trade(
timestamp=Timestamp(1595833195),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_EUR,
link='',
notes='',
)
])
# get locations
response = requests.get(
api_url_for(
rotkehlchen_api_server_with_exchanges,
'associatedlocations',
), )
result = assert_proper_response_with_result(response)
assert set(result) == {'nexo', 'binance', 'poloniex'}
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from bittrex and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'COMPLETED':
# Don't mind failed/in progress asset movements
return None
if 'source' in raw_data:
category = AssetMovementCategory.DEPOSIT
fee = Fee(ZERO)
else:
category = AssetMovementCategory.WITHDRAWAL
fee = deserialize_fee(raw_data.get('txCost', 0))
timestamp = deserialize_timestamp_from_date(
date=raw_data['completedAt'], # we only check completed orders
formatstr='iso8601',
location='bittrex',
)
asset = asset_from_bittrex(raw_data['currencySymbol'])
return AssetMovement(
location=Location.BITTREX,
category=category,
address=deserialize_asset_movement_address(
raw_data, 'cryptoAddress', asset),
transaction_id=get_key_if_has_val(raw_data, 'txId'),
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
raw_data['quantity']),
fee_asset=asset,
fee=fee,
link=str(raw_data.get('txId', '')),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found bittrex deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a bittrex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bittrex '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def _asset_movement_from_independentreserve(
raw_tx: Dict) -> Optional[AssetMovement]:
"""Convert IndependentReserve raw data to an AssetMovement
https://www.independentreserve.com/products/api#GetTransactions
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
log.debug(f'Processing raw IndependentReserve transaction: {raw_tx}')
movement_type = deserialize_asset_movement_category(raw_tx['Type'])
asset = independentreserve_asset(raw_tx['CurrencyCode'])
bitcoin_tx_id = raw_tx.get('BitcoinTransactionId')
eth_tx_id = raw_tx.get('EthereumTransactionId')
if asset == A_BTC and bitcoin_tx_id is not None:
transaction_id = raw_tx['BitcoinTransactionId']
elif eth_tx_id is not None:
transaction_id = eth_tx_id
else:
transaction_id = None
timestamp = deserialize_timestamp_from_date(
date=raw_tx['CreatedTimestampUtc'],
formatstr='iso8601',
location='IndependentReserve',
)
comment = raw_tx.get('Comment')
address = None
if comment is not None and comment.startswith('Withdrawing to'):
address = comment.rsplit()[-1]
raw_amount = raw_tx.get(
'Credit'
) if movement_type == AssetMovementCategory.DEPOSIT else raw_tx.get(
'Debit') # noqa: E501
if raw_amount is None: # skip
return None # Can end up being None for some things like this: 'Comment': 'Initial balance after Bitcoin fork' # noqa: E501
amount = deserialize_asset_amount(raw_amount)
return AssetMovement(
location=Location.INDEPENDENTRESERVE,
category=movement_type,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset, # whatever -- no fee
fee=Fee(ZERO), # we can't get fee from this exchange
link=raw_tx['CreatedTimestampUtc'] + str(amount) + str(movement_type) +
asset.identifier,
)
def _deserialize_trade(self, raw_result: List[Any]) -> Trade:
"""Process a trade result from Bitfinex and deserialize it.
The base and quote assets are instantiated using the `fee_currency_symbol`
(from raw_result[10]) over the pair (from raw_result[1]).
Known pairs format: 'tETHUST', 'tETH:UST'.
Can raise:
- DeserializationError.
- UnknownAsset
- UnsupportedAsset
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-trades
"""
amount = deserialize_asset_amount(raw_result[4])
trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL
bfx_pair = self._process_bfx_pair(raw_result[1])
if bfx_pair not in self.pair_bfx_symbols_map:
raise DeserializationError(
f'Could not deserialize bitfinex trade pair {raw_result[1]}. '
f'Raw trade: {raw_result}', )
bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[
bfx_pair]
base_asset = asset_from_bitfinex(
bitfinex_name=bfx_base_asset_symbol,
currency_map=self.currency_map,
)
quote_asset = asset_from_bitfinex(
bitfinex_name=bfx_quote_asset_symbol,
currency_map=self.currency_map,
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[10],
currency_map=self.currency_map,
)
trade = Trade(
timestamp=Timestamp(int(raw_result[2] / 1000)),
location=Location.BITFINEX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=AssetAmount(abs(amount)),
rate=deserialize_price(raw_result[5]),
fee=Fee(abs(deserialize_fee(raw_result[9]))),
fee_currency=fee_asset,
link=str(raw_result[0]),
notes='',
)
return trade
def _deserialize_asset_movement(self,
raw_result: List[Any]) -> AssetMovement:
"""Process an asset movement (i.e. deposit or withdrawal) from Bitfinex
and deserialize it.
Bitfinex support confirmed the following in regards to
DESTINATION_ADDRESS and TRANSACTION_ID:
- Fiat movement: both attributes won't show any value.
- Cryptocurrency movement: address and tx id on the blockchain.
Timestamp is from MTS_STARTED (when the movement was created), and not
from MTS_UPDATED (when it was completed/cancelled).
Can raise:
- DeserializationError.
- UnknownAsset
- UnsupportedAsset
Schema reference in:
https://docs.bitfinex.com/reference#rest-auth-movements
"""
if raw_result[9] != 'COMPLETED':
raise DeserializationError(
f'Unexpected bitfinex movement with status: {raw_result[5]}. '
f'Only completed movements are processed. Raw movement: {raw_result}',
)
fee_asset = asset_from_bitfinex(
bitfinex_name=raw_result[1],
currency_map=self.currency_map,
)
amount = deserialize_asset_amount(raw_result[12])
category = (AssetMovementCategory.DEPOSIT
if amount > ZERO else AssetMovementCategory.WITHDRAWAL)
address = None
transaction_id = None
if fee_asset.is_fiat() is False:
address = str(raw_result[16])
transaction_id = str(raw_result[20])
asset_movement = AssetMovement(
timestamp=Timestamp(int(raw_result[5] / 1000)),
location=Location.BITFINEX,
category=category,
address=address,
transaction_id=transaction_id,
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=Fee(abs(deserialize_fee(raw_result[13]))),
link=str(raw_result[0]),
)
return asset_movement
def test_deserialize_asset_movement_withdrawal(mock_bitfinex):
"""Test also both 'address' and 'transaction_id' are None for fiat
movements.
"""
mock_bitfinex.currency_map = {}
raw_result = [
13105603,
'EUR',
'Euro',
None,
None,
1569348774000,
1569348774000,
None,
None,
'COMPLETED',
None,
None,
-0.26300954,
-0.00135,
None,
None,
'DESTINATION_ADDRESS',
None,
None,
None,
'TRANSACTION_ID',
None,
]
fee_asset = A_EUR
expected_asset_movement = AssetMovement(
timestamp=Timestamp(1569348774),
location=Location.BITFINEX,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
asset=fee_asset,
amount=FVal('0.26300954'),
fee_asset=fee_asset,
fee=Fee(FVal('0.00135')),
link=str(13105603),
)
asset_movement = mock_bitfinex._deserialize_asset_movement(
raw_result=raw_result)
assert asset_movement == expected_asset_movement
def test_deserialize_asset_movement_deposit(mock_bitfinex):
mock_bitfinex.currency_map = {'WBT': 'WBTC'}
raw_result = [
13105603,
'WBT',
'Wrapped Bitcoin',
None,
None,
1569348774000,
1569348774000,
None,
None,
'COMPLETED',
None,
None,
0.26300954,
-0.00135,
None,
None,
'DESTINATION_ADDRESS',
None,
None,
None,
'TRANSACTION_ID',
None,
]
fee_asset = A_WBTC
expected_asset_movement = AssetMovement(
timestamp=Timestamp(1569348774),
location=Location.BITFINEX,
category=AssetMovementCategory.DEPOSIT,
address='DESTINATION_ADDRESS',
transaction_id='TRANSACTION_ID',
asset=fee_asset,
amount=FVal('0.26300954'),
fee_asset=fee_asset,
fee=Fee(FVal('0.00135')),
link=str(13105603),
)
asset_movement = mock_bitfinex._deserialize_asset_movement(
raw_result=raw_result)
assert asset_movement == expected_asset_movement
def test_deserialize_v2_trade_buy(mock_kucoin):
raw_result = {
'symbol': 'KCS-USDT',
'tradeId': '601da9faf1297d0007efd712',
'orderId': '601da9fa0c92050006bd83be',
'counterOrderId': '601bad620c9205000642300f',
'side': 'buy',
'liquidity': 'taker',
'forceTaker': True,
'price': 1000,
'size': '0.2',
'funds': 200,
'fee': '0.14',
'feeRate': '0.0007',
'feeCurrency': 'USDT',
'stop': '',
'tradeType': 'TRADE',
'type': 'market',
'createdAt': 1612556794259,
}
expected_trade = Trade(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
base_asset=A_KCS,
quote_asset=A_USDT,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.2')),
rate=Price(FVal('1000')),
fee=Fee(FVal('0.14')),
fee_currency=A_USDT,
link='601da9faf1297d0007efd712',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.TRADES,
)
assert trade == expected_trade
def test_deserialize_v2_trade_sell(mock_kucoin):
raw_result = {
'symbol': 'BCHSV-USDT',
'tradeId': '601da995e0ee8b00063a075c',
'orderId': '601da9950c92050006bd45c5',
'counterOrderId': '601da9950c92050006bd457d',
'side': 'sell',
'liquidity': 'taker',
'forceTaker': True,
'price': '37624.4',
'size': '0.0013',
'funds': '48.91172',
'fee': '0.034238204',
'feeRate': '0.0007',
'feeCurrency': 'USDT',
'stop': '',
'tradeType': 'TRADE',
'type': 'market',
'createdAt': 1612556794259,
}
expected_trade = Trade(
timestamp=Timestamp(1612556794),
location=Location.KUCOIN,
base_asset=A_BSV,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=A_USDT,
link='601da995e0ee8b00063a075c',
notes='',
)
trade = mock_kucoin._deserialize_trade(
raw_result=raw_result,
case=KucoinCase.TRADES,
)
assert trade == expected_trade
def test_add_asset_movements(data_dir, username, caplog):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xfoo',
transaction_id='0xboo',
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xcoo',
transaction_id='0xdoo',
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_asset_movements([movement2, movement3])
assert ('Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d'
'67ff44a32d56cef25840b3f5568e23c037fae0a') in caplog.text
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2, movement3]
def test_add_trades(data_dir, username, caplog):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(),
has_premium=True)
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_trades([trade2, trade3])
assert 'Did not add "buy trade with id a1ed19c8284940b4e59bdac941db2fd3c0ed004ddb10fdd3b9ef0a3a9b2c97bc' in caplog.text # noqa: E501
returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(),
has_premium=True)
assert returned_trades == [trade1, trade2, trade3]
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SDC,
quote_asset=A_SDT2,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {
A_USD, A_ETH, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH
} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 0
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {msg}',
)
return None
def process_trades(
db: DBHandler,
timestamp: Timestamp,
data: List[BinanceCsvRow],
) -> List[Trade]:
"""Processes multiple rows data and stores it into rotki's trades
Each row has format: {'Operation': ..., 'Change': ..., 'Coin': ...}
Change is amount, Coin is asset
If amount is negative then this asset is sold, otherwise it's bought
"""
# Because we can get mixed data (e.g. multiple Buys or Sells on a single timestamp) we need
# to group it somehow. We are doing it by grouping the highest bought with the highest
# sold value. We query usd equivalent for each amount because different Sells / Buys
# may use different assets.
# If we query price for the first time it can take long, so we would like to avoid it,
# and therefore we check if all Buys / Sells use the same asset.
# If so, we can group by original amount.
# Checking assets
same_assets = True
assets: Dict[str, Optional[Asset]] = defaultdict(lambda: None)
for row in data:
if row['Operation'] == 'Fee':
cur_operation = 'Fee'
elif row['Change'] < 0:
cur_operation = 'Sold'
else:
cur_operation = 'Bought'
assets[cur_operation] = assets[cur_operation] or row['Coin']
if assets[cur_operation] != row['Coin']:
same_assets = False
break
# Querying usd value if needed
if same_assets is False:
for row in data:
try:
price = PriceHistorian.query_historical_price(
from_asset=row['Coin'],
to_asset=A_USD,
timestamp=timestamp,
)
except NoPriceForGivenTimestamp:
# If we can't find price we can't group, so we quit the method
log.warning(f'Couldn\'t find price of {row["Coin"]} on {timestamp}')
return []
row['usd_value'] = row['Change'] * price
# Group rows depending on whether they are fee or not and then sort them by amount
rows_grouped_by_fee: Dict[bool, List[BinanceCsvRow]] = defaultdict(list)
for row in data:
is_fee = row['Operation'] == 'Fee'
rows_grouped_by_fee[is_fee].append(row)
for rows_group in rows_grouped_by_fee.values():
rows_group.sort(key=lambda x: x['Change'] if same_assets else x['usd_value'], reverse=True) # noqa: E501
# Grouping by combining the highest sold with the highest bought and the highest fee
# Using fee only we were provided with fee (checking by "True in rows_by_operation")
grouped_trade_rows = []
while len(rows_grouped_by_fee[False]) > 0:
cur_batch = [rows_grouped_by_fee[False].pop(), rows_grouped_by_fee[False].pop(0)]
if True in rows_grouped_by_fee:
cur_batch.append(rows_grouped_by_fee[True].pop())
grouped_trade_rows.append(cur_batch)
# Creating trades structures based on grouped rows data
raw_trades: List[Trade] = []
for trade_rows in grouped_trade_rows:
to_asset: Optional[Asset] = None
to_amount: Optional[AssetAmount] = None
from_asset: Optional[Asset] = None
from_amount: Optional[AssetAmount] = None
fee_asset: Optional[Asset] = None
fee_amount: Optional[Fee] = None
trade_type: Optional[TradeType] = None
for row in trade_rows:
cur_asset = row['Coin']
amount = row['Change']
if row['Operation'] == 'Fee':
fee_asset = cur_asset
fee_amount = Fee(amount)
else:
trade_type = TradeType.SELL if row['Operation'] == 'Sell' else TradeType.BUY # noqa: E501
if amount < 0:
from_asset = cur_asset
from_amount = AssetAmount(-amount)
else:
to_asset = cur_asset
to_amount = amount
# Validate that we have received proper assets and amounts.
# There can be no fee, so we don't validate it
if (
to_asset is None or from_asset is None or trade_type is None or
to_amount is None or to_amount == ZERO or
from_amount is None or from_amount == ZERO
):
log.warning(
f'Skipped binance rows {data} because '
f'it didn\'t have enough data',
)
db.msg_aggregator.add_warning('Skipped some rows because couldn\'t find amounts or it was zero') # noqa: E501
continue
rate = to_amount / from_amount
trade = Trade(
timestamp=timestamp,
location=Location.BINANCE,
trade_type=trade_type,
base_asset=to_asset,
quote_asset=from_asset,
amount=to_amount,
rate=Price(rate),
fee_currency=fee_asset,
fee=fee_amount,
link='',
notes='Imported from binance CSV file. Binance operation: Buy / Sell',
)
raw_trades.append(trade)
# Sometimes we can get absolutely identical trades (including timestamp) but the database
# allows us to add only one of them. So we combine these trades into a huge single trade
# First step: group trades
grouped_trades: Dict[TradeID, List[Trade]] = defaultdict(list)
for trade in raw_trades:
grouped_trades[trade.identifier].append(trade)
# Second step: combine them
unique_trades = []
for trades_group in grouped_trades.values():
result_trade = trades_group[0]
for trade in trades_group[1:]:
result_trade.amount = AssetAmount(result_trade.amount + trade.amount) # noqa: E501
if result_trade.fee is not None and trade.fee is not None:
result_trade.fee = Fee(result_trade.fee + trade.fee)
unique_trades.append(result_trade)
return unique_trades
def test_query_asset_movements_sandbox(
sandbox_kuckoin,
inquirer, # pylint: disable=unused-argument
):
"""Unfortunately the sandbox environment does not support deposits and
withdrawals, therefore they must be mocked.
Below a list of the movements and their timestamps in ascending mode:
Deposits:
- deposit 1 - deposit: 1612556651
- deposit 2 - deposit: 1612556652
- deposit 3 - deposit: 1612556653 -> skipped, inner deposit
Withdrawals:
- withdraw 1: 1612556651 -> skipped, inner withdraw
- withdraw 2: 1612556652
- withdraw 3: 1612556656 -> never requested
By requesting trades from 1612556651 to 1612556654 and patching the time
step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements.
"""
deposits_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1,
"fee":0.0001,
"currency":"KCS",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5a",
"status":"SUCCESS",
"remark":"movement 2 - deposit",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1000,
"fee":0.01,
"currency":"LINK",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5b@test",
"status":"SUCCESS",
"remark":"movement 1 - deposit",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
deposits_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":1,
"fee":0.1,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 4 - deposit",
"createdAt":1612556653000,
"updatedAt":1612556653000
}
]
}
}
""")
withdrawals_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a4",
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 4 - withdraw",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"id":"5c2dc64e03aa675aa263f1a3",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9",
"memo":"",
"currency":"ETH",
"amount":1,
"fee":0.01,
"walletTxId":"3e2414d82acce78d38be7fe9",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 3 - withdraw",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
withdrawals_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":0,
"pageSize":0,
"totalNum":0,
"totalPage":0,
"items":[]
}
}
""")
withdrawals_response_3 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a5",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00",
"memo":"",
"currency":"KCS",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 5 - withdraw",
"createdAt":1612556655000,
"updatedAt":1612556655000
}
]
}
}
""")
expected_asset_movements = [
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556652),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5a',
asset=A_KCS,
amount=AssetAmount(FVal('1')),
fee_asset=A_KCS,
fee=Fee(FVal('0.0001')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556651),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5b',
asset=A_LINK,
amount=AssetAmount(FVal('1000')),
fee_asset=A_LINK,
fee=Fee(FVal('0.01')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612556652),
address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt',
transaction_id=
'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4',
asset=A_BSV,
amount=AssetAmount(FVal('2.5')),
fee_asset=A_BSV,
fee=Fee(FVal('0.25')),
link='5c2dc64e03aa675aa263f1a4',
),
]
def get_endpoints_response():
results = [
f'{deposits_response_1}',
f'{deposits_response_2}',
f'{withdrawals_response_1}',
f'{withdrawals_response_2}',
# if pagination works as expected and the requesting loop is broken,
# the response below won't be processed
f'{withdrawals_response_3}',
]
for result_ in results:
yield result_
def mock_api_query_response(case, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due to not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to '
f'inability to match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def test_query_trades_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument
"""The sandbox account has 6 trades. Below a list of the trades and their
timestamps in ascending mode.
- trade 1: 1612556651 -> skipped
- trade 2: 1612556693
- trade 3: 1612556765
- trade 4: 1612556765
- trade 5: 1612556765
- trade 6: 1612556794 -> skipped
By requesting trades from 1612556693 to 1612556765, the first and last trade
should be skipped.
"""
expected_trades = [
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02934995')),
rate=Price(FVal('0.046058')),
fee=Fee(FVal('9.4625999797E-7')),
fee_currency=A_BTC,
link='601da9ddf73c300006194ec6',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02')),
rate=Price(FVal('0.04561')),
fee=Fee(FVal('6.3854E-7')),
fee_currency=A_BTC,
link='601da9ddf73c300006194ec5',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.06')),
rate=Price(FVal('0.0456')),
fee=Fee(FVal('0.0000019152')),
fee_currency=A_BTC,
link='601da9ddf73c300006194ec4',
notes='',
),
Trade(
timestamp=Timestamp(1612556693),
location=Location.KUCOIN,
base_asset=A_BTC,
quote_asset=A_USDT,
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=A_USDT,
link='601da995e0ee8b00063a075c',
notes='',
),
]
trades, _ = sandbox_kuckoin.query_online_trade_history(
start_ts=Timestamp(1612556693),
end_ts=Timestamp(1612556765),
)
assert trades == expected_trades