def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
self.query_kraken_ledgers(start_ts=start_ts, end_ts=end_ts)
filter_query = HistoryEventFilterQuery.make(
from_ts=Timestamp(start_ts),
to_ts=Timestamp(end_ts),
event_types=[
HistoryEventType.DEPOSIT,
HistoryEventType.WITHDRAWAL,
],
location=Location.KRAKEN,
location_label=self.name,
)
events = self.history_events_db.get_history_events(
filter_query=filter_query,
has_premium=True,
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(events))
movements = []
get_attr = operator.attrgetter('event_identifier')
# Create a list of lists where each sublist has the events for the same event identifier
grouped_events = [
list(g) for k, g in itertools.groupby(sorted(events, key=get_attr),
get_attr)
] # noqa: E501
for movement_events in grouped_events:
if len(movement_events) == 2:
if movement_events[0].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[0].balance.amount)
movement = movement_events[1]
elif movement_events[
1].event_subtype == HistoryEventSubType.FEE:
fee = Fee(movement_events[1].balance.amount)
movement = movement_events[0]
else:
self.msg_aggregator.add_error(
f'Failed to process deposit/withdrawal. {grouped_events}. Ignoring ...',
)
continue
else:
movement = movement_events[0]
fee = Fee(ZERO)
amount = movement.balance.amount
if movement.event_type == HistoryEventType.WITHDRAWAL:
amount = amount * -1
try:
asset = movement.asset
movement_type = movement.event_type
movements.append(
AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(
movement_type),
timestamp=ts_ms_to_sec(movement.timestamp),
address=None, # no data from kraken ledger endpoint
transaction_id=
None, # no data from kraken ledger endpoint
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=movement.event_identifier,
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement_events,
error=msg,
)
continue
return movements
def _deserialize_asset_movement(
self,
raw_movement: Dict[str, Any],
) -> AssetMovement:
"""Process a deposit/withdrawal user transaction from Bitstamp and
deserialize it.
Can raise DeserializationError.
From Bitstamp documentation, deposits/withdrawals can have a fee
(the amount is expected to be in the currency involved)
https://www.bitstamp.net/fee-schedule/
Bitstamp support confirmed the following withdrawal JSON:
{
"fee": "0.00050000",
"btc_usd": "0.00",
"datetime": "2020-12-04 09:30:00.000000",
"usd": "0.0",
"btc": "-0.50000000",
"type": "1",
"id": 123456789,
"eur": "0.0"
}
NB: any asset key not related with the pair is discarded (e.g. 'eur').
"""
type_ = raw_movement['type']
category: AssetMovementCategory
if type_ == 0:
category = AssetMovementCategory.DEPOSIT
elif type_ == 1:
category = AssetMovementCategory.WITHDRAWAL
else:
raise AssertionError(f'Unexpected Bitstamp asset movement case: {type_}.')
timestamp = deserialize_timestamp_from_bitstamp_date(raw_movement['datetime'])
trade_pair_data = self._get_trade_pair_data_from_transaction(raw_movement)
base_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.base_asset_symbol],
)
quote_asset_amount = deserialize_asset_amount(
raw_movement[trade_pair_data.quote_asset_symbol],
)
amount: FVal
fee_asset: Asset
if base_asset_amount != ZERO and quote_asset_amount == ZERO:
amount = base_asset_amount
fee_asset = trade_pair_data.base_asset
elif base_asset_amount == ZERO and quote_asset_amount != ZERO:
amount = quote_asset_amount
fee_asset = trade_pair_data.quote_asset
else:
raise DeserializationError(
'Could not deserialize Bitstamp asset movement from user transaction. '
f'Unexpected asset amount combination found in: {raw_movement}.',
)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.BITSTAMP,
category=category,
address=None, # requires query "crypto_transactions" endpoint
transaction_id=None, # requires query "crypto_transactions" endpoint
asset=fee_asset,
amount=abs(amount),
fee_asset=fee_asset,
fee=deserialize_fee(raw_movement['fee']),
link=str(raw_movement['id']),
)
return asset_movement
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
transaction_id = get_key_if_has_val(movement_data, 'txid')
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
split = movement_data['status'].split(':')
if len(split) != 2:
transaction_id = None
else:
transaction_id = split[1].lstrip()
if transaction_id == '':
transaction_id = None
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
address=deserialize_asset_movement_address(movement_data, 'address', asset),
transaction_id=transaction_id,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount_force_positive(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a poloniex '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def assert_cryptocom_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from crypto.com"""
trades = rotki.data.db.get_trades()
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
def get_trade_note(desc: str):
return f'{desc}\nSource: crypto.com (CSV import)'
expected_trades = [Trade(
timestamp=Timestamp(1595833195),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Buy ETH'),
), Trade(
timestamp=Timestamp(1596014214),
location=Location.CRYPTOCOM,
pair=TradePair('MCO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Buy MCO'),
), Trade(
timestamp=Timestamp(1596014223),
location=Location.CRYPTOCOM,
pair=TradePair('MCO_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('12.32402069')),
rate=Price(FVal('4.057117499045678736198226879')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Sign-up Bonus Unlocked'),
), Trade(
timestamp=Timestamp(1596209827),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.14445954600007045')),
rate=Price(FVal('85.28339137929999991192917299')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO -> ETH'),
), Trade(
timestamp=Timestamp(1596429934),
location=Location.CRYPTOCOM,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00061475')),
rate=Price(FVal('309.0687271248474989833265555')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Crypto Earn'),
), Trade(
timestamp=Timestamp(1596465565),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1382.306147552291')),
rate=Price(FVal('27.6439')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO/CRO Overall Swap'),
), Trade(
timestamp=Timestamp(1596730165),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_MCO'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1301.64')),
rate=Price(FVal('26.0328')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('MCO/CRO Overall Swap'),
), Trade(
timestamp=Timestamp(1599934176),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('138.256')),
rate=Price(FVal('0.1429232727693553986807082514')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Card Rebate: Deliveries'),
), Trade(
timestamp=Timestamp(1602515376),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('52.151')),
rate=Price(FVal('0.06692105616383194953116910510')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Card Cashback'),
), Trade(
timestamp=Timestamp(1602526176),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('482.2566417')),
rate=Price(FVal('0.08756748243245604635910191136')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Referral Bonus Reward'),
), Trade(
timestamp=Timestamp(1606833565),
location=Location.CRYPTOCOM,
pair=TradePair('CRO_DAI'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.007231228760408149')),
rate=Price(FVal('14.26830000900286970270179629')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes=get_trade_note('Convert Dust'),
)]
assert expected_trades == trades
expected_movements = [AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1596992965),
address=None,
transaction_id=None,
asset=A_DAI,
amount=AssetAmount(FVal('115')),
fee_asset=A_DAI,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.CRYPTOCOM,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1596993025),
asset=A_DAI,
amount=AssetAmount(FVal('115')),
fee_asset=A_DAI,
fee=Fee(ZERO),
link='',
)]
assert expected_movements == asset_movements
def test_coinbase_query_deposit_withdrawals(function_scope_coinbase):
"""Test that coinbase deposit/withdrawals history query works fine for the happy path"""
coinbase = function_scope_coinbase
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
movements = coinbase.query_online_deposits_withdrawals(
start_ts=0,
end_ts=1576726126,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(movements) == 3
expected_movements = [
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.DEPOSIT,
timestamp=1519001640,
asset=A_USD,
amount=FVal('55'),
fee_asset=A_USD,
fee=FVal('0.05'),
link='1130eaec-07d7-54c4-a72c-2e92826897df',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1485895742,
asset=A_USD,
amount=FVal('10.0'),
fee_asset=A_USD,
fee=FVal('0.01'),
link='146eaec-07d7-54c4-a72c-2e92826897df',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1566726126,
asset=A_ETH,
amount=FVal('0.05770427'),
fee_asset=A_ETH,
fee=FVal('0.00021'),
link='id1',
)
]
assert expected_movements == movements
# and now try to query within a specific range
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
movements = coinbase.query_online_deposits_withdrawals(
start_ts=0,
end_ts=1519001650,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(movements) == 2
assert movements[0].category == AssetMovementCategory.DEPOSIT
assert movements[0].timestamp == 1519001640
assert movements[1].category == AssetMovementCategory.WITHDRAWAL
assert movements[1].timestamp == 1485895742
def _deserialize_asset_movement(
self, raw_data: Dict[str, Any]) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from coinbase and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] != 'completed':
return None
payout_date = raw_data.get('payout_at', None)
if payout_date:
timestamp = deserialize_timestamp_from_date(
payout_date, 'iso8601', 'coinbase')
else:
timestamp = deserialize_timestamp_from_date(
raw_data['created_at'],
'iso8601',
'coinbase',
)
# Only get address/transaction id for "send" type of transactions
address = None
transaction_id = None
# movement_category: Union[Literal['deposit'], Literal['withdrawal']]
if 'type' in raw_data:
# Then this should be a "send" which is the way Coinbase uses to send
# crypto outside of the exchange
# https://developers.coinbase.com/api/v2?python#transaction-resource
msg = 'Non "send" type found in coinbase deposit/withdrawal processing'
assert raw_data['type'] == 'send', msg
movement_category = AssetMovementCategory.WITHDRAWAL
# Can't see the fee being charged from the "send" resource
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
# Fees dont appear in the docs but from an experiment of sending ETH
# to an address from coinbase there is the network fee in the response
fee = Fee(ZERO)
raw_network = raw_data.get('network', None)
if raw_network:
raw_fee = raw_network.get('transaction_fee', None)
if raw_fee:
# Since this is a withdrawal the fee should be the same as the moved asset
if asset != asset_from_coinbase(raw_fee['currency'],
time=timestamp):
# If not we set ZERO fee and ignore
log.error(
f'In a coinbase withdrawal of {asset.identifier} the fee'
f'is denoted in {raw_fee["currency"]}', )
else:
fee = deserialize_fee(raw_fee['amount'])
if 'network' in raw_data:
transaction_id = get_key_if_has_val(
raw_data['network'], 'hash')
if 'to' in raw_data:
address = deserialize_asset_movement_address(
raw_data['to'], 'address', asset)
else:
movement_category = deserialize_asset_movement_category(
raw_data['resource'])
amount = deserialize_asset_amount_force_positive(
raw_data['amount']['amount'])
fee = deserialize_fee(raw_data['fee']['amount'])
asset = asset_from_coinbase(raw_data['amount']['currency'],
time=timestamp)
return AssetMovement(
location=Location.COINBASE,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of a coinbase '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of coinbase '
f'asset_movement {raw_data}. Error was: {str(e)}', )
return None
def test_query_asset_movements_sandbox(
sandbox_kuckoin,
inquirer, # pylint: disable=unused-argument
):
"""Unfortunately the sandbox environment does not support deposits and
withdrawals, therefore they must be mocked.
Below a list of the movements and their timestamps in ascending mode:
Deposits:
- deposit 1 - deposit: 1612556651
- deposit 2 - deposit: 1612556652
- deposit 3 - deposit: 1612556653 -> skipped, inner deposit
Withdrawals:
- withdraw 1: 1612556651 -> skipped, inner withdraw
- withdraw 2: 1612556652
- withdraw 3: 1612556656 -> never requested
By requesting trades from 1612556651 to 1612556654 and patching the time
step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements.
"""
deposits_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1,
"fee":0.0001,
"currency":"KCS",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5a",
"status":"SUCCESS",
"remark":"movement 2 - deposit",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1000,
"fee":0.01,
"currency":"LINK",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5b@test",
"status":"SUCCESS",
"remark":"movement 1 - deposit",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
deposits_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":1,
"fee":0.1,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 4 - deposit",
"createdAt":1612556653000,
"updatedAt":1612556653000
}
]
}
}
""")
withdrawals_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a4",
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 4 - withdraw",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"id":"5c2dc64e03aa675aa263f1a3",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9",
"memo":"",
"currency":"ETH",
"amount":1,
"fee":0.01,
"walletTxId":"3e2414d82acce78d38be7fe9",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 3 - withdraw",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
withdrawals_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":0,
"pageSize":0,
"totalNum":0,
"totalPage":0,
"items":[]
}
}
""")
withdrawals_response_3 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a5",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00",
"memo":"",
"currency":"KCS",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 5 - withdraw",
"createdAt":1612556655000,
"updatedAt":1612556655000
}
]
}
}
""")
expected_asset_movements = [
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556652),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5a',
asset=A_KCS,
amount=AssetAmount(FVal('1')),
fee_asset=A_KCS,
fee=Fee(FVal('0.0001')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556651),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5b',
asset=A_LINK,
amount=AssetAmount(FVal('1000')),
fee_asset=A_LINK,
fee=Fee(FVal('0.01')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612556652),
address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt',
transaction_id=
'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4',
asset=A_BSV,
amount=AssetAmount(FVal('2.5')),
fee_asset=A_BSV,
fee=Fee(FVal('0.25')),
link='5c2dc64e03aa675aa263f1a4',
),
]
def get_endpoints_response():
results = [
f'{deposits_response_1}',
f'{deposits_response_2}',
f'{withdrawals_response_1}',
f'{withdrawals_response_2}',
# if pagination works as expected and the requesting loop is broken,
# the response below won't be processed
f'{withdrawals_response_3}',
]
for result_ in results:
yield result_
def mock_api_query_response(case, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
def test_coinbase_query_deposit_withdrawals(function_scope_coinbase):
"""Test that coinbase deposit/withdrawals history query works fine for the happy path"""
coinbase = function_scope_coinbase
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
movements = coinbase.query_online_deposits_withdrawals(
start_ts=0,
end_ts=1566726126,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(movements) == 5
expected_movements = [
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.DEPOSIT,
timestamp=1519001640,
address=None,
transaction_id=None,
asset=A_USD,
amount=FVal('55.00'),
fee_asset=A_USD,
fee=FVal('0.05'),
link='1130eaec-07d7-54c4-a72c-2e92826897df',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=1485895742,
asset=A_USD,
amount=FVal('10.00'),
fee_asset=A_USD,
fee=FVal('0.01'),
link='146eaec-07d7-54c4-a72c-2e92826897df',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
address='0x6dcD6449dbCa615e40d696328209686eA95327b2',
transaction_id=
'0x558bfa4d2a4ef598ddb92233459c00eda9e6c14cda75e6773b90208cb6938169',
timestamp=1566726126,
asset=A_ETH,
amount=FVal('0.05770427'),
fee_asset=A_ETH,
fee=FVal('0.00021'),
link='https://etherscan.io/tx/bbb',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
address='0x6dcD6449dbCa615e40d696328209686eA95327b2',
transaction_id=None,
timestamp=1566726126,
asset=A_ETH,
amount=FVal('0.05770427'),
fee_asset=A_ETH,
fee=ZERO,
link='id2',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id='ccc',
timestamp=1502554304,
asset=A_BTC,
amount=FVal('0.10181673'),
fee_asset=A_BTC,
fee=ZERO,
link='https://blockchain.info/tx/ccc',
)
]
assert expected_movements == movements
# and now try to query within a specific range
with patch.object(coinbase.session,
'get',
side_effect=mock_normal_coinbase_query):
movements = coinbase.query_online_deposits_withdrawals(
start_ts=0,
end_ts=1519001640,
)
warnings = coinbase.msg_aggregator.consume_warnings()
errors = coinbase.msg_aggregator.consume_errors()
assert len(warnings) == 0
assert len(errors) == 0
assert len(movements) == 3
assert movements[0].category == AssetMovementCategory.DEPOSIT
assert movements[0].timestamp == 1519001640
assert movements[1].category == AssetMovementCategory.WITHDRAWAL
assert movements[1].timestamp == 1485895742
assert movements[2].category == AssetMovementCategory.DEPOSIT
assert movements[2].timestamp == 1502554304
def _consume_blockfi_entry(self, csv_row: Dict[str, Any]) -> None:
"""
Process entry for BlockFi transaction history. Trades for this file are ignored
and istead should be extracted from the file containing only trades.
This method can raise:
- UnsupportedBlockFiEntry
- UnknownAsset
- DeserializationError
"""
if len(csv_row['Confirmed At']) != 0:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Confirmed At'],
formatstr='%Y-%m-%d %H:%M:%S',
location='BlockFi',
)
else:
log.debug(f'Ignoring unconfirmed BlockFi entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Cryptocurrency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Transaction Type']
# BlockFI doesn't provide information about fees
fee = Fee(ZERO)
fee_asset = A_USD # Can be whatever
if entry_type in ('Deposit', 'Wire Deposit', 'ACH Deposit'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'Wire Withdrawal', 'ACH Withdrawal'):
asset_movement = AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest Payment', 'Bonus Payment',
'Referral Bonus'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from BlockFi',
)
self.db_ledger.add_ledger_action(action)
elif entry_type == 'Trade':
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
def _consume_cointracking_entry(self, csv_row: List[str]):
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row[1] # Type
timestamp = deserialize_timestamp_from_date(
date=csv_row[9],
formatstr='%d.%m.%Y %H:%M',
location='cointracking.info',
)
notes = csv_row[8]
location = exchange_row_to_location(csv_row[6])
if row_type == 'Gift/Tip' or row_type == 'Trade':
base_asset = Asset(csv_row[3])
quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
if not quote_asset and row_type != 'Gift/Tip':
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row[2])
if csv_row[4] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row[4])
else:
quote_amount_sold = ZERO
rate = quote_amount_sold / base_amount_bought
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=Fee(
ZERO), # There are no fees when import from cointracking
fee_currency=base_asset,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row[2])
asset = Asset(csv_row[3])
else:
amount = deserialize_asset_amount(csv_row[4])
asset = Asset(csv_row[5])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=Fee(ZERO),
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def test_bitmex_api_withdrawals_deposit_and_query_after_subquery(sandbox_bitmex):
"""Test the happy case of bitmex withdrawals deposit query
This test also tests an important case where a subquery for a an in-between
time range is done first and then an encompassing range is requested. And
we test that the full query, queries the remaining timestamp ranges.
"""
# This is an initial subquery of a small range where no deposit happened.
result = sandbox_bitmex.query_deposits_withdrawals(
start_ts=1536492800,
end_ts=1536492976,
only_cache=False,
)
assert len(result) == 0
# Now after the subquery we test that the exchange engine logic properly
# queries the required start/end timestamp ranges
now = ts_now()
result = sandbox_bitmex.query_deposits_withdrawals(
start_ts=0,
end_ts=now,
only_cache=False,
)
expected_result = [
AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1536486278,
asset=A_BTC,
amount=FVal('0.46966992'),
fee_asset=A_BTC,
fee=FVal(0),
link='166b9aac-70ac-cedc-69a0-dbd12c0661bf',
),
AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1537014656,
asset=A_BTC,
amount=FVal(0.16960386),
fee_asset=A_BTC,
fee=FVal(0E-8),
link='b6c6fd2c-4d0c-b101-a41c-fa5aa1ce7ef1',
),
AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1536563759,
asset=A_BTC,
amount=FVal('0.38474377'),
fee_asset=A_BTC,
fee=FVal(0),
link='72500751-d052-5bbb-18d7-08363edef812',
),
AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.WITHDRAWAL,
address='mv4rnyY3Su5gjcDNzbMLKBQkBicCtHUtFB',
transaction_id=None,
timestamp=1536536707,
asset=A_BTC,
amount=FVal('0.00700000'),
fee_asset=A_BTC,
fee=FVal('0.00300000'),
link='bf19ca4e-e084-11f9-12cd-6ae41e26f9db',
),
]
assert result == expected_result
# also make sure that asset movements contain Asset and not strings
for movement in result:
assert isinstance(movement.asset, Asset)
def assert_nexo_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from nexo"""
ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator)
ledger_actions = ledger_db.get_ledger_actions(None, None, None)
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_actions = [LedgerAction(
identifier=3,
timestamp=Timestamp(1565888464),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('22.5653042')),
asset=symbol_to_asset_or_token('NEXO'),
rate=None,
rate_asset=None,
link='NXT0000000009',
notes='Dividend from Nexo',
), LedgerAction(
identifier=2,
timestamp=Timestamp(1597492915),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('10.3585507')),
asset=symbol_to_asset_or_token('NEXO'),
rate=None,
rate_asset=None,
link='NXT0000000007',
notes='Dividend from Nexo',
), LedgerAction(
identifier=1,
timestamp=Timestamp(1614993620),
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=AssetAmount(FVal('1')),
asset=symbol_to_asset_or_token('USDC'),
rate=None,
rate_asset=None,
link='NXT0000000002',
notes='Interest from Nexo',
)]
expected_movements = [AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1556116964),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('1')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000013',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1556122699),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('0.9995')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000012',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1558720210),
address=None,
transaction_id=None,
asset=symbol_to_asset_or_token('NEXO'),
amount=AssetAmount(FVal('1.00001')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000011',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1565912821),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('10000')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000010',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1608131364),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('2000.79')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000005',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1614366540),
address=None,
transaction_id=None,
asset=A_EUR,
amount=AssetAmount(FVal('10')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000003',
), AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1615024314),
address=None,
transaction_id=None,
asset=symbol_to_asset_or_token('USDC'),
amount=AssetAmount(FVal('1')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='NXT0000000001',
)]
assert ledger_actions == expected_actions
assert asset_movements == expected_movements
def assert_blockfi_transactions_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from blockfi"""
ledger_db = DBLedgerActions(rotki.data.db, rotki.msg_aggregator)
ledger_actions = ledger_db.get_ledger_actions(None, None, None)
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_errors()
assert len(errors) == 0
assert len(warnings) == 0
expected_actions = [LedgerAction(
identifier=3,
timestamp=Timestamp(1600293599),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.48385358')),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='Bonus Payment from BlockFi',
), LedgerAction(
identifier=2,
timestamp=Timestamp(1606953599),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.00052383')),
asset=A_BTC,
rate=None,
rate_asset=None,
link=None,
notes='Referral Bonus from BlockFi',
), LedgerAction(
identifier=1,
timestamp=Timestamp(1612051199),
action_type=LedgerActionType.INCOME,
location=Location.BLOCKFI,
amount=AssetAmount(FVal('0.56469042')),
asset=A_ETH,
rate=None,
rate_asset=None,
link=None,
notes='Interest Payment from BlockFi',
)]
assert expected_actions == ledger_actions
expected_movements = [AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1595247055),
address=None,
transaction_id=None,
asset=A_BTC,
amount=AssetAmount(FVal('1.11415058')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.BLOCKFI,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1605977971),
asset=A_ETH,
amount=AssetAmount(FVal('3')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
)]
assert expected_movements == asset_movements
def test_add_asset_movements(data_dir, username):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xfoo',
transaction_id='0xboo',
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xcoo',
transaction_id='0xdoo',
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_asset_movements([movement2, movement3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2, movement3]
def assert_poloniex_asset_movements(
to_check_list: List[Any],
deserialized: bool,
movements_to_check: Optional[Tuple[int, ...]] = None,
) -> None:
expected = [
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id=
'0xbd4da74e1a0b81c21d056c6f58a5b306de85d21ddf89992693b812bb117eace4',
timestamp=Timestamp(1468994442),
asset=A_ETH,
amount=FVal('10.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0.1')),
link='2',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id=
'2d27ae26fa9c70d6709e27ac94d4ce2fde19b3986926e9f3bfcf3e2d68354ec5',
timestamp=Timestamp(1458994442),
asset=A_BTC,
amount=FVal('5.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0.5')),
link='1',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='131rdg5Rzn6BFufnnQaHhVa5ZtRU1J2EZR',
transaction_id=
'b05bdec7430a56b5a5ed34af4a31a54859dda9b7c88a5586bc5d6540cdfbfc7a',
timestamp=Timestamp(1448994442),
asset=A_BTC,
amount=FVal('50.0'),
fee_asset=A_BTC,
fee=Fee(FVal('0')),
link='1',
),
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
address='0xB7E033598Cb94EF5A35349316D3A2e4f95f308Da',
transaction_id=
'0xf7e7eeb44edcad14c0f90a5fffb1cbb4b80e8f9652124a0838f6906ca939ccd2',
timestamp=Timestamp(1438994442),
asset=A_ETH,
amount=FVal('100.0'),
fee_asset=A_ETH,
fee=Fee(FVal('0')),
link='2',
)
]
assert_asset_movements(expected, to_check_list, deserialized,
movements_to_check)
def _consume_nexo(self, csv_row: Dict[str, Any]) -> None:
"""
Consume CSV file from NEXO.
This method can raise:
- UnsupportedNexoEntry
- UnknownAsset
- DeserializationError
"""
ignored_entries = ('ExchangeToWithdraw', 'DepositToExchange')
if 'rejected' not in csv_row['Details']:
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date / Time'],
formatstr='%Y-%m-%d %H:%M',
location='NEXO',
)
else:
log.debug(f'Ignoring rejected nexo entry {csv_row}')
return
asset = symbol_to_asset_or_token(csv_row['Currency'])
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
entry_type = csv_row['Type']
transaction = csv_row['Transaction']
if entry_type in ('Deposit', 'ExchangeDepositedOn',
'LockingTermDeposit'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type in ('Withdrawal', 'WithdrawExchanged'):
asset_movement = AssetMovement(
location=Location.NEXO,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(ZERO),
fee_asset=A_USD,
link=transaction,
)
self.db.add_asset_movements([asset_movement])
elif entry_type == 'Withdrawal Fee':
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.EXPENSE,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=None,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ('Interest', 'Bonus', 'Dividend'):
action = LedgerAction(
identifier=0, # whatever is not used at insertion
timestamp=timestamp,
action_type=LedgerActionType.INCOME,
location=Location.NEXO,
amount=amount,
asset=asset,
rate=None,
rate_asset=None,
link=transaction,
notes=f'{entry_type} from Nexo',
)
self.db_ledger.add_ledger_action(action)
elif entry_type in ignored_entries:
pass
else:
raise UnsupportedCSVEntry(
f'Unsuported entry {entry_type}. Data: {csv_row}')
'close': '2018-03-03 23:05:04',
'currency': 'DASH', # cryptocompare hourly DASH/EUR: 475.565
'fee': '0.0001',
'earned': '0.0025',
'amount': '2',
},
]
asset_movements_list = [
AssetMovement(
# before query period -- 8.915 * 0.001 = 8.915e-3
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1479510304), # 18/11/2016,
asset=A_ETH, # cryptocompare hourly ETH/EUR: 8.915
amount=FVal('95'),
fee_asset=A_ETH,
fee=Fee(FVal('0.001')),
link='krakenid1',
),
AssetMovement( # 0.0087*52.885 = 0.4600995
location=Location.KRAKEN,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1493291104), # 27/04/2017,
asset=A_ETH, # cryptocompare hourly ETH/EUR: 52.885
amount=FVal('125'),
fee_asset=A_ETH,
def test_gemini_query_deposits_withdrawals(sandbox_gemini):
"""Test that querying the asset movements endpoint works correctly
Since Gemini sandbox does not support transfers, this uses a mocked call.
"""
transfers_patch = mock_gemini_transfers(sandbox_gemini, requests.post)
with transfers_patch:
movements = sandbox_gemini.query_deposits_withdrawals(
start_ts=0,
end_ts=Timestamp(1584881354),
)
assert len(movements) == 6
expected_movements = [AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1507913541),
asset=A_USD,
amount=FVal('36'),
fee_asset=A_USD,
fee=ZERO,
link='320013281',
), AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1499990797),
asset=A_ETH,
amount=FVal('100'),
fee_asset=A_ETH,
fee=ZERO,
link='309356152',
), AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1495550176),
asset=A_BTC,
amount=FVal('1500'),
fee_asset=A_BTC,
fee=ZERO,
link='298112782',
), AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1458862076),
asset=A_USD,
amount=FVal('500'),
fee_asset=A_USD,
fee=ZERO,
link='265799530',
), AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1450403787),
asset=A_BTC,
amount=FVal('5'),
fee_asset=A_BTC,
fee=ZERO,
link='82897811',
), AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1535451930),
asset=A_USD,
amount=FVal('1'),
fee_asset=A_USD,
fee=ZERO,
link='341167014',
)]
# The deposits should be returned with the oldest first (so given list is reversed)
assert movements == expected_movements[::-1]
def test_query_asset_movements_over_limit(
rotkehlchen_api_server_with_exchanges,
start_with_valid_premium,
):
"""Test that using the asset movements query endpoint works fine"""
start_ts = 0
end_ts = 1598453214
server = rotkehlchen_api_server_with_exchanges
rotki = server.rest_api.rotkehlchen
# Make sure online kraken is not queried by setting query ranges
rotki.data.db.update_used_query_range(
name='kraken_asset_movements',
start_ts=start_ts,
end_ts=end_ts,
)
polo_entries_num = 4
# Set a ton of kraken asset movements in the DB
kraken_entries_num = FREE_ASSET_MOVEMENTS_LIMIT + 50
movements = [
AssetMovement(location=Location.KRAKEN,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=x,
asset=A_BTC,
amount=FVal(x * 100),
fee_asset=A_BTC,
fee=FVal(x),
link='') for x in range(kraken_entries_num)
]
rotki.data.db.add_asset_movements(movements)
all_movements_num = kraken_entries_num + polo_entries_num
setup = prepare_rotki_for_history_processing_test(
server.rest_api.rotkehlchen)
# Check that querying movements with/without limits works even if we query two times
for _ in range(2):
# query asset movements of polo which has less movements than the limit
with setup.polo_patch:
response = requests.get(
api_url_for(
server,
"assetmovementsresource",
),
json={'location': 'poloniex'},
)
result = assert_proper_response_with_result(response)
assert result['entries_found'] == all_movements_num
assert result[
'entries_limit'] == -1 if start_with_valid_premium else FREE_ASSET_MOVEMENTS_LIMIT # noqa: E501
assert_poloniex_asset_movements(
[x['entry'] for x in result['entries']], deserialized=True)
# now query kraken which has a ton of DB entries
response = requests.get(
api_url_for(server, "assetmovementsresource"),
json={'location': 'kraken'},
)
result = assert_proper_response_with_result(response)
if start_with_valid_premium:
assert len(result['entries']) == kraken_entries_num
assert result['entries_limit'] == -1
assert result['entries_found'] == all_movements_num
else:
assert len(result['entries']
) == FREE_ASSET_MOVEMENTS_LIMIT - polo_entries_num
assert result['entries_limit'] == FREE_ASSET_MOVEMENTS_LIMIT
assert result['entries_found'] == all_movements_num
amount = deserialize_asset_amount(raw_result['amount'])
fee = deserialize_fee(raw_result['fee'])
fee_currency_symbol = raw_result['currency']
link_id = raw_result.get('id',
'') # NB: id only exists for withdrawals
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
fee_asset = asset_from_kucoin(fee_currency_symbol)
asset_movement = AssetMovement(
timestamp=timestamp,
location=Location.KUCOIN,
category=category,
address=address,
transaction_id=transaction_id,
asset=fee_asset,
amount=amount,
fee_asset=fee_asset,
fee=fee,
link=link_id,
)
return asset_movement
@staticmethod
def _deserialize_trade(
raw_result: Dict[str, Any],
case: Literal[KucoinCase.TRADES, KucoinCase.OLD_TRADES],
) -> Trade:
"""Process a trade result and deserialize it
For the old v1 trades look here:
def assert_cointracking_import_results(rotki: Rotkehlchen):
"""A utility function to help assert on correctness of importing data from cointracking.info"""
trades = rotki.data.db.get_trades()
asset_movements = rotki.data.db.get_asset_movements()
warnings = rotki.msg_aggregator.consume_warnings()
errors = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 3
expected_trades = [Trade(
timestamp=Timestamp(1566687719),
location=Location.COINBASE,
pair=TradePair('ETH_EUR'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.05772716')),
rate=Price(FVal('190.3783245183029963712055123')),
fee=Fee(FVal("0.02")),
fee_currency=A_EUR,
link='',
notes='',
), Trade(
timestamp=Timestamp(1567418410),
location=Location.EXTERNAL,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00100000')),
rate=Price(ZERO),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='Just a small gift from someone',
), Trade(
timestamp=Timestamp(1567504805),
location=Location.EXTERNAL,
pair=TradePair('ETH_USD'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('2')),
rate=Price(ZERO),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='Sign up bonus',
)]
assert expected_trades == trades
expected_movements = [AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1565848624),
address=None,
transaction_id=None,
asset=A_XMR,
amount=AssetAmount(FVal('5')),
fee_asset=A_USD,
fee=Fee(ZERO),
link='',
), AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
address=None,
transaction_id=None,
timestamp=Timestamp(1566726155),
asset=A_ETH,
amount=AssetAmount(FVal('0.05770427')),
fee_asset=A_ETH,
fee=Fee(FVal("0.0001")),
link='',
)]
assert expected_movements == asset_movements
def test_cointracking_data_import(rotkehlchen_server):
dir_path = os.path.dirname(os.path.realpath(__file__))
filepath = os.path.join(dir_path, 'data', 'cointracking_trades_list.csv')
# Check that for unknown source we get an error
response = rotkehlchen_server.import_data_from(source='other_source',
filepath=filepath)
assert response['result'] is False
assert 'unknown location' in response['message']
# Check that the test cointracking data are imported succesfully
rotkehlchen_server.import_data_from(source='cointracking_info',
filepath=filepath)
trades = rotkehlchen_server.rotkehlchen.data.db.get_trades()
asset_movements = rotkehlchen_server.rotkehlchen.data.db.get_asset_movements(
)
warnings = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
errors = rotkehlchen_server.rotkehlchen.msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 3
expected_trades = [
Trade(
timestamp=1566687660,
location=Location.COINBASE,
pair='ETH_EUR',
trade_type=TradeType.BUY,
amount=FVal('0.05772716'),
rate=FVal('190.3783245183029963712055123'),
fee=ZERO,
fee_currency=A_ETH,
link='',
notes='',
),
Trade(
timestamp=1567418400,
location=Location.EXTERNAL,
pair='BTC_USD',
trade_type=TradeType.BUY,
amount=FVal('0.00100000'),
rate=ZERO,
fee=ZERO,
fee_currency=A_BTC,
link='',
notes='Just a small gift from someone',
)
]
assert expected_trades == trades
expected_movements = [
AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.DEPOSIT,
timestamp=1565848620,
asset=A_XMR,
amount=FVal('5'),
fee_asset=A_XMR,
fee=ZERO,
link='',
),
AssetMovement(
location=Location.COINBASE,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=1566726120,
asset=A_ETH,
amount=FVal('0.05770427'),
fee_asset=A_ETH,
fee=ZERO,
link='',
)
]
assert expected_movements == asset_movements
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a bitmex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {str(e)}', )
continue
return movements
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError('Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry',
)
def test_gemini_query_deposits_withdrawals(sandbox_gemini):
"""Test that querying the asset movements endpoint works correctly
Since Gemini sandbox does not support transfers, this uses a mocked call.
"""
transfers_patch = mock_gemini_transfers(sandbox_gemini, requests.post)
with transfers_patch:
movements = sandbox_gemini.query_deposits_withdrawals(
start_ts=0,
end_ts=Timestamp(1584881354),
only_cache=False,
)
assert len(movements) == 6
expected_movements = [
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1507913541),
address=None,
transaction_id=None,
asset=A_USD,
amount=FVal('36'),
fee_asset=A_USD,
fee=ZERO,
link='320013281',
),
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=
'605c5fa8bf99458d24d61e09941bc443ddc44839d9aaa508b14b296c0c8269b2',
timestamp=Timestamp(1499990797),
asset=A_ETH,
amount=FVal('100'),
fee_asset=A_ETH,
fee=ZERO,
link='309356152',
),
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=
'163eeee4741f8962b748289832dd7f27f754d892f5d23bf3ea6fba6e350d9ce3',
timestamp=Timestamp(1495550176),
asset=A_BTC,
amount=FVal('1500'),
fee_asset=A_BTC,
fee=ZERO,
link='298112782',
),
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=Timestamp(1458862076),
asset=A_USD,
amount=FVal('500'),
fee_asset=A_USD,
fee=ZERO,
link='265799530',
),
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.WITHDRAWAL,
address='mqjvCtt4TJfQaC7nUgLMvHwuDPXMTEUGqx',
transaction_id=
'c458b86955b80db0718cfcadbff3df3734a906367982c6eb191e61117b810bbb',
timestamp=Timestamp(1450403787),
asset=A_BTC,
amount=FVal('5'),
fee_asset=A_BTC,
fee=ZERO,
link='82897811',
),
AssetMovement(
location=Location.GEMINI,
category=AssetMovementCategory.WITHDRAWAL,
address='0xd24400ae8BfEBb18cA49Be86258a3C749cf46853',
transaction_id=
'7bffd85893ee8e72e31061a84d25c45f2c4537c2f765a1e79feb06a7294445c3',
timestamp=Timestamp(1535451930),
asset=A_USD,
amount=FVal('1'),
fee_asset=A_USD,
fee=ZERO,
link='341167014',
)
]
# The deposits should be returned with the oldest first (so given list is reversed)
assert movements == expected_movements[::-1]
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries coinbase pro for asset movements"""
log.debug('Query coinbasepro asset movements',
start_ts=start_ts,
end_ts=end_ts)
movements = []
raw_movements = []
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'withdraw'},
):
raw_movements.extend(batch)
for batch in self._paginated_query(
endpoint='transfers',
query_options={'type': 'deposit'},
):
raw_movements.extend(batch)
account_to_currency = self.create_or_return_account_to_currency_map()
for entry in raw_movements:
try:
# Check if the transaction has not been completed. If so it should be skipped
if entry.get('completed_at', None) is None:
log.warning(
f'Skipping coinbase pro deposit/withdrawal '
f'due to not having been completed: {entry}', )
continue
timestamp = coinbasepro_deserialize_timestamp(
entry, 'completed_at')
if timestamp < start_ts or timestamp > end_ts:
continue
category = deserialize_asset_movement_category(entry['type'])
asset = account_to_currency.get(entry['account_id'], None)
if asset is None:
log.warning(
f'Skipping coinbase pro asset_movement {entry} due to '
f'inability to match account id to an asset', )
continue
address = None
transaction_id = None
fee = Fee(ZERO)
if category == AssetMovementCategory.DEPOSIT:
try:
address = entry['details']['crypto_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
except KeyError:
pass
else: # withdrawal
try:
address = entry['details']['sent_to_address']
transaction_id = entry['details'][
'crypto_transaction_hash']
fee = deserialize_fee(entry['details']['fee'])
except KeyError:
pass
if transaction_id and (
asset == A_ETH or asset.asset_type
== AssetType.ETHEREUM_TOKEN): # noqa: E501
transaction_id = '0x' + transaction_id
movements.append(
AssetMovement(
location=Location.COINBASEPRO,
category=category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=deserialize_asset_amount_force_positive(
entry['amount']),
fee_asset=asset,
fee=fee,
link=str(entry['id']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring its deposit/withdrawal.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a Coinbasepro deposit/withdrawal. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro deposit/withdrawal.',
raw_asset_movement=entry,
error=msg,
)
continue
return movements
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError('Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount(csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry',
)
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'deposit'},
)
result.extend(self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict={'type': 'withdrawal'},
))
log.debug('Kraken deposit/withdrawals query result', num_results=len(result))
movements = []
for movement in result:
try:
asset = asset_from_kraken(movement['asset'])
movement_type = movement['type']
if movement_type not in ('deposit', 'withdrawal'):
# Other known types: 'transfer'
continue # Can be for moving funds from spot to stake etc.
movements.append(AssetMovement(
location=Location.KRAKEN,
category=deserialize_asset_movement_category(movement_type),
timestamp=deserialize_timestamp_from_kraken(movement['time']),
address=None, # no data from kraken ledger endpoint
transaction_id=None, # no data from kraken ledger endpoint
asset=asset,
amount=deserialize_asset_amount_force_positive(movement['amount']),
fee_asset=asset,
fee=deserialize_fee(movement['fee']),
link=str(movement['refid']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a kraken deposit/withdrawals. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a kraken deposit/withdrawal.',
raw_asset_movement=movement,
error=msg,
)
continue
return movements
def test_add_asset_movements(data_dir, username, caplog):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xfoo',
transaction_id='0xboo',
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xcoo',
transaction_id='0xdoo',
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_asset_movements([movement2, movement3])
assert ('Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d'
'67ff44a32d56cef25840b3f5568e23c037fae0a') in caplog.text
returned_movements = data.db.get_asset_movements(
filter_query=AssetMovementsFilterQuery.make(),
has_premium=True,
)
assert returned_movements == [movement1, movement2, movement3]
def _deserialize_asset_movement(
self,
raw_data: Dict[str, Any],
movement_type: AssetMovementCategory,
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from FTX and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if raw_data['status'] not in ('complete', 'confirmed'):
return None
timestamp = deserialize_timestamp_from_date(
raw_data['time'], 'iso8601', 'FTX')
amount = deserialize_asset_amount_force_positive(raw_data['size'])
asset = asset_from_ftx(raw_data['coin'])
fee = Fee(ZERO)
movement_category = movement_type
if raw_data.get('fee', None) is not None:
fee = deserialize_fee(raw_data['fee'])
address = raw_data.get('address', None)
if isinstance(address, dict):
address = raw_data['address'].get('address', None)
transaction_id = raw_data.get('txid', None)
return AssetMovement(
location=Location.FTX,
category=movement_category,
address=address,
transaction_id=transaction_id,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(raw_data['id']),
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX deposit/withdrawal with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Unexpected data encountered during deserialization of an FTX '
'asset movement. Check logs for details and open a bug report.',
)
log.error(
'Error processing FTX trade',
trade=raw_data,
error=msg,
)
return None
