def query_deposits_withdrawals(
self,
start_ts: typing.Timestamp,
end_ts: typing.Timestamp,
end_at_least_ts: typing.Timestamp,
) -> List:
# TODO: Implement cache like in other exchange calls
try:
resp = self._api_query_list('get', 'user/walletHistory')
except RemoteError as e:
msg = (
'Bitmex API request failed. Could not reach bitmex due '
'to {}'.format(e)
)
log.error(msg)
return list()
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
transaction_type = movement['transactType']
if transaction_type not in ('Deposit', 'Withdrawal'):
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = FVal(movement['amount'])
fee = ZERO
if movement['fee'] is not None:
fee = FVal(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == 'BTC':
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(AssetMovement(
exchange='bitmex',
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(fee),
))
return movements
def get_bitcoin_addresses_balances(
accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
"""Queries blockchain.info or blockstream for the balances of accounts
May raise:
- RemotError if there is a problem querying blockchain.info or blockstream
"""
source = 'blockchain.info'
balances: Dict[BTCAddress, FVal] = {}
try:
if _have_bc1_accounts(accounts):
# if 1 account is bech32 we have to query blockstream. blockchaininfo won't work
source = 'blockstream'
balances = {}
for account in accounts:
url = f'https://blockstream.info/api/address/{account}'
response_data = request_get_dict(url=url,
handle_429=True,
backoff_in_seconds=4)
stats = response_data['chain_stats']
balance = int(stats['funded_txo_sum']) - int(
stats['spent_txo_sum'])
balances[account] = satoshis_to_btc(balance)
else:
# split the list of accounts into sublists of 80 addresses per list to overcome:
# https://github.com/rotki/rotki/issues/3037
accounts_chunks = [
accounts[x:x + 80] for x in range(0, len(accounts), 80)
]
for accounts_chunk in accounts_chunks:
params = '|'.join(accounts_chunk)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
for entry in btc_resp['addresses']:
balances[entry['address']] = satoshis_to_btc(
FVal(entry['final_balance']))
except (
requests.exceptions.RequestException,
UnableToDecryptRemoteData,
requests.exceptions.Timeout,
) as e:
raise RemoteError(
f'bitcoin external API request for balances failed due to {str(e)}'
) from e # noqa: E501
except KeyError as e:
raise RemoteError(
f'Malformed response when querying bitcoin blockchain via {source}.'
f'Did not find key {e}', ) from e
return balances
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition:
"""Turn a bitmex trade returned from bitmex trade history to our common trade
history format. This only returns margin positions as bitmex only deals in
margin trading"""
close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime'])
profit_loss = AssetAmount(satoshis_to_btc(FVal(bitmex_trade['amount'])))
currency = bitmex_to_world(bitmex_trade['currency'])
fee = deserialize_fee(bitmex_trade['fee'])
notes = bitmex_trade['address']
assert currency == A_BTC, 'Bitmex trade should only deal in BTC'
log.debug(
'Processing Bitmex Trade',
sensitive_log=True,
timestamp=close_time,
profit_loss=profit_loss,
currency=currency,
fee=fee,
notes=notes,
)
return MarginPosition(
location=Location.BITMEX,
open_time=None,
close_time=close_time,
profit_loss=profit_loss,
pl_currency=currency,
fee=fee,
fee_currency=A_BTC,
notes=notes,
link=str(bitmex_trade['transactID']),
)
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition:
"""Turn a bitmex trade returned from bitmex trade history to our common trade
history format. This only returns margin positions as bitmex only deals in
margin trading"""
close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime'])
profit_loss = satoshis_to_btc(FVal(bitmex_trade['amount']))
currency = bitmex_to_world(bitmex_trade['currency'])
notes = bitmex_trade['address']
assert currency == 'BTC', 'Bitmex trade should only deal in BTC'
log.debug(
'Processing Bitmex Trade',
sensitive_log=True,
timestamp=close_time,
profit_loss=profit_loss,
currency=currency,
notes=notes,
)
return MarginPosition(
exchange='bitmex',
open_time=None,
close_time=close_time,
profit_loss=profit_loss,
pl_currency=A_BTC,
notes=notes,
)
def query_balances(self) -> Tuple[Optional[dict], str]:
try:
resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'})
except RemoteError as e:
msg = (
'Bitmex API request failed. Could not reach bitmex due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
# Bitmex shows only BTC balance
returned_balances = dict()
usd_price = Inquirer().find_usd_price(A_BTC)
# result is in satoshis
amount = satoshis_to_btc(FVal(resp['amount']))
usd_value = amount * usd_price
returned_balances[A_BTC] = dict(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Bitmex balance query result',
sensitive_log=True,
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def query_balances(self) -> Tuple[Optional[dict], str]:
try:
resp = self._api_query_dict('get', 'user/wallet',
{'currency': 'XBt'})
# Bitmex shows only BTC balance
returned_balances = {}
usd_price = Inquirer().find_usd_price(A_BTC)
except RemoteError as e:
msg = f'Bitmex API request failed due to: {str(e)}'
log.error(msg)
return None, msg
# result is in satoshis
amount = satoshis_to_btc(FVal(resp['amount']))
usd_value = amount * usd_price
returned_balances[A_BTC] = {
'amount': amount,
'usd_value': usd_value,
}
log.debug(
'Bitmex balance query result',
sensitive_log=True,
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def query_balances(self) -> ExchangeQueryBalances:
returned_balances: Dict[Asset, Balance] = {}
try:
resp = self._api_query_dict('get', 'user/wallet',
{'currency': 'XBt'})
# Bitmex shows only BTC balance
usd_price = Inquirer().find_usd_price(A_BTC)
except RemoteError as e:
msg = f'Bitmex API request failed due to: {str(e)}'
log.error(msg)
return None, msg
# result is in satoshis
try:
amount = satoshis_to_btc(deserialize_asset_amount(resp['amount']))
except DeserializationError as e:
msg = f'Bitmex API request failed. Failed to deserialized amount due to {str(e)}'
log.error(msg)
return None, msg
usd_value = amount * usd_price
returned_balances[A_BTC] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Bitmex balance query result',
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def query_btc_account_balance(account: BTCAddress) -> FVal:
"""Queries blockchain.info for the balance of account
May raise:
- InputError if the given account is not a valid BTC address
- RemotError if there is a problem querying blockchain.info
"""
try:
btc_resp = request_get_direct(
url='https://blockchain.info/q/addressbalance/%s' % account,
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
except InvalidBTCAddress:
# TODO: Move this validation into our own code and before the balance query
raise InputError(
f'The given string {account} is not a valid BTC address')
except (requests.exceptions.ConnectionError,
UnableToDecryptRemoteData) as e:
raise RemoteError(
f'blockchain.info API request failed due to {str(e)}')
return satoshis_to_btc(FVal(btc_resp)) # result is in satoshis
def query_btc_account_balance(account: BTCAddress) -> FVal:
"""Queries blockchain.info for the balance of account
May raise:
- RemotError if there is a problem querying blockchain.info or blockcypher
"""
try:
if account.lower()[0:3] == 'bc1':
url = f'https://api.blockcypher.com/v1/btc/main/addrs/{account.lower()}/balance'
response_data = request_get(url=url)
if 'balance' not in response_data:
raise RemoteError(f'Unexpected blockcypher balance response: {response_data}')
btc_resp = response_data['balance']
else:
btc_resp = request_get_direct(
url='https://blockchain.info/q/addressbalance/%s' % account,
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e:
raise RemoteError(f'bitcoin external API request failed due to {str(e)}')
return satoshis_to_btc(FVal(btc_resp)) # result is in satoshis
def _check_blockcypher_for_transactions(
accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
have_transactions = {}
new_accounts = _prepare_blockcypher_accounts(accounts)
# blockcypher's batching takes up as many api queries as the batch,
# and the api rate limit is 3 requests per second. So we should make
# sure each batch is of max size 3
# https://www.blockcypher.com/dev/bitcoin/#batching
batches = [new_accounts[x:x + 3] for x in range(0, len(new_accounts), 3)]
total_idx = 0
for batch in batches:
params = ';'.join(batch)
url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
response_data = request_get(url=url,
handle_429=True,
backoff_in_seconds=4)
if isinstance(response_data, dict):
# If only one account was requested put it in a list so the
# rest of the code works
response_data = [response_data]
for idx, entry in enumerate(response_data):
balance = satoshis_to_btc(FVal(entry['final_balance']))
# we don't use the returned address as it may be lowercased
have_transactions[accounts[total_idx +
idx]] = (entry['final_n_tx'] != 0,
balance)
total_idx += len(batch)
return have_transactions
def assert_btc_balances_result(
result: Dict[str, Any],
btc_accounts: List[str],
btc_balances: List[str],
also_eth: bool,
) -> None:
"""Asserts for correct BTC blockchain balances when mocked in tests"""
per_account = result['per_account']
if also_eth:
assert len(per_account) == 2
else:
assert len(per_account) == 1
per_account = per_account['BTC']
assert len(
per_account
) == 1 # make sure we only have standalone accounts in these tests
standalone = per_account['standalone']
msg = 'standalone results num does not match number of btc accounts'
assert len(standalone) == len(btc_accounts), msg
msg = 'given balances and accounts should have same length'
assert len(btc_accounts) == len(btc_balances), msg
for idx, account in enumerate(btc_accounts):
balance = satoshis_to_btc(FVal(btc_balances[idx]))
assert FVal(standalone[account]['amount']) == balance
if balance == ZERO:
assert FVal(standalone[account]['usd_value']) == ZERO
else:
assert FVal(standalone[account]['usd_value']) > ZERO
if 'assets' in result['totals']:
totals = result['totals']['assets']
else:
totals = result['totals']
if also_eth:
assert len(totals) >= 2 # ETH and any other tokens that may exist
else:
assert len(totals) == 1
expected_btc_total = sum(
satoshis_to_btc(FVal(balance)) for balance in btc_balances)
assert FVal(totals['BTC']['amount']) == expected_btc_total
if expected_btc_total == ZERO:
assert FVal(totals['BTC']['usd_value']) == ZERO
else:
assert FVal(totals['BTC']['usd_value']) > ZERO
def _check_blockstream_for_transactions(
accounts: List[BTCAddress],
) -> Dict[BTCAddress, Tuple[bool, FVal]]:
"""May raise connection errors or KeyError"""
have_transactions = {}
for account in accounts:
url = f'https://blockstream.info/api/address/{account}'
response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4)
stats = response_data['chain_stats']
balance = satoshis_to_btc(int(stats['funded_txo_sum']) - int(stats['spent_txo_sum']))
have_txs = stats['tx_count'] != 0
have_transactions[account] = (have_txs, balance)
return have_transactions
def _check_blockchaininfo_for_transactions(
accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]:
have_transactions = {}
params = '|'.join(accounts)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.infoq/
backoff_in_seconds=15,
)
for idx, entry in enumerate(btc_resp['addresses']):
balance = satoshis_to_btc(entry['final_balance'])
have_transactions[accounts[idx]] = (entry['n_tx'] != 0, balance)
return have_transactions
def query_balances(self) -> Tuple[Optional[dict], str]:
resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'})
# Bitmex shows only BTC balance
returned_balances = dict()
usd_price = Inquirer().find_usd_price(A_BTC)
# result is in satoshis
amount = satoshis_to_btc(FVal(resp['amount']))
usd_value = amount * usd_price
returned_balances[A_BTC] = dict(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Bitmex balance query result',
sensitive_log=True,
currency='BTC',
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a bitmex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {str(e)}', )
continue
return movements
def query_btc_accounts_balances(
accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]:
"""Queries blockchain.info for the balance of account
May raise:
- RemotError if there is a problem querying blockchain.info or blockcypher
"""
source = 'blockchain.info'
balances = {}
try:
if any(account.lower()[0:3] == 'bc1' for account in accounts):
# if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work
source = 'blockcypher.com'
# the bech32 accounts have to be given lowercase to the
# blockcypher query. No idea why.
new_accounts = []
for x in accounts:
lowered = x.lower()
if lowered[0:3] == 'bc1':
new_accounts.append(lowered)
else:
new_accounts.append(x)
# blockcypher's batching takes up as many api queries as the batch,
# and the api rate limit is 3 requests per second. So we should make
# sure each batch is of max size 3
# https://www.blockcypher.com/dev/bitcoin/#batching
batches = [
new_accounts[x:x + 3]
for x in range(0, len(new_accounts), 3)
]
total_idx = 0
for batch in batches:
params = ';'.join(batch)
url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance'
response_data = request_get(url=url,
handle_429=True,
backoff_in_seconds=4)
if isinstance(response_data, dict):
# If only one account was requested put it in a list so the
# rest of the code works
response_data = [response_data]
for idx, entry in enumerate(response_data):
# we don't use the returned address as it may be lowercased
balances[accounts[total_idx + idx]] = satoshis_to_btc(
FVal(entry['final_balance']), )
total_idx += len(batch)
else:
params = '|'.join(accounts)
btc_resp = request_get_dict(
url=f'https://blockchain.info/multiaddr?active={params}',
handle_429=True,
# If we get a 429 then their docs suggest 10 seconds
# https://blockchain.info/q
backoff_in_seconds=10,
)
for idx, entry in enumerate(btc_resp['addresses']):
balances[accounts[idx]] = satoshis_to_btc(
FVal(entry['final_balance']))
except (requests.exceptions.ConnectionError,
UnableToDecryptRemoteData) as e:
raise RemoteError(
f'bitcoin external API request failed due to {str(e)}')
except KeyError as e:
raise RemoteError(
f'Malformed response when querying bitcoin blockchain via {source}.'
f'Did not find key {e}', )
return balances
