def query_deposits_withdrawals( self, start_ts: typing.Timestamp, end_ts: typing.Timestamp, end_at_least_ts: typing.Timestamp, ) -> List: # TODO: Implement cache like in other exchange calls try: resp = self._api_query_list('get', 'user/walletHistory') except RemoteError as e: msg = ( 'Bitmex API request failed. Could not reach bitmex due ' 'to {}'.format(e) ) log.error(msg) return list() log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: transaction_type = movement['transactType'] if transaction_type not in ('Deposit', 'Withdrawal'): continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = FVal(movement['amount']) fee = ZERO if movement['fee'] is not None: fee = FVal(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == 'BTC': # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append(AssetMovement( exchange='bitmex', category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(fee), )) return movements
def get_bitcoin_addresses_balances( accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]: """Queries blockchain.info or blockstream for the balances of accounts May raise: - RemotError if there is a problem querying blockchain.info or blockstream """ source = 'blockchain.info' balances: Dict[BTCAddress, FVal] = {} try: if _have_bc1_accounts(accounts): # if 1 account is bech32 we have to query blockstream. blockchaininfo won't work source = 'blockstream' balances = {} for account in accounts: url = f'https://blockstream.info/api/address/{account}' response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4) stats = response_data['chain_stats'] balance = int(stats['funded_txo_sum']) - int( stats['spent_txo_sum']) balances[account] = satoshis_to_btc(balance) else: # split the list of accounts into sublists of 80 addresses per list to overcome: # https://github.com/rotki/rotki/issues/3037 accounts_chunks = [ accounts[x:x + 80] for x in range(0, len(accounts), 80) ] for accounts_chunk in accounts_chunks: params = '|'.join(accounts_chunk) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) for entry in btc_resp['addresses']: balances[entry['address']] = satoshis_to_btc( FVal(entry['final_balance'])) except ( requests.exceptions.RequestException, UnableToDecryptRemoteData, requests.exceptions.Timeout, ) as e: raise RemoteError( f'bitcoin external API request for balances failed due to {str(e)}' ) from e # noqa: E501 except KeyError as e: raise RemoteError( f'Malformed response when querying bitcoin blockchain via {source}.' f'Did not find key {e}', ) from e return balances
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition: """Turn a bitmex trade returned from bitmex trade history to our common trade history format. This only returns margin positions as bitmex only deals in margin trading""" close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime']) profit_loss = AssetAmount(satoshis_to_btc(FVal(bitmex_trade['amount']))) currency = bitmex_to_world(bitmex_trade['currency']) fee = deserialize_fee(bitmex_trade['fee']) notes = bitmex_trade['address'] assert currency == A_BTC, 'Bitmex trade should only deal in BTC' log.debug( 'Processing Bitmex Trade', sensitive_log=True, timestamp=close_time, profit_loss=profit_loss, currency=currency, fee=fee, notes=notes, ) return MarginPosition( location=Location.BITMEX, open_time=None, close_time=close_time, profit_loss=profit_loss, pl_currency=currency, fee=fee, fee_currency=A_BTC, notes=notes, link=str(bitmex_trade['transactID']), )
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition: """Turn a bitmex trade returned from bitmex trade history to our common trade history format. This only returns margin positions as bitmex only deals in margin trading""" close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime']) profit_loss = satoshis_to_btc(FVal(bitmex_trade['amount'])) currency = bitmex_to_world(bitmex_trade['currency']) notes = bitmex_trade['address'] assert currency == 'BTC', 'Bitmex trade should only deal in BTC' log.debug( 'Processing Bitmex Trade', sensitive_log=True, timestamp=close_time, profit_loss=profit_loss, currency=currency, notes=notes, ) return MarginPosition( exchange='bitmex', open_time=None, close_time=close_time, profit_loss=profit_loss, pl_currency=A_BTC, notes=notes, )
def query_balances(self) -> Tuple[Optional[dict], str]: try: resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'}) except RemoteError as e: msg = ( 'Bitmex API request failed. Could not reach bitmex due ' 'to {}'.format(e) ) log.error(msg) return None, msg # Bitmex shows only BTC balance returned_balances = dict() usd_price = Inquirer().find_usd_price(A_BTC) # result is in satoshis amount = satoshis_to_btc(FVal(resp['amount'])) usd_value = amount * usd_price returned_balances[A_BTC] = dict( amount=amount, usd_value=usd_value, ) log.debug( 'Bitmex balance query result', sensitive_log=True, currency='BTC', amount=amount, usd_value=usd_value, ) return returned_balances, ''
def query_balances(self) -> Tuple[Optional[dict], str]: try: resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'}) # Bitmex shows only BTC balance returned_balances = {} usd_price = Inquirer().find_usd_price(A_BTC) except RemoteError as e: msg = f'Bitmex API request failed due to: {str(e)}' log.error(msg) return None, msg # result is in satoshis amount = satoshis_to_btc(FVal(resp['amount'])) usd_value = amount * usd_price returned_balances[A_BTC] = { 'amount': amount, 'usd_value': usd_value, } log.debug( 'Bitmex balance query result', sensitive_log=True, currency='BTC', amount=amount, usd_value=usd_value, ) return returned_balances, ''
def query_balances(self) -> ExchangeQueryBalances: returned_balances: Dict[Asset, Balance] = {} try: resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'}) # Bitmex shows only BTC balance usd_price = Inquirer().find_usd_price(A_BTC) except RemoteError as e: msg = f'Bitmex API request failed due to: {str(e)}' log.error(msg) return None, msg # result is in satoshis try: amount = satoshis_to_btc(deserialize_asset_amount(resp['amount'])) except DeserializationError as e: msg = f'Bitmex API request failed. Failed to deserialized amount due to {str(e)}' log.error(msg) return None, msg usd_value = amount * usd_price returned_balances[A_BTC] = Balance( amount=amount, usd_value=usd_value, ) log.debug( 'Bitmex balance query result', currency='BTC', amount=amount, usd_value=usd_value, ) return returned_balances, ''
def query_btc_account_balance(account: BTCAddress) -> FVal: """Queries blockchain.info for the balance of account May raise: - InputError if the given account is not a valid BTC address - RemotError if there is a problem querying blockchain.info """ try: btc_resp = request_get_direct( url='https://blockchain.info/q/addressbalance/%s' % account, handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) except InvalidBTCAddress: # TODO: Move this validation into our own code and before the balance query raise InputError( f'The given string {account} is not a valid BTC address') except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e: raise RemoteError( f'blockchain.info API request failed due to {str(e)}') return satoshis_to_btc(FVal(btc_resp)) # result is in satoshis
def query_btc_account_balance(account: BTCAddress) -> FVal: """Queries blockchain.info for the balance of account May raise: - RemotError if there is a problem querying blockchain.info or blockcypher """ try: if account.lower()[0:3] == 'bc1': url = f'https://api.blockcypher.com/v1/btc/main/addrs/{account.lower()}/balance' response_data = request_get(url=url) if 'balance' not in response_data: raise RemoteError(f'Unexpected blockcypher balance response: {response_data}') btc_resp = response_data['balance'] else: btc_resp = request_get_direct( url='https://blockchain.info/q/addressbalance/%s' % account, handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e: raise RemoteError(f'bitcoin external API request failed due to {str(e)}') return satoshis_to_btc(FVal(btc_resp)) # result is in satoshis
def _check_blockcypher_for_transactions( accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]: have_transactions = {} new_accounts = _prepare_blockcypher_accounts(accounts) # blockcypher's batching takes up as many api queries as the batch, # and the api rate limit is 3 requests per second. So we should make # sure each batch is of max size 3 # https://www.blockcypher.com/dev/bitcoin/#batching batches = [new_accounts[x:x + 3] for x in range(0, len(new_accounts), 3)] total_idx = 0 for batch in batches: params = ';'.join(batch) url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance' response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4) if isinstance(response_data, dict): # If only one account was requested put it in a list so the # rest of the code works response_data = [response_data] for idx, entry in enumerate(response_data): balance = satoshis_to_btc(FVal(entry['final_balance'])) # we don't use the returned address as it may be lowercased have_transactions[accounts[total_idx + idx]] = (entry['final_n_tx'] != 0, balance) total_idx += len(batch) return have_transactions
def assert_btc_balances_result( result: Dict[str, Any], btc_accounts: List[str], btc_balances: List[str], also_eth: bool, ) -> None: """Asserts for correct BTC blockchain balances when mocked in tests""" per_account = result['per_account'] if also_eth: assert len(per_account) == 2 else: assert len(per_account) == 1 per_account = per_account['BTC'] assert len( per_account ) == 1 # make sure we only have standalone accounts in these tests standalone = per_account['standalone'] msg = 'standalone results num does not match number of btc accounts' assert len(standalone) == len(btc_accounts), msg msg = 'given balances and accounts should have same length' assert len(btc_accounts) == len(btc_balances), msg for idx, account in enumerate(btc_accounts): balance = satoshis_to_btc(FVal(btc_balances[idx])) assert FVal(standalone[account]['amount']) == balance if balance == ZERO: assert FVal(standalone[account]['usd_value']) == ZERO else: assert FVal(standalone[account]['usd_value']) > ZERO if 'assets' in result['totals']: totals = result['totals']['assets'] else: totals = result['totals'] if also_eth: assert len(totals) >= 2 # ETH and any other tokens that may exist else: assert len(totals) == 1 expected_btc_total = sum( satoshis_to_btc(FVal(balance)) for balance in btc_balances) assert FVal(totals['BTC']['amount']) == expected_btc_total if expected_btc_total == ZERO: assert FVal(totals['BTC']['usd_value']) == ZERO else: assert FVal(totals['BTC']['usd_value']) > ZERO
def _check_blockstream_for_transactions( accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]: """May raise connection errors or KeyError""" have_transactions = {} for account in accounts: url = f'https://blockstream.info/api/address/{account}' response_data = request_get_dict(url=url, handle_429=True, backoff_in_seconds=4) stats = response_data['chain_stats'] balance = satoshis_to_btc(int(stats['funded_txo_sum']) - int(stats['spent_txo_sum'])) have_txs = stats['tx_count'] != 0 have_transactions[account] = (have_txs, balance) return have_transactions
def _check_blockchaininfo_for_transactions( accounts: List[BTCAddress], ) -> Dict[BTCAddress, Tuple[bool, FVal]]: have_transactions = {} params = '|'.join(accounts) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.infoq/ backoff_in_seconds=15, ) for idx, entry in enumerate(btc_resp['addresses']): balance = satoshis_to_btc(entry['final_balance']) have_transactions[accounts[idx]] = (entry['n_tx'] != 0, balance) return have_transactions
def query_balances(self) -> Tuple[Optional[dict], str]: resp = self._api_query_dict('get', 'user/wallet', {'currency': 'XBt'}) # Bitmex shows only BTC balance returned_balances = dict() usd_price = Inquirer().find_usd_price(A_BTC) # result is in satoshis amount = satoshis_to_btc(FVal(resp['amount'])) usd_value = amount * usd_price returned_balances[A_BTC] = dict( amount=amount, usd_value=usd_value, ) log.debug( 'Bitmex balance query result', sensitive_log=True, currency='BTC', amount=amount, usd_value=usd_value, ) return returned_balances, ''
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List: resp = self._api_query_list('get', 'user/walletHistory') log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: try: transaction_type = movement['transactType'] if transaction_type == 'Deposit': transaction_type = AssetMovementCategory.DEPOSIT elif transaction_type == 'Withdrawal': transaction_type = AssetMovementCategory.WITHDRAWAL else: continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == A_BTC: # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append( AssetMovement( location=Location.BITMEX, category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(movement['transactID']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bitmex deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a bitmex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of bitmex ' f'asset_movement {movement}. Error was: {str(e)}', ) continue return movements
def query_btc_accounts_balances( accounts: List[BTCAddress]) -> Dict[BTCAddress, FVal]: """Queries blockchain.info for the balance of account May raise: - RemotError if there is a problem querying blockchain.info or blockcypher """ source = 'blockchain.info' balances = {} try: if any(account.lower()[0:3] == 'bc1' for account in accounts): # if 1 account is bech32 we have to query blockcypher. blockchaininfo won't work source = 'blockcypher.com' # the bech32 accounts have to be given lowercase to the # blockcypher query. No idea why. new_accounts = [] for x in accounts: lowered = x.lower() if lowered[0:3] == 'bc1': new_accounts.append(lowered) else: new_accounts.append(x) # blockcypher's batching takes up as many api queries as the batch, # and the api rate limit is 3 requests per second. So we should make # sure each batch is of max size 3 # https://www.blockcypher.com/dev/bitcoin/#batching batches = [ new_accounts[x:x + 3] for x in range(0, len(new_accounts), 3) ] total_idx = 0 for batch in batches: params = ';'.join(batch) url = f'https://api.blockcypher.com/v1/btc/main/addrs/{params}/balance' response_data = request_get(url=url, handle_429=True, backoff_in_seconds=4) if isinstance(response_data, dict): # If only one account was requested put it in a list so the # rest of the code works response_data = [response_data] for idx, entry in enumerate(response_data): # we don't use the returned address as it may be lowercased balances[accounts[total_idx + idx]] = satoshis_to_btc( FVal(entry['final_balance']), ) total_idx += len(batch) else: params = '|'.join(accounts) btc_resp = request_get_dict( url=f'https://blockchain.info/multiaddr?active={params}', handle_429=True, # If we get a 429 then their docs suggest 10 seconds # https://blockchain.info/q backoff_in_seconds=10, ) for idx, entry in enumerate(btc_resp['addresses']): balances[accounts[idx]] = satoshis_to_btc( FVal(entry['final_balance'])) except (requests.exceptions.ConnectionError, UnableToDecryptRemoteData) as e: raise RemoteError( f'bitcoin external API request failed due to {str(e)}') except KeyError as e: raise RemoteError( f'Malformed response when querying bitcoin blockchain via {source}.' f'Did not find key {e}', ) return balances