def test_bitstamp_exchange_assets_are_known(mock_bitstamp):
request_url = f'{mock_bitstamp.base_uri}/v2/trading-pairs-info'
try:
response = requests.get(request_url)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bitstamp get request at {request_url} connection error: {str(e)}.',
) from e
if response.status_code != 200:
raise RemoteError(
f'Bitstamp query responded with error status code: {response.status_code} '
f'and text: {response.text}', )
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Bitstamp returned invalid JSON response: {response.text}') from e
# Extract the unique symbols from the exchange pairs
pairs = [raw_result.get('name') for raw_result in response_list]
symbols = set()
for pair in pairs:
symbols.update(set(pair.split('/')))
for symbol in symbols:
try:
asset_from_bitstamp(symbol)
except UnknownAsset as e:
test_warnings.warn(
UserWarning(
f'Found unknown asset {e.asset_name} in {mock_bitstamp.name}. '
f'Support for it has to be added', ))
def _query_exchange_pairs(self) -> ExchangePairsResponse:
"""Query and return the list of the exchange (trades) pairs in
`<ExchangePairsResponse>.pairs`.
Otherwise populate <ExchangePairsResponse> with data that each endpoint
can process as an unsuccessful request.
"""
was_successful = True
pairs = []
response = self._api_query('configs_list_pair_exchange')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(f'{self.name} exchange pairs list query failed. Check further logs')
else:
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} exchange pairs list returned invalid JSON response. '
f'Check further logs',
)
else:
pairs = [
pair for pair in response_list[0]
if not pair.startswith(BITFINEX_EXCHANGE_TEST_ASSETS) and
not pair.endswith(BITFINEX_EXCHANGE_TEST_ASSETS)
]
return ExchangePairsResponse(
success=was_successful,
response=response,
pairs=pairs,
)
def _public_api_query(
self,
endpoint: str,
) -> List[Any]:
"""Performs a Gemini API Query for a public endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Raises RemoteError if something went wrong with connecting or reading from the exchange
"""
response = self._query_continuously(method='get', endpoint=endpoint)
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Gemini query at {response.url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
try:
json_ret = jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Gemini query at {response.url} '
f'returned invalid JSON response: {response.text}',
) from e
return json_ret
def _query_currencies(self) -> CurrenciesResponse:
"""Query and return the list of all the currencies supported in
`<CurrenciesResponse>.currencies`.
Otherwise populate <CurrenciesResponse> with data that each endpoint
can process as an unsuccessful request.
"""
was_successful = True
currencies = []
response = self._api_query('configs_list_currency')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(f'{self.name} currencies list query failed. Check further logs')
else:
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} currencies list returned invalid JSON response. '
f'Check further logs',
)
else:
currencies = [
currency for currency in response_list[0]
if currency not in set(BITFINEX_EXCHANGE_TEST_ASSETS)
]
return CurrenciesResponse(
success=was_successful,
response=response,
currencies=currencies,
)
def test_bitpanda_exchange_assets_are_known():
"""Since normal Bitpanda API has no endpoint listing supposrted assets
https://developers.bitpanda.com/platform/#bitpanda-public-api
Bitpanda PRO leasts some of the same assets but not all. So this test catches some,
but unfortunately not all assets
"""
request_url = 'https://api.exchange.bitpanda.com/public/v1/currencies'
try:
response = requests.get(request_url)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bitpanda get request at {request_url} connection error: {str(e)}.',
) from e
if response.status_code != 200:
raise RemoteError(
f'Bitpanda query responded with error status code: {response.status_code} '
f'and text: {response.text}', )
try:
response_list = jsonloads_list(response.text)
except json.JSONDecodeError as e:
raise RemoteError(
f'Bitpanda returned invalid JSON response: {response.text}') from e
for entry in response_list:
try:
asset_from_bitpanda(entry['code'])
except UnknownAsset as e:
test_warnings.warn(
UserWarning(
f'Found unknown asset {e.asset_name} in bitpanda. '
f'Support for it has to be added', ))
def api_query(
self,
endpoint: str,
method: Literal['get', 'put', 'delete'] = 'get',
options: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
"""
Queries Bittrex api v3 for given endpoint, method and options
"""
given_options = options.copy() if options else {}
backoff = self.initial_backoff
request_url = self.uri + endpoint
if given_options:
# iso8601 dates need special handling in bittrex since they can't parse them urlencoded
# https://github.com/Bittrex/bittrex.github.io/issues/72#issuecomment-498335240
start_date = given_options.pop('startDate', None)
end_date = given_options.pop('endDate', None)
request_url += '?' + urlencode(given_options)
if start_date is not None:
request_url += f'&startDate={start_date}'
if end_date is not None:
request_url += f'&endDate={end_date}'
while True:
response = self._single_api_query(
request_url=request_url,
options=given_options,
method=method,
public_endpoint=endpoint in BITTREX_V3_PUBLIC_ENDPOINTS,
)
should_backoff = (response.status_code
== HTTPStatus.TOO_MANY_REQUESTS
and backoff < self.backoff_limit)
if should_backoff:
log.debug('Got 429 from Bittrex. Backing off', seconds=backoff)
gevent.sleep(backoff)
backoff = backoff * 2
continue
# else we got a result
break
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Bittrex query responded with error status code: {response.status_code}'
f' and text: {response.text}', )
try:
result = jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Bittrex returned invalid JSON response: {response.text}'
) from e
return result
def _query_currency_map(self) -> CurrencyMapResponse:
"""Query the list that maps standard currency symbols with the version
of the Bitfinex API. If the request is successful and the list format
as well, return it as dict in `<CurrencyMapResponse>.currency_map`.
Otherwise populate <CurrencyMapResponse> with data that each endpoint
can process as an unsuccessful request.
API result format is: [[[<bitfinex_symbol>, <symbol>], ...]]
May raise IndexError if the list is empty.
"""
was_successful = True
currency_map = {}
response = self._api_query('configs_map_currency_symbol')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(
f'{self.name} currency map query failed. Check further logs')
else:
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} currency map returned invalid JSON response. Check further logs',
)
else:
currency_map = {
bfx_symbol: symbol
for bfx_symbol, symbol in response_list[0]
if bfx_symbol not in set(BITFINEX_EXCHANGE_TEST_ASSETS)
}
currency_map.update(BITFINEX_TO_WORLD)
return CurrencyMapResponse(
success=was_successful,
response=response,
currency_map=currency_map,
)
def _api_query(
self,
endpoint: str,
request_method: Literal['GET', 'POST'] = 'GET',
options: Optional[Dict[str, Any]] = None,
query_options: Optional[Dict[str, Any]] = None,
) -> Tuple[List[Any], Optional[str]]:
"""Performs a coinbase PRO API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Returns a tuple of the result and optional pagination cursor.
Raises RemoteError if something went wrong with connecting or reading from the exchange
Raises CoinbaseProPermissionError if the API Key does not have sufficient
permissions for the endpoint
"""
request_url = f'/{endpoint}'
timestamp = str(int(time.time()))
if options:
stringified_options = json.dumps(options, separators=(',', ':'))
else:
stringified_options = ''
options = {}
if query_options:
request_url += '?' + urlencode(query_options)
message = timestamp + request_method + request_url + stringified_options
if 'products' not in endpoint:
try:
signature = hmac.new(
b64decode(self.secret),
message.encode(),
hashlib.sha256,
).digest()
except binascii.Error as e:
raise RemoteError('Provided API Secret is invalid') from e
self.session.headers.update({
'CB-ACCESS-SIGN':
b64encode(signature).decode('utf-8'),
'CB-ACCESS-TIMESTAMP':
timestamp,
})
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
log.debug(
'Coinbase Pro API query',
request_method=request_method,
request_url=request_url,
options=options,
)
full_url = self.base_uri + request_url
try:
response = self.session.request(
request_method.lower(),
full_url,
data=stringified_options,
timeout=DEFAULT_TIMEOUT_TUPLE,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at '
f'{full_url} connection error: {str(e)}', ) from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
# Backoff a bit by sleeping. Sleep more, the more retries have been made
backoff_secs = QUERY_RETRY_TIMES / retries_left
log.debug(
f'Backing off coinbase pro api query for {backoff_secs} secs'
)
gevent.sleep(backoff_secs)
retries_left -= 1
else:
# get out of the retry loop, we did not get 429 complaint
break
json_ret: Union[List[Any], Dict[str, Any]]
if response.status_code == HTTPStatus.BAD_REQUEST:
json_ret = jsonloads_dict(response.text)
if json_ret['message'] == 'invalid signature':
raise CoinbaseProPermissionError(
f'While doing {request_method} at {endpoint} endpoint the API secret '
f'created an invalid signature.', )
# else do nothing and a generic remote error will be thrown below
elif response.status_code == HTTPStatus.FORBIDDEN:
raise CoinbaseProPermissionError(
f'API key does not have permission for {endpoint}', )
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
try:
json_ret = jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} '
f'returned invalid JSON response: {response.text}', ) from e
return json_ret, response.headers.get('cb-after', None)
def _api_query(self,
command: str,
req: Optional[Dict] = None) -> Union[Dict, List]:
"""An api query to poloniex. May make multiple requests
Can raise:
- RemoteError if there is a problem reaching poloniex or with the returned response
"""
if req is None:
req = {}
log.debug(
'Poloniex API query',
command=command,
post_data=req,
)
tries = QUERY_RETRY_TIMES
while tries >= 0:
try:
response = self._single_query(command, req)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Poloniex API request failed due to {str(e)}') from e
if response is None:
if tries >= 1:
backoff_seconds = 20 / tries
log.debug(
f'Got a recoverable poloniex error. '
f'Backing off for {backoff_seconds}', )
gevent.sleep(backoff_seconds)
tries -= 1
continue
else:
break
if response is None:
raise RemoteError(
f'Got a recoverable poloniex error and did not manage to get a '
f'request through even after {QUERY_RETRY_TIMES} '
f'incremental backoff retries', )
result: Union[Dict, List]
try:
if command == 'returnLendingHistory':
result = jsonloads_list(response.text)
else:
# For some reason poloniex can also return [] for an empty trades result
if response.text == '[]':
result = {}
else:
result = jsonloads_dict(response.text)
result = _post_process(result)
except JSONDecodeError as e:
raise RemoteError(
f'Poloniex returned invalid JSON response: {response.text}'
) from e
if isinstance(result, dict) and 'error' in result:
raise RemoteError(
'Poloniex query for "{}" returned error: {}'.format(
command,
result['error'],
))
return result
def query_balances(self) -> ExchangeQueryBalances:
"""Return the account exchange balances on Bitfinex
The wallets endpoint returns a list where each item is a currency wallet.
Each currency wallet has type (i.e. exchange, margin, funding), currency,
balance, etc. Currencies (tickers) are in Bitfinex format and must be
standardized.
Endpoint documentation:
https://docs.bitfinex.com/reference#rest-auth-wallets
"""
self.first_connection()
response = self._api_query('wallets')
if response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=response,
case='balances',
)
return result, msg
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError as e:
msg = f'{self.name} returned invalid JSON response: {response.text}.'
log.error(msg)
raise RemoteError(msg) from e
# Wallet items indices
currency_index = 1
balance_index = 2
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
for wallet in response_list:
if len(wallet) < API_WALLET_MIN_RESULT_LENGTH:
log.error(
f'Error processing a {self.name} balance result. '
f'Found less items than expected',
wallet=wallet,
)
self.msg_aggregator.add_error(
f'Failed to deserialize a {self.name} balance result. '
f'Check logs for details. Ignoring it.',
)
continue
if wallet[balance_index] <= 0:
continue # bitfinex can show small negative balances for some coins. Ignore
try:
asset = asset_from_bitfinex(
bitfinex_name=wallet[currency_index],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'unknown' if isinstance(e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} {self.name} asset {e.asset_name} due to: {str(e)}. '
f'Ignoring its balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} {asset.name} balance result due to inability '
f'to query USD price: {str(e)}. Skipping balance result.',
)
continue
try:
amount = deserialize_asset_amount(wallet[balance_index])
except DeserializationError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} {asset.name} balance result due to inability '
f'to deserialize asset amount due to {str(e)}. Skipping balance result.',
)
continue
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return dict(assets_balance), ''
# Unexpected JSON dict case, better to log it
msg = f'Unexpected {self.name} {case} unsuccessful response JSON'
log.error(msg, error_response=error_response)
self.msg_aggregator.add_error(
f'Got remote error while querying {self.name} {case}: {msg}',
)
return [] # type: ignore # bug list nothing
return self._process_unsuccessful_response(
response=response,
case=case_,
)
try:
response_list = jsonloads_list(response.text)
except JSONDecodeError:
msg = f'{self.name} {case} returned invalid JSON response: {response.text}.'
log.error(msg)
self.msg_aggregator.add_error(
f'Got remote error while querying {self.name} {case}: {msg}',
)
return [] # type: ignore # bug list nothing
results_ = self._deserialize_api_query_paginated_results(
case=case_,
options=call_options,
raw_results=response_list,
processed_result_ids=processed_result_ids,
)
results.extend(cast(Iterable, results_))
def get_coins_list(self) -> List[Dict[str, Any]]:
response_data = self._query('coins')
return jsonloads_list(response_data)
def test_jsonloads_list():
result = jsonloads_list('["foo", "boo", 3]')
assert result == ["foo", "boo", 3]
with pytest.raises(JSONDecodeError) as e:
jsonloads_list('{"foo": 1, "boo": "value"}')
assert 'Returned json is not a list' in str(e.value)
