def test_bitstamp_exchange_assets_are_known(mock_bitstamp):
request_url = f'{mock_bitstamp.base_uri}/v2/trading-pairs-info'
try:
response = requests.get(request_url)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Bitstamp get request at {request_url} connection error: {str(e)}.',
) from e
if response.status_code != 200:
raise RemoteError(
f'Bitstamp query responded with error status code: {response.status_code} '
f'and text: {response.text}',
)
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(f'Bitstamp returned invalid JSON response: {response.text}') from e
# Extract the unique symbols from the exchange pairs
pairs = [raw_result.get('name') for raw_result in response_list]
symbols = set()
for pair in pairs:
symbols.update(set(pair.split('/')))
for symbol in symbols:
try:
asset_from_bitstamp(symbol)
except UnknownAsset as e:
test_warnings.warn(UserWarning(
f'Found unknown asset {e.asset_name} in {mock_bitstamp.name}. '
f'Support for it has to be added',
))
def _get_tom_pool_fee_rewards_from_api(self) -> FeeRewards:
"""Do a GET request to the Tom pool fee rewards API.
"""
fee_rewards: FeeRewards = []
try:
response = self.session.get(TOM_POOL_FEE_REWARDS_API_URL)
except requests.exceptions.RequestException as e:
msg = (
f'AdEx get request at {TOM_POOL_FEE_REWARDS_API_URL} connection error: {str(e)}.'
)
self.msg_aggregator.add_error(
f'Got remote error while querying AdEx fee rewards API: {msg}',
)
return fee_rewards
if response.status_code != HTTPStatus.OK:
msg = (
f'AdEx fee rewards API query responded with error status code: '
f'{response.status_code} and text: {response.text}.')
self.msg_aggregator.add_error(
f'Got remote error while querying AdEx fee rewards API: {msg}',
)
return fee_rewards
try:
fee_rewards = rlk_jsonloads_list(response.text)
except JSONDecodeError:
msg = f'AdEx fee rewards API returned invalid JSON response: {response.text}.'
self.msg_aggregator.add_error(
f'Got remote error while querying AdEx fee rewards API: {msg}',
)
return fee_rewards
return fee_rewards
def _public_api_query(
self,
endpoint: str,
) -> List[Any]:
"""Performs a Gemini API Query for a public endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Raises RemoteError if something went wrong with connecting or reading from the exchange
"""
response = self._query_continuously(method='get', endpoint=endpoint)
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Gemini query at {response.url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
try:
json_ret = rlk_jsonloads_list(response.text)
except JSONDecodeError:
raise RemoteError(
f'Gemini query at {response.url} '
f'returned invalid JSON response: {response.text}', )
return json_ret
def _query_exchange_pairs(self) -> ExchangePairsResponse:
"""Query and return the list of the exchange (trades) pairs in
`<ExchangePairsResponse>.pairs`.
Otherwise populate <ExchangePairsResponse> with data that each endpoint
can process as an unsuccessful request.
"""
was_successful = True
pairs = []
response = self._api_query('configs_list_pair_exchange')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(
f'{self.name} exchange pairs list query failed. Check further logs'
)
else:
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} exchange pairs list returned invalid JSON response. '
f'Check further logs', )
else:
pairs = [
pair for pair in response_list[0]
if not pair.startswith(BITFINEX_EXCHANGE_TEST_ASSETS)
and not pair.endswith(BITFINEX_EXCHANGE_TEST_ASSETS)
]
return ExchangePairsResponse(
success=was_successful,
response=response,
pairs=pairs,
)
def _query_currencies(self) -> CurrenciesResponse:
"""Query and return the list of all the currencies supported in
`<CurrenciesResponse>.currencies`.
Otherwise populate <CurrenciesResponse> with data that each endpoint
can process as an unsuccessful request.
"""
was_successful = True
currencies = []
response = self._api_query('configs_list_currency')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(
f'{self.name} currencies list query failed. Check further logs'
)
else:
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} currencies list returned invalid JSON response. '
f'Check further logs', )
else:
currencies = [
currency for currency in response_list[0]
if currency not in set(BITFINEX_EXCHANGE_TEST_ASSETS)
]
return CurrenciesResponse(
success=was_successful,
response=response,
currencies=currencies,
)
def api_query( # noqa: F811
self,
endpoint: str,
method: Literal['get', 'put', 'delete'] = 'get',
options: Optional[Dict[str, Any]] = None,
) -> List[Dict[str, Any]]:
"""
Queries Bittrex api v3 for given endpoint, method and options
"""
given_options = options.copy() if options else {}
backoff = self.initial_backoff
request_url = self.uri + endpoint
if given_options:
# iso8601 dates need special handling in bittrex since they can't parse them urlencoded
# https://github.com/Bittrex/bittrex.github.io/issues/72#issuecomment-498335240
start_date = given_options.pop('startDate', None)
end_date = given_options.pop('endDate', None)
request_url += '?' + urlencode(given_options)
if start_date is not None:
request_url += f'&startDate={start_date}'
if end_date is not None:
request_url += f'&endDate={end_date}'
while True:
response = self._single_api_query(
request_url=request_url,
options=given_options,
method=method,
public_endpoint=endpoint in BITTREX_V3_PUBLIC_ENDPOINTS,
)
should_backoff = (response.status_code
== HTTPStatus.TOO_MANY_REQUESTS
and backoff < self.backoff_limit)
if should_backoff:
log.debug('Got 429 from Bittrex. Backing off', seconds=backoff)
gevent.sleep(backoff)
backoff = backoff * 2
continue
# else we got a result
break
if response.status_code != HTTPStatus.OK:
self._check_for_system_clock_not_synced_error(response)
raise RemoteError(
f'Bittrex query responded with error status code: {response.status_code}'
f' and text: {response.text}', )
try:
result = rlk_jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Bittrex returned invalid JSON response: {response.text}'
) from e
return result
def _api_query(self,
command: str,
req: Optional[Dict] = None) -> Union[Dict, List]:
if req is None:
req = {}
if command == 'returnTicker' or command == 'returnCurrencies':
log.debug(f'Querying poloniex for {command}')
ret = self.session.get(self.public_uri + command)
return rlk_jsonloads(ret.text)
req['command'] = command
with self.lock:
# Protect this region with a lock since poloniex will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
req['nonce'] = int(time.time() * 1000)
post_data = str.encode(urlencode(req))
sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
self.session.headers.update({'Sign': sign})
log.debug(
'Poloniex private API query',
command=command,
post_data=req,
)
ret = self.session.post('https://poloniex.com/tradingApi', req)
if ret.status_code != 200:
raise RemoteError(
f'Poloniex query responded with error status code: {ret.status_code}'
f' and text: {ret.text}', )
try:
if command == 'returnLendingHistory':
return rlk_jsonloads_list(ret.text)
else:
# For some reason poloniex can also return [] for an empty trades result
if ret.text == '[]':
return {}
else:
result = rlk_jsonloads_dict(ret.text)
return _post_process(result)
except JSONDecodeError:
raise RemoteError(
f'Poloniex returned invalid JSON response: {ret.text}')
def _query_currency_map(self) -> CurrencyMapResponse:
"""Query the list that maps standard currency symbols with the version
of the Bitfinex API. If the request is successful and the list format
as well, return it as dict in `<CurrencyMapResponse>.currency_map`.
Otherwise populate <CurrencyMapResponse> with data that each endpoint
can process as an unsuccessful request.
API result format is: [[[<bitfinex_symbol>, <symbol>], ...]]
May raise IndexError if the list is empty.
"""
was_successful = True
currency_map = {}
response = self._api_query('configs_map_currency_symbol')
if response.status_code != HTTPStatus.OK:
was_successful = False
log.error(
f'{self.name} currency map query failed. Check further logs')
else:
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError:
was_successful = False
log.error(
f'{self.name} currency map returned invalid JSON response. Check further logs',
)
else:
currency_map = {
bfx_symbol: symbol
for bfx_symbol, symbol in response_list[0]
if bfx_symbol not in set(BITFINEX_EXCHANGE_TEST_ASSETS)
}
currency_map.update(BITFINEX_TO_WORLD)
return CurrencyMapResponse(
success=was_successful,
response=response,
currency_map=currency_map,
)
def _api_query(self, command: str, req: Optional[Dict] = None) -> Union[Dict, List]:
"""An api query to poloniex. May make multiple requests
Can raise:
- RemoteError if there is a problem reaching poloniex or with the returned response
"""
if req is None:
req = {}
log.debug(
'Poloniex API query',
command=command,
post_data=req,
)
tries = QUERY_RETRY_TIMES
while tries >= 0:
try:
response = self._single_query(command, req)
except requests.exceptions.ConnectionError as e:
raise RemoteError(f'Poloniex API request failed due to {str(e)}')
if response is None:
if tries >= 1:
backoff_seconds = 20 / tries
log.debug(
f'Got a recoverable poloniex error. '
f'Backing off for {backoff_seconds}',
)
gevent.sleep(backoff_seconds)
tries -= 1
continue
else:
break
if response is None:
raise RemoteError(
f'Got a recoverable poloniex error and did not manage to get a '
f'request through even after {QUERY_RETRY_TIMES} '
f'incremental backoff retries',
)
result: Union[Dict, List]
try:
if command == 'returnLendingHistory':
result = rlk_jsonloads_list(response.text)
else:
# For some reason poloniex can also return [] for an empty trades result
if response.text == '[]':
result = {}
else:
result = rlk_jsonloads_dict(response.text)
result = _post_process(result)
except JSONDecodeError:
raise RemoteError(f'Poloniex returned invalid JSON response: {response.text}')
if isinstance(result, dict) and 'error' in result:
raise RemoteError(
'Poloniex query for "{}" returned error: {}'.format(
command,
result['error'],
))
return result
def get_coins_list(self) -> List[Dict[str, Any]]:
response_data = self._query('coins')
return rlk_jsonloads_list(response_data)
call_options = options.copy()
limit = options.get('limit', API_MAX_LIMIT)
results: Union[List[Trade], List[AssetMovement]] = [] # type: ignore
while True:
response = self._api_query(
endpoint=endpoint,
method='post',
options=call_options,
)
if response.status_code != HTTPStatus.OK:
return self._process_unsuccessful_response(
response=response,
case=response_case,
)
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError:
msg = f'Bitstamp returned invalid JSON response: {response.text}.'
log.error(msg)
self.msg_aggregator.add_error(
f'Got remote error while querying Bistamp trades: {msg}',
)
no_results: Union[List[Trade], List[AssetMovement]] = [] # type: ignore
return no_results
has_results = False
is_result_timesamp_gt_end_ts = False
result: Union[Trade, AssetMovement]
for raw_result in response_list:
if raw_result['type'] not in raw_result_type_filter:
continue
def query_balances(self) -> ExchangeQueryBalances:
"""Return the account exchange balances on Bitfinex
The wallets endpoint returns a list where each item is a currency wallet.
Each currency wallet has type (i.e. exchange, margin, funding), currency,
balance, etc. Currencies (tickers) are in Bitfinex format and must be
standardized.
Endpoint documentation:
https://docs.bitfinex.com/reference#rest-auth-wallets
"""
self.first_connection()
response = self._api_query('wallets')
if response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=response,
case='balances',
)
return result, msg
try:
response_list = rlk_jsonloads_list(response.text)
except JSONDecodeError as e:
msg = f'{self.name} returned invalid JSON response: {response.text}.'
log.error(msg)
raise RemoteError(msg) from e
# Wallet items indices
currency_index = 1
balance_index = 2
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
for wallet in response_list:
if len(wallet) < API_WALLET_MIN_RESULT_LENGTH or wallet[
balance_index] <= 0:
log.error(
f'Error processing a {self.name} balance result. '
f'Found less items than expected',
wallet=wallet,
)
self.msg_aggregator.add_error(
f'Failed to deserialize a {self.name} balance result. '
f'Check logs for details. Ignoring it.', )
continue
try:
asset = asset_from_bitfinex(
bitfinex_name=wallet[currency_index],
currency_map=self.currency_map,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'unknown' if isinstance(
e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} {self.name} asset {e.asset_name} due to: {str(e)}. '
f'Ignoring its balance query.', )
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance result.', )
continue
amount = FVal(wallet[balance_index])
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return dict(assets_balance), ''
def _api_query(
self,
endpoint: str,
request_method: Literal['GET', 'POST'] = 'GET',
options: Optional[Dict[str, Any]] = None,
query_options: Optional[Dict[str, Any]] = None,
) -> Tuple[List[Any], Optional[str]]:
"""Performs a coinbase PRO API Query for endpoint
You can optionally provide extra arguments to the endpoint via the options argument.
Returns a tuple of the result and optional pagination cursor.
Raises RemoteError if something went wrong with connecting or reading from the exchange
Raises CoinbaseProPermissionError if the API Key does not have sufficient
permissions for the endpoint
"""
request_url = f'/{endpoint}'
timestamp = str(int(time.time()))
if options:
stringified_options = json.dumps(options, separators=(',', ':'))
else:
stringified_options = ''
options = {}
if query_options:
request_url += '?' + urlencode(query_options)
message = timestamp + request_method + request_url + stringified_options
log.debug(
'Coinbase Pro API query',
request_method=request_method,
request_url=request_url,
options=options,
)
if 'products' not in endpoint:
try:
signature = hmac.new(
b64decode(self.secret),
message.encode(),
hashlib.sha256,
).digest()
except binascii.Error as e:
raise RemoteError('Provided API Secret is invalid') from e
self.session.headers.update({
'CB-ACCESS-SIGN':
b64encode(signature).decode('utf-8'),
'CB-ACCESS-TIMESTAMP':
timestamp,
})
retries_left = QUERY_RETRY_TIMES
while retries_left > 0:
full_url = self.base_uri + request_url
try:
response = self.session.request(
request_method.lower(),
full_url,
data=stringified_options,
)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at '
f'{full_url} connection error: {str(e)}', ) from e
if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
# Backoff a bit by sleeping. Sleep more, the more retries have been made
gevent.sleep(QUERY_RETRY_TIMES / retries_left)
retries_left -= 1
else:
# get out of the retry loop, we did not get 429 complaint
break
json_ret: Union[List[Any], Dict[str, Any]]
if response.status_code == HTTPStatus.BAD_REQUEST:
json_ret = rlk_jsonloads_dict(response.text)
if json_ret['message'] == 'invalid signature':
raise CoinbaseProPermissionError(
f'While doing {request_method} at {endpoint} endpoint the API secret '
f'created an invalid signature.', )
# else do nothing and a generic remote error will be thrown below
elif response.status_code == HTTPStatus.FORBIDDEN:
raise CoinbaseProPermissionError(
f'API key does not have permission for {endpoint}', )
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} responded with error '
f'status code: {response.status_code} and text: {response.text}',
)
try:
json_ret = rlk_jsonloads_list(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Coinbase Pro {request_method} query at {full_url} '
f'returned invalid JSON response: {response.text}', ) from e
return json_ret, response.headers.get('cb-after', None)
