def api_query(self, method: str, req: Optional[dict] = None) -> dict:
# Pretty ugly ... mock a kraken remote eror
if self.remote_errors:
raise RemoteError('Kraken remote error')
if self.use_original_kraken:
return super().api_query(method, req)
if method == 'Balance':
if self.random_balance_data:
return generate_random_kraken_balance_response()
# else
return self.balance_data_return
if method == 'TradesHistory':
assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
if self.random_trade_data:
return generate_random_kraken_trades_data(
start=req['start'],
end=req['end'],
tradeable_pairs=list(self.tradeable_pairs.keys()),
)
# else
return rlk_jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
if method == 'Ledgers':
assert req, 'Should have given arguments for kraken Ledgers endpoint call'
ledger_type = req['type']
if self.random_ledgers_data:
return generate_random_kraken_ledger_data(
start=req['start'],
end=req['end'],
ledger_type=ledger_type,
)
# else use specific data
if ledger_type in ('deposit', 'withdrawal'):
data = json.loads(
KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type
== 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, )
new_data: Dict[str, Any] = {'ledger': {}}
for key, val in data['ledger'].items():
try:
ts = int(val['time'])
except ValueError:
ts = req[
'start'] # can happen for tests of invalid data -- let it through
if ts < req['start'] or ts > req['end']:
continue
new_data['ledger'][key] = val
new_data['count'] = len(new_data['ledger'])
response = json.dumps(new_data)
else:
raise AssertionError(
'Unknown ledger type at kraken ledgers mock query')
return rlk_jsonloads_dict(response)
# else
return super().api_query(method, req)
def _request_chain_metadata(self) -> Dict[str, Any]:
"""Subscan API metadata documentation:
https://docs.api.subscan.io/#metadata
"""
response = self._request_explorer_api(endpoint='metadata')
if response.status_code != HTTPStatus.OK:
message = (
f'{self.chain} chain metadata request was not successful. '
f'Response status code: {response.status_code}. '
f'Response text: {response.text}.'
)
log.error(message)
raise RemoteError(message)
try:
result = rlk_jsonloads_dict(response.text)
except JSONDecodeError as e:
message = (
f'{self.chain} chain metadata request returned invalid JSON '
f'response: {response.text}.'
)
log.error(message)
raise RemoteError(message) from e
log.debug(f'{self.chain} subscan API metadata', result=result)
return result
def _save_cached_price(
self,
from_asset: Asset,
to_asset: Asset,
date: str,
price: Price,
) -> None:
price_history_dir = get_or_make_price_history_dir(self.data_directory)
filename = (
price_history_dir /
f'{PRICE_HISTORY_FILE_PREFIX }{from_asset.identifier}_{to_asset.identifier}.json'
)
data: Dict[str, Price] = {}
if filename.is_file():
with open(filename, 'r') as f:
try:
data = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
data = {}
if not isinstance(data, dict):
data = {}
data[date] = price
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(data))
def query_ipinfo() -> Optional[GeolocationData]:
"""
50,000 requests per month tied to the API Key
https://ipinfo.io/developers
"""
try:
response = requests.get(
'https://ipinfo.io/json?token=16ab40aad9bd5b',
timeout=LOCATION_DATA_QUERY_TIMEOUT,
)
except requests.exceptions.RequestException:
return None
if response.status_code != HTTPStatus.OK:
return None
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
return None
return GeolocationData(
country_code=json_ret.get('country', 'unknown'),
city=json_ret.get('city', 'unknown'),
)
def query_ipstack() -> Optional[GeolocationData]:
"""
10,000 requests per month tied to the API Key
https://ipstack.com/
"""
try:
response = requests.get(
'http://api.ipstack.com/check?access_key=affd920d6e1008a614900dbc31d52fa6',
timeout=LOCATION_DATA_QUERY_TIMEOUT,
)
except requests.exceptions.RequestException:
return None
if response.status_code != HTTPStatus.OK:
return None
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
return None
return GeolocationData(
country_code=json_ret.get('country_code', 'unknown'),
city=json_ret.get('city', 'unknown'),
)
def get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
# TODO: Both here and in cryptocompare the cache funcionality is the same
# Extract the caching part into its own function somehow and abstract it
# away
invalidate_cache = True
coinlist_cache_path = os.path.join(self.data_directory, 'cmc_coinlist.json')
if os.path.isfile(coinlist_cache_path):
log.info('Found coinmarketcap coinlist cache', path=coinlist_cache_path)
with open(coinlist_cache_path, 'r') as f:
try:
file_data = rlk_jsonloads_dict(f.read())
now = ts_now()
invalidate_cache = False
# If we got a cache and it's over a month old then requery coinmarketcap
if file_data['time'] < now and now - file_data['time'] > 2629800:
log.info('Coinmarketcap coinlist cache is now invalidated')
invalidate_cache = True
data = file_data['data']
except JSONDecodeError:
invalidate_cache = True
if invalidate_cache:
data = self._get_cryptocyrrency_map()
# Also save the cache
with open(coinlist_cache_path, 'w') as f:
now = ts_now()
log.info('Writing coinmarketcap coinlist cache', timestamp=now)
write_data = {'time': now, 'data': data}
f.write(rlk_jsondumps(write_data))
else:
# in any case take the data
data = file_data['data']
return data
def _check_and_get_response(response: Response, method: str) -> dict:
"""Checks the kraken response and if it's succesfull returns the result. If there
is an error an exception is raised"""
if response.status_code in (520, 525, 504):
raise RecoverableRequestError('kraken', 'Usual kraken 5xx shenanigans')
elif response.status_code != 200:
raise RemoteError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
result = rlk_jsonloads_dict(response.text)
if result['error']:
if isinstance(result['error'], list):
error = result['error'][0]
else:
error = result['error']
if 'Rate limit exceeded' in error:
raise RecoverableRequestError('kraken', 'Rate limited exceeded')
else:
raise RemoteError(error)
return result['result']
def __new__(
cls,
data_dir: Path = None,
cryptocompare: 'Cryptocompare' = None,
coingecko: 'Coingecko' = None,
) -> 'Inquirer':
if Inquirer.__instance is not None:
return Inquirer.__instance
assert data_dir, 'arguments should be given at the first instantiation'
assert cryptocompare, 'arguments should be given at the first instantiation'
assert coingecko, 'arguments should be given at the first instantiation'
Inquirer.__instance = object.__new__(cls)
Inquirer.__instance._data_directory = data_dir
Inquirer._cryptocompare = cryptocompare
Inquirer._coingecko = coingecko
Inquirer._cached_current_price = {}
# Make price history directory if it does not exist
price_history_dir = get_or_make_price_history_dir(data_dir)
filename = price_history_dir / 'price_history_forex.json'
try:
with open(filename, 'r') as f:
# we know price_history_forex contains a dict
data = rlk_jsonloads_dict(f.read())
Inquirer.__instance._cached_forex_data = data
except (OSError, JSONDecodeError):
Inquirer.__instance._cached_forex_data = {}
return Inquirer.__instance
def query_cryptocompare_for_fiat_price(asset: Asset) -> Price:
log.debug('Get usd price from cryptocompare', asset=asset)
cc_asset_str = asset.to_cryptocompare()
resp = retry_calls(
5,
'find_usd_price',
'requests.get',
requests.get,
u'https://min-api.cryptocompare.com/data/price?'
'fsym={}&tsyms=USD'.format(cc_asset_str),
)
if resp.status_code != 200:
raise RemoteError(
'Cant reach cryptocompare to get USD value of {}'.format(asset))
resp = rlk_jsonloads_dict(resp.text)
# If there is an error in the response skip this token
if 'USD' not in resp:
error_message = ''
if resp['Response'] == 'Error':
error_message = resp['Message']
log.error(
'Cryptocompare usd price query failed',
asset=asset,
error=error_message,
)
return Price(ZERO)
price = Price(FVal(resp['USD']))
log.debug('Got usd price from cryptocompare', asset=asset, price=price)
return price
def __new__(
cls,
data_dir: FilePath = None,
cryptocompare: 'Cryptocompare' = None,
) -> 'Inquirer':
if Inquirer.__instance is not None:
return Inquirer.__instance
assert data_dir, 'arguments should be given at the first instantiation'
assert cryptocompare, 'arguments should be given at the first instantiation'
Inquirer.__instance = object.__new__(cls)
Inquirer.__instance._data_directory = data_dir
Inquirer._cryptocompare = cryptocompare
filename = os.path.join(data_dir, 'price_history_forex.json')
try:
with open(filename, 'r') as f:
# we know price_history_forex contains a dict
data = rlk_jsonloads_dict(f.read())
Inquirer.__instance._cached_forex_data = data
except (OSError, JSONDecodeError):
Inquirer.__instance._cached_forex_data = {}
return Inquirer.__instance
def test_kucoin_exchange_assets_are_known(mock_kucoin):
request_url = f'{mock_kucoin.base_uri}/api/v1/currencies'
try:
response = requests.get(request_url)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Kucoin get request at {request_url} connection error: {str(e)}.',
) from e
if response.status_code != HTTPStatus.OK:
raise RemoteError(
f'Kucoin query responded with error status code: {response.status_code} '
f'and text: {response.text}', )
try:
response_dict = rlk_jsonloads_dict(response.text)
except JSONDecodeError as e:
raise RemoteError(
f'Kucoin returned invalid JSON response: {response.text}') from e
# Extract the unique symbols from the exchange pairs
unsupported_assets = set(UNSUPPORTED_KUCOIN_ASSETS)
for entry in response_dict['data']:
symbol = entry['currency']
try:
asset_from_kucoin(symbol)
except UnsupportedAsset:
assert symbol in unsupported_assets
except UnknownAsset as e:
test_warnings.warn(
UserWarning(
f'Found unknown asset {e.asset_name} in kucoin. '
f'Support for it has to be added', ))
def query_historical_fiat_exchange_rates(
from_fiat_currency: Asset,
to_fiat_currency: Asset,
timestamp: Timestamp,
) -> Optional[Price]:
assert from_fiat_currency.is_fiat(), 'fiat currency should have been provided'
assert to_fiat_currency.is_fiat(), 'fiat currency should have been provided'
date = timestamp_to_date(timestamp, formatstr='%Y-%m-%d')
instance = Inquirer()
rate = instance._get_cached_forex_data(date, from_fiat_currency, to_fiat_currency)
if rate:
return rate
log.debug(
'Querying exchangeratesapi',
from_fiat_currency=from_fiat_currency.identifier,
to_fiat_currency=to_fiat_currency.identifier,
timestamp=timestamp,
)
query_str = (
f'https://api.exchangeratesapi.io/{date}?'
f'base={from_fiat_currency.identifier}'
)
resp = retry_calls(
times=5,
location='query_exchangeratesapi',
handle_429=False,
backoff_in_seconds=0,
method_name='requests.get',
function=requests.get,
# function's arguments
url=query_str,
)
if resp.status_code != 200:
return None
try:
result = rlk_jsonloads_dict(resp.text)
except JSONDecodeError:
return None
if 'rates' not in result or to_fiat_currency.identifier not in result['rates']:
return None
if date not in instance._cached_forex_data:
instance._cached_forex_data[date] = {}
if from_fiat_currency not in instance._cached_forex_data[date]:
instance._cached_forex_data[date][from_fiat_currency] = {}
for key, value in result['rates'].items():
instance._cached_forex_data[date][from_fiat_currency][key] = FVal(value)
rate = Price(FVal(result['rates'][to_fiat_currency.identifier]))
log.debug('Exchangeratesapi query succesful', rate=rate)
return rate
def api_query(self, method: str, req: Optional[dict] = None) -> dict:
# Pretty ugly ... mock a kraken remote eror
if self.remote_errors:
raise RemoteError('Kraken remote error')
if self.use_original_kraken:
return super().api_query(method, req)
if method == 'Balance':
if self.random_balance_data:
return generate_random_kraken_balance_response()
# else
return self.balance_data_return
elif method == 'TradesHistory':
assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
if self.random_trade_data:
return generate_random_kraken_trades_data(
start=req['start'],
end=req['end'],
tradeable_pairs=list(self.tradeable_pairs.keys()),
)
# else
return rlk_jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
elif method == 'Ledgers':
assert req, 'Should have given arguments for kraken Ledgers endpoint call'
ledger_type = req['type']
if self.random_ledgers_data:
return generate_random_kraken_ledger_data(
start=req['start'],
end=req['end'],
ledger_type=ledger_type,
)
# else use specific data
if ledger_type == 'deposit':
response = KRAKEN_SPECIFIC_DEPOSITS_RESPONSE
elif ledger_type == 'withdrawal':
response = KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE
else:
raise AssertionError(
'Unknown ledger type at kraken ledgers mock query')
return rlk_jsonloads_dict(response)
return super().api_query(method, req)
def _query(
self,
module: Literal['validator'],
endpoint: Literal['balancehistory', 'performance', 'eth1'],
encoded_args: str,
) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
"""
May raise:
- RemoteError due to problems querying beaconcha.in API
"""
if endpoint == 'eth1':
query_str = f'{self.url}{module}/{endpoint}/{encoded_args}'
else:
query_str = f'{self.url}{module}/{encoded_args}/{endpoint}'
times = QUERY_RETRY_TIMES
backoff_in_seconds = 10
while True:
try:
response = self.session.get(query_str)
except requests.exceptions.RequestException as e:
raise RemoteError(f'Querying {query_str} failed due to {str(e)}')
if response.status_code == 429:
if times == 0:
raise RemoteError(
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text} after 5 retries',
)
# We got rate limited. Let's try incremental backoff
gevent.sleep(backoff_in_seconds * (QUERY_RETRY_TIMES - times + 1))
continue
else:
break
if response.status_code != 200:
raise RemoteError(
f'Beaconchain API request {response.url} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}',
)
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Beaconchain API returned invalid JSON response: {response.text}')
if json_ret.get('status') != 'OK':
raise RemoteError(f'Beaconchain API returned non-OK status. Response: {json_ret}')
if 'data' not in json_ret:
raise RemoteError(f'Beaconchain API did not contain a data key. Response: {json_ret}')
return json_ret['data']
def query_balances(self) -> Tuple[Optional[Dict[Asset, Balance]], str]:
"""Return the account balances on Bistamp
The balance endpoint returns a dict where the keys (str) are related to
assets and the values (str) amounts. The keys that end with `_balance`
contain the exact amount of an asset the account is holding (available
amount + orders amount, per asset).
"""
response = self._api_query('balance')
if response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=response,
case='balances',
)
return result, msg
try:
response_dict = rlk_jsonloads_dict(response.text)
except JSONDecodeError as e:
msg = f'Bitstamp returned invalid JSON response: {response.text}.'
log.error(msg)
raise RemoteError(msg) from e
asset_balance: Dict[Asset, Balance] = {}
for entry, amount in response_dict.items():
amount = FVal(amount)
if not entry.endswith('_balance') or amount == ZERO:
continue
symbol = entry.split('_')[0] # If no `_`, defaults to entry
try:
asset = Asset(symbol)
except (UnknownAsset, UnsupportedAsset) as e:
log.error(str(e))
asset_tag = 'unknown' if isinstance(
e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} Bistamp asset {e.asset_name}. Ignoring its balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
log.error(str(e))
self.msg_aggregator.add_error(
f'Error processing Bitstamp balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry.', )
continue
asset_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
return asset_balance, ''
def get_jsonfile_contents_or_empty_dict(filepath: FilePath) -> Dict:
if not os.path.isfile(filepath):
return dict()
with open(filepath, 'r') as infile:
try:
data = rlk_jsonloads_dict(infile.read())
except json.decoder.JSONDecodeError:
data = dict()
return data
def api_query( # noqa: F811
self,
method: str,
options: Optional[Dict[str, Any]] = None,
) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
"""
Queries Bittrex with given method and options
"""
if not options:
options = {}
nonce = str(ts_now_in_ms())
method_type = 'public'
if method in BITTREX_MARKET_METHODS:
method_type = 'market'
elif method in BITTREX_ACCOUNT_METHODS:
method_type = 'account'
request_url = self.uri + method_type + '/' + method + '?'
if method_type != 'public':
request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&'
request_url += urlencode(options)
signature = hmac.new(
self.secret,
request_url.encode(),
hashlib.sha512,
).hexdigest()
self.session.headers.update({'apisign': signature})
log.debug('Bittrex API query', request_url=request_url)
try:
response = self.session.get(request_url)
except requests.exceptions.ConnectionError as e:
raise RemoteError(f'Bittrex API request failed due to {str(e)}')
if response.status_code != 200:
raise RemoteError(
f'Bittrex query responded with error status code: {response.status_code}'
f' and text: {response.text}', )
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(
f'Bittrex returned invalid JSON response: {response.text}')
if json_ret['success'] is not True:
raise RemoteError(json_ret['message'])
result = json_ret['result']
assert isinstance(result, dict) or isinstance(result, list)
return result
def query_historical_fiat_exchange_rates(
from_fiat_currency: FiatAsset,
to_fiat_currency: FiatAsset,
timestamp: Timestamp,
) -> Optional[Price]:
date = tsToDate(timestamp, formatstr='%Y-%m-%d')
instance = Inquirer()
rate = instance._get_cached_forex_data(date, from_fiat_currency,
to_fiat_currency)
if rate:
return rate
log.debug(
'Querying exchangeratesapi',
from_fiat_currency=from_fiat_currency,
to_fiat_currency=to_fiat_currency,
timestamp=timestamp,
)
query_str = (f'https://api.exchangeratesapi.io/{date}?'
f'base={from_fiat_currency}')
resp = retry_calls(
5,
'query_exchangeratesapi',
'requests.get',
requests.get,
query_str,
)
if resp.status_code != 200:
return None
try:
result = rlk_jsonloads_dict(resp.text)
except JSONDecodeError:
return None
if 'rates' not in result or to_fiat_currency not in result['rates']:
return None
if date not in instance._cached_forex_data:
instance._cached_forex_data[date] = {}
if from_fiat_currency not in instance._cached_forex_data[date]:
instance._cached_forex_data[date][from_fiat_currency] = {}
for key, value in result['rates'].items():
instance._cached_forex_data[date][from_fiat_currency][key] = FVal(
value)
rate = Price(FVal(result['rates'][to_fiat_currency]))
log.debug('Exchangeratesapi query succesful', rate=rate)
return rate
def get_poap_airdrop_data(name: str, data_dir: Path) -> Dict[str, Any]:
airdrops_dir = data_dir / 'airdrops_poap'
airdrops_dir.mkdir(parents=True, exist_ok=True)
filename = airdrops_dir / f'{name}.json'
if not filename.is_file():
# if not cached, get it from the gist
try:
request = requests.get(POAP_AIRDROPS[name][0])
except requests.exceptions.RequestException as e:
raise RemoteError(f'POAP airdrops Gist request failed due to {str(e)}') from e
try:
json_data = rlk_jsonloads_dict(request.content.decode('utf-8'))
except JSONDecodeError as e:
raise RemoteError(f'POAP airdrops Gist contains an invalid JSON {str(e)}') from e
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(json_data))
infile = open(filename, 'r')
data_dict = rlk_jsonloads_dict(infile.read())
return data_dict
def _process_dict_response(response: requests.Response) -> Dict:
"""Processess a dict response returned from the Rotkehlchen server and returns
the result for success or raises RemoteError if an error happened"""
if response.status_code not in HANDLABLE_STATUS_CODES:
raise RemoteError(
f'Unexpected status response({response.status_code}) from '
'rotkehlchen server', )
result_dict = rlk_jsonloads_dict(response.text)
if 'error' in result_dict:
raise RemoteError(result_dict['error'])
return result_dict
def _check_for_system_clock_not_synced_error(response: Response) -> None:
if response.status_code == HTTPStatus.UNAUTHORIZED:
try:
result = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Bittrex returned invalid JSON response: {response.text}')
if result.get('code', None) == 'INVALID_TIMESTAMP':
raise SystemClockNotSyncedError(
current_time=str(datetime.now()),
remote_server='Bittrex',
)
return None
def _get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
start = 1
limit = 5000
result: List[Dict[str, Any]] = []
while True:
response_data = rlk_jsonloads_dict(
self._query(
f'v1/cryptocurrency/map?start={start}&limit={limit}'), )
result.extend(response_data['data'])
if len(response_data['data']) != limit:
break
return result
def _query(self, path: str) -> Dict:
response = requests.get(f'{self.prefix}{path}')
if response.status_code != 200:
raise RemoteError(
f'Github API request {response.url} for {path} failed '
f'with HTTP status code {response.status_code} and text '
f'{response.text}',
)
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(f'Github returned invalid JSON response: {response.text}')
return json_ret
def api_query(
self,
method: str,
options: Optional[Dict] = None,
) -> Union[List, Dict]:
"""
Queries Bittrex with given method and options
"""
if not options:
options = {}
nonce = str(int(time.time() * 1000))
method_type = 'public'
if method in BITTREX_MARKET_METHODS:
method_type = 'market'
elif method in BITTREX_ACCOUNT_METHODS:
method_type = 'account'
request_url = self.uri + method_type + '/' + method + '?'
if method_type != 'public':
request_url += 'apikey=' + self.api_key.decode(
) + "&nonce=" + nonce + '&'
request_url += urlencode(options)
signature = hmac.new(
self.secret,
request_url.encode(),
hashlib.sha512,
).hexdigest()
self.session.headers.update({'apisign': signature})
log.debug('Bittrex API query', request_url=request_url)
response = self.session.get(request_url)
if response.status_code != 200:
raise RemoteError(
f'Bittrex query responded with error status code: {response.status_code}'
f' and text: {response.text}', )
try:
json_ret = rlk_jsonloads_dict(response.text)
except JSONDecodeError:
raise RemoteError(
f'Bittrex returned invalid JSON response: {response.text}')
if json_ret['success'] is not True:
raise RemoteError(json_ret['message'])
return json_ret['result']
def _api_query(self,
command: str,
req: Optional[Dict] = None) -> Union[Dict, List]:
if req is None:
req = {}
if command == 'returnTicker' or command == 'returnCurrencies':
log.debug(f'Querying poloniex for {command}')
ret = self.session.get(self.public_uri + command)
return rlk_jsonloads(ret.text)
req['command'] = command
with self.lock:
# Protect this region with a lock since poloniex will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
req['nonce'] = int(time.time() * 1000)
post_data = str.encode(urlencode(req))
sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
self.session.headers.update({'Sign': sign})
log.debug(
'Poloniex private API query',
command=command,
post_data=req,
)
ret = self.session.post('https://poloniex.com/tradingApi', req)
if ret.status_code != 200:
raise RemoteError(
f'Poloniex query responded with error status code: {ret.status_code}'
f' and text: {ret.text}', )
try:
if command == 'returnLendingHistory':
return rlk_jsonloads_list(ret.text)
else:
# For some reason poloniex can also return [] for an empty trades result
if ret.text == '[]':
return {}
else:
result = rlk_jsonloads_dict(ret.text)
return _post_process(result)
except JSONDecodeError:
raise RemoteError(
f'Poloniex returned invalid JSON response: {ret.text}')
def _check_and_get_response(response: Response,
method: str) -> Union[str, Dict]:
"""Checks the kraken response and if it's succesfull returns the result.
If there is recoverable error a string is returned explaining the error
May raise:
- RemoteError if there is an unrecoverable/unexpected remote error
"""
if response.status_code in (520, 525, 504):
log.debug(f'Kraken returned status code {response.status_code}')
return 'Usual kraken 5xx shenanigans'
if response.status_code != 200:
raise RemoteError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
try:
decoded_json = rlk_jsonloads_dict(response.text)
except json.decoder.JSONDecodeError as e:
raise RemoteError(f'Invalid JSON in Kraken response. {e}') from e
error = decoded_json.get('error', None)
if error:
if isinstance(error, list) and len(error) != 0:
error = error[0]
if 'Rate limit exceeded' in error:
log.debug(f'Kraken: Got rate limit exceeded error: {error}')
return 'Rate limited exceeded'
# else
raise RemoteError(error)
result = decoded_json.get('result', None)
if result is None:
if method == 'Balance':
return {}
raise RemoteError(f'Missing result in kraken response for {method}')
return result
def read_info_at_start(self) -> DBStartupAction:
dbinfo = None
action = DBStartupAction.NOTHING
filepath = os.path.join(self.user_data_dir, DBINFO_FILENAME)
if not os.path.exists(filepath):
return action
with open(filepath, 'r') as f:
try:
dbinfo = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
log.warning(
'dbinfo.json file is corrupt. Does not contain expected keys'
)
return action
current_md5_hash = self.get_md5hash()
if not dbinfo:
return action
if 'sqlcipher_version' not in dbinfo or 'md5_hash' not in dbinfo:
log.warning(
'dbinfo.json file is corrupt. Does not contain expected keys')
return action
if dbinfo['md5_hash'] != current_md5_hash:
log.warning(
'dbinfo.json contains an outdated hash. Was data changed outside the program?',
)
return action
if dbinfo['sqlcipher_version'] == 3 and self.sqlcipher_version == 3:
return DBStartupAction.NOTHING
if dbinfo['sqlcipher_version'] == 4 and self.sqlcipher_version == 4:
return DBStartupAction.NOTHING
if dbinfo['sqlcipher_version'] == 3 and self.sqlcipher_version == 4:
return DBStartupAction.UPGRADE_3_4
if dbinfo['sqlcipher_version'] == 4 and self.sqlcipher_version == 3:
return DBStartupAction.STUCK_4_3
raise ValueError('Unexpected values at dbinfo.json')
def test_dbinfo_is_written_at_shutdown(rotkehlchen_server):
"""Test that when rotkehlchen shuts down dbinfo is written"""
r = rotkehlchen_server.rotkehlchen
filepath = os.path.join(r.data.user_data_dir, 'dbinfo.json')
sqlcipher_version = r.data.db.sqlcipher_version
# Using rotkehlchen instance's shutdown and not server's since the
# server is not mocked well here for this.
r.shutdown()
assert os.path.exists(filepath), 'dbinfo.json was not written'
with open(filepath, 'r') as f:
try:
dbinfo = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
raise AssertionError('Could not decode dbinfo.json')
assert dbinfo['sqlcipher_version'] == sqlcipher_version
assert 'md5_hash' in dbinfo
def _check_and_get_response(response: Response,
method: str) -> Union[str, Dict]:
"""Checks the kraken response and if it's succesfull returns the result.
If there is recoverable error a string is returned explaining the error
May raise:
- RemoteError if there is an unrecoverable/unexpected remote error
"""
if response.status_code in (520, 525, 504):
log.debug(f'Kraken returned status code {response.status_code}')
return 'Usual kraken 5xx shenanigans'
elif response.status_code != 200:
raise RemoteError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
try:
decoded_json = rlk_jsonloads_dict(response.text)
except json.decoder.JSONDecodeError as e:
raise RemoteError(f'Invalid JSON in Kraken response. {e}')
try:
if decoded_json['error']:
if isinstance(decoded_json['error'], list):
error = decoded_json['error'][0]
else:
error = decoded_json['error']
if 'Rate limit exceeded' in error:
log.debug(f'Kraken: Got rate limit exceeded error: {error}')
return 'Rate limited exceeded'
else:
raise RemoteError(error)
result = decoded_json['result']
except KeyError as e:
raise RemoteError(
f'Unexpected format of Kraken response. Missing key: {e}')
return result
def check_trades_cache(
self,
start_ts: Timestamp,
end_ts: Timestamp,
special_name: Optional[str] = None,
) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
trades_file = self._get_cachefile_name(special_name)
trades: Dict[str, Dict[str, Any]] = dict()
if os.path.isfile(trades_file):
with open(trades_file, 'r') as f:
try:
trades = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
pass
# no need to query again
if data_up_todate(trades, start_ts, end_ts):
return trades['data']
return None
