コード例 #1
0
ファイル: kraken.py プロジェクト: zalam003/rotki
    def api_query(self, method: str, req: Optional[dict] = None) -> dict:
        # Pretty ugly ... mock a kraken remote eror
        if self.remote_errors:
            raise RemoteError('Kraken remote error')

        if self.use_original_kraken:
            return super().api_query(method, req)

        if method == 'Balance':
            if self.random_balance_data:
                return generate_random_kraken_balance_response()
            # else
            return self.balance_data_return
        if method == 'TradesHistory':
            assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
            if self.random_trade_data:
                return generate_random_kraken_trades_data(
                    start=req['start'],
                    end=req['end'],
                    tradeable_pairs=list(self.tradeable_pairs.keys()),
                )
            # else
            return rlk_jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
        if method == 'Ledgers':
            assert req, 'Should have given arguments for kraken Ledgers endpoint call'
            ledger_type = req['type']
            if self.random_ledgers_data:
                return generate_random_kraken_ledger_data(
                    start=req['start'],
                    end=req['end'],
                    ledger_type=ledger_type,
                )

            # else use specific data
            if ledger_type in ('deposit', 'withdrawal'):
                data = json.loads(
                    KRAKEN_SPECIFIC_DEPOSITS_RESPONSE if ledger_type
                    == 'deposit' else KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE, )
                new_data: Dict[str, Any] = {'ledger': {}}
                for key, val in data['ledger'].items():
                    try:
                        ts = int(val['time'])
                    except ValueError:
                        ts = req[
                            'start']  # can happen for tests of invalid data -- let it through
                    if ts < req['start'] or ts > req['end']:
                        continue
                    new_data['ledger'][key] = val

                new_data['count'] = len(new_data['ledger'])
                response = json.dumps(new_data)
            else:
                raise AssertionError(
                    'Unknown ledger type at kraken ledgers mock query')

            return rlk_jsonloads_dict(response)
        # else
        return super().api_query(method, req)
コード例 #2
0
    def _request_chain_metadata(self) -> Dict[str, Any]:
        """Subscan API metadata documentation:
        https://docs.api.subscan.io/#metadata
        """
        response = self._request_explorer_api(endpoint='metadata')
        if response.status_code != HTTPStatus.OK:
            message = (
                f'{self.chain} chain metadata request was not successful. '
                f'Response status code: {response.status_code}. '
                f'Response text: {response.text}.'
            )
            log.error(message)
            raise RemoteError(message)
        try:
            result = rlk_jsonloads_dict(response.text)
        except JSONDecodeError as e:
            message = (
                f'{self.chain} chain metadata request returned invalid JSON '
                f'response: {response.text}.'
            )
            log.error(message)
            raise RemoteError(message) from e

        log.debug(f'{self.chain} subscan API metadata', result=result)
        return result
コード例 #3
0
ファイル: coingecko.py プロジェクト: rkalis/rotki
    def _save_cached_price(
        self,
        from_asset: Asset,
        to_asset: Asset,
        date: str,
        price: Price,
    ) -> None:
        price_history_dir = get_or_make_price_history_dir(self.data_directory)
        filename = (
            price_history_dir /
            f'{PRICE_HISTORY_FILE_PREFIX }{from_asset.identifier}_{to_asset.identifier}.json'
        )
        data: Dict[str, Price] = {}
        if filename.is_file():
            with open(filename, 'r') as f:
                try:
                    data = rlk_jsonloads_dict(f.read())
                except JSONDecodeError:
                    data = {}

        if not isinstance(data, dict):
            data = {}

        data[date] = price
        with open(filename, 'w') as outfile:
            outfile.write(rlk_jsondumps(data))
コード例 #4
0
ファイル: usage_analytics.py プロジェクト: sponnet/rotki
def query_ipinfo() -> Optional[GeolocationData]:
    """
    50,000 requests per month tied to the API Key
    https://ipinfo.io/developers
    """
    try:
        response = requests.get(
            'https://ipinfo.io/json?token=16ab40aad9bd5b',
            timeout=LOCATION_DATA_QUERY_TIMEOUT,
        )
    except requests.exceptions.RequestException:
        return None

    if response.status_code != HTTPStatus.OK:
        return None

    try:
        json_ret = rlk_jsonloads_dict(response.text)
    except JSONDecodeError:
        return None

    return GeolocationData(
        country_code=json_ret.get('country', 'unknown'),
        city=json_ret.get('city', 'unknown'),
    )
コード例 #5
0
ファイル: usage_analytics.py プロジェクト: sponnet/rotki
def query_ipstack() -> Optional[GeolocationData]:
    """
    10,000 requests per month tied to the API Key
    https://ipstack.com/
    """
    try:
        response = requests.get(
            'http://api.ipstack.com/check?access_key=affd920d6e1008a614900dbc31d52fa6',
            timeout=LOCATION_DATA_QUERY_TIMEOUT,
        )
    except requests.exceptions.RequestException:
        return None

    if response.status_code != HTTPStatus.OK:
        return None

    try:
        json_ret = rlk_jsonloads_dict(response.text)
    except JSONDecodeError:
        return None

    return GeolocationData(
        country_code=json_ret.get('country_code', 'unknown'),
        city=json_ret.get('city', 'unknown'),
    )
コード例 #6
0
ファイル: coinmarketcap.py プロジェクト: davbre/rotki
    def get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
        # TODO: Both here and in cryptocompare the cache funcionality is the same
        # Extract the caching part into its own function somehow and abstract it
        # away
        invalidate_cache = True
        coinlist_cache_path = os.path.join(self.data_directory, 'cmc_coinlist.json')
        if os.path.isfile(coinlist_cache_path):
            log.info('Found coinmarketcap coinlist cache', path=coinlist_cache_path)
            with open(coinlist_cache_path, 'r') as f:
                try:
                    file_data = rlk_jsonloads_dict(f.read())
                    now = ts_now()
                    invalidate_cache = False

                    # If we got a cache and it's over a month old then requery coinmarketcap
                    if file_data['time'] < now and now - file_data['time'] > 2629800:
                        log.info('Coinmarketcap coinlist cache is now invalidated')
                        invalidate_cache = True
                        data = file_data['data']
                except JSONDecodeError:
                    invalidate_cache = True

        if invalidate_cache:
            data = self._get_cryptocyrrency_map()
            # Also save the cache
            with open(coinlist_cache_path, 'w') as f:
                now = ts_now()
                log.info('Writing coinmarketcap coinlist cache', timestamp=now)
                write_data = {'time': now, 'data': data}
                f.write(rlk_jsondumps(write_data))
        else:
            # in any case take the data
            data = file_data['data']

        return data
コード例 #7
0
def _check_and_get_response(response: Response, method: str) -> dict:
    """Checks the kraken response and if it's succesfull returns the result. If there
    is an error an exception is raised"""
    if response.status_code in (520, 525, 504):
        raise RecoverableRequestError('kraken', 'Usual kraken 5xx shenanigans')
    elif response.status_code != 200:
        raise RemoteError(
            'Kraken API request {} for {} failed with HTTP status '
            'code: {}'.format(
                response.url,
                method,
                response.status_code,
            ))

    result = rlk_jsonloads_dict(response.text)
    if result['error']:
        if isinstance(result['error'], list):
            error = result['error'][0]
        else:
            error = result['error']

        if 'Rate limit exceeded' in error:
            raise RecoverableRequestError('kraken', 'Rate limited exceeded')
        else:
            raise RemoteError(error)

    return result['result']
コード例 #8
0
ファイル: inquirer.py プロジェクト: yakkomajuri/rotki
    def __new__(
            cls,
            data_dir: Path = None,
            cryptocompare: 'Cryptocompare' = None,
            coingecko: 'Coingecko' = None,
    ) -> 'Inquirer':
        if Inquirer.__instance is not None:
            return Inquirer.__instance

        assert data_dir, 'arguments should be given at the first instantiation'
        assert cryptocompare, 'arguments should be given at the first instantiation'
        assert coingecko, 'arguments should be given at the first instantiation'

        Inquirer.__instance = object.__new__(cls)

        Inquirer.__instance._data_directory = data_dir
        Inquirer._cryptocompare = cryptocompare
        Inquirer._coingecko = coingecko
        Inquirer._cached_current_price = {}
        # Make price history directory if it does not exist
        price_history_dir = get_or_make_price_history_dir(data_dir)
        filename = price_history_dir / 'price_history_forex.json'
        try:
            with open(filename, 'r') as f:
                # we know price_history_forex contains a dict
                data = rlk_jsonloads_dict(f.read())
                Inquirer.__instance._cached_forex_data = data
        except (OSError, JSONDecodeError):
            Inquirer.__instance._cached_forex_data = {}

        return Inquirer.__instance
コード例 #9
0
def query_cryptocompare_for_fiat_price(asset: Asset) -> Price:
    log.debug('Get usd price from cryptocompare', asset=asset)
    cc_asset_str = asset.to_cryptocompare()
    resp = retry_calls(
        5,
        'find_usd_price',
        'requests.get',
        requests.get,
        u'https://min-api.cryptocompare.com/data/price?'
        'fsym={}&tsyms=USD'.format(cc_asset_str),
    )

    if resp.status_code != 200:
        raise RemoteError(
            'Cant reach cryptocompare to get USD value of {}'.format(asset))

    resp = rlk_jsonloads_dict(resp.text)

    # If there is an error in the response skip this token
    if 'USD' not in resp:
        error_message = ''
        if resp['Response'] == 'Error':
            error_message = resp['Message']

        log.error(
            'Cryptocompare usd price query failed',
            asset=asset,
            error=error_message,
        )
        return Price(ZERO)

    price = Price(FVal(resp['USD']))
    log.debug('Got usd price from cryptocompare', asset=asset, price=price)
    return price
コード例 #10
0
    def __new__(
        cls,
        data_dir: FilePath = None,
        cryptocompare: 'Cryptocompare' = None,
    ) -> 'Inquirer':
        if Inquirer.__instance is not None:
            return Inquirer.__instance

        assert data_dir, 'arguments should be given at the first instantiation'
        assert cryptocompare, 'arguments should be given at the first instantiation'

        Inquirer.__instance = object.__new__(cls)

        Inquirer.__instance._data_directory = data_dir
        Inquirer._cryptocompare = cryptocompare
        filename = os.path.join(data_dir, 'price_history_forex.json')
        try:
            with open(filename, 'r') as f:
                # we know price_history_forex contains a dict
                data = rlk_jsonloads_dict(f.read())
                Inquirer.__instance._cached_forex_data = data
        except (OSError, JSONDecodeError):
            Inquirer.__instance._cached_forex_data = {}

        return Inquirer.__instance
コード例 #11
0
ファイル: test_kucoin.py プロジェクト: zalam003/rotki
def test_kucoin_exchange_assets_are_known(mock_kucoin):
    request_url = f'{mock_kucoin.base_uri}/api/v1/currencies'
    try:
        response = requests.get(request_url)
    except requests.exceptions.RequestException as e:
        raise RemoteError(
            f'Kucoin get request at {request_url} connection error: {str(e)}.',
        ) from e

    if response.status_code != HTTPStatus.OK:
        raise RemoteError(
            f'Kucoin query responded with error status code: {response.status_code} '
            f'and text: {response.text}', )
    try:
        response_dict = rlk_jsonloads_dict(response.text)
    except JSONDecodeError as e:
        raise RemoteError(
            f'Kucoin returned invalid JSON response: {response.text}') from e

    # Extract the unique symbols from the exchange pairs
    unsupported_assets = set(UNSUPPORTED_KUCOIN_ASSETS)
    for entry in response_dict['data']:
        symbol = entry['currency']
        try:
            asset_from_kucoin(symbol)
        except UnsupportedAsset:
            assert symbol in unsupported_assets
        except UnknownAsset as e:
            test_warnings.warn(
                UserWarning(
                    f'Found unknown asset {e.asset_name} in kucoin. '
                    f'Support for it has to be added', ))
コード例 #12
0
ファイル: inquirer.py プロジェクト: yakkomajuri/rotki
    def query_historical_fiat_exchange_rates(
            from_fiat_currency: Asset,
            to_fiat_currency: Asset,
            timestamp: Timestamp,
    ) -> Optional[Price]:
        assert from_fiat_currency.is_fiat(), 'fiat currency should have been provided'
        assert to_fiat_currency.is_fiat(), 'fiat currency should have been provided'
        date = timestamp_to_date(timestamp, formatstr='%Y-%m-%d')
        instance = Inquirer()
        rate = instance._get_cached_forex_data(date, from_fiat_currency, to_fiat_currency)
        if rate:
            return rate

        log.debug(
            'Querying exchangeratesapi',
            from_fiat_currency=from_fiat_currency.identifier,
            to_fiat_currency=to_fiat_currency.identifier,
            timestamp=timestamp,
        )

        query_str = (
            f'https://api.exchangeratesapi.io/{date}?'
            f'base={from_fiat_currency.identifier}'
        )
        resp = retry_calls(
            times=5,
            location='query_exchangeratesapi',
            handle_429=False,
            backoff_in_seconds=0,
            method_name='requests.get',
            function=requests.get,
            # function's arguments
            url=query_str,
        )

        if resp.status_code != 200:
            return None

        try:
            result = rlk_jsonloads_dict(resp.text)
        except JSONDecodeError:
            return None

        if 'rates' not in result or to_fiat_currency.identifier not in result['rates']:
            return None

        if date not in instance._cached_forex_data:
            instance._cached_forex_data[date] = {}

        if from_fiat_currency not in instance._cached_forex_data[date]:
            instance._cached_forex_data[date][from_fiat_currency] = {}

        for key, value in result['rates'].items():
            instance._cached_forex_data[date][from_fiat_currency][key] = FVal(value)

        rate = Price(FVal(result['rates'][to_fiat_currency.identifier]))
        log.debug('Exchangeratesapi query succesful', rate=rate)
        return rate
コード例 #13
0
    def api_query(self, method: str, req: Optional[dict] = None) -> dict:
        # Pretty ugly ... mock a kraken remote eror
        if self.remote_errors:
            raise RemoteError('Kraken remote error')

        if self.use_original_kraken:
            return super().api_query(method, req)

        if method == 'Balance':
            if self.random_balance_data:
                return generate_random_kraken_balance_response()
            # else
            return self.balance_data_return
        elif method == 'TradesHistory':
            assert req, 'Should have given arguments for kraken TradesHistory endpoint call'
            if self.random_trade_data:
                return generate_random_kraken_trades_data(
                    start=req['start'],
                    end=req['end'],
                    tradeable_pairs=list(self.tradeable_pairs.keys()),
                )
            # else
            return rlk_jsonloads_dict(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
        elif method == 'Ledgers':
            assert req, 'Should have given arguments for kraken Ledgers endpoint call'
            ledger_type = req['type']
            if self.random_ledgers_data:
                return generate_random_kraken_ledger_data(
                    start=req['start'],
                    end=req['end'],
                    ledger_type=ledger_type,
                )

            # else use specific data
            if ledger_type == 'deposit':
                response = KRAKEN_SPECIFIC_DEPOSITS_RESPONSE
            elif ledger_type == 'withdrawal':
                response = KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE
            else:
                raise AssertionError(
                    'Unknown ledger type at kraken ledgers mock query')

            return rlk_jsonloads_dict(response)

        return super().api_query(method, req)
コード例 #14
0
ファイル: beaconchain.py プロジェクト: djibix/rotki
    def _query(
            self,
            module: Literal['validator'],
            endpoint: Literal['balancehistory', 'performance', 'eth1'],
            encoded_args: str,
    ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
        """
        May raise:
        - RemoteError due to problems querying beaconcha.in API
        """
        if endpoint == 'eth1':
            query_str = f'{self.url}{module}/{endpoint}/{encoded_args}'
        else:
            query_str = f'{self.url}{module}/{encoded_args}/{endpoint}'
        times = QUERY_RETRY_TIMES
        backoff_in_seconds = 10

        while True:
            try:
                response = self.session.get(query_str)
            except requests.exceptions.RequestException as e:
                raise RemoteError(f'Querying {query_str} failed due to {str(e)}')

            if response.status_code == 429:
                if times == 0:
                    raise RemoteError(
                        f'Beaconchain API request {response.url} failed '
                        f'with HTTP status code {response.status_code} and text '
                        f'{response.text} after 5 retries',
                    )

                # We got rate limited. Let's try incremental backoff
                gevent.sleep(backoff_in_seconds * (QUERY_RETRY_TIMES - times + 1))
                continue
            else:
                break

        if response.status_code != 200:
            raise RemoteError(
                f'Beaconchain API request {response.url} failed '
                f'with HTTP status code {response.status_code} and text '
                f'{response.text}',
            )

        try:
            json_ret = rlk_jsonloads_dict(response.text)
        except JSONDecodeError:
            raise RemoteError(f'Beaconchain API returned invalid JSON response: {response.text}')

        if json_ret.get('status') != 'OK':
            raise RemoteError(f'Beaconchain API returned non-OK status. Response: {json_ret}')

        if 'data' not in json_ret:
            raise RemoteError(f'Beaconchain API did not contain a data key. Response: {json_ret}')

        return json_ret['data']
コード例 #15
0
ファイル: bitstamp.py プロジェクト: philhug/rotki
    def query_balances(self) -> Tuple[Optional[Dict[Asset, Balance]], str]:
        """Return the account balances on Bistamp

        The balance endpoint returns a dict where the keys (str) are related to
        assets and the values (str) amounts. The keys that end with `_balance`
        contain the exact amount of an asset the account is holding (available
        amount + orders amount, per asset).
        """
        response = self._api_query('balance')

        if response.status_code != HTTPStatus.OK:
            result, msg = self._process_unsuccessful_response(
                response=response,
                case='balances',
            )
            return result, msg
        try:
            response_dict = rlk_jsonloads_dict(response.text)
        except JSONDecodeError as e:
            msg = f'Bitstamp returned invalid JSON response: {response.text}.'
            log.error(msg)
            raise RemoteError(msg) from e

        asset_balance: Dict[Asset, Balance] = {}
        for entry, amount in response_dict.items():
            amount = FVal(amount)
            if not entry.endswith('_balance') or amount == ZERO:
                continue

            symbol = entry.split('_')[0]  # If no `_`, defaults to entry
            try:
                asset = Asset(symbol)
            except (UnknownAsset, UnsupportedAsset) as e:
                log.error(str(e))
                asset_tag = 'unknown' if isinstance(
                    e, UnknownAsset) else 'unsupported'
                self.msg_aggregator.add_warning(
                    f'Found {asset_tag} Bistamp asset {e.asset_name}. Ignoring its balance query.',
                )
                continue
            try:
                usd_price = Inquirer().find_usd_price(asset=asset)
            except RemoteError as e:
                log.error(str(e))
                self.msg_aggregator.add_error(
                    f'Error processing Bitstamp balance result due to inability to '
                    f'query USD price: {str(e)}. Skipping balance entry.', )
                continue

            asset_balance[asset] = Balance(
                amount=amount,
                usd_value=amount * usd_price,
            )

        return asset_balance, ''
コード例 #16
0
ファイル: misc.py プロジェクト: smfang/rotkehlchen
def get_jsonfile_contents_or_empty_dict(filepath: FilePath) -> Dict:
    if not os.path.isfile(filepath):
        return dict()

    with open(filepath, 'r') as infile:
        try:
            data = rlk_jsonloads_dict(infile.read())
        except json.decoder.JSONDecodeError:
            data = dict()

    return data
コード例 #17
0
ファイル: bittrex.py プロジェクト: yairash/rotki
    def api_query(  # noqa: F811
        self,
        method: str,
        options: Optional[Dict[str, Any]] = None,
    ) -> Union[List[Dict[str, Any]], Dict[str, Any]]:
        """
        Queries Bittrex with given method and options
        """
        if not options:
            options = {}
        nonce = str(ts_now_in_ms())
        method_type = 'public'

        if method in BITTREX_MARKET_METHODS:
            method_type = 'market'
        elif method in BITTREX_ACCOUNT_METHODS:
            method_type = 'account'

        request_url = self.uri + method_type + '/' + method + '?'

        if method_type != 'public':
            request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&'

        request_url += urlencode(options)
        signature = hmac.new(
            self.secret,
            request_url.encode(),
            hashlib.sha512,
        ).hexdigest()
        self.session.headers.update({'apisign': signature})
        log.debug('Bittrex API query', request_url=request_url)
        try:
            response = self.session.get(request_url)
        except requests.exceptions.ConnectionError as e:
            raise RemoteError(f'Bittrex API request failed due to {str(e)}')

        if response.status_code != 200:
            raise RemoteError(
                f'Bittrex query responded with error status code: {response.status_code}'
                f' and text: {response.text}', )

        try:
            json_ret = rlk_jsonloads_dict(response.text)
        except JSONDecodeError:
            raise RemoteError(
                f'Bittrex returned invalid JSON response: {response.text}')

        if json_ret['success'] is not True:
            raise RemoteError(json_ret['message'])

        result = json_ret['result']
        assert isinstance(result, dict) or isinstance(result, list)
        return result
コード例 #18
0
    def query_historical_fiat_exchange_rates(
        from_fiat_currency: FiatAsset,
        to_fiat_currency: FiatAsset,
        timestamp: Timestamp,
    ) -> Optional[Price]:
        date = tsToDate(timestamp, formatstr='%Y-%m-%d')
        instance = Inquirer()
        rate = instance._get_cached_forex_data(date, from_fiat_currency,
                                               to_fiat_currency)
        if rate:
            return rate

        log.debug(
            'Querying exchangeratesapi',
            from_fiat_currency=from_fiat_currency,
            to_fiat_currency=to_fiat_currency,
            timestamp=timestamp,
        )

        query_str = (f'https://api.exchangeratesapi.io/{date}?'
                     f'base={from_fiat_currency}')
        resp = retry_calls(
            5,
            'query_exchangeratesapi',
            'requests.get',
            requests.get,
            query_str,
        )

        if resp.status_code != 200:
            return None

        try:
            result = rlk_jsonloads_dict(resp.text)
        except JSONDecodeError:
            return None

        if 'rates' not in result or to_fiat_currency not in result['rates']:
            return None

        if date not in instance._cached_forex_data:
            instance._cached_forex_data[date] = {}

        if from_fiat_currency not in instance._cached_forex_data[date]:
            instance._cached_forex_data[date][from_fiat_currency] = {}

        for key, value in result['rates'].items():
            instance._cached_forex_data[date][from_fiat_currency][key] = FVal(
                value)

        rate = Price(FVal(result['rates'][to_fiat_currency]))
        log.debug('Exchangeratesapi query succesful', rate=rate)
        return rate
コード例 #19
0
def get_poap_airdrop_data(name: str, data_dir: Path) -> Dict[str, Any]:
    airdrops_dir = data_dir / 'airdrops_poap'
    airdrops_dir.mkdir(parents=True, exist_ok=True)
    filename = airdrops_dir / f'{name}.json'
    if not filename.is_file():
        # if not cached, get it from the gist
        try:
            request = requests.get(POAP_AIRDROPS[name][0])
        except requests.exceptions.RequestException as e:
            raise RemoteError(f'POAP airdrops Gist request failed due to {str(e)}') from e

        try:
            json_data = rlk_jsonloads_dict(request.content.decode('utf-8'))
        except JSONDecodeError as e:
            raise RemoteError(f'POAP airdrops Gist contains an invalid JSON {str(e)}') from e

        with open(filename, 'w') as outfile:
            outfile.write(rlk_jsondumps(json_data))

    infile = open(filename, 'r')
    data_dict = rlk_jsonloads_dict(infile.read())
    return data_dict
コード例 #20
0
ファイル: premium.py プロジェクト: georgerobescu/rotkehlchen
def _process_dict_response(response: requests.Response) -> Dict:
    """Processess a dict response returned from the Rotkehlchen server and returns
    the result for success or raises RemoteError if an error happened"""
    if response.status_code not in HANDLABLE_STATUS_CODES:
        raise RemoteError(
            f'Unexpected status response({response.status_code}) from '
            'rotkehlchen server', )

    result_dict = rlk_jsonloads_dict(response.text)
    if 'error' in result_dict:
        raise RemoteError(result_dict['error'])

    return result_dict
コード例 #21
0
ファイル: bittrex.py プロジェクト: zachmeador/rotki
    def _check_for_system_clock_not_synced_error(response: Response) -> None:
        if response.status_code == HTTPStatus.UNAUTHORIZED:
            try:
                result = rlk_jsonloads_dict(response.text)
            except JSONDecodeError:
                raise RemoteError(f'Bittrex returned invalid JSON response: {response.text}')

            if result.get('code', None) == 'INVALID_TIMESTAMP':
                raise SystemClockNotSyncedError(
                    current_time=str(datetime.now()),
                    remote_server='Bittrex',
                )
        return None
コード例 #22
0
ファイル: coinmarketcap.py プロジェクト: sponnet/rotki
    def _get_cryptocyrrency_map(self) -> List[Dict[str, Any]]:
        start = 1
        limit = 5000
        result: List[Dict[str, Any]] = []
        while True:
            response_data = rlk_jsonloads_dict(
                self._query(
                    f'v1/cryptocurrency/map?start={start}&limit={limit}'), )
            result.extend(response_data['data'])
            if len(response_data['data']) != limit:
                break

        return result
コード例 #23
0
    def _query(self, path: str) -> Dict:
        response = requests.get(f'{self.prefix}{path}')

        if response.status_code != 200:
            raise RemoteError(
                f'Github API request {response.url} for {path} failed '
                f'with HTTP status code {response.status_code} and text '
                f'{response.text}',
            )

        try:
            json_ret = rlk_jsonloads_dict(response.text)
        except JSONDecodeError:
            raise RemoteError(f'Github returned invalid JSON response: {response.text}')
        return json_ret
コード例 #24
0
    def api_query(
        self,
        method: str,
        options: Optional[Dict] = None,
    ) -> Union[List, Dict]:
        """
        Queries Bittrex with given method and options
        """
        if not options:
            options = {}
        nonce = str(int(time.time() * 1000))
        method_type = 'public'

        if method in BITTREX_MARKET_METHODS:
            method_type = 'market'
        elif method in BITTREX_ACCOUNT_METHODS:
            method_type = 'account'

        request_url = self.uri + method_type + '/' + method + '?'

        if method_type != 'public':
            request_url += 'apikey=' + self.api_key.decode(
            ) + "&nonce=" + nonce + '&'

        request_url += urlencode(options)
        signature = hmac.new(
            self.secret,
            request_url.encode(),
            hashlib.sha512,
        ).hexdigest()
        self.session.headers.update({'apisign': signature})
        log.debug('Bittrex API query', request_url=request_url)
        response = self.session.get(request_url)

        if response.status_code != 200:
            raise RemoteError(
                f'Bittrex query responded with error status code: {response.status_code}'
                f' and text: {response.text}', )

        try:
            json_ret = rlk_jsonloads_dict(response.text)
        except JSONDecodeError:
            raise RemoteError(
                f'Bittrex returned invalid JSON response: {response.text}')

        if json_ret['success'] is not True:
            raise RemoteError(json_ret['message'])
        return json_ret['result']
コード例 #25
0
ファイル: poloniex.py プロジェクト: Traunquilizer/rotkehlchen
    def _api_query(self,
                   command: str,
                   req: Optional[Dict] = None) -> Union[Dict, List]:
        if req is None:
            req = {}

        if command == 'returnTicker' or command == 'returnCurrencies':
            log.debug(f'Querying poloniex for {command}')
            ret = self.session.get(self.public_uri + command)
            return rlk_jsonloads(ret.text)

        req['command'] = command
        with self.lock:
            # Protect this region with a lock since poloniex will reject
            # non-increasing nonces. So if two greenlets come in here at
            # the same time one of them will fail
            req['nonce'] = int(time.time() * 1000)
            post_data = str.encode(urlencode(req))

            sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
            self.session.headers.update({'Sign': sign})

            log.debug(
                'Poloniex private API query',
                command=command,
                post_data=req,
            )
            ret = self.session.post('https://poloniex.com/tradingApi', req)

        if ret.status_code != 200:
            raise RemoteError(
                f'Poloniex query responded with error status code: {ret.status_code}'
                f' and text: {ret.text}', )

        try:
            if command == 'returnLendingHistory':
                return rlk_jsonloads_list(ret.text)
            else:
                # For some reason poloniex can also return [] for an empty trades result
                if ret.text == '[]':
                    return {}
                else:
                    result = rlk_jsonloads_dict(ret.text)
                    return _post_process(result)
        except JSONDecodeError:
            raise RemoteError(
                f'Poloniex returned invalid JSON response: {ret.text}')
コード例 #26
0
def _check_and_get_response(response: Response,
                            method: str) -> Union[str, Dict]:
    """Checks the kraken response and if it's succesfull returns the result.

    If there is recoverable error a string is returned explaining the error
    May raise:
    - RemoteError if there is an unrecoverable/unexpected remote error
    """
    if response.status_code in (520, 525, 504):
        log.debug(f'Kraken returned status code {response.status_code}')
        return 'Usual kraken 5xx shenanigans'
    if response.status_code != 200:
        raise RemoteError(
            'Kraken API request {} for {} failed with HTTP status '
            'code: {}'.format(
                response.url,
                method,
                response.status_code,
            ))

    try:
        decoded_json = rlk_jsonloads_dict(response.text)
    except json.decoder.JSONDecodeError as e:
        raise RemoteError(f'Invalid JSON in Kraken response. {e}') from e

    error = decoded_json.get('error', None)
    if error:
        if isinstance(error, list) and len(error) != 0:
            error = error[0]

        if 'Rate limit exceeded' in error:
            log.debug(f'Kraken: Got rate limit exceeded error: {error}')
            return 'Rate limited exceeded'

        # else
        raise RemoteError(error)

    result = decoded_json.get('result', None)
    if result is None:
        if method == 'Balance':
            return {}

        raise RemoteError(f'Missing result in kraken response for {method}')

    return result
コード例 #27
0
    def read_info_at_start(self) -> DBStartupAction:
        dbinfo = None
        action = DBStartupAction.NOTHING
        filepath = os.path.join(self.user_data_dir, DBINFO_FILENAME)

        if not os.path.exists(filepath):
            return action

        with open(filepath, 'r') as f:
            try:
                dbinfo = rlk_jsonloads_dict(f.read())
            except JSONDecodeError:
                log.warning(
                    'dbinfo.json file is corrupt. Does not contain expected keys'
                )
                return action
        current_md5_hash = self.get_md5hash()

        if not dbinfo:
            return action

        if 'sqlcipher_version' not in dbinfo or 'md5_hash' not in dbinfo:
            log.warning(
                'dbinfo.json file is corrupt. Does not contain expected keys')
            return action

        if dbinfo['md5_hash'] != current_md5_hash:
            log.warning(
                'dbinfo.json contains an outdated hash. Was data changed outside the program?',
            )
            return action

        if dbinfo['sqlcipher_version'] == 3 and self.sqlcipher_version == 3:
            return DBStartupAction.NOTHING

        if dbinfo['sqlcipher_version'] == 4 and self.sqlcipher_version == 4:
            return DBStartupAction.NOTHING

        if dbinfo['sqlcipher_version'] == 3 and self.sqlcipher_version == 4:
            return DBStartupAction.UPGRADE_3_4

        if dbinfo['sqlcipher_version'] == 4 and self.sqlcipher_version == 3:
            return DBStartupAction.STUCK_4_3

        raise ValueError('Unexpected values at dbinfo.json')
コード例 #28
0
ファイル: test_api.py プロジェクト: buendiadas/rotkehlchen
def test_dbinfo_is_written_at_shutdown(rotkehlchen_server):
    """Test that when rotkehlchen shuts down dbinfo is written"""
    r = rotkehlchen_server.rotkehlchen
    filepath = os.path.join(r.data.user_data_dir, 'dbinfo.json')
    sqlcipher_version = r.data.db.sqlcipher_version
    # Using rotkehlchen instance's shutdown and not server's since the
    # server is not mocked well here for this.
    r.shutdown()

    assert os.path.exists(filepath), 'dbinfo.json was not written'
    with open(filepath, 'r') as f:
        try:
            dbinfo = rlk_jsonloads_dict(f.read())
        except JSONDecodeError:
            raise AssertionError('Could not decode dbinfo.json')

    assert dbinfo['sqlcipher_version'] == sqlcipher_version
    assert 'md5_hash' in dbinfo
コード例 #29
0
def _check_and_get_response(response: Response,
                            method: str) -> Union[str, Dict]:
    """Checks the kraken response and if it's succesfull returns the result.

    If there is recoverable error a string is returned explaining the error
    May raise:
    - RemoteError if there is an unrecoverable/unexpected remote error
    """
    if response.status_code in (520, 525, 504):
        log.debug(f'Kraken returned status code {response.status_code}')
        return 'Usual kraken 5xx shenanigans'
    elif response.status_code != 200:
        raise RemoteError(
            'Kraken API request {} for {} failed with HTTP status '
            'code: {}'.format(
                response.url,
                method,
                response.status_code,
            ))

    try:
        decoded_json = rlk_jsonloads_dict(response.text)
    except json.decoder.JSONDecodeError as e:
        raise RemoteError(f'Invalid JSON in Kraken response. {e}')

    try:
        if decoded_json['error']:
            if isinstance(decoded_json['error'], list):
                error = decoded_json['error'][0]
            else:
                error = decoded_json['error']

            if 'Rate limit exceeded' in error:
                log.debug(f'Kraken: Got rate limit exceeded error: {error}')
                return 'Rate limited exceeded'
            else:
                raise RemoteError(error)

        result = decoded_json['result']
    except KeyError as e:
        raise RemoteError(
            f'Unexpected format of Kraken response. Missing key: {e}')

    return result
コード例 #30
0
ファイル: exchange.py プロジェクト: buendiadas/rotkehlchen
    def check_trades_cache(
        self,
        start_ts: Timestamp,
        end_ts: Timestamp,
        special_name: Optional[str] = None,
    ) -> Optional[Union[List[Dict[str, Any]], Dict[str, Any]]]:
        trades_file = self._get_cachefile_name(special_name)
        trades: Dict[str, Dict[str, Any]] = dict()
        if os.path.isfile(trades_file):
            with open(trades_file, 'r') as f:
                try:
                    trades = rlk_jsonloads_dict(f.read())
                except JSONDecodeError:
                    pass

                # no need to query again
                if data_up_todate(trades, start_ts, end_ts):
                    return trades['data']

        return None