Ejemplo n.º 1
0
    def _api_query(  # noqa: F811
        self,
        endpoint: str,
        options: Optional[Dict[str, Any]],
    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
        querystr = self.base_url + endpoint
        if options is not None:
            querystr += '?' + urlencode(options)

        logger.debug(f'Querying loopring {querystr}')
        try:
            response = self.session.get(querystr)
        except requests.exceptions.RequestException as e:
            raise RemoteError(
                f'Loopring api query {querystr} failed due to {str(e)}') from e
        if response.status_code == HTTPStatus.BAD_REQUEST:
            try:
                json_ret = rlk_jsonloads(response.text)
            except JSONDecodeError as e:
                raise RemoteError(
                    f'Loopring API {response.url} returned invalid '
                    f'JSON response: {response.text}', ) from e

            if isinstance(json_ret, dict):
                result_info = json_ret.get('resultInfo', None)
                if result_info:
                    code = result_info.get('code', None)
                    if code and code == 104002:
                        raise LoopringAPIKeyMismatch()
            # else just let it hit the generic remote error below

        if response.status_code != HTTPStatus.OK:
            raise RemoteError(
                f'Loopring API request {response.url} failed '
                f'with HTTP status code {response.status_code} and text '
                f'{response.text}', )

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError as e:
            raise RemoteError(
                f'Loopring API {response.url} returned invalid '
                f'JSON response: {response.text}', ) from e

        if isinstance(json_ret, dict) and 'code' in json_ret:
            code = json_ret['code']
            msg = json_ret.get('msg', 'no message')
            raise RemoteError(
                f'Loopring API {response.url} returned an error '
                f'with code: {code} and message: {msg}', )

        return json_ret
Ejemplo n.º 2
0
    def _query(
        self,
        module: str,
        subpath: Optional[str] = None,
        options: Optional[Dict[str, Any]] = None,
    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
        """Performs a coingecko query

        May raise:
        - RemoteError if there is a problem querying coingecko
        """
        if options is None:
            options = {}
        url = f'https://api.coingecko.com/api/v3/{module}/'
        if subpath:
            url += subpath

        logger.debug(f'Querying coingecko: {url}?{urlencode(options)}')
        tries = COINGECKO_QUERY_RETRY_TIMES
        while tries >= 0:
            try:
                response = self.session.get(f'{url}?{urlencode(options)}')
            except requests.exceptions.RequestException as e:
                raise RemoteError(
                    f'Coingecko API request failed due to {str(e)}') from e

            if response.status_code == 429:
                # Coingecko allows only 100 calls per minute. If you get 429 it means you
                # exceeded this and are throttled until the next minute window
                # backoff and retry 4 times =  2.5 + 3.33 + 5 + 10 = at most 20.8 secs
                if tries >= 1:
                    backoff_seconds = 10 / tries
                    log.debug(
                        f'Got rate limited by coingecko. '
                        f'Backing off for {backoff_seconds}', )
                    gevent.sleep(backoff_seconds)
                    tries -= 1
                    continue

                # else
                log.debug(
                    f'Got rate limited by coingecko and did not manage to get a '
                    f'request through even after {COINGECKO_QUERY_RETRY_TIMES} '
                    f'incremental backoff retries', )

            break

        if response.status_code != 200:
            msg = (
                f'Coingecko API request {response.url} failed with HTTP status '
                f'code: {response.status_code}')
            raise RemoteError(msg)

        try:
            decoded_json = rlk_jsonloads(response.text)
        except json.decoder.JSONDecodeError as e:
            msg = f'Invalid JSON in Coingecko response. {e}'
            raise RemoteError(msg) from e

        return decoded_json
Ejemplo n.º 3
0
def request_get(
        url: str,
        timeout: int = ALL_REMOTES_TIMEOUT,
        handle_429: bool = False,
        backoff_in_seconds: Union[int, float] = 0,
) -> Union[Dict, List]:
    # TODO make this a bit more smart. Perhaps conditional on the type of request.
    # Not all requests would need repeated attempts
    response = retry_calls(
        times=QUERY_RETRY_TIMES,
        location='',
        handle_429=handle_429,
        backoff_in_seconds=backoff_in_seconds,
        method_name=url,
        function=requests.get,
        # function's arguments
        url=url,
        timeout=timeout,
    )

    if response.status_code != 200:
        if 'https://blockchain.info/q/addressbalance' in url and response.status_code == 500:
            # For some weird reason blockchain.info returns
            # 500 server error when giving invalid account
            raise InvalidBTCAddress('Invalid BTC address given to blockchain.info')

    try:
        result = rlk_jsonloads(response.text)
    except json.decoder.JSONDecodeError:
        raise UnableToDecryptRemoteData(f'{url} returned malformed json')

    return result
Ejemplo n.º 4
0
def request_get(
        url: str,
        timeout: int = ALL_REMOTES_TIMEOUT,
        handle_429: bool = False,
        backoff_in_seconds: Union[int, float] = 0,
) -> Union[Dict, List]:
    # TODO make this a bit more smart. Perhaps conditional on the type of request.
    # Not all requests would need repeated attempts
    response = retry_calls(
        times=5,
        location='',
        handle_429=handle_429,
        backoff_in_seconds=backoff_in_seconds,
        method_name=url,
        function=requests.get,
        # function's arguments
        url=url,
        timeout=timeout,
    )

    if response.status_code != 200:
        raise RemoteError('Get {} returned status code {}'.format(url, response.status_code))

    try:
        result = rlk_jsonloads(response.text)
    except json.decoder.JSONDecodeError:
        raise ValueError('{} returned malformed json'.format(url))

    return result
Ejemplo n.º 5
0
def do_read_manual_margin_positions(
        user_directory: FilePath) -> List[MarginPosition]:
    manual_margin_path = os.path.join(user_directory, MANUAL_MARGINS_LOGFILE)
    if os.path.isfile(manual_margin_path):
        with open(manual_margin_path, 'r') as f:
            margin_data = rlk_jsonloads(f.read())
    else:
        margin_data = []
        logger.info(
            'Could not find manual margins log file at {}'.format(
                manual_margin_path), )

    # Now turn the manual margin data to our MarginPosition format
    # The poloniex manual data format is:
    # { "open_time": unix_timestamp, "close_time": unix_timestamp,
    #   "btc_profit_loss": floating_point_number for profit or loss,
    #   "notes": "optional string with notes on the margin position"
    # }
    margin_positions = list()
    for position in margin_data:
        margin_positions.append(
            MarginPosition(
                exchange='poloniex',
                open_time=position['open_time'],
                close_time=position['close_time'],
                profit_loss=FVal(position['btc_profit_loss']),
                pl_currency=A_BTC,
                notes=position['notes'],
            ), )

    return margin_positions
Ejemplo n.º 6
0
    def _query(
        self,
        module: str,
        subpath: Optional[str] = None,
        options: Optional[Dict[str, Any]] = None,
    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
        """Performs a coingecko query

        May raise:
        - RemoteError if there is a problem querying coingecko
        """
        if options is None:
            options = {}
        url = f'https://api.coingecko.com/api/v3/{module}/'
        if subpath:
            url += subpath
        try:
            response = self.session.get(f'{url}?{urlencode(options)}')
        except requests.exceptions.RequestException as e:
            raise RemoteError(f'Coingecko API request failed due to {str(e)}')

        if response.status_code != 200:
            raise RemoteError(
                f'Coingecko API request {response.url} failed with HTTP status '
                f'code: {response.status_code}', )

        try:
            decoded_json = rlk_jsonloads(response.text)
        except json.decoder.JSONDecodeError as e:
            raise RemoteError(f'Invalid JSON in Kraken response. {e}')

        return decoded_json
Ejemplo n.º 7
0
Archivo: misc.py Proyecto: jbrit/rotki
def request_get(
    url: str,
    timeout: int = ALL_REMOTES_TIMEOUT,
    handle_429: bool = False,
    backoff_in_seconds: Union[int, float] = 0,
) -> Union[Dict, List]:
    """
    May raise:
    - UnableToDecryptRemoteData from request_get
    - Remote error if the get request fails
    """
    # TODO make this a bit more smart. Perhaps conditional on the type of request.
    # Not all requests would need repeated attempts
    response = retry_calls(
        times=QUERY_RETRY_TIMES,
        location='',
        handle_429=handle_429,
        backoff_in_seconds=backoff_in_seconds,
        method_name=url,
        function=requests.get,
        # function's arguments
        url=url,
        timeout=timeout,
    )

    try:
        result = rlk_jsonloads(response.text)
    except json.decoder.JSONDecodeError:
        raise UnableToDecryptRemoteData(f'{url} returned malformed json')

    return result
Ejemplo n.º 8
0
    def query_private(self, method: str, req: Optional[dict] = None) -> dict:
        self.first_connection()
        if method == 'Balance':
            if self.random_balance_data:
                return generate_random_kraken_balance_response()
            # else
            return self.balance_data_return
        elif method == 'TradesHistory':
            if self.random_trade_data:
                return generate_random_kraken_trades_data(
                    start=req['start'],
                    end=req['end'],
                    tradeable_pairs=list(self.tradeable_pairs.keys()),
                )
            # else
            return rlk_jsonloads(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
        elif method == 'Ledgers':
            if self.random_ledgers_data:
                return generate_random_kraken_ledger_data(
                    start=req['start'],
                    end=req['end'],
                    ledger_type=req['type'],
                )
            # else
            return self.ledger_data_generate_cb(
                start=req['start'],
                end=req['end'],
                ledger_type=req['type'],
            )

        return super().query_private(method, req)
Ejemplo n.º 9
0
def test_decoding():
    strdata = (
        '{"a": 3.14, "b":5, "c": "foo", "d": "5.42323143", "e": { "u1": "3.221"}, '
        '"f": [2.1, "boo", 3, "4.2324"]}')

    data = rlk_jsonloads(strdata)
    assert isinstance(data['a'], FVal)
    assert data['a'] == FVal('3.14')
    assert isinstance(data['b'], int)
    assert data['b'] == 5
    assert isinstance(data['c'], (str, bytes))
    assert data['c'] == 'foo'
    assert isinstance(data['d'], FVal)
    assert data['d'] == FVal('5.42323143')

    assert isinstance(data['e']['u1'], FVal)
    assert data['e']['u1'] == FVal('3.221')

    assert isinstance(data['f'][0], FVal)
    assert data['f'][0] == FVal('2.1')
    assert isinstance(data['f'][1], (str, bytes))
    assert data['f'][1] == "boo"
    assert isinstance(data['f'][2], int)
    assert data['f'][2] == 3
    assert isinstance(data['f'][3], FVal)
    assert data['f'][3] == FVal('4.2324')
Ejemplo n.º 10
0
def test_rlk_jsonloads():
    data = '{"a": "5.4", "b": "foo", "c": 32.1, "d": 5, "e": [1, "a", "5.1"]}'
    result = rlk_jsonloads(data)
    assert result == {
        'a': FVal('5.4'),
        'b': 'foo',
        'c': FVal('32.1'),
        'd': 5,
        'e': [1, 'a', FVal('5.1')],
    }
Ejemplo n.º 11
0
    def attempt_connect(self, ethrpc_port: int, mainnet_check=True) -> Tuple[bool, str]:
        if self.rpc_port == ethrpc_port and self.connected:
            # We are already connected
            return True, 'Already connected to an ethereum node'

        if self.web3:
            del self.web3

        try:
            self.web3 = Web3(HTTPProvider('http://localhost:{}'.format(ethrpc_port)))
        except requests.exceptions.ConnectionError:
            log.warning('Could not connect to a local ethereum node. Will use etherscan only')
            self.connected = False
            return False, 'Failed to connect to ethereum node at port {}'.format(ethrpc_port)

        if self.web3.isConnected():
            dir_path = os.path.dirname(os.path.realpath(__file__))
            with open(os.path.join(dir_path, 'data', 'token_abi.json'), 'r') as f:
                self.token_abi = rlk_jsonloads(f.read())

            # Also make sure we are actually connected to the Ethereum mainnet
            if mainnet_check:
                genesis_hash = self.web3.eth.getBlock(0)['hash'].hex()  # pylint: disable=no-member
                target = '0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3'
                if genesis_hash != target:
                    log.warning(
                        'Connected to a local ethereum node but it is not on the ethereum mainnet',
                    )
                    self.connected = False
                    message = (
                        'Connected to ethereum node at port {} but it is not on '
                        'the ethereum mainnet'.format(ethrpc_port)
                    )
                    return False, message
                if self.web3.eth.syncing:  # pylint: disable=no-member
                    current_block = self.web3.eth.syncing.currentBlock  # pylint: disable=no-member
                    latest_block = self.web3.eth.syncing.highestBlock  # pylint: disable=no-member
                    return self.is_synchronized(current_block, latest_block)
                else:
                    current_block = self.web3.eth.blockNumber  # pylint: disable=no-member
                    latest_block = self.query_eth_highest_block()
                    if latest_block is None:
                        return False, 'Could not query latest block from blockcypher.'
                    return self.is_synchronized(current_block, latest_block)

            self.connected = True
            return True, ''
        else:
            log.warning('Could not connect to a local ethereum node. Will use etherscan only')
            self.connected = False
            message = 'Failed to connect to ethereum node at port {}'.format(ethrpc_port)

        # If we get here we did not connnect
        return False, message
Ejemplo n.º 12
0
def test_rlk_jsonloads():
    data = '{"a": "5.4", "b": "foo", "c": 32.1, "d": 5, "e": [1, "a", "5.1"], "f": "37451082560000003241"}'  # noqa: E501
    result = rlk_jsonloads(data)
    assert result == {
        'a': FVal('5.4'),
        'b': 'foo',
        'c': FVal('32.1'),
        'd': 5,
        'e': [1, 'a', FVal('5.1')],
        'f': 37451082560000003241,
    }
Ejemplo n.º 13
0
    def query_private(self, method: str, req: Optional[dict] = None) -> dict:
        # Pretty ugly ... mock a kraken remote eror
        if self.remote_errors:
            raise RemoteError('Kraken remote error')

        if method == 'Balance':
            if self.random_balance_data:
                return generate_random_kraken_balance_response()
            # else
            return self.balance_data_return
        elif method == 'TradesHistory':
            if self.random_trade_data:
                return generate_random_kraken_trades_data(
                    start=req['start'],
                    end=req['end'],
                    tradeable_pairs=list(self.tradeable_pairs.keys()),
                )
            # else
            return rlk_jsonloads(KRAKEN_SPECIFIC_TRADES_HISTORY_RESPONSE)
        elif method == 'Ledgers':
            ledger_type = req['type']
            if self.random_ledgers_data:
                return generate_random_kraken_ledger_data(
                    start=req['start'],
                    end=req['end'],
                    ledger_type=ledger_type,
                )

            # else use specific data
            if ledger_type == 'deposit':
                response = KRAKEN_SPECIFIC_DEPOSITS_RESPONSE
            elif ledger_type == 'withdrawal':
                response = KRAKEN_SPECIFIC_WITHDRAWALS_RESPONSE
            else:
                raise AssertionError(
                    'Unknown ledger type at kraken ledgers mock query')

            return rlk_jsonloads(response)

        return super().query_private(method, req)
Ejemplo n.º 14
0
    def query_private(self, method: str, req: Optional[dict] = None) -> dict:
        self.first_connection()
        if method == 'Balance':
            return generate_random_kraken_balance_response()
        elif method == 'TradesHistory':
            trades_num = random.randint(1, 49)
            start = req['start']
            end = req['end']

            # Trades is a dict with txid as the key
            trades = {}
            for _ in range(trades_num):
                trade = generate_random_kraken_trade_data(
                    list(self.tradeable_pairs.keys()),
                    start,
                    end,
                )
                trades[trade['ordertxid']] = trade

            response_str = json.dumps({'trades': trades, 'count': trades_num})
            return rlk_jsonloads(response_str)
        elif method == 'Ledgers':
            ledgers_num = random.randint(1, 49)
            start = req['start']
            end = req['end']
            ledger_type = req['type']
            # Ledgers is a dict with txid as the key
            ledgers = {}
            for _ in range(ledgers_num):
                ledger = generate_random_kraken_ledger_data(
                    start_ts=start,
                    end_ts=end,
                    ledger_type=ledger_type,
                )
                ledgers[ledger['refid']] = ledger

            response_str = json.dumps({'ledger': ledgers, 'count': ledgers_num})
            return rlk_jsonloads(response_str)

        return super().query_private(method, req)
Ejemplo n.º 15
0
def generate_random_kraken_ledger_data(start: Timestamp, end: Timestamp, ledger_type):
    ledgers_num = random.randint(1, 49)
    # Ledgers is a dict with txid as the key
    ledgers = {}
    for _ in range(ledgers_num):
        ledger = generate_random_single_kraken_ledger_data(
            start_ts=start,
            end_ts=end,
            ledger_type=ledger_type,
        )
        ledgers[ledger['refid']] = ledger

    response_str = json.dumps({'ledger': ledgers, 'count': ledgers_num})
    return rlk_jsonloads(response_str)
Ejemplo n.º 16
0
    def _api_query(self,
                   command: str,
                   req: Optional[Dict] = None) -> Union[Dict, List]:
        if req is None:
            req = {}

        if command == 'returnTicker' or command == 'returnCurrencies':
            log.debug(f'Querying poloniex for {command}')
            ret = self.session.get(self.public_uri + command)
            return rlk_jsonloads(ret.text)

        req['command'] = command
        with self.lock:
            # Protect this region with a lock since poloniex will reject
            # non-increasing nonces. So if two greenlets come in here at
            # the same time one of them will fail
            req['nonce'] = int(time.time() * 1000)
            post_data = str.encode(urlencode(req))

            sign = hmac.new(self.secret, post_data, hashlib.sha512).hexdigest()
            self.session.headers.update({'Sign': sign})

            log.debug(
                'Poloniex private API query',
                command=command,
                post_data=req,
            )
            ret = self.session.post('https://poloniex.com/tradingApi', req)

        if ret.status_code != 200:
            raise RemoteError(
                f'Poloniex query responded with error status code: {ret.status_code}'
                f' and text: {ret.text}', )

        try:
            if command == 'returnLendingHistory':
                return rlk_jsonloads_list(ret.text)
            else:
                # For some reason poloniex can also return [] for an empty trades result
                if ret.text == '[]':
                    return {}
                else:
                    result = rlk_jsonloads_dict(ret.text)
                    return _post_process(result)
        except JSONDecodeError:
            raise RemoteError(
                f'Poloniex returned invalid JSON response: {ret.text}')
Ejemplo n.º 17
0
def generate_random_kraken_trades_data(
    start: Timestamp,
    end: Timestamp,
    tradeable_pairs: List[str],
):
    trades_num = random.randint(1, 49)

    # Trades is a dict with txid as the key
    trades = {}
    for _ in range(trades_num):
        trade = generate_random_kraken_trade_data(
            tradeable_pairs,
            start,
            end,
        )
        trades[trade['ordertxid']] = trade

    response_str = json.dumps({'trades': trades, 'count': trades_num})
    return rlk_jsonloads(response_str)
Ejemplo n.º 18
0
def request_get(uri: str, timeout: int = ALL_REMOTES_TIMEOUT) -> Union[Dict, List]:
    # TODO make this a bit more smart. Perhaps conditional on the type of request.
    # Not all requests would need repeated attempts
    response = retry_calls(
        5,
        '',
        uri,
        requests.get,
        uri,
        str(timeout),
    )

    if response.status_code != 200:
        raise RemoteError('Get {} returned status code {}'.format(uri, response.status_code))

    try:
        result = rlk_jsonloads(response.text)
    except json.decoder.JSONDecodeError:
        raise ValueError('{} returned malformed json'.format(uri))

    return result
Ejemplo n.º 19
0
def main():
    arg_parser = aggregator_args()
    args = arg_parser.parse_args()
    msg_aggregator = MessagesAggregator()
    user_data_dir = Path(default_data_directory()) / args.db_user
    database = DBHandler(
        user_data_dir=user_data_dir,
        password=args.db_password,
        msg_aggregator=msg_aggregator,
    )
    our_data = AssetResolver().assets
    paprika = CoinPaprika()
    cmc = None
    cmc_list = None
    root_path = os.path.dirname(
        os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
    data_directory = f'{Path.home()}/.rotkehlchen'
    if args.cmc_api_key:
        cmc = Coinmarketcap(
            data_directory=data_directory,
            api_key=args.cmc_api_key,
        )
        cmc_list = cmc.get_cryptocyrrency_map()

    cryptocompare = Cryptocompare(data_directory=data_directory,
                                  database=database)
    paprika_coins_list = paprika.get_coins_list()
    cryptocompare_coins_map = cryptocompare.all_coins()

    if args.input_file:
        if not os.path.isfile(args.input_file):
            print(f'Given input file {args.input_file} is not a file')
            sys.exit(1)

        with open(args.input_file, 'r') as f:
            input_data = rlk_jsonloads(f.read())

        given_symbols = set(input_data.keys())
        current_symbols = set(our_data.keys())
        if not given_symbols.isdisjoint(current_symbols):
            print(
                f'The following given input symbols already exist in the '
                f'all_assets.json file {given_symbols.intersection(current_symbols)}',
            )
            sys.exit(1)

        # If an input file is given, iterate only its assets and perform checks
        for asset_symbol in input_data.keys():
            input_data = process_asset(
                our_data=input_data,
                asset_symbol=asset_symbol,
                paprika_coins_list=paprika_coins_list,
                paprika=paprika,
                cmc_list=cmc_list,
                cryptocompare_coins_map=cryptocompare_coins_map,
                always_keep_our_time=args.always_keep_our_time,
            )

        # and now combine the two dictionaries to get the final one. Note that no
        # checks are perfomed for what was in all_assets.json before the script
        # ran in this case
        our_data = {**our_data, **input_data}

    else:

        # Iterate all of the assets of the all_assets.json file and perform checks
        for asset_symbol in our_data.keys():
            our_data = process_asset(
                our_data=our_data,
                asset_symbol=asset_symbol,
                paprika_coins_list=paprika_coins_list,
                paprika=paprika,
                cmc_list=cmc_list,
                cryptocompare_coins_map=cryptocompare_coins_map,
                always_keep_our_time=args.always_keep_our_time,
            )

    # Finally overwrite the all_assets.json with the modified assets
    with open(
            os.path.join(root_path, 'rotkehlchen', 'data', 'all_assets.json'),
            'w') as f:
        f.write(json.dumps(
            our_data,
            sort_keys=True,
            indent=4,
        ), )
Ejemplo n.º 20
0
    def attempt_connect(
        self,
        ethrpc_endpoint: str,
        mainnet_check: bool = True,
    ) -> Tuple[bool, str]:
        message = ''
        if self.rpc_endpoint == ethrpc_endpoint and self.connected:
            # We are already connected
            return True, 'Already connected to an ethereum node'

        if self.web3:
            del self.web3

        try:
            parsed_eth_rpc_endpoint = urlparse(ethrpc_endpoint)
            if not parsed_eth_rpc_endpoint.scheme:
                ethrpc_endpoint = f"http://{ethrpc_endpoint}"
            provider = HTTPProvider(
                endpoint_uri=ethrpc_endpoint,
                request_kwargs={'timeout': self.eth_rpc_timeout},
            )
            self.web3 = Web3(provider)
            self.web3.middleware_onion.inject(http_retry_request_middleware,
                                              layer=0)
        except requests.exceptions.ConnectionError:
            log.warning(
                'Could not connect to an ethereum node. Will use etherscan only'
            )
            self.connected = False
            return False, f'Failed to connect to ethereum node at endpoint {ethrpc_endpoint}'

        if self.web3.isConnected():
            dir_path = os.path.dirname(
                os.path.dirname(os.path.dirname(os.path.realpath(__file__))), )
            with open(os.path.join(dir_path, 'data', 'token_abi.json'),
                      'r') as f:
                self.token_abi = rlk_jsonloads(f.read())

            # Also make sure we are actually connected to the Ethereum mainnet
            synchronized = True
            msg = ''
            if mainnet_check:
                chain_id = self.web3.eth.chainId
                if chain_id != 1:
                    message = (
                        f'Connected to ethereum node at endpoint {ethrpc_endpoint} but '
                        f'it is not on the ethereum mainnet. The chain id '
                        f'the node is in is {chain_id}.')
                    log.warning(message)
                    self.connected = False
                    return False, message

                if self.web3.eth.syncing:  # pylint: disable=no-member
                    current_block = self.web3.eth.syncing.currentBlock  # pylint: disable=no-member
                    latest_block = self.web3.eth.syncing.highestBlock  # pylint: disable=no-member
                    synchronized, msg = self.is_synchronized(
                        current_block, latest_block)
                else:
                    current_block = self.web3.eth.blockNumber  # pylint: disable=no-member
                    latest_block = self.query_eth_highest_block()
                    if latest_block is None:
                        msg = 'Could not query latest block'
                        log.warning(msg)
                        synchronized = False
                    else:
                        synchronized, msg = self.is_synchronized(
                            current_block, latest_block)

            if not synchronized:
                self.msg_aggregator.add_warning(
                    'You are using an ethereum node but we could not verify that it is '
                    'synchronized in the ethereum mainnet. Balances and other queries '
                    'may be incorrect.', )

            self.connected = True
            log.info(f'Connected to ethereum node at {ethrpc_endpoint}')
            return True, ''
        else:
            log.warning(
                'Could not connect to an ethereum node. Will use etherscan only'
            )
            self.connected = False
            message = f'Failed to connect to ethereum node at endpoint {ethrpc_endpoint}'

        # If we get here we did not connnect
        return False, message
Ejemplo n.º 21
0
    def get_history(self, start_ts, end_ts, end_at_least_ts=None):
        """Gets or creates trades and loans history from start_ts to end_ts or if
        `end_at_least` is given and we have a cache history which satisfies it we
        return the cache
        """
        if end_at_least_ts is None:
            end_at_least_ts = end_ts

        log.info(
            'Get or create trade history',
            start_ts=start_ts,
            end_ts=end_ts,
            end_at_least_ts=end_at_least_ts,
        )

        historyfile_path = os.path.join(self.user_directory,
                                        TRADES_HISTORYFILE)
        if os.path.isfile(historyfile_path):
            with open(historyfile_path, 'r') as infile:
                try:
                    history_json_data = rlk_jsonloads(infile.read())
                except JSONDecodeError:
                    pass

                all_history_okay = data_up_todate(history_json_data, start_ts,
                                                  end_at_least_ts)
                poloniex_history_okay = True
                if self.poloniex is not None:
                    poloniex_history_okay = self.poloniex.check_trades_cache(
                        start_ts,
                        end_at_least_ts,
                    ) is not None
                kraken_history_okay = True
                if self.kraken is not None:
                    kraken_history_okay = self.kraken.check_trades_cache(
                        start_ts,
                        end_at_least_ts,
                    ) is not None
                bittrex_history_okay = True
                if self.bittrex is not None:
                    bittrex_history_okay = self.bittrex.check_trades_cache(
                        start_ts,
                        end_at_least_ts,
                    ) is not None
                bitmex_history_okay = True
                if self.bitmex is not None:
                    bitmex_history_okay = self.bitmex.check_trades_cache(
                        start_ts,
                        end_at_least_ts,
                    ) is not None
                binance_history_okay = True
                if self.binance is not None:
                    binance_history_okay = self.binance.check_trades_cache(
                        start_ts,
                        end_at_least_ts,
                    ) is not None

                if not self.read_manual_margin_positions:
                    marginfile_path = os.path.join(self.user_directory,
                                                   MARGIN_HISTORYFILE)
                    margin_file_contents = get_jsonfile_contents_or_empty_dict(
                        marginfile_path)
                    margin_history_is_okay = data_up_todate(
                        margin_file_contents,
                        start_ts,
                        end_at_least_ts,
                    )
                else:
                    margin_history_is_okay = True
                    margin_file_contents = do_read_manual_margin_positions(
                        self.user_directory, )

                loansfile_path = os.path.join(self.user_directory,
                                              LOANS_HISTORYFILE)
                loan_file_contents = get_jsonfile_contents_or_empty_dict(
                    loansfile_path)
                loan_history_is_okay = data_up_todate(
                    loan_file_contents,
                    start_ts,
                    end_at_least_ts,
                )

                assetmovementsfile_path = os.path.join(
                    self.user_directory,
                    ASSETMOVEMENTS_HISTORYFILE,
                )
                asset_movements_contents = get_jsonfile_contents_or_empty_dict(
                    assetmovementsfile_path, )
                asset_movements_history_is_okay = data_up_todate(
                    asset_movements_contents,
                    start_ts,
                    end_at_least_ts,
                )

                eth_tx_log_path = os.path.join(self.user_directory,
                                               ETHEREUM_TX_LOGFILE)
                eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
                    eth_tx_log_path)
                eth_tx_log_history_history_is_okay = data_up_todate(
                    eth_tx_log_contents,
                    start_ts,
                    end_at_least_ts,
                )

                if (all_history_okay and poloniex_history_okay
                        and kraken_history_okay and bittrex_history_okay
                        and bitmex_history_okay and binance_history_okay
                        and margin_history_is_okay and loan_history_is_okay
                        and asset_movements_history_is_okay
                        and eth_tx_log_history_history_is_okay):

                    log.info(
                        'Using cached history',
                        start_ts=start_ts,
                        end_ts=end_ts,
                        end_at_least_ts=end_at_least_ts,
                    )

                    history_trades = trades_from_dictlist(
                        history_json_data['data'],
                        start_ts,
                        end_ts,
                    )
                    if not self.read_manual_margin_positions:
                        margin_trades = trades_from_dictlist(
                            margin_file_contents['data'],
                            start_ts,
                            end_ts,
                        )
                    else:
                        margin_trades = margin_file_contents

                    eth_transactions = transactions_from_dictlist(
                        eth_tx_log_contents['data'],
                        start_ts,
                        end_ts,
                    )
                    asset_movements = asset_movements_from_dictlist(
                        asset_movements_contents['data'],
                        start_ts,
                        end_ts,
                    )

                    history_trades = include_external_trades(
                        self.db,
                        start_ts,
                        end_ts,
                        history_trades,
                    )

                    # make sure that this is the same as what is returned
                    # from create_history
                    return (
                        '',
                        history_trades,
                        margin_trades,
                        loan_file_contents['data'],
                        asset_movements,
                        eth_transactions,
                    )

        return self.create_history(start_ts, end_ts, end_at_least_ts)
Ejemplo n.º 22
0
    def api_query(self, method: str, options: Optional[Dict] = None) -> Union[List, Dict]:
        if not options:
            options = {}

        backoff = self.initial_backoff

        while True:
            with self.lock:
                # Protect this region with a lock since binance will reject
                # non-increasing nonces. So if two greenlets come in here at
                # the same time one of them will fail
                if method in V3_ENDPOINTS or method in WAPI_ENDPOINTS:
                    api_version = 3
                    # Recommended recvWindows is 5000 but we get timeouts with it
                    options['recvWindow'] = 10000
                    options['timestamp'] = str(int(time.time() * 1000))
                    signature = hmac.new(
                        self.secret,
                        urlencode(options).encode('utf-8'),
                        hashlib.sha256,
                    ).hexdigest()
                    options['signature'] = signature
                elif method in V1_ENDPOINTS:
                    api_version = 1
                else:
                    raise ValueError('Unexpected binance api method {}'.format(method))

                apistr = 'wapi/' if method in WAPI_ENDPOINTS else 'api/'
                request_url = f'{self.uri}{apistr}v{str(api_version)}/{method}?'
                request_url += urlencode(options)

                log.debug('Binance API request', request_url=request_url)

                response = self.session.get(request_url)

            limit_ban = response.status_code == 429 and backoff > self.backoff_limit
            if limit_ban or response.status_code not in (200, 429):
                code = 'no code found'
                msg = 'no message found'
                try:
                    result = rlk_jsonloads(response.text)
                    if isinstance(result, dict):
                        code = result.get('code', code)
                        msg = result.get('msg', msg)
                except JSONDecodeError:
                    pass

                raise RemoteError(
                    'Binance API request {} for {} failed with HTTP status '
                    'code: {}, error code: {} and error message: {}'.format(
                        response.url,
                        method,
                        response.status_code,
                        code,
                        msg,
                    ))
            elif response.status_code == 429:
                if backoff > self.backoff_limit:
                    break
                # Binance has limits and if we hit them we should backoff
                # https://github.com/binance-exchange/binance-official-api-docs/blob/master/rest-api.md#limits
                log.debug('Got 429 from Binance. Backing off', seconds=backoff)
                gevent.sleep(backoff)
                backoff = backoff * 2
                continue
            else:
                # success
                break

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError:
            raise RemoteError(f'Binance returned invalid JSON response: {response.text}')
        return json_ret
Ejemplo n.º 23
0
    def _api_query(
        self,
        verb: Literal['get', 'post'],
        path: str,
        options: Optional[Dict] = None,
    ) -> Dict:
        """
        Queries Bitcoin.de with the given verb for the given path and options
        """
        assert verb in ('get', 'post'), (
            'Given verb {} is not a valid HTTP verb'.format(verb))

        request_path_no_args = '/v4/' + path

        data = ''
        if not options:
            request_path = request_path_no_args
        else:
            request_path = request_path_no_args + '?' + urlencode(options)

        nonce = str(int(time.time() * 1000))
        request_url = self.uri + request_path

        self._generate_signature(
            request_type=verb.upper(),
            url=request_url,
            nonce=nonce,
        )

        headers = {
            'x-api-nonce': nonce,
        }
        if data != '':
            headers.update({
                'Content-Type': 'application/json',
                'Content-Length': str(len(data)),
            })

        log.debug('Bitcoin.de API Query', verb=verb, request_url=request_url)

        try:
            response = getattr(self.session, verb)(request_url,
                                                   data=data,
                                                   headers=headers)
        except requests.exceptions.RequestException as e:
            raise RemoteError(
                f'Bitcoin.de API request failed due to {str(e)}') from e

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError as exc:
            raise RemoteError(
                'Bitcoin.de returned invalid JSON response') from exc

        if response.status_code not in (200, 401):
            if isinstance(json_ret, dict) and 'errors' in json_ret:
                for error in json_ret['errors']:
                    if error.get('field') == 'X-API-KEY' and error.get(
                            'code') == 1:
                        raise RemoteError(
                            'Provided API Key is in invalid Format')

                    if error.get('code') == 3:
                        raise RemoteError('Provided API Key is invalid')

                raise RemoteError(json_ret['errors'])

            raise RemoteError(
                'Bitcoin.de api request for {} failed with HTTP status code {}'
                .format(
                    response.url,
                    response.status_code,
                ), )

        if not isinstance(json_ret, dict):
            raise RemoteError('Bitcoin.de returned invalid non-dict response')

        return json_ret
Ejemplo n.º 24
0
    def _api_query(
            self,
            verb: Literal['get', 'post'],
            path: str,
            options: Optional[Dict] = None,
            authenticated: bool = True,
    ) -> Any:
        """
        Queries ICONOMI with the given verb for the given path and options
        """
        assert verb in ('get', 'post'), (
            'Given verb {} is not a valid HTTP verb'.format(verb)
        )

        request_path_no_args = '/v1/' + path

        data = ''
        if not options:
            request_path = request_path_no_args
        else:
            request_path = request_path_no_args + '?' + urlencode(options)

        timestamp = str(int(time.time() * 1000))
        request_url = self.uri + request_path

        headers = {}
        if authenticated:
            signature = self._generate_signature(
                request_type=verb.upper(),
                request_path=request_path_no_args,
                timestamp=timestamp,
            )
            headers.update({
                'ICN-SIGN': signature,
                'ICN-TIMESTAMP': timestamp,
                'ICN-API-KEY': self.api_key,
            })

        if data != '':
            headers.update({
                'Content-Type': 'application/json',
                'Content-Length': str(len(data)),
            })

        log.debug('ICONOMI API Query', verb=verb, request_url=request_url)

        try:
            response = getattr(self.session, verb)(
                request_url,
                data=data,
                timeout=30,
                headers=headers,
            )
        except requests.exceptions.RequestException as e:
            raise RemoteError(f'ICONOMI API request failed due to {str(e)}') from e

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError as exc:
            raise RemoteError('ICONOMI returned invalid JSON response') from exc

        if response.status_code not in (200, 201):
            if isinstance(json_ret, dict) and 'message' in json_ret:
                raise RemoteError(json_ret['message'])

            raise RemoteError(
                'ICONOMI api request for {} failed with HTTP status code {}'.format(
                    response.url,
                    response.status_code,
                ),
            )

        return json_ret
Ejemplo n.º 25
0
    def _api_query(
        self,
        verb: str,
        path: str,
        options: Optional[Dict] = None,
    ) -> Union[List, Dict]:
        """
        Queries Bitmex with the given verb for the given path and options
        """
        assert verb in ('get', 'post', 'push'), (
            'Given verb {} is not a valid HTTP verb'.format(verb))

        # 20 seconds expiration
        expires = int(time.time()) + 20

        request_path_no_args = '/api/v1/' + path

        data = ''
        if not options:
            request_path = request_path_no_args
        else:
            request_path = request_path_no_args + '?' + urlencode(options)

        if path in BITMEX_PRIVATE_ENDPOINTS:
            self._generate_signature(
                verb=verb,
                path=request_path,
                expires=expires,
                data=data,
            )

        self.session.headers.update({
            'api-expires': str(expires),
        })
        if data != '':
            self.session.headers.update({
                'Content-Type': 'application/json',
                'Content-Length': str(len(data)),
            })

        request_url = self.uri + request_path
        log.debug('Bitmex API Query', verb=verb, request_url=request_url)

        response = getattr(self.session, verb)(request_url, data=data)

        if response.status_code not in (200, 401):
            raise RemoteError(
                'Bitmex api request for {} failed with HTTP status code {}'.
                format(
                    response.url,
                    response.status_code,
                ), )

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError:
            raise RemoteError('Bitmex returned invalid JSON response')

        if isinstance(json_ret, dict) and 'error' in json_ret:
            raise RemoteError(json_ret['error']['message'])

        return json_ret
Ejemplo n.º 26
0
    def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
        """Gets all the cached history data instead of querying all external sources
        to create the history through create_history()

        Can raise:
            - HistoryCacheInvalid:
                If any of the cache files are corrupt in any way, missing or
                do not cover the given time range
        """
        if end_at_least_ts is None:
            end_at_least_ts = end_ts

        historyfile_path = os.path.join(self.user_directory,
                                        TRADES_HISTORYFILE)
        if not os.path.isfile(historyfile_path):
            raise HistoryCacheInvalid()

        with open(historyfile_path, 'r') as infile:
            try:
                history_json_data = rlk_jsonloads(infile.read())
            except JSONDecodeError:
                pass

        if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
            raise HistoryCacheInvalid('Historical trades cache invalid')
        try:
            history_trades = trades_from_dictlist(
                given_trades=history_json_data['data'],
                start_ts=start_ts,
                end_ts=end_ts,
                location='historical trades',
                msg_aggregator=self.msg_aggregator,
            )
        except KeyError:
            raise HistoryCacheInvalid('Historical trades cache invalid')
        history_trades = maybe_add_external_trades_to_history(
            db=self.db,
            start_ts=start_ts,
            end_ts=end_ts,
            history=history_trades,
            msg_aggregator=self.msg_aggregator,
        )

        kraken_okay = (self.kraken is None or self.kraken.check_trades_cache(
            start_ts,
            end_at_least_ts,
        ) is not None)
        if not kraken_okay:
            raise HistoryCacheInvalid('Kraken cache is invalid')

        bittrex_okay = (self.bittrex is None
                        or self.bittrex.check_trades_cache(
                            start_ts,
                            end_at_least_ts,
                        ) is not None)
        if not bittrex_okay:
            raise HistoryCacheInvalid('Bittrex cache is invalid')

        binance_okay = (self.binance is None
                        or self.binance.check_trades_cache(
                            start_ts,
                            end_at_least_ts,
                        ) is not None)
        if not binance_okay:
            raise HistoryCacheInvalid('Binance cache is invalid')

        bitmex_okay = (self.bitmex is None or self.bitmex.check_trades_cache(
            start_ts,
            end_at_least_ts,
        ) is not None)
        if not bitmex_okay:
            raise HistoryCacheInvalid('Bitmex cache is invalid')

        # Poloniex specific
        loan_data = []
        if self.poloniex:
            if not self.poloniex.check_trades_cache(start_ts, end_at_least_ts):
                raise HistoryCacheInvalid('Poloniex cache is invalid')

            loansfile_path = os.path.join(self.user_directory,
                                          LOANS_HISTORYFILE)
            loan_file_contents = get_jsonfile_contents_or_empty_dict(
                loansfile_path)
            loan_history_is_okay = data_up_todate(
                loan_file_contents,
                start_ts,
                end_at_least_ts,
            )
            if not loan_history_is_okay:
                raise HistoryCacheInvalid('Poloniex loan cache is invalid')
            loan_data = loan_file_contents['data']

        # margin positions that have been manually input
        if not self.read_manual_margin_positions:
            marginfile_path = os.path.join(self.user_directory,
                                           MARGIN_HISTORYFILE)
            margin_file_contents = get_jsonfile_contents_or_empty_dict(
                marginfile_path)
            margin_history_is_okay = data_up_todate(
                margin_file_contents,
                start_ts,
                end_at_least_ts,
            )
            if not margin_history_is_okay:
                raise HistoryCacheInvalid('Margin Positions cache is invalid')

            try:
                margin_trades = trades_from_dictlist(
                    given_trades=margin_file_contents['data'],
                    start_ts=start_ts,
                    end_ts=end_ts,
                    location='Margin position trades',
                    msg_aggregator=self.msg_aggregator,
                )
            except KeyError:
                raise HistoryCacheInvalid('Margin Positions cache is invalid')

        else:
            margin_trades = do_read_manual_margin_positions(
                self.user_directory, )

        asset_movements = self._get_cached_asset_movements(
            start_ts=start_ts,
            end_ts=end_ts,
            end_at_least_ts=end_at_least_ts,
        )

        eth_tx_log_path = os.path.join(self.user_directory,
                                       ETHEREUM_TX_LOGFILE)
        eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
            eth_tx_log_path)
        eth_tx_log_history_is_okay = data_up_todate(
            eth_tx_log_contents,
            start_ts,
            end_at_least_ts,
        )
        if not eth_tx_log_history_is_okay:
            raise HistoryCacheInvalid('Ethereum transactions cache is invalid')

        try:
            eth_transactions = transactions_from_dictlist(
                eth_tx_log_contents['data'],
                start_ts,
                end_ts,
            )
        except KeyError:
            raise HistoryCacheInvalid('Ethereum transactions cache is invalid')

        # make sure that this is the same as what is returned
        # from create_history, except for the first argument
        return (
            history_trades,
            margin_trades,
            loan_data,
            asset_movements,
            eth_transactions,
        )
Ejemplo n.º 27
0
    def attempt_connect(self,
                        ethrpc_endpoint: str,
                        mainnet_check=True) -> Tuple[bool, str]:
        message = ''
        if self.rpc_endpoint == ethrpc_endpoint and self.connected:
            # We are already connected
            return True, 'Already connected to an ethereum node'

        if self.web3:
            del self.web3

        try:
            parsed_eth_rpc_endpoint = urlparse(ethrpc_endpoint)
            if not parsed_eth_rpc_endpoint.scheme:
                ethrpc_endpoint = f"http://{ethrpc_endpoint}"
            provider = HTTPProvider(ethrpc_endpoint)
            self.web3 = Web3(provider)
        except requests.exceptions.ConnectionError:
            log.warning(
                'Could not connect to an ethereum node. Will use etherscan only'
            )
            self.connected = False
            return False, f'Failed to connect to ethereum node at endpoint {ethrpc_endpoint}'

        if self.web3.isConnected():
            dir_path = os.path.dirname(os.path.realpath(__file__))
            with open(os.path.join(dir_path, 'data', 'token_abi.json'),
                      'r') as f:
                self.token_abi = rlk_jsonloads(f.read())

            # Also make sure we are actually connected to the Ethereum mainnet
            synchronized = True
            msg = ''
            if mainnet_check:
                genesis_hash = self.web3.eth.getBlock(0)['hash'].hex()  # pylint: disable=no-member
                target = '0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3'
                if genesis_hash != target:
                    log.warning(
                        'Connected to an ethereum node but it is not on the ethereum mainnet',
                    )
                    self.connected = False
                    message = (
                        f'Connected to ethereum node at endpoint {ethrpc_endpoint} but '
                        f'it is not on the ethereum mainnet')
                    return False, message

                if self.web3.eth.syncing:  # pylint: disable=no-member
                    current_block = self.web3.eth.syncing.currentBlock  # pylint: disable=no-member
                    latest_block = self.web3.eth.syncing.highestBlock  # pylint: disable=no-member
                    synchronized, msg = self.is_synchronized(
                        current_block, latest_block)
                else:
                    current_block = self.web3.eth.blockNumber  # pylint: disable=no-member
                    latest_block = self.query_eth_highest_block()
                    if latest_block is None:
                        return False, 'Could not query latest block from blockcypher.'
                    synchronized, msg = self.is_synchronized(
                        current_block, latest_block)

            if not synchronized:
                return False, msg

            self.connected = True
            log.info(f'Connected to ethereum node at {ethrpc_endpoint}')
            return True, ''
        else:
            log.warning(
                'Could not connect to an ethereum node. Will use etherscan only'
            )
            self.connected = False
            message = f'Failed to connect to ethereum node at endpoint {ethrpc_endpoint}'

        # If we get here we did not connnect
        return False, message
Ejemplo n.º 28
0
    def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
        """Gets all the cached history data instead of querying all external sources
        to create the history through create_history()

        Can raise:
            - HistoryCacheInvalid:
                If any of the cache files are corrupt in any way, missing or
                do not cover the given time range
        """
        if end_at_least_ts is None:
            end_at_least_ts = end_ts

        historyfile_path = os.path.join(self.user_directory,
                                        TRADES_HISTORYFILE)
        if not os.path.isfile(historyfile_path):
            raise HistoryCacheInvalid()

        with open(historyfile_path, 'r') as infile:
            try:
                history_json_data = rlk_jsonloads(infile.read())
            except JSONDecodeError:
                pass

        if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
            raise HistoryCacheInvalid('Historical trades cache invalid')
        try:
            history_trades = trades_from_dictlist(
                given_trades=history_json_data['data'],
                start_ts=start_ts,
                end_ts=end_ts,
                location='historical trades',
                msg_aggregator=self.msg_aggregator,
            )
        except (KeyError, DeserializationError):
            raise HistoryCacheInvalid('Historical trades cache invalid')

        history_trades = maybe_add_external_trades_to_history(
            db=self.db,
            start_ts=start_ts,
            end_ts=end_ts,
            history=history_trades,
            msg_aggregator=self.msg_aggregator,
        )

        # Check the cache of each exchange
        poloniex = None
        for _, exchange in self.exchange_manager.connected_exchanges.items():
            if exchange.name == 'poloniex':
                poloniex = exchange
            if not exchange.check_trades_cache(start_ts, end_at_least_ts):
                raise HistoryCacheInvalid(f'{exchange.name} cache is invalid')

        # Poloniex specific
        loan_data = []
        if poloniex:
            loansfile_path = os.path.join(self.user_directory,
                                          LOANS_HISTORYFILE)
            loan_file_contents = get_jsonfile_contents_or_empty_dict(
                loansfile_path)
            loan_history_is_okay = data_up_todate(
                loan_file_contents,
                start_ts,
                end_at_least_ts,
            )
            if not loan_history_is_okay:
                raise HistoryCacheInvalid('Poloniex loan cache is invalid')
            loan_data = loan_file_contents['data']

        asset_movements = self._get_cached_asset_movements(
            start_ts=start_ts,
            end_ts=end_ts,
            end_at_least_ts=end_at_least_ts,
        )

        eth_tx_log_path = os.path.join(self.user_directory,
                                       ETHEREUM_TX_LOGFILE)
        eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
            eth_tx_log_path)
        eth_tx_log_history_is_okay = data_up_todate(
            eth_tx_log_contents,
            start_ts,
            end_at_least_ts,
        )
        if not eth_tx_log_history_is_okay:
            raise HistoryCacheInvalid('Ethereum transactions cache is invalid')

        try:
            eth_transactions = transactions_from_dictlist(
                eth_tx_log_contents['data'],
                start_ts,
                end_ts,
            )
        except KeyError:
            raise HistoryCacheInvalid('Ethereum transactions cache is invalid')

        # make sure that this is the same as what is returned
        # from create_history, except for the first argument
        return (
            history_trades,
            loan_data,
            asset_movements,
            eth_transactions,
        )
Ejemplo n.º 29
0
            raise AssertionError(f'Unexpected {self.name} case: {case}')

        call_options = options.copy()
        limit = options['limit']
        results: Union[List[Trade], List[AssetMovement]] = [
        ]  # type: ignore # bug list nothing
        processed_result_ids: Set[int] = set()
        retries_left = API_REQUEST_RETRY_TIMES
        while retries_left >= 0:
            response = self._api_query(
                endpoint=endpoint,
                options=call_options,
            )
            if response.status_code != HTTPStatus.OK:
                try:
                    error_response = rlk_jsonloads(response.text)
                except JSONDecodeError:
                    msg = f'{self.name} {case} returned an invalid JSON response: {response.text}.'
                    log.error(msg, options=call_options)
                    self.msg_aggregator.add_error(
                        f'Got remote error while querying {self.name} {case}: {msg}',
                    )
                    return []  # type: ignore # bug list nothing

                # Check if the rate limits have been hit (response JSON as dict)
                if isinstance(error_response, dict):
                    if error_response.get(
                            'error', None) == API_RATE_LIMITS_ERROR_MESSAGE:
                        if retries_left == 0:
                            msg = (
                                f'{self.name} {case} request failed after retrying '
Ejemplo n.º 30
0
    def api_query(
        self,
        api_type: BINANCE_API_TYPE,
        method: str,
        options: Optional[Dict] = None,
    ) -> Union[List, Dict]:
        """Performs a binance api query

        May raise:
         - RemoteError
         - BinancePermissionError
        """
        call_options = options.copy() if options else {}

        while True:
            with self.nonce_lock:
                # Protect this region with a lock since binance will reject
                # non-increasing nonces. So if two greenlets come in here at
                # the same time one of them will fail
                if 'signature' in call_options:
                    del call_options['signature']

                is_v3_api_method = api_type == 'api' and method in V3_METHODS
                is_new_futures_api = api_type in ('fapi', 'dapi')
                call_needs_signature = (
                    (api_type == 'fapi' and method in FAPI_METHODS)
                    or (api_type == 'dapi' and method in FAPI_METHODS)
                    or  # same as fapi
                    (api_type == 'sapi' and method in SAPI_METHODS) or
                    (api_type == 'wapi'
                     and method in WAPI_METHODS) or is_v3_api_method)
                if call_needs_signature:
                    if api_type in ('sapi', 'dapi'):
                        api_version = 1
                    elif api_type == 'fapi':
                        api_version = 2
                    elif api_type == 'wapi' or is_v3_api_method:
                        api_version = 3
                    else:
                        raise AssertionError(
                            f'Should never get to signed binance api call for '
                            f'api_type: {api_type} and method {method}', )

                    # Recommended recvWindows is 5000 but we get timeouts with it
                    call_options['recvWindow'] = 10000
                    call_options['timestamp'] = str(ts_now_in_ms() +
                                                    self.offset_ms)
                    signature = hmac.new(
                        self.secret,
                        urlencode(call_options).encode('utf-8'),
                        hashlib.sha256,
                    ).hexdigest()
                    call_options['signature'] = signature
                elif api_type == 'api' and method in V1_METHODS:
                    api_version = 1
                else:
                    raise AssertionError(
                        f'Unexpected {self.name} API method {method}')

                api_subdomain = api_type if is_new_futures_api else 'api'
                request_url = (
                    f'https://{api_subdomain}.{self.uri}{api_type}/v{str(api_version)}/{method}?'
                )
                request_url += urlencode(call_options)
                log.debug(f'{self.name} API request', request_url=request_url)
                try:
                    response = self.session.get(request_url)
                except requests.exceptions.RequestException as e:
                    raise RemoteError(
                        f'{self.name} API request failed due to {str(e)}',
                    ) from e

            if response.status_code not in (200, 418, 429):
                code = 'no code found'
                msg = 'no message found'
                try:
                    result = rlk_jsonloads(response.text)
                    if isinstance(result, dict):
                        code = result.get('code', code)
                        msg = result.get('msg', msg)
                except JSONDecodeError:
                    pass

                exception_class: Union[Type[RemoteError],
                                       Type[BinancePermissionError]]
                if response.status_code == 401 and code == REJECTED_MBX_KEY:
                    # Either API key permission error or if futures/dapi then not enables yet
                    exception_class = BinancePermissionError
                else:
                    exception_class = RemoteError

                raise exception_class(
                    '{} API request {} for {} failed with HTTP status '
                    'code: {}, error code: {} and error message: {}'.format(
                        self.name,
                        response.url,
                        method,
                        response.status_code,
                        code,
                        msg,
                    ))

            if response.status_code in (418, 429):
                # Binance has limits and if we hit them we should backoff.
                # A Retry-After header is sent with a 418 or 429 responses and
                # will give the number of seconds required to wait, in the case
                # of a 429, to prevent a ban, or, in the case of a 418, until
                # the ban is over.
                # https://binance-docs.github.io/apidocs/spot/en/#limits
                retry_after = int(response.headers.get('retry-after', '0'))
                log.debug(
                    f'Got status code {response.status_code} from {self.name}. Backing off',
                    seconds=retry_after,
                )
                if retry_after > RETRY_AFTER_LIMIT:
                    raise RemoteError(
                        '{} API request {} for {} failed with HTTP status '
                        'code: {} due to a too long retry after value (> {})'.
                        format(
                            self.name,
                            response.url,
                            method,
                            response.status_code,
                            RETRY_AFTER_LIMIT,
                        ))

                gevent.sleep(retry_after)
                continue

            # else success
            break

        try:
            json_ret = rlk_jsonloads(response.text)
        except JSONDecodeError as e:
            raise RemoteError(
                f'{self.name} returned invalid JSON response: {response.text}',
            ) from e
        return json_ret