Пример #1
0
    def get_bibtex(self, osti_id: str) -> Union[str, None]:
        """
        GET request for Explorer, get bibtex
        Args:
            osti_id: OSTI ID

        Returns:
            bibtex or None if entry wiht that OSTI ID do not exist
        """
        payload = {"osti_id": osti_id}
        header = {"Accept": "application/x-bibtex"}
        try:
            r = requests.get(
                url=self.config.endpoint,
                auth=(self.config.username, self.config.password),
                params=payload,
                headers=header,
            )
        except Exception:
            raise HTTPError(f"Failed to request for OSTI ID = {osti_id}")
        if r.status_code == 200:
            if r.content.decode() == "":
                return None
            return r.content.decode()
        else:
            raise HTTPError(f"Query for OSTI ID = {osti_id} failed")
Пример #2
0
    def retrieve(self):
        """Retrieve credential value and its expiry from IAM EC2."""
        # Get role names.
        creds_path = "/latest/meta-data/iam/security-credentials"
        url = self._endpoint + creds_path
        res = self._http_client.urlopen("GET", url)
        if res.status != 200:
            raise HTTPError(
                "request failed with status {0}".format(res.status), )
        role_names = res.data.decode("utf-8").split("\n")
        if not role_names:
            raise ResponseError("no role names found in response")

        # Get credentials of first role.
        url = self._endpoint + creds_path + "/" + role_names[0]
        res = self._http_client.urlopen("GET", url)
        if res.status != 200:
            raise HTTPError(
                "request failed with status {0}".format(res.status), )
        data = json.loads(res.data)
        if data["Code"] != "Success":
            raise ResponseError(
                "credential retrieval failed with code {0}".format(
                    data["Code"]), )

        try:
            expiration = datetime.strptime(data["Expiration"], RFC3339NANO)
        except ValueError:
            expiration = datetime.strptime(data["Expiration"], RFC3339)
        return Value(
            data["AccessKeyId"],
            data["SecretAccessKey"],
            session_token=data["Token"],
        ), expiration - self._expiry_delta
Пример #3
0
    def post(self, data: bytes) -> List[ELinkPostResponseModel]:
        """
        Post xml. The xml is assumed to be in the format that ELINK wants. Note that this xml may contain
        multiple records, and therefore ELink may response with multiple objects

        Args:
            data: data to post

        Returns:
            Elink Response.
        """
        r = requests.post(
            self.config.endpoint,
            auth=(self.config.username, self.config.password),
            data=data,
        )
        self.logger.debug("Your data has been posted")
        if r.status_code != 200:
            self.logger.error(f"POST for {data} failed")
            raise HTTPError(f"POST for {data} failed")
        else:
            self.logger.debug("Parsing Elink Response")
            content: Dict[str, Dict[str,
                                    ELinkPostResponseModel]] = parse(r.content)
            if content["records"] is None:
                raise HTTPError(
                    f"POST for {data} failed because there's no data to post")
            to_return = []
            for _, elink_responses in content["records"].items():
                if type(elink_responses) == list:
                    """
                    This is the case where you posted multiple items
                    """
                    for elink_response in elink_responses:
                        e = self.parse_obj_to_elink_post_response_model(
                            elink_response)
                        if e is not None:
                            self.logger.debug(
                                f"Received mp-id=[{e.accession_num}] - OSTI-ID = [{e.osti_id}]"
                            )
                            to_return.append(e)
                else:
                    """
                    This is the case where you posted only one item
                    """
                    e = self.parse_obj_to_elink_post_response_model(
                        elink_responses)
                    if e is not None:
                        to_return.append(e)
            return to_return
Пример #4
0
    def write(self, rows: Iterable[WriterTableRow]):
        response = self.__pool.urlopen(
            "POST",
            "/?" + urlencode({
                **self.__options,
                "query":
                f"INSERT INTO {self.__table_name} FORMAT JSONEachRow",
            }),
            headers={
                "Connection": "keep-alive",
                "Accept-Encoding": "gzip,deflate"
            },
            body=self._prepare_chunks(rows),
            chunked=True,
        )

        if response.status != 200:
            # XXX: This should be switched to just parse the JSON body after
            # https://github.com/yandex/ClickHouse/issues/6272 is available.
            content = response.data.decode("utf8")
            details = CLICKHOUSE_ERROR_RE.match(content)
            if details is not None:
                code, type, message = details.groups()
                raise ClickhouseError(int(code), message)
            else:
                raise HTTPError(
                    f"Received unexpected {response.status} response: {content}"
                )
Пример #5
0
def test_connection_recovery(livedata, mocker):
    watch_mock = mocker.Mock(wraps=live.watcher.watch)
    cancel_mock = mocker.Mock(wraps=live.watcher.cancel)
    live.watcher.watch = watch_mock
    live.watcher.cancel = cancel_mock
    livedata._cache = None
    livedata._ttl = datetime.timedelta(seconds=1)
    livedata._retry_interval = datetime.timedelta(seconds=2)
    livedata.is_stale = lambda: True
    livedata._db.child = mocker.Mock(side_effect=HTTPError('Test error'))

    livedata.restart()
    time.sleep(3)

    watch_mock.assert_any_call(livedata.get_metawatcher_name(),
                               callee.functions.Callable(),
                               livedata.get_data_silent,
                               interval=livedata._retry_interval)

    livedata._db.child = mocker.Mock()
    livedata.is_stale = lambda: False

    time.sleep(3)

    cancel_mock.assert_any_call('meta_{}'.format(id(livedata)))
    assert isinstance(livedata._cache, data.FirebaseData)
Пример #6
0
    async def get_top_tags(self, session: aiohttp.ClientSession,
                           artist_index: int,
                           artist_name: str) -> (int, Dict[str, Any]):
        params = {
            'artist': artist_name,
            'api_key': self.client_id,
            'method': 'artist.getTopTags',
            'format': 'json'
        }

        url = self.base_url + "?" + "&".join(
            ["=".join([k, quote_plus(v)]) for k, v in params.items()])
        with async_timeout.timeout(5):
            async with session.get(url) as resp:
                try:
                    res = await resp.json()
                    if "toptags" in res:
                        tags = {}
                        for item in res["toptags"]["tag"]:
                            tags[item["name"]] = item["count"]
                        print(f"Tagged {artist_name} at {artist_index}")
                        return artist_index, tags
                    if "error" in res:
                        message = res.get('message', res['error'])
                        if "Rate" in message:
                            raise RateLimited(message)
                        else:
                            raise HTTPError(message)
                except Exception as e:
                    if "could not be found" not in str(e):
                        raise e
        return artist_index, []
Пример #7
0
    def get_token(data: dict) -> Token:
        """
        Get user token with credentials
        :param data email + password or refresh_token
        """
        local = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
        data.update({
            'get_secure_url': 1,
            'client_id': PixivConstant.CLIENT_ID,
            'client_secret': PixivConstant.CLIENT_SECRET,
        })
        headers = {
            'User-Agent':
            'PixivAndroidApp/5.0.115 (Android 6.0; PixivBot)',
            'X-Client-Time':
            local,
            'X-Client-Hash':
            hashlib.md5(
                (local +
                 PixivConstant.HASH_SECRET).encode('utf8')).hexdigest(),
            'Accept-Language':
            'en-US',
        }
        res = HTTPClient.client.post(
            'https://oauth.secure.pixiv.net/auth/token',
            data=data,
            headers=headers)

        if res.status_code == 200:
            res = json.loads(res.text, object_hook=Token.object_hook)
            del res['response']
            return Token(**res)
        else:
            raise HTTPError(res.status_code, res.text)
Пример #8
0
    def get_multiple_bibtex_helper(self,
                                   osti_ids: List[str]) -> Dict[str, str]:
        """
        Get multiple bibtex, assuming that I can send all osti_ids at once
        Args:
            osti_ids: OSTI ID

        Returns:
            return OSTI -> bibtex
        """
        payload = {"rows": len(osti_ids)}
        header = {"Accept": "application/x-bibtex"}
        r = requests.get(
            url=self.config.endpoint + "?osti_id=" + "%20OR%20".join(osti_ids),
            auth=(self.config.username, self.config.password),
            params=payload,
            headers=header,
        )
        if r.status_code == 200:
            if r.content.decode() == "":
                return dict()
            result = self.parse_bibtex(r.content.decode())
            return result
        else:
            raise HTTPError(f"Query for OSTI IDs = {osti_ids} failed")
Пример #9
0
    def test_request_reprocess_text_exception(self):
        document = AttachmentFile(
            external_id='exception_documentcloud_id',
            url=
            'https://www.documentcloud.org/documents/exception_documentcloud_id-CRID-234-CR.html'
        )

        with patch(
                'document_cloud.documentcloud_session.DocumentCloudSession.post',
                return_value=Mock(status_code=200)):
            session = DocumentCloudSession(self.log_func)

        with patch(
                'document_cloud.documentcloud_session.DocumentCloudSession.post',
                side_effect=HTTPError('Invalid request')) as post_mock:
            requested, success = session._request_reprocess_text(document)

            expect(post_mock).to.be.called_with(
                'https://www.documentcloud.org/documents/exception_documentcloud_id/reprocess_text',
                headers={
                    'x-requested-with': 'XMLHttpRequest',
                    'accept': 'application/json, text/javascript, */*; q=0.01'
                })
            expect(self.log_func).to.be.called_with(
                '[ERROR] when sending reprocess request for '
                'https://www.documentcloud.org/documents/exception_documentcloud_id-CRID-234-CR.html: Invalid request'
            )
            expect(requested).to.be.false()
            expect(success).to.be.false()
Пример #10
0
    def get(self, mpid_or_ostiid: str) -> Union[None, ELinkGetResponseModel]:
        """
        get a single ELinkGetResponseModel from mpid

        Args:
            mpid_or_ostiid: mpid to query

        Returns:
            ELinkGetResponseModel
        """
        key = ("site_unique_id" if "mp-" in mpid_or_ostiid
               or "mvc-" in mpid_or_ostiid else "osti_id")
        payload = {key: mpid_or_ostiid}
        self.logger.debug("GET from {} w/i payload = {} ...".format(
            self.config.endpoint, payload))
        r = requests.get(
            self.config.endpoint,
            auth=(self.config.username, self.config.password),
            params=payload,
        )
        if r.status_code == 200:
            elink_response_xml = r.content
            return ELinkGetResponseModel.parse_obj(
                parse(elink_response_xml)["records"]["record"])
        else:
            msg = f"Error code from GET is {r.status_code}"
            self.logger.error(msg)
            raise HTTPError(msg)
Пример #11
0
 def test_resync_http_error(self, m_sleep):
     self.driver._init_received.set()
     with patch.object(self.driver, "get_etcd_connection") as m_get:
         with patch("calico.etcddriver.driver.monotonic_time") as m_time:
             m_time.side_effect = iter([1, 10, RuntimeError()])
             m_get.side_effect = HTTPError()
             self.assertRaises(RuntimeError, self.driver._resync_and_merge)
Пример #12
0
    def test_later_exceptions(self):
        log.debug("test_later_read_exceptions")

        client = stub_etcd.Client()
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(stub_etcd.EtcdException())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(ReadTimeoutError("pool", "url", "message"))
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(SocketTimeout())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(ConnectTimeoutError())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(HTTPError())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(HTTPException())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(stub_etcd.EtcdClusterIdChanged())
        client.add_read_result(key="/bloop", value="value")
        client.add_read_exception(stub_etcd.EtcdEventIndexCleared())
        elector = election.Elector(client,
                                   "test_basic",
                                   "/bloop",
                                   interval=5,
                                   ttl=15)
        self._wait_and_stop(client, elector)
Пример #13
0
    def download_data(
        self, keys: List[str]
    ) -> Tuple[Dict[str, ELinkGetResponseModel], Dict[str, dict]]:
        """
        Download data from elink and explorer given a set of accession numbers
        Args:
            keys: accession numbers

        Returns:
            Elink and bibtex record in mp_id -> record dictionary format
        """
        elink_records = self.elink_adapter.get_multiple(mp_ids=keys,
                                                        chunk_size=100)
        elink_records_dict = ELinkAdapter.list_to_dict(
            elink_records)  # mp_id -> elink_record
        self.log_info_msg(
            f"Found and downloaded [{len(elink_records_dict)}] records from ELink."
        )
        try:
            self.log_info_msg("Downloading Bibtex")
            bibtex_dict_raw = self.explorer_adapter.get_multiple_bibtex(
                osti_ids=[r.osti_id for r in elink_records], chunk_size=100)
            bibtex_dict = dict()
            for elink in elink_records_dict.values():
                if elink.osti_id in bibtex_dict_raw:
                    bibtex_dict[elink.accession_num] = bibtex_dict_raw[
                        elink.osti_id]
            self.log_info_msg(
                f"Found and downloaded [{len(bibtex_dict)}] records from Explorer."
            )
        except HTTPError:
            bibtex_dict = dict()
        except Exception as e:
            raise HTTPError(f"Downloading Bibtex Failed {e}")
        return elink_records_dict, bibtex_dict
Пример #14
0
 def fetch_data(self, url):
     if url:
         site = req.get(url)
         if site.status_code != 200:
             raise HTTPError(
                 f'Request error with status code {site.status_code}')
         return site
     raise ValueError('URL must be present')
Пример #15
0
class TestPickle(object):
    @pytest.mark.parametrize('exception', [
        HTTPError(None),
        MaxRetryError(None, None, None),
        LocationParseError(None),
        ConnectTimeoutError(None),
        HTTPError('foo'),
        HTTPError('foo', IOError('foo')),
        MaxRetryError(HTTPConnectionPool('localhost'), '/', None),
        LocationParseError('fake location'),
        ClosedPoolError(HTTPConnectionPool('localhost'), None),
        EmptyPoolError(HTTPConnectionPool('localhost'), None),
        ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None),
    ])
    def test_exceptions(self, exception):
        result = pickle.loads(pickle.dumps(exception))
        assert isinstance(result, type(exception))
Пример #16
0
    def retrieve(self):
        """
            Retrieve credential value and its expiry from IAM EC2 instance role
            or ECS task role.
        """
        if not self._is_ecs_task:
            # Get role names and get the first role for EC2.
            creds_path = "/latest/meta-data/iam/security-credentials"
            url = self._endpoint + creds_path
            res = self._http_client.urlopen("GET", url)
            if res.status != 200:
                raise HTTPError(
                    "request failed with status {0}".format(res.status), )
            role_names = res.data.decode("utf-8").split("\n")
            if not role_names:
                raise ResponseError("no role names found in response")
            credentials_url = self._endpoint + creds_path + "/" + role_names[0]
        else:
            # This URL directly gives the credentials for an ECS task
            relative_url_var = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
            creds_path = os.environ.get(relative_url_var) or ""
            credentials_url = self._endpoint + creds_path

        # Get credentials of role.
        res = self._http_client.urlopen("GET", credentials_url)
        if res.status != 200:
            raise HTTPError(
                "request failed with status {0}".format(res.status), )
        data = json.loads(res.data)

        # Note the response in ECS does not include the "Code" key.
        if not self._is_ecs_task and data["Code"] != "Success":
            raise ResponseError(
                "credential retrieval failed with code {0}".format(
                    data["Code"]), )

        try:
            expiration = datetime.strptime(data["Expiration"], RFC3339NANO)
        except ValueError:
            expiration = datetime.strptime(data["Expiration"], RFC3339)
        return Value(
            data["AccessKeyId"],
            data["SecretAccessKey"],
            session_token=data["Token"],
        ), expiration - self._expiry_delta
Пример #17
0
    def exec(self, method, args, is_batch=False):
        """ Execute a method against steemd RPC.

            Warning: Auto-retry on failure, including broadcasting a tx.
        """
        body = self.rpc_body(method, args, is_batch)

        tries = 0
        while tries < 50:
            tries += 1
            try:
                response = self.request(body=body)
                if response.status != 200:
                    raise HTTPError(response.status, "non-200 response")

                response_data = response.data.decode('utf-8')
                result = json.loads(response_data)
                assert result, "result entirely blank"

                if 'error' in result:
                    raise RPCError.build(result['error'], method, args)

                if not is_batch:
                    assert isinstance(result, dict), "result was not a dict"
                    assert 'result' in result, "response with no result key"
                    return result['result']

                # sanity-checking of batch results
                assert isinstance(result, list), "batch result must be list"
                assert len(args) == len(result), "batch result len mismatch"
                for i, item in enumerate(result):
                    id1, id2 = [i + 1, item['id']]
                    assert id1 == id2, "got id %s, expected %s" % (id2, id1)
                    if 'error' in item:
                        raise RPCError.build(item['error'], method, args[i], i)
                    assert 'result' in item, "batch[%d] result empty" % i
                return [item['result'] for item in result]

            except (AssertionError, RPCErrorFatal) as e:
                raise e

            except (MaxRetryError, ConnectionResetError, ReadTimeoutError,
                    RemoteDisconnected, ProtocolError, RPCError,
                    HTTPError) as e:
                logging.error("%s failed, try %d. %s", method, tries, repr(e))

            except json.decoder.JSONDecodeError as e:
                logging.error("invalid JSON returned: %s", response_data)

            except Exception as e:
                logging.error('Unexpected %s: %s', e.__class__.__name__, e)

            if tries % 2 == 0:
                self.next_node()
            time.sleep(tries / 10)

        raise Exception("abort %s after %d tries" % (method, tries))
def test_FUNC_call_WITH_not_existing_url_EXPECT_not_reached() -> None:
    """
    Check whether the call failed.

    :return: No return.
    """
    with pook.use():
        mock = pook.get("https://unreachable.com").error(HTTPError()).persist()
        with pytest.raises(NotReachedError):
            HttpCall.call('GET', 'https://unreachable.com')
Пример #19
0
 def get(cls, url):
     headers = {
         'Content-Type': 'application/json',
     }
     res = requests.get(url,
                        headers=headers,
                        timeout=RESTClient.default_timeout)
     if not res.ok:
         raise HTTPError("url={} status_code={} text={}".format(
             url, res.status_code, res.text))
     return res
Пример #20
0
class TestPickle(object):
    @pytest.mark.parametrize(
        "exception",
        [
            HTTPError(None),
            MaxRetryError(None, None, None),
            LocationParseError(None),
            ConnectTimeoutError(None),
            HTTPError("foo"),
            HTTPError("foo", IOError("foo")),
            MaxRetryError(HTTPConnectionPool("localhost"), "/", None),
            LocationParseError("fake location"),
            ClosedPoolError(HTTPConnectionPool("localhost"), None),
            EmptyPoolError(HTTPConnectionPool("localhost"), None),
            ReadTimeoutError(HTTPConnectionPool("localhost"), "/", None),
        ],
    )
    def test_exceptions(self, exception):
        result = pickle.loads(pickle.dumps(exception))
        assert isinstance(result, type(exception))
def test_FUNC_call_WITH_network_failures_EXPECT_correct_number_of_retries(
) -> None:
    """
    Check if the retry logic works correctly.

    :return: No return.
    """
    with pook.use():
        mock = pook.get("https://unreachable.com").error(HTTPError()).times(5)
        with pytest.raises(NotReachedError):
            HttpCall.call('GET', 'https://unreachable.com')
        assert mock.total_matches == 5
Пример #22
0
    def test_exceptions_with_objects(self):
        assert self.verify_pickling(HTTPError('foo'))

        assert self.verify_pickling(HTTPError('foo', IOError('foo')))

        assert self.verify_pickling(
            MaxRetryError(HTTPConnectionPool('localhost'), '/', None))

        assert self.verify_pickling(LocationParseError('fake location'))

        assert self.verify_pickling(
            ClosedPoolError(HTTPConnectionPool('localhost'), None))

        assert self.verify_pickling(
            EmptyPoolError(HTTPConnectionPool('localhost'), None))

        assert self.verify_pickling(
            HostChangedError(HTTPConnectionPool('localhost'), '/', None))

        assert self.verify_pickling(
            ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None))
Пример #23
0
def validated_json_payload(response):
    """Asserts that the HTTP response was successful and valid JSON."""
    if response.status != 200:
        raise HTTPError(response.status, "non-200 response")

    try:
        data = response.data.decode('utf-8')
        payload = json.loads(data)
    except Exception as e:
        raise Exception("JSON error %s: %s" % (str(e), data[0:1024]))

    return payload
Пример #24
0
 def test_exceptions_with_objects(self):
     assert self.cycle(HTTPError('foo'))
     assert self.cycle(
         MaxRetryError(HTTPConnectionPool('localhost'), '/', None))
     assert self.cycle(LocationParseError('fake location'))
     assert self.cycle(
         ClosedPoolError(HTTPConnectionPool('localhost'), None))
     assert self.cycle(EmptyPoolError(HTTPConnectionPool('localhost'),
                                      None))
     assert self.cycle(
         HostChangedError(HTTPConnectionPool('localhost'), '/', None))
     assert self.cycle(
         TimeoutError(HTTPConnectionPool('localhost'), '/', None))
Пример #25
0
    def get_multiple_helper(self,
                            mp_ids: List[str]) -> List[ELinkGetResponseModel]:
        """
        get a list of elink responses from mpid-s
        Args:
            mp_ids: list of mpids

        Returns:
            list of ELinkGetResponseModel
        """
        if len(mp_ids) == 0:
            return []
        payload = {
            "accession_num": "(" + " ".join(mp_ids) + ")",
            "rows": len(mp_ids)
        }
        r = requests.get(
            self.config.endpoint,
            auth=(self.config.username, self.config.password),
            params=payload,
        )
        if r.status_code == 200:
            elink_response_xml = r.content
            result = []
            try:
                num_found = parse(elink_response_xml)["records"]["@numfound"]
                if num_found == "1":
                    ordered_dict: dict = parse(
                        elink_response_xml)["records"]["record"]
                    if "contributors" in ordered_dict:
                        ordered_dict.pop("contributors")
                    result.append(
                        ELinkGetResponseModel.parse_obj(ordered_dict))
                elif num_found == "0":
                    return []
                else:
                    result: List[ELinkGetResponseModel] = []
                    for record in parse(
                            elink_response_xml)["records"]["record"]:
                        if "contributors" in record:
                            record.pop("contributors")
                        result.append(ELinkGetResponseModel.parse_obj(record))
            except Exception as e:
                self.logger.error(
                    f"Cannot parse returned xml. Error: {e} \n{elink_response_xml}"
                )
            return result
        else:
            msg = f"Error code from GET is {r.status_code}: {r.content}"
            self.logger.error(msg)
            raise HTTPError(msg)
Пример #26
0
    def test_request_reprocess_missing_text_not_update_count_when_cannot_sending_requests(
            self, _):
        AttachmentFileFactory(
            file_type='document',
            text_content='',
            external_id='DOCUMENTCLOUD_empty_text_id',
            source_type=AttachmentSourceType.DOCUMENTCLOUD,
        )
        AttachmentFileFactory(
            file_type='document',
            text_content='',
            external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id',
            source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD,
            reprocess_text_count=1,
        )

        AttachmentFileFactory(
            file_type='document',
            text_content='',
            external_id='DOCUMENTCLOUD_tries_enough_id',
            source_type=AttachmentSourceType.DOCUMENTCLOUD,
            reprocess_text_count=3,
        )

        with patch(
                'document_cloud.documentcloud_session.DocumentCloudSession.post',
                return_value=Mock(status_code=200)):
            session = DocumentCloudSession(self.log_func)

        with patch(
                'document_cloud.documentcloud_session.DocumentCloudSession.post',
                side_effect=HTTPError('Invalid request')):
            session.request_reprocess_missing_text_documents()

        expect(self.log_func).to.be.any_call(
            'Sent reprocessing text requests: 0 success, 2 failure, 1 skipped for 3 no-text documents'
        )

        expect(
            AttachmentFile.objects.get(
                external_id='DOCUMENTCLOUD_empty_text_id').reprocess_text_count
        ).to.equal(0)
        expect(
            AttachmentFile.objects.get(
                external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id').
            reprocess_text_count).to.equal(1)
        expect(
            AttachmentFile.objects.get(
                external_id='DOCUMENTCLOUD_tries_enough_id').
            reprocess_text_count).to.equal(3)
def test_FUNC_call_WITH_network_failures_EXPECT_correct_time_waiting() -> None:
    """
    Check if the retry logic works correctly.

    :return: No return.
    """
    with pook.use():
        mock = pook.get("https://unreachable.com").error(HTTPError()).times(5)
        with pytest.raises(NotReachedError):
            start = time.time()
            HttpCall.call('GET', 'https://unreachable.com')
        end = time.time()
        time_taken = end - start
        # HttpCall uses an exponentiation algorithm that should result in about 8 seconds long total wait time.
        assert 7.5 < time_taken < 8.5
Пример #28
0
 def __make_request(self,
                    api_route: str,
                    method: str = 'get') -> Dict[str, Any]:
     try:
         response: HTTPResponse = self.__pool.request(method,
                                                      api_route,
                                                      release_conn=True,
                                                      redirect=True)
         res_data = json.loads(response.data)
         if response.status != 200:
             raise HTTPError(response.status, res_data['message'])
         return res_data
     except (NewConnectionError, MaxRetryError):
         sys.exit("""Failed to connect. Exiting...""")
     except HTTPError as err:
         sys.exit(err)
Пример #29
0
 async def get_host_status(host: str) -> StatusMessage:
     s = requests.Session()
     retries = Retry(total=5,
                     backoff_factor=0.5,
                     status_forcelist=[500, 502, 503, 504])
     s.mount("http://", HTTPAdapter(max_retries=retries))
     url = "http://{}:{}".format(host, StatusServer.port)
     resp = s.get(url, timeout=1.0)
     if resp.ok:
         status_message = StatusMessage(**resp.json())
         return status_message
     else:
         raise AlgorithmError(
             message=
             "Could not get status for host {} status code: {} response: {}"
             .format(host, resp.status_code, resp.text),
             caused_by=HTTPError(),
         )
Пример #30
0
    def join(self, timeout: Optional[float] = None) -> None:
        response = self.__result.result(timeout)
        logger.debug("Received response for %r.", self)

        if response.status != 200:
            # XXX: This should be switched to just parse the JSON body after
            # https://github.com/yandex/ClickHouse/issues/6272 is available.
            content = response.data.decode("utf8")
            details = CLICKHOUSE_ERROR_RE.match(content)
            if details is not None:
                code = int(details["code"])
                message = details["message"]
                row = int(details["row"]) if details["row"] is not None else None
                raise ClickhouseWriterError(code, message, row)
            else:
                raise HTTPError(
                    f"Received unexpected {response.status} response: {content}"
                )