def get_bibtex(self, osti_id: str) -> Union[str, None]: """ GET request for Explorer, get bibtex Args: osti_id: OSTI ID Returns: bibtex or None if entry wiht that OSTI ID do not exist """ payload = {"osti_id": osti_id} header = {"Accept": "application/x-bibtex"} try: r = requests.get( url=self.config.endpoint, auth=(self.config.username, self.config.password), params=payload, headers=header, ) except Exception: raise HTTPError(f"Failed to request for OSTI ID = {osti_id}") if r.status_code == 200: if r.content.decode() == "": return None return r.content.decode() else: raise HTTPError(f"Query for OSTI ID = {osti_id} failed")
def retrieve(self): """Retrieve credential value and its expiry from IAM EC2.""" # Get role names. creds_path = "/latest/meta-data/iam/security-credentials" url = self._endpoint + creds_path res = self._http_client.urlopen("GET", url) if res.status != 200: raise HTTPError( "request failed with status {0}".format(res.status), ) role_names = res.data.decode("utf-8").split("\n") if not role_names: raise ResponseError("no role names found in response") # Get credentials of first role. url = self._endpoint + creds_path + "/" + role_names[0] res = self._http_client.urlopen("GET", url) if res.status != 200: raise HTTPError( "request failed with status {0}".format(res.status), ) data = json.loads(res.data) if data["Code"] != "Success": raise ResponseError( "credential retrieval failed with code {0}".format( data["Code"]), ) try: expiration = datetime.strptime(data["Expiration"], RFC3339NANO) except ValueError: expiration = datetime.strptime(data["Expiration"], RFC3339) return Value( data["AccessKeyId"], data["SecretAccessKey"], session_token=data["Token"], ), expiration - self._expiry_delta
def post(self, data: bytes) -> List[ELinkPostResponseModel]: """ Post xml. The xml is assumed to be in the format that ELINK wants. Note that this xml may contain multiple records, and therefore ELink may response with multiple objects Args: data: data to post Returns: Elink Response. """ r = requests.post( self.config.endpoint, auth=(self.config.username, self.config.password), data=data, ) self.logger.debug("Your data has been posted") if r.status_code != 200: self.logger.error(f"POST for {data} failed") raise HTTPError(f"POST for {data} failed") else: self.logger.debug("Parsing Elink Response") content: Dict[str, Dict[str, ELinkPostResponseModel]] = parse(r.content) if content["records"] is None: raise HTTPError( f"POST for {data} failed because there's no data to post") to_return = [] for _, elink_responses in content["records"].items(): if type(elink_responses) == list: """ This is the case where you posted multiple items """ for elink_response in elink_responses: e = self.parse_obj_to_elink_post_response_model( elink_response) if e is not None: self.logger.debug( f"Received mp-id=[{e.accession_num}] - OSTI-ID = [{e.osti_id}]" ) to_return.append(e) else: """ This is the case where you posted only one item """ e = self.parse_obj_to_elink_post_response_model( elink_responses) if e is not None: to_return.append(e) return to_return
def write(self, rows: Iterable[WriterTableRow]): response = self.__pool.urlopen( "POST", "/?" + urlencode({ **self.__options, "query": f"INSERT INTO {self.__table_name} FORMAT JSONEachRow", }), headers={ "Connection": "keep-alive", "Accept-Encoding": "gzip,deflate" }, body=self._prepare_chunks(rows), chunked=True, ) if response.status != 200: # XXX: This should be switched to just parse the JSON body after # https://github.com/yandex/ClickHouse/issues/6272 is available. content = response.data.decode("utf8") details = CLICKHOUSE_ERROR_RE.match(content) if details is not None: code, type, message = details.groups() raise ClickhouseError(int(code), message) else: raise HTTPError( f"Received unexpected {response.status} response: {content}" )
def test_connection_recovery(livedata, mocker): watch_mock = mocker.Mock(wraps=live.watcher.watch) cancel_mock = mocker.Mock(wraps=live.watcher.cancel) live.watcher.watch = watch_mock live.watcher.cancel = cancel_mock livedata._cache = None livedata._ttl = datetime.timedelta(seconds=1) livedata._retry_interval = datetime.timedelta(seconds=2) livedata.is_stale = lambda: True livedata._db.child = mocker.Mock(side_effect=HTTPError('Test error')) livedata.restart() time.sleep(3) watch_mock.assert_any_call(livedata.get_metawatcher_name(), callee.functions.Callable(), livedata.get_data_silent, interval=livedata._retry_interval) livedata._db.child = mocker.Mock() livedata.is_stale = lambda: False time.sleep(3) cancel_mock.assert_any_call('meta_{}'.format(id(livedata))) assert isinstance(livedata._cache, data.FirebaseData)
async def get_top_tags(self, session: aiohttp.ClientSession, artist_index: int, artist_name: str) -> (int, Dict[str, Any]): params = { 'artist': artist_name, 'api_key': self.client_id, 'method': 'artist.getTopTags', 'format': 'json' } url = self.base_url + "?" + "&".join( ["=".join([k, quote_plus(v)]) for k, v in params.items()]) with async_timeout.timeout(5): async with session.get(url) as resp: try: res = await resp.json() if "toptags" in res: tags = {} for item in res["toptags"]["tag"]: tags[item["name"]] = item["count"] print(f"Tagged {artist_name} at {artist_index}") return artist_index, tags if "error" in res: message = res.get('message', res['error']) if "Rate" in message: raise RateLimited(message) else: raise HTTPError(message) except Exception as e: if "could not be found" not in str(e): raise e return artist_index, []
def get_token(data: dict) -> Token: """ Get user token with credentials :param data email + password or refresh_token """ local = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00') data.update({ 'get_secure_url': 1, 'client_id': PixivConstant.CLIENT_ID, 'client_secret': PixivConstant.CLIENT_SECRET, }) headers = { 'User-Agent': 'PixivAndroidApp/5.0.115 (Android 6.0; PixivBot)', 'X-Client-Time': local, 'X-Client-Hash': hashlib.md5( (local + PixivConstant.HASH_SECRET).encode('utf8')).hexdigest(), 'Accept-Language': 'en-US', } res = HTTPClient.client.post( 'https://oauth.secure.pixiv.net/auth/token', data=data, headers=headers) if res.status_code == 200: res = json.loads(res.text, object_hook=Token.object_hook) del res['response'] return Token(**res) else: raise HTTPError(res.status_code, res.text)
def get_multiple_bibtex_helper(self, osti_ids: List[str]) -> Dict[str, str]: """ Get multiple bibtex, assuming that I can send all osti_ids at once Args: osti_ids: OSTI ID Returns: return OSTI -> bibtex """ payload = {"rows": len(osti_ids)} header = {"Accept": "application/x-bibtex"} r = requests.get( url=self.config.endpoint + "?osti_id=" + "%20OR%20".join(osti_ids), auth=(self.config.username, self.config.password), params=payload, headers=header, ) if r.status_code == 200: if r.content.decode() == "": return dict() result = self.parse_bibtex(r.content.decode()) return result else: raise HTTPError(f"Query for OSTI IDs = {osti_ids} failed")
def test_request_reprocess_text_exception(self): document = AttachmentFile( external_id='exception_documentcloud_id', url= 'https://www.documentcloud.org/documents/exception_documentcloud_id-CRID-234-CR.html' ) with patch( 'document_cloud.documentcloud_session.DocumentCloudSession.post', return_value=Mock(status_code=200)): session = DocumentCloudSession(self.log_func) with patch( 'document_cloud.documentcloud_session.DocumentCloudSession.post', side_effect=HTTPError('Invalid request')) as post_mock: requested, success = session._request_reprocess_text(document) expect(post_mock).to.be.called_with( 'https://www.documentcloud.org/documents/exception_documentcloud_id/reprocess_text', headers={ 'x-requested-with': 'XMLHttpRequest', 'accept': 'application/json, text/javascript, */*; q=0.01' }) expect(self.log_func).to.be.called_with( '[ERROR] when sending reprocess request for ' 'https://www.documentcloud.org/documents/exception_documentcloud_id-CRID-234-CR.html: Invalid request' ) expect(requested).to.be.false() expect(success).to.be.false()
def get(self, mpid_or_ostiid: str) -> Union[None, ELinkGetResponseModel]: """ get a single ELinkGetResponseModel from mpid Args: mpid_or_ostiid: mpid to query Returns: ELinkGetResponseModel """ key = ("site_unique_id" if "mp-" in mpid_or_ostiid or "mvc-" in mpid_or_ostiid else "osti_id") payload = {key: mpid_or_ostiid} self.logger.debug("GET from {} w/i payload = {} ...".format( self.config.endpoint, payload)) r = requests.get( self.config.endpoint, auth=(self.config.username, self.config.password), params=payload, ) if r.status_code == 200: elink_response_xml = r.content return ELinkGetResponseModel.parse_obj( parse(elink_response_xml)["records"]["record"]) else: msg = f"Error code from GET is {r.status_code}" self.logger.error(msg) raise HTTPError(msg)
def test_resync_http_error(self, m_sleep): self.driver._init_received.set() with patch.object(self.driver, "get_etcd_connection") as m_get: with patch("calico.etcddriver.driver.monotonic_time") as m_time: m_time.side_effect = iter([1, 10, RuntimeError()]) m_get.side_effect = HTTPError() self.assertRaises(RuntimeError, self.driver._resync_and_merge)
def test_later_exceptions(self): log.debug("test_later_read_exceptions") client = stub_etcd.Client() client.add_read_result(key="/bloop", value="value") client.add_read_exception(stub_etcd.EtcdException()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(ReadTimeoutError("pool", "url", "message")) client.add_read_result(key="/bloop", value="value") client.add_read_exception(SocketTimeout()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(ConnectTimeoutError()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(HTTPError()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(HTTPException()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(stub_etcd.EtcdClusterIdChanged()) client.add_read_result(key="/bloop", value="value") client.add_read_exception(stub_etcd.EtcdEventIndexCleared()) elector = election.Elector(client, "test_basic", "/bloop", interval=5, ttl=15) self._wait_and_stop(client, elector)
def download_data( self, keys: List[str] ) -> Tuple[Dict[str, ELinkGetResponseModel], Dict[str, dict]]: """ Download data from elink and explorer given a set of accession numbers Args: keys: accession numbers Returns: Elink and bibtex record in mp_id -> record dictionary format """ elink_records = self.elink_adapter.get_multiple(mp_ids=keys, chunk_size=100) elink_records_dict = ELinkAdapter.list_to_dict( elink_records) # mp_id -> elink_record self.log_info_msg( f"Found and downloaded [{len(elink_records_dict)}] records from ELink." ) try: self.log_info_msg("Downloading Bibtex") bibtex_dict_raw = self.explorer_adapter.get_multiple_bibtex( osti_ids=[r.osti_id for r in elink_records], chunk_size=100) bibtex_dict = dict() for elink in elink_records_dict.values(): if elink.osti_id in bibtex_dict_raw: bibtex_dict[elink.accession_num] = bibtex_dict_raw[ elink.osti_id] self.log_info_msg( f"Found and downloaded [{len(bibtex_dict)}] records from Explorer." ) except HTTPError: bibtex_dict = dict() except Exception as e: raise HTTPError(f"Downloading Bibtex Failed {e}") return elink_records_dict, bibtex_dict
def fetch_data(self, url): if url: site = req.get(url) if site.status_code != 200: raise HTTPError( f'Request error with status code {site.status_code}') return site raise ValueError('URL must be present')
class TestPickle(object): @pytest.mark.parametrize('exception', [ HTTPError(None), MaxRetryError(None, None, None), LocationParseError(None), ConnectTimeoutError(None), HTTPError('foo'), HTTPError('foo', IOError('foo')), MaxRetryError(HTTPConnectionPool('localhost'), '/', None), LocationParseError('fake location'), ClosedPoolError(HTTPConnectionPool('localhost'), None), EmptyPoolError(HTTPConnectionPool('localhost'), None), ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None), ]) def test_exceptions(self, exception): result = pickle.loads(pickle.dumps(exception)) assert isinstance(result, type(exception))
def retrieve(self): """ Retrieve credential value and its expiry from IAM EC2 instance role or ECS task role. """ if not self._is_ecs_task: # Get role names and get the first role for EC2. creds_path = "/latest/meta-data/iam/security-credentials" url = self._endpoint + creds_path res = self._http_client.urlopen("GET", url) if res.status != 200: raise HTTPError( "request failed with status {0}".format(res.status), ) role_names = res.data.decode("utf-8").split("\n") if not role_names: raise ResponseError("no role names found in response") credentials_url = self._endpoint + creds_path + "/" + role_names[0] else: # This URL directly gives the credentials for an ECS task relative_url_var = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" creds_path = os.environ.get(relative_url_var) or "" credentials_url = self._endpoint + creds_path # Get credentials of role. res = self._http_client.urlopen("GET", credentials_url) if res.status != 200: raise HTTPError( "request failed with status {0}".format(res.status), ) data = json.loads(res.data) # Note the response in ECS does not include the "Code" key. if not self._is_ecs_task and data["Code"] != "Success": raise ResponseError( "credential retrieval failed with code {0}".format( data["Code"]), ) try: expiration = datetime.strptime(data["Expiration"], RFC3339NANO) except ValueError: expiration = datetime.strptime(data["Expiration"], RFC3339) return Value( data["AccessKeyId"], data["SecretAccessKey"], session_token=data["Token"], ), expiration - self._expiry_delta
def exec(self, method, args, is_batch=False): """ Execute a method against steemd RPC. Warning: Auto-retry on failure, including broadcasting a tx. """ body = self.rpc_body(method, args, is_batch) tries = 0 while tries < 50: tries += 1 try: response = self.request(body=body) if response.status != 200: raise HTTPError(response.status, "non-200 response") response_data = response.data.decode('utf-8') result = json.loads(response_data) assert result, "result entirely blank" if 'error' in result: raise RPCError.build(result['error'], method, args) if not is_batch: assert isinstance(result, dict), "result was not a dict" assert 'result' in result, "response with no result key" return result['result'] # sanity-checking of batch results assert isinstance(result, list), "batch result must be list" assert len(args) == len(result), "batch result len mismatch" for i, item in enumerate(result): id1, id2 = [i + 1, item['id']] assert id1 == id2, "got id %s, expected %s" % (id2, id1) if 'error' in item: raise RPCError.build(item['error'], method, args[i], i) assert 'result' in item, "batch[%d] result empty" % i return [item['result'] for item in result] except (AssertionError, RPCErrorFatal) as e: raise e except (MaxRetryError, ConnectionResetError, ReadTimeoutError, RemoteDisconnected, ProtocolError, RPCError, HTTPError) as e: logging.error("%s failed, try %d. %s", method, tries, repr(e)) except json.decoder.JSONDecodeError as e: logging.error("invalid JSON returned: %s", response_data) except Exception as e: logging.error('Unexpected %s: %s', e.__class__.__name__, e) if tries % 2 == 0: self.next_node() time.sleep(tries / 10) raise Exception("abort %s after %d tries" % (method, tries))
def test_FUNC_call_WITH_not_existing_url_EXPECT_not_reached() -> None: """ Check whether the call failed. :return: No return. """ with pook.use(): mock = pook.get("https://unreachable.com").error(HTTPError()).persist() with pytest.raises(NotReachedError): HttpCall.call('GET', 'https://unreachable.com')
def get(cls, url): headers = { 'Content-Type': 'application/json', } res = requests.get(url, headers=headers, timeout=RESTClient.default_timeout) if not res.ok: raise HTTPError("url={} status_code={} text={}".format( url, res.status_code, res.text)) return res
class TestPickle(object): @pytest.mark.parametrize( "exception", [ HTTPError(None), MaxRetryError(None, None, None), LocationParseError(None), ConnectTimeoutError(None), HTTPError("foo"), HTTPError("foo", IOError("foo")), MaxRetryError(HTTPConnectionPool("localhost"), "/", None), LocationParseError("fake location"), ClosedPoolError(HTTPConnectionPool("localhost"), None), EmptyPoolError(HTTPConnectionPool("localhost"), None), ReadTimeoutError(HTTPConnectionPool("localhost"), "/", None), ], ) def test_exceptions(self, exception): result = pickle.loads(pickle.dumps(exception)) assert isinstance(result, type(exception))
def test_FUNC_call_WITH_network_failures_EXPECT_correct_number_of_retries( ) -> None: """ Check if the retry logic works correctly. :return: No return. """ with pook.use(): mock = pook.get("https://unreachable.com").error(HTTPError()).times(5) with pytest.raises(NotReachedError): HttpCall.call('GET', 'https://unreachable.com') assert mock.total_matches == 5
def test_exceptions_with_objects(self): assert self.verify_pickling(HTTPError('foo')) assert self.verify_pickling(HTTPError('foo', IOError('foo'))) assert self.verify_pickling( MaxRetryError(HTTPConnectionPool('localhost'), '/', None)) assert self.verify_pickling(LocationParseError('fake location')) assert self.verify_pickling( ClosedPoolError(HTTPConnectionPool('localhost'), None)) assert self.verify_pickling( EmptyPoolError(HTTPConnectionPool('localhost'), None)) assert self.verify_pickling( HostChangedError(HTTPConnectionPool('localhost'), '/', None)) assert self.verify_pickling( ReadTimeoutError(HTTPConnectionPool('localhost'), '/', None))
def validated_json_payload(response): """Asserts that the HTTP response was successful and valid JSON.""" if response.status != 200: raise HTTPError(response.status, "non-200 response") try: data = response.data.decode('utf-8') payload = json.loads(data) except Exception as e: raise Exception("JSON error %s: %s" % (str(e), data[0:1024])) return payload
def test_exceptions_with_objects(self): assert self.cycle(HTTPError('foo')) assert self.cycle( MaxRetryError(HTTPConnectionPool('localhost'), '/', None)) assert self.cycle(LocationParseError('fake location')) assert self.cycle( ClosedPoolError(HTTPConnectionPool('localhost'), None)) assert self.cycle(EmptyPoolError(HTTPConnectionPool('localhost'), None)) assert self.cycle( HostChangedError(HTTPConnectionPool('localhost'), '/', None)) assert self.cycle( TimeoutError(HTTPConnectionPool('localhost'), '/', None))
def get_multiple_helper(self, mp_ids: List[str]) -> List[ELinkGetResponseModel]: """ get a list of elink responses from mpid-s Args: mp_ids: list of mpids Returns: list of ELinkGetResponseModel """ if len(mp_ids) == 0: return [] payload = { "accession_num": "(" + " ".join(mp_ids) + ")", "rows": len(mp_ids) } r = requests.get( self.config.endpoint, auth=(self.config.username, self.config.password), params=payload, ) if r.status_code == 200: elink_response_xml = r.content result = [] try: num_found = parse(elink_response_xml)["records"]["@numfound"] if num_found == "1": ordered_dict: dict = parse( elink_response_xml)["records"]["record"] if "contributors" in ordered_dict: ordered_dict.pop("contributors") result.append( ELinkGetResponseModel.parse_obj(ordered_dict)) elif num_found == "0": return [] else: result: List[ELinkGetResponseModel] = [] for record in parse( elink_response_xml)["records"]["record"]: if "contributors" in record: record.pop("contributors") result.append(ELinkGetResponseModel.parse_obj(record)) except Exception as e: self.logger.error( f"Cannot parse returned xml. Error: {e} \n{elink_response_xml}" ) return result else: msg = f"Error code from GET is {r.status_code}: {r.content}" self.logger.error(msg) raise HTTPError(msg)
def test_request_reprocess_missing_text_not_update_count_when_cannot_sending_requests( self, _): AttachmentFileFactory( file_type='document', text_content='', external_id='DOCUMENTCLOUD_empty_text_id', source_type=AttachmentSourceType.DOCUMENTCLOUD, ) AttachmentFileFactory( file_type='document', text_content='', external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id', source_type=AttachmentSourceType.PORTAL_COPA_DOCUMENTCLOUD, reprocess_text_count=1, ) AttachmentFileFactory( file_type='document', text_content='', external_id='DOCUMENTCLOUD_tries_enough_id', source_type=AttachmentSourceType.DOCUMENTCLOUD, reprocess_text_count=3, ) with patch( 'document_cloud.documentcloud_session.DocumentCloudSession.post', return_value=Mock(status_code=200)): session = DocumentCloudSession(self.log_func) with patch( 'document_cloud.documentcloud_session.DocumentCloudSession.post', side_effect=HTTPError('Invalid request')): session.request_reprocess_missing_text_documents() expect(self.log_func).to.be.any_call( 'Sent reprocessing text requests: 0 success, 2 failure, 1 skipped for 3 no-text documents' ) expect( AttachmentFile.objects.get( external_id='DOCUMENTCLOUD_empty_text_id').reprocess_text_count ).to.equal(0) expect( AttachmentFile.objects.get( external_id='PORTAL_COPA_DOCUMENTCLOUD_empty_text_id'). reprocess_text_count).to.equal(1) expect( AttachmentFile.objects.get( external_id='DOCUMENTCLOUD_tries_enough_id'). reprocess_text_count).to.equal(3)
def test_FUNC_call_WITH_network_failures_EXPECT_correct_time_waiting() -> None: """ Check if the retry logic works correctly. :return: No return. """ with pook.use(): mock = pook.get("https://unreachable.com").error(HTTPError()).times(5) with pytest.raises(NotReachedError): start = time.time() HttpCall.call('GET', 'https://unreachable.com') end = time.time() time_taken = end - start # HttpCall uses an exponentiation algorithm that should result in about 8 seconds long total wait time. assert 7.5 < time_taken < 8.5
def __make_request(self, api_route: str, method: str = 'get') -> Dict[str, Any]: try: response: HTTPResponse = self.__pool.request(method, api_route, release_conn=True, redirect=True) res_data = json.loads(response.data) if response.status != 200: raise HTTPError(response.status, res_data['message']) return res_data except (NewConnectionError, MaxRetryError): sys.exit("""Failed to connect. Exiting...""") except HTTPError as err: sys.exit(err)
async def get_host_status(host: str) -> StatusMessage: s = requests.Session() retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]) s.mount("http://", HTTPAdapter(max_retries=retries)) url = "http://{}:{}".format(host, StatusServer.port) resp = s.get(url, timeout=1.0) if resp.ok: status_message = StatusMessage(**resp.json()) return status_message else: raise AlgorithmError( message= "Could not get status for host {} status code: {} response: {}" .format(host, resp.status_code, resp.text), caused_by=HTTPError(), )
def join(self, timeout: Optional[float] = None) -> None: response = self.__result.result(timeout) logger.debug("Received response for %r.", self) if response.status != 200: # XXX: This should be switched to just parse the JSON body after # https://github.com/yandex/ClickHouse/issues/6272 is available. content = response.data.decode("utf8") details = CLICKHOUSE_ERROR_RE.match(content) if details is not None: code = int(details["code"]) message = details["message"] row = int(details["row"]) if details["row"] is not None else None raise ClickhouseWriterError(code, message, row) else: raise HTTPError( f"Received unexpected {response.status} response: {content}" )