def test_reload_unexpected_url(self): """ If the error URL is not the root, the error is propagated. """ error = HTTPError(URL, 503, "Service Unavailable", {}, None) error.url = "/foo" self.fakes.jenkins.responses[urljoin(URL, "/reload")] = error self.assertRaises(HTTPError, self.api.reload)
def test_reload(self): """ The reload method POSTs a request to the '/reload' URL, expecting a 503 on the homepage (which happens after redirection). """ error = HTTPError(URL, 503, "Service Unavailable", {}, None) error.url = URL self.fakes.jenkins.responses[urljoin(URL, "/reload")] = error self.api.reload()
def _except(self, kwargs): # rh_logger.logger.report_event(kwargs['msg']) raise HTTPError(self.request.uri, 400, kwargs['msg'], [], None)
def test_segmented_only_catches_404(cipher_signature): stream = cipher_signature.streams.filter(adaptive=True)[0] with mock.patch('pytube.request.head') as mock_head: mock_head.side_effect = HTTPError('', 403, 'Forbidden', '', '') with pytest.raises(HTTPError): stream.download()
def test_head_url_500(check_docker): expected_exception = HTTPError(code=500, fp=None, url='url', msg='msg', hdrs=[]) with patch('check_docker.check_docker.HTTPSHandler.https_open', side_effect=expected_exception), \ pytest.raises(HTTPError): check_docker.head_url(url='https://example.com/test')
def redirect_response(self, req, fp, code, msg, headers, newurl): if code in (301, 302, 303, 307): raise HTTPError(req.get_full_url(), code, msg, headers, fp)
def _load_data(): raise HTTPError(url=None, code=412, msg='Simulated mock error', hdrs=None, fp=None)
def mock_urlopen_raising_HTTPError(self, url, data, timeout): """Mock implementation of urllib2.urlopen raising HTTPError""" raise HTTPError(None, 404, "Failure", None, None)
def _raise_http_error(url, req): """ Raises HTTP error for 'url' and 'req' """ # Indent req.text to pretty print it later indented_lines = ['\t' + l for l in req.text.splitlines(True)] msg = '\n' + ''.join(indented_lines) raise HTTPError(url, req.status_code, msg, req.headers, None)
def make_request(self, path, data=None, ajax=False, debug=True, force_login=False): url = path if path.startswith("http") else self.url + path logger.error(f" make_request ----------> url: {url}") if ajax: url += f"{('&' if '?' in url else '?')}force_ajax=true" self._session.headers['X_REQUESTED_WITH'] = "XMLHttpRequest" cookie_value = self._session.cookies.get(settings.SESSION_COOKIE_NAME) if force_login and cookie_value: self.response_cookies += f"; {settings.SESSION_COOKIE_NAME}={cookie_value}" if self.csrf_token: self._session.headers['X-CSRFToken'] = self.csrf_token if self.response_cookies: self._session.headers['cookie'] = self.response_cookies if data: for name, value in data.items(): if isinstance(value, IOBase): data[name] = (os.path.basename(value.name), value) encoder = MultipartEncoder(fields=data) self._session.headers['Content-Type'] = encoder.content_type self._session.mount(f"{urlsplit(url).scheme}://", self._adapter) self._session.verify = False self._action = getattr(self._session, 'post', None) _retry = 0 _not_done = True while _not_done and _retry < 3: try: response = self._action( url=url, data=encoder, headers=self._session.headers, timeout=10, stream=False) _not_done = False except (ProtocolError, ConnectionError, ConnectionResetError): time.sleep(1.0) _not_done = True finally: _retry += 1 else: self._session.mount(f"{urlsplit(url).scheme}://", self._adapter) self._session.verify = False self._action = getattr(self._session, 'get', None) _retry = 0 _not_done = True while _not_done and _retry < 3: try: response = self._action( url=url, data=None, headers=self._session.headers, timeout=10, stream=False) _not_done = False except (ProtocolError, ConnectionError, ConnectionResetError): time.sleep(1.0) _not_done = True finally: _retry += 1 try: response.raise_for_status() except requests.exceptions.HTTPError as ex: message = '' if hasattr(ex, 'message'): if debug: logger.error(f'error in request to {path}') logger.error(ex.message) message = ex.message[ex.message.index(':') + 2:] else: message = str(ex) raise HTTPError(url, response.status_code, message, response.headers, None) logger.error(f" make_request ----------> response: {response}") return response
def test_download_file_response_code(mocker, test_image_url, code, reason): side_effect = HTTPError(test_image_url, code, reason, {}, None) mocker.patch(make_mock_target("misc", "urlopen"), side_effect=side_effect) with pytest.raises(RuntimeError): misc.download_file(test_image_url)
def test_build_request_status_http_error(self): status = self.do_collect_views_error( 1, "foo", HTTPError("", 500, "kaputt", None, None)) self.assertEqual(status[("ci.sbb.ch", "foo")].request_status, RequestStatus.ERROR)
def test_collect_views_http_error(self): self.do_collect_views_error(1, view_name=self.view_name_1, error=HTTPError(self.url, 500, None, None, None))
def test_build_request_status_not_found(self): status = self.do_collect_jobs_error( "foo", HTTPError(self.url, 404, None, None, None)) self.assertEqual(RequestStatus.NOT_FOUND, status[("ci.sbb.ch", "foo")].request_status)
def test_build_request_status_error(self): status = self.do_collect_jobs_error( "foo", HTTPError(self.url, 500, None, None, None)) self.assertEqual(RequestStatus.ERROR, status[("ci.sbb.ch", "foo")].request_status)
def _perform(self, url: str, curl_obj: pycurl.Curl = None, headers: dict = None, postfields: dict = None, skip_auth=False) -> bytes: if not skip_auth: self._wait_authenticated() if not curl_obj: curl_obj = pycurl.Curl() if postfields: postfields_dict = urlencode(postfields) _set_postfields(curl_obj, postfields_dict) logger.debug("url={url}, headers={headers}", url=url, headers=headers) if not headers: headers = self.BASE_HEADERS.copy() headers_list = self._headers_to_list(headers) logger.debug("prepared headers={h}", h=headers_list) buffer = BytesIO() curl_obj.setopt(pycurl.WRITEFUNCTION, buffer.write) curl_obj.setopt(pycurl.HEADERFUNCTION, self._header_function) curl_obj.setopt(pycurl.BUFFERSIZE, 102400) curl_obj.setopt(pycurl.URL, url) curl_obj.setopt(pycurl.HTTPHEADER, headers_list) curl_obj.setopt(pycurl.USERAGENT, CURL_USERAGENT) curl_obj.setopt(pycurl.MAXREDIRS, 50) curl_obj.setopt(pycurl.ACCEPT_ENCODING, "") curl_obj.setopt(pycurl.TCP_KEEPALIVE, 1) curl_obj.setopt(pycurl.FOLLOWLOCATION, True) curl_obj.setopt(pycurl.ENCODING, "gzip, deflate") try: curl_obj.perform() except pycurl.error as e: logger.debug(e, exc_info=True) raise status = curl_obj.getinfo(pycurl.HTTP_CODE) logger.debug("HTTP status: {s}", s=status) curl_obj.close() if status != HTTPStatus.OK: hdrs = {} try: hdrs = {k: v[-1] for k, v in self._headers.items()} except (IndexError, KeyError): pass phrase = "error" try: phrase = http.client.responses[status] logger.error("HTTP status error: {s}", s=status) except KeyError: pass raise HTTPError(url=url, msg=phrase, code=status, hdrs=hdrs, fp=None) # Server changing maps will trigger sessionid change, # keep track of latest sessionid in response headers. sessionid = self._find_sessionid() if sessionid and self._auth_data: self._auth_data.sessionid = sessionid return buffer.getvalue()
def urlopen(request): raise HTTPError(request.url, 418, 'Not implemented yet', {}, None)
def redirect_request(self, req: URLRequest, fp: IO[str], code: int, msg: str, headers: Mapping[str, str], newurl: str) -> Optional[URLRequest]: raise HTTPError(newurl, code, msg, headers, fp)
def _post(url, form: Dict[str, Any], headers: Dict[str, Any], timeout=5) -> str: response = requests.post(url, json=form, headers=headers, timeout=timeout) if response.status_code != 200: raise HTTPError(url, response.status_code, response.text, response.headers, None) return response.text
def raise_http_error(self, url, *args, **kw): raise HTTPError(url, 500, 'kaboom: %s %s' % (pp(args), pp(kw)), dict(), None)
def autoretry_backoff_jitter_task(self, url): self.iterations += 1 if "error" in url: fp = tempfile.TemporaryFile() raise HTTPError(url, '500', 'Error', '', fp) return url
def test_is_new_contributor_nonexistent_user(self): self.mocks['api_req'].side_effect = HTTPError(None, 422, None, None, None) assert self.is_new_contributor() self.assert_api_req_call()
{ "path": "src/main.c", "status": "modified", "additions": 1, "deletions": 1 } ] } }"""), ), # valid response ("CVE-1970-1000", 200, b"", None, io.BytesIO(b"")), # empty response ( "CVE-1970-1000", 404, b"vcs not found", HTTPError("/", 404, "vcs not found", {}, io.StringIO("vcs not found")), None, ), # not found ( "CVE-1970-1000", 400, b"something wrong", URLError("something wrong"), None, ), # url error ("CVE-1970-1500", 404, b"not found", None, "None"), # no commit ("CVE-1970-9000", 404, b"not found", None, None), # cve not found ("9999", 404, b"not found", None, None), # id not found ("Invalid ID", 404, b"not found", None, None), # invalid id ]
def test_is_new_contributor_error(self): self.mocks['api_req'].side_effect = HTTPError(None, 403, None, None, None) with pytest.raises(HTTPError): self.is_new_contributor() self.assert_api_req_call()
def main(): hostnames = args.hostname.split('.') if len(hostnames) < 3: msg = 'Hostname "{}" is not a fully-qualified host name of form "HOST.DOMAIN.TOP".'.format( args.hostname) raise Exception(msg) if not args.ip: try: with urlopen("http://ipv4.icanhazip.com/") as f: resp = f.read() if sys.version_info > (3, ): resp = resp.decode('utf-8') args.ip = resp.strip() except URLError: msg = 'Unable to automatically obtain IP address from http://ipv4.icanhazip.com/.' raise Exception(msg) ips = args.ip.split('.') if len(ips)!=4 or \ not ips[0].isdigit() or not ips[1].isdigit() or not ips[2].isdigit() or not ips[3].isdigit() or \ int(ips[0])>255 or int(ips[1])>255 or int(ips[2])>255 or int(ips[3])>255: msg = '"{}" is not valid IP address.'.format(args.ip) raise Exception(msg) url = 'https://api.godaddy.com/v1/domains/{}/records/A/{}'.format( '.'.join(hostnames[1:]), hostnames[0]) data = json.dumps([{ "data": args.ip, "ttl": args.ttl, "name": hostnames[0], "type": "A" }]) if sys.version_info > (3, ): data = data.encode('utf-8') req = Request(url, method='PUT', data=data) req.add_header("Content-Type", "application/json") req.add_header("Accept", "application/json") if args.key and args.secret: req.add_header("Authorization", "sso-key {}:{}".format(args.key, args.secret)) try: with urlopen(req) as f: resp = f.read() if sys.version_info > (3, ): resp = resp.decode('utf-8') resp = json.loads(resp) except HTTPError(e): if e.code == 400: msg = 'Unable to set IP address: GoDaddy API URL ({}) was malformed.'.format( req.full_url) elif e.code == 401: if args.key and args.secret: msg = '''Unable to set IP address: --key or --secret option incorrect. Correct values can be obtained from from https://developer.godaddy.com/keys/ and are ideally placed in a % file.''' else: msg = '''Unable to set IP address: --key or --secret option missing. Correct values can be obtained from from https://developer.godaddy.com/keys/ and are ideally placed in a % file.''' elif e.code == 403: msg = '''Unable to set IP address: customer identified by --key and --secret options denied permission. Correct values can be obtained from from https://developer.godaddy.com/keys/ and are ideally placed in a % file.''' elif e.code == 404: msg = 'Unable to set IP address: {} not found at GoDaddy.'.format( args.hostname) elif e.code == 422: msg = 'Unable to set IP address: "{}" has invalid domain or lacks A record.'.format( args.hostname) elif e.code == 429: msg = 'Unable to set IP address: too many requests to GoDaddy within brief period.' else: msg = 'Unable to set IP address: GoDaddy API failure because "{}".'.format( e.reason) raise Exception(msg) except URLError(e): msg = 'Unable to set IP address: GoDaddy API failure because "{}".'.format( e.reason) raise Exception(msg) print('IP address for {} set to {}.'.format(args.hostname, args.ip))
def import_source_locations( vial_http: urllib3.connectionpool.ConnectionPool, import_run_id: str, import_locations: Iterable[load.ImportSourceLocation], import_batch_size: int = IMPORT_BATCH_SIZE, ) -> ImportSourceLocationsResult: """Import source locations""" created = set() updated = set() path_and_query = f"/api/importSourceLocations?import_run_id={import_run_id}" logger.info("Contacting VIAL: POST %s", path_and_query) batches = 0 for import_locations_batch in misc.batch(import_locations, import_batch_size): encoded_locs = [] for loc in import_locations_batch: if loc.match and loc.match.action == "new": created.add(loc.source_uid) else: updated.add(loc.source_uid) loc_json = orjson.dumps(loc.dict(exclude_none=True)) encoded_locs.append(loc_json) encoded_ndjson = b"\n".join(encoded_locs) try: rsp = vial_http.request( "POST", path_and_query, headers={**vial_http.headers, "Content-Type": "application/x-ndjson"}, body=encoded_ndjson, ) except Exception as e: logger.error( "Error while importing locations: %s (...) %s: %s", encoded_ndjson[:100], encoded_ndjson[-100:], e, ) raise if rsp.status != 200: raise HTTPError( f"/api/importSourceLocations?import_run_id={import_run_id}", rsp.status, rsp.data[:100], dict(rsp.headers), None, ) batches += 1 if batches % 5 == 0: logger.info( "Submitted %d batches of up to %d records to VIAL.", batches, import_batch_size, ) logger.info("Submitted %d total batches to VIAL.", batches) return ImportSourceLocationsResult(created=created, updated=updated)
def test_segmented_stream_on_404(cipher_signature): stream = cipher_signature.streams.filter(adaptive=True)[0] with mock.patch('pytube.request.head') as mock_head: with mock.patch('pytube.request.urlopen') as mock_url_open: # Mock the responses to YouTube mock_url_open_object = mock.Mock() # These are our 4 "segments" of a dash stream # The first explains how many pieces there are, and # the rest are those pieces responses = [ b'Raw_data\r\nSegment-Count: 3', b'a', b'b', b'c', ] joined_responses = b''.join(responses) # We create response headers to match the segments response_headers = [{ 'content-length': len(r), 'Content-Range': '0-%s/%s' % (str(len(r)), str(len(r))) } for r in responses] # Request order for stream: # Filesize: # 1. head(url) -> 404 # 2. get(url&sn=0) # 3. head(url&sn=[1,2,3]) # Download: # 4. info(url) -> 404 # 5. get(url&sn=0) # 6. get(url&sn=[1,2,3]) # Handle filesize requests mock_head.side_effect = [ HTTPError('', 404, 'Not Found', '', ''), *response_headers[1:], ] # Each response must be followed by None, to break iteration # in the stream() function mock_url_open_object.read.side_effect = [ responses[0], None, responses[0], None, responses[1], None, responses[2], None, responses[3], None, ] # This handles the HEAD requests to get content-length mock_url_open_object.info.side_effect = [ HTTPError('', 404, 'Not Found', '', ''), *response_headers ] mock_url_open.return_value = mock_url_open_object with mock.patch('builtins.open', new_callable=mock.mock_open) as mock_open: file_handle = mock_open.return_value.__enter__.return_value fp = stream.download() full_content = b'' for call in file_handle.write.call_args_list: args, kwargs = call full_content += b''.join(args) assert full_content == joined_responses mock_open.assert_called_once_with(fp, 'wb')
def fail(*ignore): raise HTTPError('url', 403, 'Forbidden', 'Forbidden', mock.Mock())
def http_error_302(self, req, fp, code, msg, headers): raise HTTPError(req.full_url, code, msg, headers, fp)
def raise_url_error(url, filename, md5sum=None): raise HTTPError('message', None, None, None, None)
def get_mask(self): # TODO: implement this raise HTTPError(self.request.uri, 501, "The server does not yet support /api/mask requests", [], None)
def process_entries(self, omimids, transform, included_fields=None, graph=None, limit=None): """ Given a list of omim ids, this will use the omim API to fetch the entries, according to the ```included_fields``` passed as a parameter. If a transformation function is supplied, this will iterate over each entry, and either add the results to the supplied ```graph``` or will return a set of processed entries that the calling function can further iterate. If no ```included_fields``` are provided, this will simply fetch the basic entry from omim, that is ALL fields, which includes an entry's: prefix, mimNumber, status, and titles. :param omimids: the set of omim entry ids to fetch using their API :param transform: Function to transform each omim entry when looping :param included_fields: A set of what fields are required to retrieve from the API :param graph: the graph to add the transformed data into """ omimparams = {} reponse_batches = [] # add the included_fields as parameters if included_fields is not None and included_fields: omimparams['include'] = ','.join(included_fields) # not expecting any, but keeping just in case cleanomimids = [o.split(':')[-1] for o in omimids] diff = set(omimids) - set(cleanomimids) if diff: LOG.warning('OMIM has %i dirty bits see"\n %s', len(diff), str(diff)) omimids = cleanomimids cleanomimids = [] # WIP: check if we can use a cached copy of the json records # maybe if exists raw/omim/_<iso-date>.json use that # in the meanwhile, to bypass (in case of emergencies) # with open('raw/omim/_2019-05-01.json', 'r') as cachefile: # reponse_batches = json.load(cachefile) if True: # False: acc = 0 # for counting # note that you can only do request batches of 20 # see info about "Limits" at http://omim.org/help/api # TODO 2017 May seems a majority of many groups of 20 # are producing python None for RDF triple Objects groupsize = 20 if not self.test_mode and limit is not None: # just in case the limit is larger than the number of records, maxit = limit if limit > len(omimids): maxit = len(omimids) else: maxit = len(omimids) while acc < maxit: end = min((maxit, acc + groupsize)) # iterate through the omim ids list, # and fetch from the OMIM api in batches of 20 if self.test_mode: intersect = list( set([str(i) for i in self.test_ids]) & set(omimids[acc:end])) # some of the test ids are in the omimids if intersect: LOG.info("found test ids: %s", intersect) omimparams.update({'mimNumber': ','.join(intersect)}) else: acc += groupsize continue else: omimparams.update( {'mimNumber': ','.join(omimids[acc:end])}) url = OMIMAPI + urllib.parse.urlencode(omimparams) try: req = urllib.request.urlopen(url) except HTTPError as err: # URLError? LOG.warning('fetching: %s', url) error_msg = err.read() if re.search(r'The API key: .* is invalid', str(error_msg)): msg = "API Key not valid" raise HTTPError(url, err.code, msg, err.hdrs, err.fp) LOG.error("Failed with: %s", str(error_msg)) break resp = req.read().decode() acc += groupsize # gather all batches reponse_batches.append(json.loads(resp)) # snag a copy of all the batches with open('./raw/omim/_' + date.today().isoformat() + '.json', 'w') as writer: json.dump(reponse_batches, writer) LOG.info("begin transforming the %i blocks of (20) records", len(reponse_batches)) for myjson in reponse_batches: for entery in myjson['omim']['entryList']: # apply the data transformation, and save it to the graph transform(entery, graph)