def get(self, data=None): logger.debug("GET %s", self.url) req = urllib.request.Request(url=self.url, data=data, headers=self.headers) try: if self.opener: request = self.opener.open(req, timeout=self.timeout) else: request = urllib.request.urlopen(req, timeout=self.timeout) except (urllib.error.HTTPError, CertificateError) as error: if error.code == 401: raise UnauthorizedAccess("Access to %s denied" % self.url) else: raise HTTPError("Request to %s failed: %s" % (self.url, error)) except (socket.timeout, urllib.error.URLError) as error: raise HTTPError("Unable to connect to server %s: %s" % (self.url, error)) if request.getcode() > 200: logger.debug("Server responded with status code %s", request.getcode()) try: self.total_size = int(request.info().get("Content-Length").strip()) except AttributeError: logger.warning("Failed to read response's content length") self.total_size = 0 self.response_headers = request.getheaders() self.status_code = request.getcode() if self.status_code > 299: logger.warning("Request responded with code %s", self.status_code) self.content = b"".join(self._iter_chunks(request)) self.info = request.info() request.close() return self
def get(self, data=None): logger.debug("GET %s", self.url) req = urllib.request.Request(url=self.url, data=data, headers=self.headers) try: if self.opener: request = self.opener.open(req, timeout=self.timeout) else: request = urllib.request.urlopen(req, timeout=self.timeout) except (urllib.error.HTTPError, CertificateError) as error: raise HTTPError("Unavailable url %s: %s" % (self.url, error)) except (socket.timeout, urllib.error.URLError) as error: raise HTTPError("Unable to connect to server %s: %s" % (self.url, error)) if request.getcode() > 200: logger.debug("Server responded with status code %s", request.getcode()) try: total_size = request.info().get("Content-Length").strip() total_size = int(total_size) except AttributeError: logger.warning("Failed to read response's content length") total_size = 0 self.response_headers = request.getheaders() self.status_code = request.getcode() if self.status_code > 299: logger.warning("Request responded with code %s", self.status_code) chunks = [] while 1: if self.stop_request and self.stop_request.is_set(): self.content = "" return self try: chunk = request.read(self.buffer_size) except socket.timeout: logger.error("Request timed out") self.content = "" return self self.downloaded_size += len(chunk) if self.thread_queue: self.thread_queue.put( (chunk, self.downloaded_size, total_size)) else: chunks.append(chunk) if not chunk: break request.close() self.content = b"".join(chunks) self.info = request.info() return self
def do_GET(self): workerPort = random.choice(WORKERS) workerUrl = 'http://127.0.0.1:' + workerPort + self.path print('Forwarding request to %s' % workerUrl) response = bytes('', 'utf-8') try: request = urllib.request.urlopen(workerUrl) self.send_response(request.code) for header in request.getheaders(): if header[0] == 'Content-type': self.send_header(header[0], header[1]) self.end_headers() response = request.read() except: self.send_response(504) self.end_headers() finally: self.wfile.write(response)
def main(): url = 'http://httpbin.org/xml' request = urllib.request.urlopen(url=url) # Print status code from server print('STATUS ----------') print(request.status) print('-----------------') # Print headers from server print('HEADERS ----------') print(request.getheaders()) print('-----------------') # Print body from server print('BODY ----------') # need to decode to utf8 string as body (by default) is in raw bytes print(request.read().decode('utf-8')) print('-----------------')
def get(self, data=None): logger.debug("GET %s", self.url) try: req = urllib.request.Request(url=self.url, data=data, headers=self.headers) except ValueError as ex: raise HTTPError("Failed to create HTTP request to %s: %s" % (self.url, ex)) from ex try: if self.opener: request = self.opener.open(req, timeout=self.timeout) else: request = urllib.request.urlopen(req, timeout=self.timeout) # pylint: disable=consider-using-with except (urllib.error.HTTPError, CertificateError) as error: if error.code == 401: raise UnauthorizedAccess("Access to %s denied" % self.url) from error raise HTTPError("%s" % error, code=error.code) from error except (socket.timeout, urllib.error.URLError) as error: raise HTTPError("Unable to connect to server %s: %s" % (self.url, error)) from error self.response_headers = request.getheaders() self.status_code = request.getcode() if self.status_code > 299: logger.warning("Request responded with code %s", self.status_code) try: self.total_size = int(request.info().get("Content-Length").strip()) except AttributeError: self.total_size = 0 self.content = b"".join(self._iter_chunks(request)) self.info = request.info() request.close() return self