def get(self, data=None):
logger.debug("GET %s", self.url)
req = urllib.request.Request(url=self.url,
data=data,
headers=self.headers)
try:
if self.opener:
request = self.opener.open(req, timeout=self.timeout)
else:
request = urllib.request.urlopen(req, timeout=self.timeout)
except (urllib.error.HTTPError, CertificateError) as error:
if error.code == 401:
raise UnauthorizedAccess("Access to %s denied" % self.url)
else:
raise HTTPError("Request to %s failed: %s" % (self.url, error))
except (socket.timeout, urllib.error.URLError) as error:
raise HTTPError("Unable to connect to server %s: %s" %
(self.url, error))
if request.getcode() > 200:
logger.debug("Server responded with status code %s",
request.getcode())
try:
self.total_size = int(request.info().get("Content-Length").strip())
except AttributeError:
logger.warning("Failed to read response's content length")
self.total_size = 0
self.response_headers = request.getheaders()
self.status_code = request.getcode()
if self.status_code > 299:
logger.warning("Request responded with code %s", self.status_code)
self.content = b"".join(self._iter_chunks(request))
self.info = request.info()
request.close()
return self
def get(self, data=None):
logger.debug("GET %s", self.url)
req = urllib.request.Request(url=self.url,
data=data,
headers=self.headers)
try:
if self.opener:
request = self.opener.open(req, timeout=self.timeout)
else:
request = urllib.request.urlopen(req, timeout=self.timeout)
except (urllib.error.HTTPError, CertificateError) as error:
raise HTTPError("Unavailable url %s: %s" % (self.url, error))
except (socket.timeout, urllib.error.URLError) as error:
raise HTTPError("Unable to connect to server %s: %s" %
(self.url, error))
if request.getcode() > 200:
logger.debug("Server responded with status code %s",
request.getcode())
try:
total_size = request.info().get("Content-Length").strip()
total_size = int(total_size)
except AttributeError:
logger.warning("Failed to read response's content length")
total_size = 0
self.response_headers = request.getheaders()
self.status_code = request.getcode()
if self.status_code > 299:
logger.warning("Request responded with code %s", self.status_code)
chunks = []
while 1:
if self.stop_request and self.stop_request.is_set():
self.content = ""
return self
try:
chunk = request.read(self.buffer_size)
except socket.timeout:
logger.error("Request timed out")
self.content = ""
return self
self.downloaded_size += len(chunk)
if self.thread_queue:
self.thread_queue.put(
(chunk, self.downloaded_size, total_size))
else:
chunks.append(chunk)
if not chunk:
break
request.close()
self.content = b"".join(chunks)
self.info = request.info()
return self
def do_GET(self):
workerPort = random.choice(WORKERS)
workerUrl = 'http://127.0.0.1:' + workerPort + self.path
print('Forwarding request to %s' % workerUrl)
response = bytes('', 'utf-8')
try:
request = urllib.request.urlopen(workerUrl)
self.send_response(request.code)
for header in request.getheaders():
if header[0] == 'Content-type':
self.send_header(header[0], header[1])
self.end_headers()
response = request.read()
except:
self.send_response(504)
self.end_headers()
finally:
self.wfile.write(response)
def main():
url = 'http://httpbin.org/xml'
request = urllib.request.urlopen(url=url)
# Print status code from server
print('STATUS ----------')
print(request.status)
print('-----------------')
# Print headers from server
print('HEADERS ----------')
print(request.getheaders())
print('-----------------')
# Print body from server
print('BODY ----------')
# need to decode to utf8 string as body (by default) is in raw bytes
print(request.read().decode('utf-8'))
print('-----------------')
def get(self, data=None):
logger.debug("GET %s", self.url)
try:
req = urllib.request.Request(url=self.url,
data=data,
headers=self.headers)
except ValueError as ex:
raise HTTPError("Failed to create HTTP request to %s: %s" %
(self.url, ex)) from ex
try:
if self.opener:
request = self.opener.open(req, timeout=self.timeout)
else:
request = urllib.request.urlopen(req, timeout=self.timeout) # pylint: disable=consider-using-with
except (urllib.error.HTTPError, CertificateError) as error:
if error.code == 401:
raise UnauthorizedAccess("Access to %s denied" %
self.url) from error
raise HTTPError("%s" % error, code=error.code) from error
except (socket.timeout, urllib.error.URLError) as error:
raise HTTPError("Unable to connect to server %s: %s" %
(self.url, error)) from error
self.response_headers = request.getheaders()
self.status_code = request.getcode()
if self.status_code > 299:
logger.warning("Request responded with code %s", self.status_code)
try:
self.total_size = int(request.info().get("Content-Length").strip())
except AttributeError:
self.total_size = 0
self.content = b"".join(self._iter_chunks(request))
self.info = request.info()
request.close()
return self