def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if hasattr(value, 'read'):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
raise TypeError
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if (
request.has_header('Content-Type') and
request.get_header('Content-Type').find(
'multipart/form-data') != 0
):
six.print_(
"Replacing %s with %s" % (
request.get_header('content-type'),
'multipart/form-data'
)
)
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def http_request(self, request):
data = request.get_data()
if isinstance(data, dict):
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if isinstance(value, file) or hasattr(value, "file") or isinstance(value, io.StringIO):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise SqlmapDataException("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = "multipart/form-data; boundary=%s" % boundary
#if (request.has_header("Content-Type") and request.get_header("Content-Type").find("multipart/form-data") != 0):
# print "Replacing %s with %s" % (request.get_header("content-type"), "multipart/form-data")
request.add_unredirected_header("Content-Type", contenttype)
request.add_data(data)
return request
def _make_token_request(self, data):
headers = {
'Content-Type':
'application/x-www-form-urlencoded',
'Authorization':
'Basic ' +
base64.b64encode(b'codalab_cli_client:').decode('utf-8'),
'X-Requested-With':
'XMLHttpRequest',
}
headers.update(self._extra_headers)
request = urllib.request.Request(
self._address + '/rest/oauth2/token',
data=urllib.parse.urlencode(data).encode('utf-8'),
)
for k, v in headers.items():
request.add_unredirected_header(k, v)
try:
response = urlopen_with_retry(request)
result = json.loads(response.read().decode())
return result
except urllib.error.HTTPError as e:
if e.code == 401:
return None
elif e.code == 404:
raise LoginPermissionError("Invalid username or password.")
raise e
def http_request(request):
data = request.data
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in data.items():
if type(value) == io.BufferedReader:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError
if len(v_files) == 0:
data = urllib.urlencode(v_vars, doseq)
else:
boundary, data = multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if( request.has_header('Content-Type') and request.get_header['Content-Type'].find('multipart/form-data') != 0):
print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.data = data.encode('utf-8')
return request
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for (key, value) in list(data.items()):
if type(value) == io.BufferedReader:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
# raise TypeError("not a valid non-string sequence or mapping object %d" % traceback)
logger.debug("not a valid non-string sequence or mapping object : %s" % traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, self.doseq).encode()
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = "multipart/form-data; boundary=%s" % boundary
request.add_unredirected_header("Content-Type", contenttype)
request.add_data(data)
return request
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if type(value) == io.BufferedReader:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
#raise TypeError("not a valid non-string sequence or mapping object %d" % traceback)
logger.debug("not a valid non-string sequence or mapping object : %s" % traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, self.doseq).encode()
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def http_request(self, request):
try:
data = request.get_data()
except AttributeError:
data = request.data
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for (key, value) in list(data.items()):
if hasattr(value, 'read'):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
raise TypeError
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if (request.has_header('Content-Type') and request.get_header(
'Content-Type').find('multipart/form-data') != 0):
six.print_("Replacing %s with %s" %
(request.get_header('content-type'),
'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
try:
request.add_data(data)
except AttributeError:
request.data = data
return request
def http_request(self, request):
data = request.get_data()
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if type(value) == file:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if request.has_header('Content-type'):
pass # print "Replacing %s with %s" % (request.get_header('content-type'), contenttype)
request.add_unredirected_header('Content-type', contenttype)
request.add_data(data)
return request
def submit(self, filename):
"""Submit a file to Fossology for analysis.
:param filename: path to the file to submit.
:type filename: str.
:returns: the submission identifier.
:rtype: int
"""
url = self._url + '?mod=upload_file'
with open(filename, 'rb') as f:
variables = [
('folder', '1'),
('description', ''),
('name', ''),
('Check_agent_copyright', '1'),
('Check_agent_mimetype', '1'),
('Check_agent_nomos', '1'),
('Check_agent_pkgagent', '0'),
]
files = [('getfile', os.path.basename(filename), f.read())]
ct, data = encode_multipart_formdata(variables, files)
request = urllib.request.Request(url, data)
request.add_unredirected_header('Content-Type', ct)
request.add_unredirected_header('Content-Length', len(data))
response = self._opener.open(request)
doc = BeautifulSoup(response.read().decode('utf-8'))
expr = re.compile(r'^/\?mod=showjobs&upload=(\d+)')
for tag in doc.find_all('a'):
if tag.has_attr('href'):
match = expr.match(tag['href'])
if match:
return int(match.group(1))
raise FossologyError(_('Upload failed'))
def http_request(self, request):
print(request.__dict__)
data = request.data
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for key, value in data.items():
if isinstance(value,IOBase):
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object")
if len(v_files) == 0:
data = urllib.urlencode(v_vars, doseq)
else:
boundary, data = MultipartPostHandler.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if(request.has_header('Content-Type')
and request.get_header('Content-Type').find('multipart/form-data') != 0):
print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.data = data.encode('utf-8')
return request
def __GetJSONMetadataValue(self, url):
try:
request = urllib.request.Request(url)
request.add_unredirected_header('Metadata-Flavor', 'Google')
data = urllib.request.urlopen(request).read()
return json.loads(data)
except (urllib.error.URLError, urllib.error.HTTPError, IOError):
return None
def GetNumericProjectId():
"""Get the numeric project ID for this VM."""
try:
request = urllib.request.Request(NUMERIC_PROJECT_ID_URL)
request.add_unredirected_header("Metadata-Flavor", "Google")
return urllib.request.urlopen(request).read()
except (urllib.error.URLError, urllib.error.HTTPError, IOError) as e:
return None
def http_request(self, request):
if not request.has_header(self.AUTH_HEADER):
url = request.get_full_url()
user, password = self._creds_mgr.get_credentials(url)
if user is not None and password is not None:
creds = base64.b64encode("%s:%s" % (user, password))
auth = "Basic %s" % creds
request.add_unredirected_header(self.AUTH_HEADER, auth)
return request
def http_request(self, request):
if not request.has_header('Cookie'):
request.add_unredirected_header('Cookie', self.cookie)
request.add_header('Cookie', self.cookie)
else:
cookie = request.get_header('Cookie')
request.add_unredirected_header('Cookie', cookie + '; ' + cookie)
self.cookiejar.add_cookie_header(request)
return request
def process_oauth2(self, request, response, www_authenticate_header):
# This keep infinite auth loops from happening
full_url = request.full_url
self.auth_failure_tracker[full_url] += 1
if self.auth_failure_tracker[full_url] > 1:
raise HTTPError(full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url),
response.headers, response)
auth_token = self._get_outh2_token(www_authenticate_header)
request.add_unredirected_header('Authorization', 'Bearer ' + auth_token)
return self.parent.open(request, timeout=request.timeout)
def process_oauth2(self, request, response, www_authenticate_header):
# This keep infinite auth loops from happening
full_url = request.full_url
self.auth_failure_tracker[full_url] += 1
if self.auth_failure_tracker[full_url] > 1:
raise HTTPError(
full_url, 401,
"Stopping Oauth2 failure loop for {}".format(full_url),
response.headers, response)
auth_token = self._get_outh2_token(www_authenticate_header)
request.add_unredirected_header('Authorization',
'Bearer ' + auth_token)
return self.parent.open(request, timeout=request.timeout)
def _add_referer_header(self, request, origin_request=True):
if self.request is None:
return request
scheme = request.get_type()
original_scheme = self.request.get_type()
if scheme not in ["http", "https"]:
return request
if not origin_request and not self.request.has_header("Referer"):
return request
if (self._handle_referer and original_scheme in ["http", "https"]
and not (original_scheme == "https" and scheme != "https")):
# strip URL fragment (RFC 2616 14.36)
parts = _rfc3986.urlsplit(self.request.get_full_url())
parts = parts[:-1] + (None, )
referer = _rfc3986.urlunsplit(parts)
request.add_unredirected_header("Referer", referer)
return request
def ReadPublicEndpoints(self):
"""Fetch list of public endpoint IPs from metadata server."""
try:
# If the connection gets abandoned, ensure we don't hang more than
# 70 seconds.
url = PUBLIC_ENDPOINT_URL_PREFIX + self.last_etag
request = urllib.request.Request(url)
request.add_unredirected_header('Metadata-Flavor', 'Google')
u = self.urllib2.urlopen(request, timeout=70)
addrs_data = u.read()
headers = u.info().dict
self.last_etag = headers.get('etag', self.default_last_etag)
except urllib.error.HTTPError as h:
self.ResetEtag()
# 404 is treated like an empty list, for backward compatibility.
if h.code == 404:
return []
raise h
return self.ParseIPAddrs(addrs_data)
def _add_referer_header(self, request, origin_request=True):
if self.request is None:
return request
scheme = request.type
original_scheme = self.request.type
if scheme not in ["http", "https"]:
return request
if not origin_request and not self.request.has_header("Referer"):
return request
if (self._handle_referer and
original_scheme in ["http", "https"] and
not (original_scheme == "https" and scheme != "https")):
# strip URL fragment (RFC 2616 14.36)
parts = urlsplit(self.request.get_full_url())
parts = parts[:-1]+(None,)
referer = urlunsplit(parts)
request.add_unredirected_header("Referer", referer)
return request
def http_request(self, request):
data = request.get_data()
def isfiledata(p_str):
import re
r_c = re.compile("^f'(.*)'$")
rert = r_c.search(str(p_str))
#rert = re.search("^f'(.*)'$", p_str)
if rert:
return rert.group(1)
else:
return None
if data is not None and type(data) != str:
v_files = []
v_vars = []
try:
for(key, value) in list(data.items()):
if isfiledata(value): # type(value) == file:
v_files.append((key, value))
else:
v_vars.append((key, value))
except TypeError:
systype, value, traceback = sys.exc_info()
raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback)
if len(v_files) == 0:
data = urllib.parse.urlencode(v_vars, doseq)
else:
boundary, data = self.multipart_encode(v_vars, v_files)
contenttype = 'multipart/form-data; boundary=%s' % boundary
if(request.has_header('Content-Type')
and request.get_header('Content-Type').find('multipart/form-data') != 0):
print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data'))
request.add_unredirected_header('Content-Type', contenttype)
request.add_data(data)
return request
def GetMetadata(self, url_path, recursive=False, timeout=None):
"""Retrieves instance metadata.
Args:
url_path: The path of the metadata url after the api version.
http://169.254.169.254/computeMetadata/v1/url_path
recursive: If set, returns the tree of metadata starting at url_path as
a json string.
timeout: How long to wait for blocking operations (in seconds).
A value of None uses urllib2's default timeout.
Returns:
The metadata returned based on the url path.
"""
# Use the latest version of the metadata.
suffix = ""
if recursive:
suffix = "?recursive=true"
url = "{0}{1}{2}".format(METADATA_V1_URL_PREFIX, url_path, suffix)
request = urllib.request.Request(url)
request.add_unredirected_header("Metadata-Flavor", "Google")
return self.Get(request, timeout=timeout)
def do_request_(self, request):
host = request.get_host()
if not host:
raise URLError('no host given')
if request.has_data(): # POST
data = request.get_data()
if not request.has_header('Content-type'):
request.add_unredirected_header(
'Content-type', 'application/x-www-form-urlencoded')
scheme, sel = urllib.parse.splittype(request.get_selector())
sel_host, sel_path = urllib.parse.splithost(sel)
if not request.has_header('Host'):
request.add_unredirected_header('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
return request
def do_request_(self, request):
host = request.get_host()
if not host:
raise URLError('no host given')
if request.has_data(): # POST
data = request.get_data()
if not request.has_header('Content-type'):
request.add_unredirected_header(
'Content-type',
'application/x-www-form-urlencoded')
scheme, sel = urllib.parse.splittype(request.get_selector())
sel_host, sel_path = urllib.parse.splithost(sel)
if not request.has_header('Host'):
request.add_unredirected_header('Host', sel_host or host)
for name, value in self.parent.addheaders:
name = name.capitalize()
if not request.has_header(name):
request.add_unredirected_header(name, value)
return request
def http_request(url,
json_string,
username=None,
password=None,
timeout=None,
additional_headers=None,
content_type=None,
cookies=None,
gzipped=None,
ssl_context=None,
debug=None):
"""
Fetch data from webserver (POST request)
:param json_string: JSON-String
:param username: If *username* is given, BASE authentication will be used.
:param timeout: Specifies a timeout in seconds for blocking operations
like the connection attempt (if not specified, the global default
timeout setting will be used).
See: https://github.com/gerold-penz/python-jsonrpc/pull/6
:param additional_headers: Dictionary with additional headers
See: https://github.com/gerold-penz/python-jsonrpc/issues/5
:param content_type: Possibility to change the content-type header.
:param cookies: Possibility to add simple cookie-items as key-value pairs.
The key and the value of each cookie-item must be a bytestring.
Unicode is not allowed here.
:param gzipped: If `True`, the JSON-String will be gzip-compressed.
:param ssl_context: Specifies custom TLS/SSL settings for connection.
Python > 2.7.9
See: https://docs.python.org/2/library/ssl.html#client-side-operation
:param debug: If `True` --> *logging.debug*
"""
# Debug
if debug:
logging.debug("Client-->Server: {json_string}".format(
json_string=repr(json_string)))
# Create request and add data
request = Request(url)
if gzipped:
# Compress content (SpooledTemporaryFile to reduce memory usage)
spooled_file = tools.SpooledFile()
tools.gzip_str_to_file(json_string, spooled_file)
del json_string
request.add_header("Content-Encoding", "gzip")
request.add_header("Accept-Encoding", "gzip")
spooled_file.seek(0)
request.data = spooled_file
else:
request.data = json_string
# Content Type
request.add_header("Content-Type", content_type or "application/json")
# Authorization
if username:
base64string = base64.b64encode(
("%s:%s" % (username, password)).encode('ascii')).strip()
request.add_unredirected_header(
"Authorization", "Basic %s" % (base64string.decode('ascii'), ))
# Cookies
if cookies:
cookie = http.cookies.SimpleCookie(cookies)
request.add_header("Cookie", cookie.output(header="", sep=";"))
# Additional headers (overrides other headers)
if additional_headers:
for key, val in list(additional_headers.items()):
request.add_header(key, val)
# Send request to server
try:
if ssl_context:
try:
response = urlopen(request,
timeout=timeout,
context=ssl_context)
except TypeError as err:
if "context" in str(err):
raise NotImplementedError(
"SSL-Context needs Python >= 2.7.9")
else:
raise
else:
response = urlopen(request, timeout=timeout)
except HTTPError as err:
if debug:
retval = err.read()
logging.debug(
"Client<--Server: {retval}".format(retval=repr(retval)))
raise
# Analyze response and return result
try:
if "gzip" in response.headers.get("Content-Encoding", ""):
response_file = tools.SpooledFile(source_file=response)
if debug:
retval = tools.gunzip_file(response_file)
logging.debug(
"Client<--Server: {retval}".format(retval=repr(retval)))
return retval
return tools.gunzip_file(response_file)
else:
if debug:
retval = response.read()
logging.debug(
"Client<--Server: {retval}".format(retval=repr(retval)))
return retval
return response.read()
finally:
response.close()
def http_request(self, request):
if ((self.referer is not None) and not request.has_header("Referer")):
request.add_unredirected_header("Referer", self.referer)
return request
def http_request(self, request):
if ((self.referer is not None) and
not request.has_header("Referer")):
request.add_unredirected_header("Referer", self.referer)
return request
def HttpGet(url, headers=None):
request = urllib.request.Request(url)
if headers:
for key in headers.keys():
request.add_unredirected_header(key, headers[key])
return urllib.request.urlopen(request).read()
def _request_add_headers(self, request):
for name, value in list(self.headers.items()):
request.add_header(name, value)
request.add_unredirected_header(name, value)
return self
