def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if hasattr(value, 'read'): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: raise TypeError if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if ( request.has_header('Content-Type') and request.get_header('Content-Type').find( 'multipart/form-data') != 0 ): six.print_( "Replacing %s with %s" % ( request.get_header('content-type'), 'multipart/form-data' ) ) request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def http_request(self, request): data = request.get_data() if isinstance(data, dict): v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if isinstance(value, file) or hasattr(value, "file") or isinstance(value, io.StringIO): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise SqlmapDataException("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = "multipart/form-data; boundary=%s" % boundary #if (request.has_header("Content-Type") and request.get_header("Content-Type").find("multipart/form-data") != 0): # print "Replacing %s with %s" % (request.get_header("content-type"), "multipart/form-data") request.add_unredirected_header("Content-Type", contenttype) request.add_data(data) return request
def _make_token_request(self, data): headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'Authorization': 'Basic ' + base64.b64encode(b'codalab_cli_client:').decode('utf-8'), 'X-Requested-With': 'XMLHttpRequest', } headers.update(self._extra_headers) request = urllib.request.Request( self._address + '/rest/oauth2/token', data=urllib.parse.urlencode(data).encode('utf-8'), ) for k, v in headers.items(): request.add_unredirected_header(k, v) try: response = urlopen_with_retry(request) result = json.loads(response.read().decode()) return result except urllib.error.HTTPError as e: if e.code == 401: return None elif e.code == 404: raise LoginPermissionError("Invalid username or password.") raise e
def http_request(request): data = request.data if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in data.items(): if type(value) == io.BufferedReader: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError if len(v_files) == 0: data = urllib.urlencode(v_vars, doseq) else: boundary, data = multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if( request.has_header('Content-Type') and request.get_header['Content-Type'].find('multipart/form-data') != 0): print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) request.data = data.encode('utf-8') return request
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for (key, value) in list(data.items()): if type(value) == io.BufferedReader: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() # raise TypeError("not a valid non-string sequence or mapping object %d" % traceback) logger.debug("not a valid non-string sequence or mapping object : %s" % traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, self.doseq).encode() else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = "multipart/form-data; boundary=%s" % boundary request.add_unredirected_header("Content-Type", contenttype) request.add_data(data) return request
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if type(value) == io.BufferedReader: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() #raise TypeError("not a valid non-string sequence or mapping object %d" % traceback) logger.debug("not a valid non-string sequence or mapping object : %s" % traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, self.doseq).encode() else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def http_request(self, request): try: data = request.get_data() except AttributeError: data = request.data if data is not None and type(data) != str: v_files = [] v_vars = [] try: for (key, value) in list(data.items()): if hasattr(value, 'read'): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: raise TypeError if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if (request.has_header('Content-Type') and request.get_header( 'Content-Type').find('multipart/form-data') != 0): six.print_("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) try: request.add_data(data) except AttributeError: request.data = data return request
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if type(value) == file: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if request.has_header('Content-type'): pass # print "Replacing %s with %s" % (request.get_header('content-type'), contenttype) request.add_unredirected_header('Content-type', contenttype) request.add_data(data) return request
def submit(self, filename): """Submit a file to Fossology for analysis. :param filename: path to the file to submit. :type filename: str. :returns: the submission identifier. :rtype: int """ url = self._url + '?mod=upload_file' with open(filename, 'rb') as f: variables = [ ('folder', '1'), ('description', ''), ('name', ''), ('Check_agent_copyright', '1'), ('Check_agent_mimetype', '1'), ('Check_agent_nomos', '1'), ('Check_agent_pkgagent', '0'), ] files = [('getfile', os.path.basename(filename), f.read())] ct, data = encode_multipart_formdata(variables, files) request = urllib.request.Request(url, data) request.add_unredirected_header('Content-Type', ct) request.add_unredirected_header('Content-Length', len(data)) response = self._opener.open(request) doc = BeautifulSoup(response.read().decode('utf-8')) expr = re.compile(r'^/\?mod=showjobs&upload=(\d+)') for tag in doc.find_all('a'): if tag.has_attr('href'): match = expr.match(tag['href']) if match: return int(match.group(1)) raise FossologyError(_('Upload failed'))
def http_request(self, request): print(request.__dict__) data = request.data if data is not None and type(data) != str: v_files = [] v_vars = [] try: for key, value in data.items(): if isinstance(value,IOBase): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object") if len(v_files) == 0: data = urllib.urlencode(v_vars, doseq) else: boundary, data = MultipartPostHandler.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if(request.has_header('Content-Type') and request.get_header('Content-Type').find('multipart/form-data') != 0): print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) request.data = data.encode('utf-8') return request
def __GetJSONMetadataValue(self, url): try: request = urllib.request.Request(url) request.add_unredirected_header('Metadata-Flavor', 'Google') data = urllib.request.urlopen(request).read() return json.loads(data) except (urllib.error.URLError, urllib.error.HTTPError, IOError): return None
def GetNumericProjectId(): """Get the numeric project ID for this VM.""" try: request = urllib.request.Request(NUMERIC_PROJECT_ID_URL) request.add_unredirected_header("Metadata-Flavor", "Google") return urllib.request.urlopen(request).read() except (urllib.error.URLError, urllib.error.HTTPError, IOError) as e: return None
def http_request(self, request): if not request.has_header(self.AUTH_HEADER): url = request.get_full_url() user, password = self._creds_mgr.get_credentials(url) if user is not None and password is not None: creds = base64.b64encode("%s:%s" % (user, password)) auth = "Basic %s" % creds request.add_unredirected_header(self.AUTH_HEADER, auth) return request
def http_request(self, request): if not request.has_header('Cookie'): request.add_unredirected_header('Cookie', self.cookie) request.add_header('Cookie', self.cookie) else: cookie = request.get_header('Cookie') request.add_unredirected_header('Cookie', cookie + '; ' + cookie) self.cookiejar.add_cookie_header(request) return request
def process_oauth2(self, request, response, www_authenticate_header): # This keep infinite auth loops from happening full_url = request.full_url self.auth_failure_tracker[full_url] += 1 if self.auth_failure_tracker[full_url] > 1: raise HTTPError(full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url), response.headers, response) auth_token = self._get_outh2_token(www_authenticate_header) request.add_unredirected_header('Authorization', 'Bearer ' + auth_token) return self.parent.open(request, timeout=request.timeout)
def process_oauth2(self, request, response, www_authenticate_header): # This keep infinite auth loops from happening full_url = request.full_url self.auth_failure_tracker[full_url] += 1 if self.auth_failure_tracker[full_url] > 1: raise HTTPError( full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url), response.headers, response) auth_token = self._get_outh2_token(www_authenticate_header) request.add_unredirected_header('Authorization', 'Bearer ' + auth_token) return self.parent.open(request, timeout=request.timeout)
def _add_referer_header(self, request, origin_request=True): if self.request is None: return request scheme = request.get_type() original_scheme = self.request.get_type() if scheme not in ["http", "https"]: return request if not origin_request and not self.request.has_header("Referer"): return request if (self._handle_referer and original_scheme in ["http", "https"] and not (original_scheme == "https" and scheme != "https")): # strip URL fragment (RFC 2616 14.36) parts = _rfc3986.urlsplit(self.request.get_full_url()) parts = parts[:-1] + (None, ) referer = _rfc3986.urlunsplit(parts) request.add_unredirected_header("Referer", referer) return request
def ReadPublicEndpoints(self): """Fetch list of public endpoint IPs from metadata server.""" try: # If the connection gets abandoned, ensure we don't hang more than # 70 seconds. url = PUBLIC_ENDPOINT_URL_PREFIX + self.last_etag request = urllib.request.Request(url) request.add_unredirected_header('Metadata-Flavor', 'Google') u = self.urllib2.urlopen(request, timeout=70) addrs_data = u.read() headers = u.info().dict self.last_etag = headers.get('etag', self.default_last_etag) except urllib.error.HTTPError as h: self.ResetEtag() # 404 is treated like an empty list, for backward compatibility. if h.code == 404: return [] raise h return self.ParseIPAddrs(addrs_data)
def _add_referer_header(self, request, origin_request=True): if self.request is None: return request scheme = request.type original_scheme = self.request.type if scheme not in ["http", "https"]: return request if not origin_request and not self.request.has_header("Referer"): return request if (self._handle_referer and original_scheme in ["http", "https"] and not (original_scheme == "https" and scheme != "https")): # strip URL fragment (RFC 2616 14.36) parts = urlsplit(self.request.get_full_url()) parts = parts[:-1]+(None,) referer = urlunsplit(parts) request.add_unredirected_header("Referer", referer) return request
def http_request(self, request): data = request.get_data() def isfiledata(p_str): import re r_c = re.compile("^f'(.*)'$") rert = r_c.search(str(p_str)) #rert = re.search("^f'(.*)'$", p_str) if rert: return rert.group(1) else: return None if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if isfiledata(value): # type(value) == file: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if(request.has_header('Content-Type') and request.get_header('Content-Type').find('multipart/form-data') != 0): print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def GetMetadata(self, url_path, recursive=False, timeout=None): """Retrieves instance metadata. Args: url_path: The path of the metadata url after the api version. http://169.254.169.254/computeMetadata/v1/url_path recursive: If set, returns the tree of metadata starting at url_path as a json string. timeout: How long to wait for blocking operations (in seconds). A value of None uses urllib2's default timeout. Returns: The metadata returned based on the url path. """ # Use the latest version of the metadata. suffix = "" if recursive: suffix = "?recursive=true" url = "{0}{1}{2}".format(METADATA_V1_URL_PREFIX, url_path, suffix) request = urllib.request.Request(url) request.add_unredirected_header("Metadata-Flavor", "Google") return self.Get(request, timeout=timeout)
def do_request_(self, request): host = request.get_host() if not host: raise URLError('no host given') if request.has_data(): # POST data = request.get_data() if not request.has_header('Content-type'): request.add_unredirected_header( 'Content-type', 'application/x-www-form-urlencoded') scheme, sel = urllib.parse.splittype(request.get_selector()) sel_host, sel_path = urllib.parse.splithost(sel) if not request.has_header('Host'): request.add_unredirected_header('Host', sel_host or host) for name, value in self.parent.addheaders: name = name.capitalize() if not request.has_header(name): request.add_unredirected_header(name, value) return request
def http_request(url, json_string, username=None, password=None, timeout=None, additional_headers=None, content_type=None, cookies=None, gzipped=None, ssl_context=None, debug=None): """ Fetch data from webserver (POST request) :param json_string: JSON-String :param username: If *username* is given, BASE authentication will be used. :param timeout: Specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used). See: https://github.com/gerold-penz/python-jsonrpc/pull/6 :param additional_headers: Dictionary with additional headers See: https://github.com/gerold-penz/python-jsonrpc/issues/5 :param content_type: Possibility to change the content-type header. :param cookies: Possibility to add simple cookie-items as key-value pairs. The key and the value of each cookie-item must be a bytestring. Unicode is not allowed here. :param gzipped: If `True`, the JSON-String will be gzip-compressed. :param ssl_context: Specifies custom TLS/SSL settings for connection. Python > 2.7.9 See: https://docs.python.org/2/library/ssl.html#client-side-operation :param debug: If `True` --> *logging.debug* """ # Debug if debug: logging.debug("Client-->Server: {json_string}".format( json_string=repr(json_string))) # Create request and add data request = Request(url) if gzipped: # Compress content (SpooledTemporaryFile to reduce memory usage) spooled_file = tools.SpooledFile() tools.gzip_str_to_file(json_string, spooled_file) del json_string request.add_header("Content-Encoding", "gzip") request.add_header("Accept-Encoding", "gzip") spooled_file.seek(0) request.data = spooled_file else: request.data = json_string # Content Type request.add_header("Content-Type", content_type or "application/json") # Authorization if username: base64string = base64.b64encode( ("%s:%s" % (username, password)).encode('ascii')).strip() request.add_unredirected_header( "Authorization", "Basic %s" % (base64string.decode('ascii'), )) # Cookies if cookies: cookie = http.cookies.SimpleCookie(cookies) request.add_header("Cookie", cookie.output(header="", sep=";")) # Additional headers (overrides other headers) if additional_headers: for key, val in list(additional_headers.items()): request.add_header(key, val) # Send request to server try: if ssl_context: try: response = urlopen(request, timeout=timeout, context=ssl_context) except TypeError as err: if "context" in str(err): raise NotImplementedError( "SSL-Context needs Python >= 2.7.9") else: raise else: response = urlopen(request, timeout=timeout) except HTTPError as err: if debug: retval = err.read() logging.debug( "Client<--Server: {retval}".format(retval=repr(retval))) raise # Analyze response and return result try: if "gzip" in response.headers.get("Content-Encoding", ""): response_file = tools.SpooledFile(source_file=response) if debug: retval = tools.gunzip_file(response_file) logging.debug( "Client<--Server: {retval}".format(retval=repr(retval))) return retval return tools.gunzip_file(response_file) else: if debug: retval = response.read() logging.debug( "Client<--Server: {retval}".format(retval=repr(retval))) return retval return response.read() finally: response.close()
def http_request(self, request): if ((self.referer is not None) and not request.has_header("Referer")): request.add_unredirected_header("Referer", self.referer) return request
def HttpGet(url, headers=None): request = urllib.request.Request(url) if headers: for key in headers.keys(): request.add_unredirected_header(key, headers[key]) return urllib.request.urlopen(request).read()
def _request_add_headers(self, request): for name, value in list(self.headers.items()): request.add_header(name, value) request.add_unredirected_header(name, value) return self