def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if type(value) == file: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if request.has_header('Content-type'): pass # print "Replacing %s with %s" % (request.get_header('content-type'), contenttype) request.add_unredirected_header('Content-type', contenttype) request.add_data(data) return request
def http_request(self, request): data = request.get_data() if isinstance(data, dict): v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if isinstance(value, file) or hasattr(value, "file") or isinstance(value, io.StringIO): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise SqlmapDataException("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = "multipart/form-data; boundary=%s" % boundary #if (request.has_header("Content-Type") and request.get_header("Content-Type").find("multipart/form-data") != 0): # print "Replacing %s with %s" % (request.get_header("content-type"), "multipart/form-data") request.add_unredirected_header("Content-Type", contenttype) request.add_data(data) return request
def http_request(self, request): try: data = request.get_data() except AttributeError: data = request.data if data is not None and type(data) != str: v_files = [] v_vars = [] try: for (key, value) in list(data.items()): if hasattr(value, 'read'): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: raise TypeError if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if (request.has_header('Content-Type') and request.get_header( 'Content-Type').find('multipart/form-data') != 0): six.print_("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) try: request.add_data(data) except AttributeError: request.data = data return request
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if type(value) == io.BufferedReader: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() #raise TypeError("not a valid non-string sequence or mapping object %d" % traceback) logger.debug("not a valid non-string sequence or mapping object : %s" % traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, self.doseq).encode() else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def http_request(self, request): print(request.__dict__) data = request.data if data is not None and type(data) != str: v_files = [] v_vars = [] try: for key, value in data.items(): if isinstance(value,IOBase): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object") if len(v_files) == 0: data = urllib.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if(request.has_header('Content-Type') and request.get_header('Content-Type').find('multipart/form-data') != 0): print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def test_post_local(self, filename, ccd_data, use_base64=False): """ filename : The name of the file attachment ccd_data : The xml to post (send) use_base64 : If true, encode the encrypted XML as base64. Maybe necessary for 3DES """ attachments = "" payload = """--%s Content-Disposition: attachment; name="%s"; filename="%s.xml" Content-Type: text/xml %s""" payload_uuid = str(uuid.uuid4()) #try: for data in ccd_data: if len(attachments) > 0: attachments += "\r\n\r\n" if use_base64: attachments += payload % (payload_uuid, filename, filename, base64.b64encode(data)) else: attachments += payload % (payload_uuid, filename, filename, data) http = urllib.request.HTTPHandler(debuglevel=1) opener = urllib.request.build_opener(http) urllib.request.install_opener(opener) request = urllib.request.Request(self._url) request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid) request.add_header("User-Agent", "synthesis") request.add_data(attachments) response = urllib.request.urlopen(request).read() # check for some sign of success within the response if response[0:4] == "202:": return (True, response) else: return (False, response)
def glsrequest(uri, method, data=None): ''' Returns xml node tree as Element instance. 'uri' may be absolute or relative to _BASEURI. 'method' in ('GET', 'POST', 'PUT') 'data' can be a string or Element instance ''' if method not in {'GET', 'POST', 'PUT'}: raise GlslibException(MSGUNSUPPORTEDMETHOD % method) if not uri.startswith(_BASEURI): uri = _BASEURI.rstrip('/') + '/' + uri.lstrip('/') request = urllib.request.Request(uri) request.add_header("Authorization", "Basic %s" % _AUTHSTR) if etree.iselement(data): # tostring generates bytestring (as required for data) data = etree.tostring(data) request.add_header('Content-Type', 'application/xml') request.add_data(data) request.get_method = lambda: method msg = '%s %s\n%s\n%s' % (request.get_method(), request.get_full_url(), request.headers, data.decode('utf-8') if data else '') logger.debug(msg) try: r = urllib.request.urlopen(request) return etree.XML(r.read()) except urllib.error.HTTPError as httperr: logger.error(httperr.read()) raise except urllib.error.URLError as urlerr: logger.error(request.get_full_url()) raise
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if hasattr(value, 'read'): v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: raise TypeError if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if ( request.has_header('Content-Type') and request.get_header('Content-Type').find( 'multipart/form-data') != 0 ): six.print_( "Replacing %s with %s" % ( request.get_header('content-type'), 'multipart/form-data' ) ) request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
def http_request(self, request): data = request.get_data() if data is not None and type(data) != str: v_files = [] v_vars = [] try: for (key, value) in list(data.items()): if type(value) == io.BufferedReader: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() # raise TypeError("not a valid non-string sequence or mapping object %d" % traceback) logger.debug("not a valid non-string sequence or mapping object : %s" % traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, self.doseq).encode() else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = "multipart/form-data; boundary=%s" % boundary request.add_unredirected_header("Content-Type", contenttype) request.add_data(data) return request
def post(self, filename, ccd_data, use_base64=False): """ filename : The name of the file attachment ccd_data : The xml to post (send) use_base64 : If true, encode the encrypted XML as base64. Maybe necessary for 3DES """ attachments = "" payload = """--%s Content-Disposition: attachment; name="%s"; filename="%s.xml" Content-Type: text/xml %s""" payload_uuid = str(uuid.uuid4()) try: for data in ccd_data: if len(attachments) > 0: attachments += "\r\n\r\n" if use_base64: attachments += payload % (payload_uuid, filename, filename, base64.b64encode(data)) else: attachments += payload % (payload_uuid, filename, filename, data) #import pdb; pdb.set_trace() if self._url.find("https") == 0: https = urllib.request.HTTPSHandler(debuglevel=1) opener = urllib.request.build_opener(https) urllib.request.install_opener(opener) request = urllib.request.Request(self._url) request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid) request.add_header("User-Agent", "synthesis") request.add_data(attachments) if PRINT_HTTP_POST: print(request.header_items()) print(request.get_data()) print(request.get_method()) print(request.get_host()) return (True, "True") response = urllib2.urlopen(request).read() else: print("**** POSTING TO LOCAL SERVER ****") request = urllib.request.Request(self._url) request.add_header("Content-Type", "multipart/form-data; boundary=%s" % payload_uuid) request.add_header("User-Agent", "synthesis") request.add_data(attachments) if PRINT_HTTP_POST: print(request.header_items()) print(request.get_data()) print(request.get_method()) print(request.get_host()) return (True, "True") response = urllib2.urlopen(request).read() # check for some sign of success within the response if response[0:4] == "202:": return (True, response) else: return (False, response) except Exception as err: return (False, "An error occurred while performing an HTTP-POST or receiving the response: (%s)" % str(err))
def _open(self, method, path, headers=None, data=None): """Perform an HTTP request. Args: method: The HTTP method (GET, PUT, POST, DELETE). path: The HTTP path to retrieve. headers: A dictionary of HTTP headers to add. data: The data to send as the body of the request. Returns: A Response object. """ url = urllib.parse.urljoin(self.site, path) self.log.info('%s %s', method, url) request = self._request(url) request.set_method(method) if headers: for key, value in headers.items(): request.add_header(key, value) if self.auth: # Insert basic authentication header request.add_header('Authorization', 'Basic {}'.format(self.auth.decode())) if request.headers: header_string = '\n'.join([':'.join((k, v)) for k, v in request.headers.items()]) self.log.debug('request-headers:%s', header_string) if data: request.add_header('Content-Type', self.format.mime_type) request.add_data(data) self.log.debug('request-body:%s', request.get_data()) elif method in ['POST', 'PUT']: # Some web servers need a content length on all POST/PUT operations request.add_header('Content-Type', self.format.mime_type) request.add_header('Content-Length', '0') if self.timeout and not _urllib_has_timeout(): # Hack around lack of timeout option in python < 2.6 old_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self.timeout) try: http_response = None try: http_response = self._handle_error(self._urlopen(request)) except urllib.error.HTTPError as err: http_response = self._handle_error(err) except urllib.error.URLError as err: raise Error(err, url) response = Response.from_httpresponse(http_response) self.log.debug('Response(code=%d, headers=%s, msg="%s")', response.code, response.headers, response.msg) finally: if http_response: http_response.close() if self.timeout and not _urllib_has_timeout(): socket.setdefaulttimeout(old_timeout) self.log.info('--> %d %s %db', response.code, response.msg, len(response.body)) return response
def request(self, host, handler, request_body, verbose): self.verbose = verbose url = 'http://{0}{1}'.format(host, handler) request = urllib.Request(url) request.add_data(request_body) request.add_header("User-Agent", self.user_agent) request.add_header("Content-Type", "text/html") f = urllib.urlopen(request) return self.parse_response(f)
def request(self, host, handler, request_body, verbose): self.verbose = verbose url = "http://{0}{1}".format(host, handler) request = urllib.Request(url) request.add_data(request_body) request.add_header("User-Agent", self.user_agent) request.add_header("Content-Type", "text/html") f = urllib.urlopen(request) return self.parse_response(f)
def run(self, params=[], auth=True, console=True): params.insert(1, self.auth_token) if params[0] != 'session.list': params.insert(2, self.console_id) #print(params) request = self.set_request() query_params = msgpack.packb(params) request.add_data(query_params) response = msgpack.unpackb(urllib.request.urlopen(request).read()) return response
def post(self, path, headers, body): uri = "%s/%s" % (self.url.geturl(), path) if self.debug: print("post: uri=%s, headers=%s" % (uri, headers)) request = urllib2.Request(uri) for header in headers: request.add_header(header, headers[header]) request.add_data(body) resp = urllib2.urlopen(request, cafile=self.cafile) if self.debug: print("resp: %s" % resp.info()) return resp
def check_app_updates(app_info_list, raw_result=False): # This function expects a list of dicts with, at a minimum, the 'adam-id' key set # This ID is the unique product identifier for an app on the App Store and can be found in the store URL # when viewing the app's page in a web browser, like so: # https://itunes.apple.com/us/app/evernote/id406056744 # ^^^^^^^^^ # The other 3 keys that will be passed in the search are: CFBundleIdentifier, CFBundleShortVersionString, and # installed-version-identifier (explained earlier). Lack of any of these keys will result in a filler value # being provided which, as long as the adam-id is present, the App Store update mechanism doesn't seem to # care about. update_url = 'https://su.itunes.apple.com/WebObjects/MZSoftwareUpdate.woa/wa/availableSoftwareUpdatesExtended' request = urllib.request.Request(update_url) # Headers # # This sets us to the US store. See: # http://blogs.oreilly.com/iphone/2008/08/scraping-appstore-reviews.html # http://secrets.blacktree.com/edit?id=129761 request.add_header('X-Apple-Store-Front', '143441-1,13') # This indicates we're sending an XML plist request.add_header('Content-Type', 'application/x-apple-plist') # This seems to be the minimum string to be recognized as a valid app store update checker # Normally, it's of the form: User-Agent: MacAppStore/1.3 (Macintosh; OS X 10.9) AppleWebKit/537.71 request.add_header('User-Agent', 'MacAppStore') # Build up the plist local_software = [] for an_app in app_info_list: app_entry = { 'CFBundleIdentifier': an_app.get('CFBundleIdentifier', '.'), 'CFBundleShortVersionString': an_app.get('CFBundleShortVersionString', '0'), 'adam-id': an_app['adam-id'], 'installed-version-identifier': an_app.get('installed-version-identifier', 0) } local_software.append(app_entry) plist_dict = {'local-software': local_software} plist_str = dump_plist(plist_dict) request.add_data(plist_str) # Build the connection response_handle = urllib.request.urlopen(request) try: response = response_handle.read() except HTTPError as e: raise ProcessorError("Invalid adam-id %s" % e) response_handle.close() # Currently returning the raw response # Initial analysis: # - It appears that applications that need updating will be under the 'incompatible-items' key # - 'version-external-identifiers' is a list of historical versions, in order # - 'version-external-identifier' is the current version # - 'current-version' is the CFBundleShortVersionString # - 'bundle-id' is the CFBundleIdentifier # - 'preflight' is a *very* interesting 'pfpkg'. See details at bottom. return load_plist(response)
def http_request(self, request): try: data = request.get_data() except AttributeError: data = request.data if data is not None and type(data) != str: data = urllib.parse.urlencode(data, doseq).encode("utf-8") try: request.add_data(data) except AttributeError: request.data = data return request
def get_file_data_request(self, url, file_field_name=None, file_data=None, **fields): msg = MultipartMimeFormData() msg.add_csv_file(file_field_name, "import.csv", file_data) for field, value in fields.items(): msg.add_field(field, value) body = str(msg).encode('utf-8') request = urllib.request.Request(url) request.add_header("Content-Type", msg.get_content_type()) request.add_header("Content-Length", len(body)) request.add_header("Referer", self.referer) request.add_data(body) return request
def create_console(self): options = ['console.create'] options.insert(1, self.auth_token) request = self.set_request() query_params = msgpack.packb(options) request.add_data(query_params) response = msgpack.unpackb(urllib.request.urlopen(request).read()) #print(response) if response.get(b'id') is None: print("ERROR!!!!!") exit() print("#Console #"+str(response.get(b'id')).replace("b'", "").replace("'","")+" Created") return response.get(b'id')
def _github_api_request(self, url, data=None, method=None, authenticate=False): logging.debug("Making github API request {0}".format(url)) request = urllib.request.Request(url) if method: request.get_method = lambda: method if data == "": # Workaround for PUTs requiring data, even if you have nothing to pass request.add_data(data) elif data: request.add_data(json.dumps(data)) # Manually adding authentication data # Basic works in curl, but urllib2 doesn't # probably because github's API doesn't send a www-authenticate header if authenticate or self._github_have_authorization(): from base64 import encodestring auth = self._github_authorization() if ":" in auth: # username:password base64string = encodestring(auth).replace('\n', '') request.add_header("Authorization", "Basic {0}".format(base64string)) else: # token request.add_header("Authorization", "Bearer {0}".format(auth)) try: response = urllib.request.urlopen(request) except IOError as e: raise Error("GitHub API failure: " + str(e)) if response.code == 204: # 204 = No content return None json_parsed = json.load(response) link_headers = response.info().getallmatchingheaders("Link") if link_headers: logging.debug("Found a Link header in response, analyzing...") link_header = link_headers[0].lstrip("Link:") links_raw = link_header.split(",") links_split_raw = [link.split(";") for link in links_raw] links_split_proc = [(l[1].strip().lstrip('rel="').rstrip('"'), l[0].strip().lstrip("<").rstrip(">")) for l in links_split_raw] links_dict = dict((k, v) for (k, v) in links_split_proc) if "next" in links_dict: logging.debug("Link with rel=\"next\" found, recursing to deal with pagination") rest = self._github_api_request(links_dict["next"], data, method, authenticate) json_parsed += rest return json_parsed
def __call__(self, post=False, *args, **kargs): if len(args): raise TypeError('Only keyword arguments are allowed') if type(post) is not bool: raise TypeError('post argument can only be True or False') form = _MultiPartForm() add_form = False for (k, v) in kargs.items(): if isinstance(v, File): add_form = True form.add_file(k, v.get_filename(), v.content) if post: url = self._urlbase for k, v in self._mkarg(kargs).items(): form.add_field(k, v) add_form = True else: url = self.geturl(**kargs) request = urllib.request.Request(url) if add_form: body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', str(len(body))) request.add_data(body) self._api.update_request(request) retry = self._api.max_retries while True: retry -= 1 try: ret = urllib.request.urlopen(request, timeout=self._api.timeout).read() break except urllib.error.HTTPError as e: raise APIError(e.code, url, e.read()) except (socket.error, urllib.error.URLError) as e: if retry < 0: raise e _print_debug('caught error: {}; retrying'.format(e)) time.sleep(self._api.retry_delay) if self._api.decode_result: try: ret = json.loads(ret.decode()) except: raise APIError(-1, url, 'json decode error, value={0!r}'.format(ret)) return ret
def fetch(self, url, debug=False, data=None): """ Fetches a web page. If data is None, a GET request will be done, else it will be a POST request using data (must be encoded using urllib.parse.urlencode). """ u = None opener = None request = None try: # Building opener if self.proxy != None: if debug: print(("Using proxy %s" % self.proxy)) proxy = urllib.request.ProxyHandler({'http': self.proxy}) opener = urllib.request.build_opener(proxy) #urllib2.install_opener(opener) #urllib2.urlopen('http://www.google.com') else: opener = urllib.request.build_opener() # Building request request = urllib.request.Request(url) request.add_header( "User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox" ) request.add_data(data) # Opening URL u = opener.open(request) except Exception as e: if debug: print(("Couldn't fetch %s" % url)) print(e) self.notfound += 1 self.current_url = None return None try: l = u.read() self.current_url = u.geturl() except Exception as e: if debug: print("Couldn't read data from socket") print(e) self.notfound += 1 self.current_url = None return None return l
def __call__(self, url, params): data = json.dumps({ "jsonrpc": "2.0", "method": "call", "params": params, "id": random.randint(0, 1000000000), }) request = urllib.request.Request(url='/'.join([self._root_url, url])) request.add_header('Content-Type', 'application/json') request.add_data(data) response = self._opener.open(request, timeout=self._timeout) if not self._deserialize: return response return json.load(response)
def __call__(self, post = False, *args, **kargs): if len(args): raise TypeError('Only keyword arguments are allowed') if type(post) is not bool: raise TypeError('post argument can only be True or False') form = _MultiPartForm() add_form = False for (k, v) in kargs.items(): if isinstance(v, File): add_form = True form.add_file(k, v.get_filename(), v.content) if post: url = self._urlbase for k, v in self._mkarg(kargs).items(): form.add_field(k, v) add_form = True else: url = self.geturl(**kargs) request = urllib.request.Request(url) if add_form: body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', str(len(body))) request.add_data(body) self._api.update_request(request) retry = self._api.max_retries while True: retry -= 1 try: ret = urllib.request.urlopen(request, timeout = self._api.timeout).read() break except urllib.error.HTTPError as e: raise APIError(e.code, url, e.read()) except (socket.error, urllib.error.URLError) as e: if retry < 0: raise e _print_debug('caught error: {}; retrying'.format(e)) time.sleep(self._api.retry_delay) if self._api.decode_result: try: ret = json.loads(ret.decode()) except: raise APIError(-1, url, 'json decode error, value={0!r}'.format(ret)) return ret
def mylogin(self): options = ['auth.login', self.user, self.password] token = None request = self.set_request() query_params = msgpack.packb(options) request.add_data(query_params) response = msgpack.unpackb(urllib.request.urlopen(request).read()) #print(response) if response.get(b'result') == b'success': print("#Authed") token = response.get(b'token') else: print("#GTFO!!!!!!") exit() return token
def marshall(self, args, files, request): """ Marshalls the arguments to the CGI script as multi-part/form-data, not the default application/x-www-form-url-encoded. This improves the transfer of the large inputs and eases command line invocation of the CGI script. """ (contentType, body) = self.encode(args, files) request.add_header(b'Content-Type', contentType) if PY2: request.add_header('Content-Length', str(len(body))) request.add_data(body) elif PY3: request.data = body return
def __send_request(self, method, uri, data): """Summary. Args: method (TYPE): Description uri (TYPE): Description data (TYPE): Description Returns: TYPE: Description Raises: APIError: Description """ url = self.__url + uri request = urllib.request.Request(url) if (method == 'POST'): request.add_data(json.dumps(data)) auth = base64.b64encode('%s:%s' % (self.user, self.password)) request.add_header('Authorization', 'Basic %s' % auth) request.add_header('Content-Type', 'application/json') if self.debug: handler = urllib.request.HTTPHandler(debuglevel=1) opener = urllib.request.build_opener(handler) urllib.request.install_opener(opener) e = None try: response = urllib.request.urlopen(request).read() except urllib.error.HTTPError as e: response = e.read() if response: result = json.loads(response) else: result = {} if e is not None: if result and 'error' in result: error = '"' + result['error'] + '"' else: error = 'No additional error message received' raise APIError('TestRail API returned HTTP %s (%s)' % (e.code, error)) if isinstance(result, list): return [AttrDict(x) for x in result] return AttrDict(result)
def request(self, host, handler, request_body, verbose): self.verbose = verbose url = self.schema + "://" + host + handler request = urllib.request.Request(url) request.add_data(request_body) # Note: 'Host' and 'Content-Length' are added automatically request.add_header("User-Agent", self.user_agent) request.add_header("Content-Type", "text/xml") # Important proxy_handler = urllib.request.ProxyHandler() opener = urllib.request.build_opener(proxy_handler) f = opener.open(request) return self.parse_response(f)
def fetch(self, url, debug=False, data=None): """ Fetches a web page. If data is None, a GET request will be done, else it will be a POST request using data (must be encoded using urllib.parse.urlencode). """ u = None opener = None request = None try: # Building opener if self.proxy != None: if debug: print(("Using proxy %s" % self.proxy)) proxy= urllib.request.ProxyHandler({'http': self.proxy}) opener = urllib.request.build_opener(proxy) #urllib2.install_opener(opener) #urllib2.urlopen('http://www.google.com') else: opener = urllib.request.build_opener() # Building request request = urllib.request.Request(url) request.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.6) Gecko/20050512 Firefox") request.add_data(data) # Opening URL u = opener.open(request) except Exception as e: if debug: print(("Couldn't fetch %s" % url)) print(e) self.notfound += 1 self.current_url = None return None try: l = u.read() self.current_url = u.geturl() except Exception as e: if debug: print("Couldn't read data from socket") print(e) self.notfound += 1 self.current_url = None return None return l
def _action(self, params, body=None, content_type=None): # about token, see https://github.com/bittorrent/webui/wiki/TokenSystem url = ( self.base_url + '?token=' + self.token + '&' + urllib.parse.urlencode(params) ) request = urllib.request.Request(url) if body: request.add_data(body) request.add_header('Content-length', len(body)) if content_type: request.add_header('Content-type', content_type) try: response = self.opener.open(request) return response.code, json.loads(response.read().decode('utf-8')) except urllib.error.HTTPError: raise
def get_trusted_ticket_for_user(self, username, site='default', ip=None): trusted_url = self.tableau_server_url + "/trusted" opener = urllib.request.build_opener(urllib.request.HTTPHandler) request = urllib.request.Request(trusted_url) post_data = "username={}".format(username) if site.lower() != 'default': post_data += "&target_site={}".format(site) request.add_data(post_data) trusted_ticket_response = opener.open(request) try: ticket = trusted_ticket_response.read() if ticket == '-1' or not ticket: raise NoResultsException('Ticket generation was not complete.') else: return ticket except urllib.error.HTTPError as e: if e.code >= 500: raise raw_error_response = e.fp.read()
def MultipartPost(url, fields): """ Post form data specified by fields (a dictionary) to the specified url. Returns a urllib2-style file object. """ content_type, body = encode_multipart_formdata(fields) request = Request(url) request.add_header('User-Agent', BROWSER_MASQUERADE) request.add_header('Content-Type', content_type) request.add_header('Content-Length', str(len(body))) if usingPython2: request.add_data(body) else: # Python 3: add_data method is deprecated; body must be # converted from Python 3 string to bytes for Request object request.data = body.encode('utf-8') connection = urlopen(request) return connection
def sendSpeechData(self, vid, data): url = self._baseAsrUrl + 'voices/' + self._uuid headers = { 'Content-Type': 'multipart/form-data', 'X-Token': self._token } form_data = "" form_data += self._boundary + "\r\n" form_data += "Content-Disposition: form-data;name=\"voice_id\"\r\n\r\n" form_data += str(vid) + "\r\n" form_data += self._boundary + "\r\n" form_data += "Content-Disposition: form-data;name=\"voice\"\r\n" form_data += "Content-Type: application/octet-stream\r\n\r\n" form_data += data form_data += "\r\n" form_data += self._boundary + "\r\n" request = urllib.request.Request(url) request.add_header('X-Token', self._token) request.add_header('Content-Type', 'multipart/form-data') request.add_data(bytearray(form_data)) request.get_method = lambda: 'PUT' try: result = urllib.request.urlopen(request) except urllib.error.HTTPError as e: print('Error code:', e.code) print('Reason:', e.reason) return False except urllib.error.URLError as e: print('URLErroe reason:', e.reason) return False else: response = result.read() res = response.decode('utf-8') if res: return res return False
def getUrlFd(url, headers=None, data=None, timeout=None): """getUrlFd(url, headers=None, data=None, timeout=None) Opens the given url and returns a file object. Headers and data are a dict and string, respectively, as per urllib.request.Request's arguments.""" if headers is None: headers = defaultHeaders if minisix.PY3 and isinstance(data, str): data = data.encode() try: if not isinstance(url, Request): (scheme, loc, path, query, frag) = urlsplit(url) (user, host) = splituser(loc) url = urlunsplit((scheme, host, path, query, '')) request = Request(url, headers=headers, data=data) if user: request.add_header('Authorization', 'Basic %s' % base64.b64encode(user)) else: request = url request.add_data(data) httpProxy = force(proxy) if httpProxy: request.set_proxy(httpProxy, 'http') fd = urlopen(request, timeout=timeout) return fd except socket.timeout as e: raise Error(TIMED_OUT) except sockerrors as e: raise Error(strError(e)) except InvalidURL as e: raise Error('Invalid URL: %s' % e) except HTTPError as e: raise Error(strError(e)) except URLError as e: raise Error(strError(e.reason)) # Raised when urllib doesn't recognize the url type except ValueError as e: raise Error(strError(e))
def http_request(self, request): data = request.get_data() def isfiledata(p_str): import re r_c = re.compile("^f'(.*)'$") rert = r_c.search(str(p_str)) #rert = re.search("^f'(.*)'$", p_str) if rert: return rert.group(1) else: return None if data is not None and type(data) != str: v_files = [] v_vars = [] try: for(key, value) in list(data.items()): if isfiledata(value): # type(value) == file: v_files.append((key, value)) else: v_vars.append((key, value)) except TypeError: systype, value, traceback = sys.exc_info() raise TypeError("not a valid non-string sequence or mapping object").with_traceback(traceback) if len(v_files) == 0: data = urllib.parse.urlencode(v_vars, doseq) else: boundary, data = self.multipart_encode(v_vars, v_files) contenttype = 'multipart/form-data; boundary=%s' % boundary if(request.has_header('Content-Type') and request.get_header('Content-Type').find('multipart/form-data') != 0): print("Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')) request.add_unredirected_header('Content-Type', contenttype) request.add_data(data) return request
urlPost = 'https://api.medium.com/v1/users/%s/posts' % authorId values = dict() values['title'] = "Hola Medium!" values['contentFormath'] = "html" values[ 'content'] = "<h1>Hola Medium!</h1><p>Probando el API de Medium desde Pythonista.</p>" values['publishStatus'] = "draft" data = json.dumps(values) request = urllib.request.Request(urlPost) request.add_header("Authorization", "Bearer %s" % integrationToken) request.add_header("Content-Type", contentTypeHeader) request.add_header("Accept", acceptHeader) request.add_header("Accept-Charset", acceptCharsetHeader) request.add_data(data) try: response = urllib.request.urlopen(request) except urllib.error.HTTPError as e: if e.code == 400: print("Incorrect fields. Bad request") print(e.reason) elif e.code == 401: print("The integration token is invalid") print(e.reason) elif e.code == 403: print("User without permission to publish.") print(e.reason) elif e.code == 404: print("User unknown") print(e.reason)
def upload(self, filename=None, jpegData=None, **arg): """Upload a file to flickr. Be extra careful you spell the parameters correctly, or you will get a rather cryptic "Invalid Signature" error on the upload! Supported parameters: One of filename or jpegData must be specified by name when calling this method: filename -- name of a file to upload jpegData -- array of jpeg data to upload api_key auth_token title description tags -- space-delimited list of tags, "tag1 tag2 tag3" is_public -- "1" or "0" is_friend -- "1" or "0" is_family -- "1" or "0" """ if filename == None and jpegData == None or \ filename != None and jpegData != None: raise UploadException("filename OR jpegData must be specified") # verify key names for a in list(arg.keys()): if a != "api_key" and a != "auth_token" and a != "title" and \ a != "description" and a != "tags" and a != "is_public" and \ a != "is_friend" and a != "is_family": sys.stderr.write("FlickrAPI: warning: unknown parameter " \ "\"%s\" sent to FlickrAPI.upload\n" % (a)) arg["api_sig"] = self.__sign(arg) url = "http://" + FlickrAPI.flickrHost + FlickrAPI.flickrUploadForm # construct POST data boundary = mimetools.choose_boundary() body = "" # required params for a in ('api_key', 'auth_token', 'api_sig'): body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\""+a+"\"\r\n\r\n" body += "%s\r\n" % (arg[a]) # optional params for a in ('title', 'description', 'tags', 'is_public', \ 'is_friend', 'is_family'): if a in arg: body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\""+a+"\"\r\n\r\n" body += "%s\r\n" % (arg[a]) body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\"photo\";" body += " filename=\"%s\"\r\n" % filename body += "Content-Type: image/jpeg\r\n\r\n" #print body if filename != None: fp = file(filename, "rb") data = fp.read() fp.close() else: data = jpegData postData = body.encode("utf_8") + data + \ ("--%s--" % (boundary)).encode("utf_8") request = urllib.request.Request(url) request.add_data(postData) request.add_header("Content-Type", \ "multipart/form-data; boundary=%s" % boundary) response = urllib.request.urlopen(request) rspXML = response.read() return XMLNode.parseXML(rspXML)
def http_request( url, json_string, username = None, password = None, timeout = None, additional_headers = None, content_type = None, cookies = None, gzipped = None, ssl_context = None, debug = None ): """ Fetch data from webserver (POST request) :param json_string: JSON-String :param username: If *username* is given, BASE authentication will be used. :param timeout: Specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used). See: https://github.com/gerold-penz/python-jsonrpc/pull/6 :param additional_headers: Dictionary with additional headers See: https://github.com/gerold-penz/python-jsonrpc/issues/5 :param content_type: Possibility to change the content-type header. :param cookies: Possibility to add simple cookie-items as key-value pairs. The key and the value of each cookie-item must be a bytestring. Unicode is not allowed here. :param gzipped: If `True`, the JSON-String will be gzip-compressed. :param ssl_context: Specifies custom TLS/SSL settings for connection. Python > 2.7.9 See: https://docs.python.org/2/library/ssl.html#client-side-operation :param debug: If `True` --> *logging.debug* """ # Debug if debug: logging.debug("Client-->Server: {json_string}".format(json_string=repr(json_string))) # Create request and add data request = urllib2.Request(url) if gzipped: # Compress content (SpooledTemporaryFile to reduce memory usage) spooled_file = tools.SpooledFile() tools.gzip_str_to_file(json_string, spooled_file) del json_string request.add_header("Content-Encoding", "gzip") request.add_header("Accept-Encoding", "gzip") spooled_file.seek(0) if six.PY2: request.add_data(spooled_file) else: request.data = spooled_file else: if six.PY2: request.add_data(json_string) else: request.data = json_string # Content Type request.add_header("Content-Type", content_type or "application/json") # Authorization if username: base64string = base64.b64encode("%s:%s" % (username, password)).strip() request.add_unredirected_header("Authorization", "Basic %s" % base64string) # Cookies if cookies: cookie = Cookie.SimpleCookie(cookies) request.add_header("Cookie", cookie.output(header = "", sep = ";")) # Additional headers (overrides other headers) if additional_headers: for key, val in six.iteritems(additional_headers): request.add_header(key, val) # Send request to server http_error_exception = urllib2.HTTPError if six.PY2 else urllib.error.HTTPError try: if ssl_context: try: response = urllib2.urlopen( request, timeout=timeout, context=ssl_context ) except TypeError as err: if "context" in unicode(err): raise NotImplementedError("SSL-Context needs Python >= 2.7.9") else: raise else: response = urllib2.urlopen(request, timeout=timeout) except http_error_exception as err: if debug: retval = err.read() logging.debug("Client<--Server: {retval}".format(retval=repr(retval))) raise # Analyze response and return result try: if "gzip" in response.headers.get("Content-Encoding", ""): response_file = tools.SpooledFile(source_file = response) if debug: retval = tools.gunzip_file(response_file) logging.debug("Client<--Server: {retval}".format(retval = repr(retval))) return retval return tools.gunzip_file(response_file) else: if debug: retval = response.read() logging.debug("Client<--Server: {retval}".format(retval=repr(retval))) return retval return response.read() finally: response.close()
def add_to_request(self, request): body = str(self).encode('utf-8') request.add_header('Content-type', self.get_content_type()) request.add_header('Content-length', str(len(body))) request.add_data(body)
def upload(self, filename=None, jpegData=None, **arg): """Upload a file to flickr. Be extra careful you spell the parameters correctly, or you will get a rather cryptic "Invalid Signature" error on the upload! Supported parameters: One of filename or jpegData must be specified by name when calling this method: filename -- name of a file to upload jpegData -- array of jpeg data to upload api_key auth_token title description tags -- space-delimited list of tags, "tag1 tag2 tag3" is_public -- "1" or "0" is_friend -- "1" or "0" is_family -- "1" or "0" """ if filename == None and jpegData == None or \ filename != None and jpegData != None: raise UploadException("filename OR jpegData must be specified") # verify key names for a in list(arg.keys()): if a != "api_key" and a != "auth_token" and a != "title" and \ a != "description" and a != "tags" and a != "is_public" and \ a != "is_friend" and a != "is_family": sys.stderr.write("FlickrAPI: warning: unknown parameter " \ "\"%s\" sent to FlickrAPI.upload\n" % (a)) arg["api_sig"] = self.__sign(arg) url = "https://" + FlickrAPI.flickrHost + FlickrAPI.flickrUploadForm # construct POST data boundary = email.generator._make_boundary() body = "" # required params for a in ('api_key', 'auth_token', 'api_sig'): body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n" body += "%s\r\n" % (arg[a]) # optional params for a in ('title', 'description', 'tags', 'is_public', \ 'is_friend', 'is_family'): if a in arg: body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\"" + a + "\"\r\n\r\n" body += "%s\r\n" % (arg[a]) body += "--%s\r\n" % (boundary) body += "Content-Disposition: form-data; name=\"photo\";" body += " filename=\"%s\"\r\n" % filename body += "Content-Type: image/jpeg\r\n\r\n" #print body if filename != None: fp = file(filename, "rb") data = fp.read() fp.close() else: data = jpegData postData = body.encode("utf_8") + data + \ ("--%s--" % (boundary)).encode("utf_8") request = urllib.request.Request(url) request.add_data(postData) request.add_header("Content-Type", \ "multipart/form-data; boundary=%s" % boundary) response = urllib.request.urlopen(request) rspXML = response.read() return XMLNode.parseXML(rspXML)
base64_consumer_key_secret = base64.b64encode( urllib.parse.quote(CONSUMER_KEY) + b':' + urllib.parse.quote(CONSUMER_SECRET)) # # Step 2: Obtain a bearer token # # note: the following line won't verify server certificate; to do so you'll have to # use python3 and specify cafile & capauth request = urllib.request.Request("https://api.twitter.com/oauth2/token") request.add_header('Authorization', b'Basic ' + base64_consumer_key_secret) request.add_header("Content-Type", b'application/x-www-form-urlencoded;charset=UTF-8') request.add_data(b'grant_type=client_credentials') resp = urllib.request.urlopen(request) data = json.load(resp) if data['token_type'] != 'bearer': throw("Bad token_type: " + data['token_type']) access_token = data['access_token'] print(("access_token: " + access_token)) print('') # # Step 3: Authenticate API requests with the bearer token # request = urllib.request.Request(
#!/usr/env/bin python3 # -*- coding:utf-8 -*- import urllib.request, urllib.error, http.cookiejar # 创建cookie容器 cj = http.cookiejar.CookieJar() # 创建一个opener opener = urllib.request.build_opener(urllib2.HTTPCookieProcessor(cj)) # 给urllib2安装opener urllib.request.install_opener(opener) # 创建Request对象 request = urllib.request.Request(url) # 添加数据 request.add_data('a', '1') # 添加http的header request.add_header('User-Agent', 'Mozilla/5.0') # 发送请求获取结果 response = urllib.request.urlopen(request)
def plugin(srv, item): """ addrs: (method, url, dict(params), list(username, password), json) """ srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target) method = item.addrs[0] url = item.addrs[1] params = item.addrs[2] timeout = item.config.get('timeout', 60) auth = None try: username, password = item.addrs[3] auth = base64.encodestring('%s:%s' % (username, password)).replace( '\n', '') except: pass tojson = None try: tojson = item.addrs[4] except: pass # Try and transform the URL. Use original URL if it's not possible try: url = url.format(**item.data) except: pass if params is not None: for key in list(params.keys()): # { 'q' : '@message' } # Quoted field, starts with '@'. Do not use .format, instead grab # the item's [message] and inject as parameter value. if params[key].startswith('@'): # "@message" params[key] = item.get(params[key][1:], "NOP") else: try: params[key] = params[key].format( **item.data).encode('utf-8') except Exception as e: srv.logging.debug("Parameter %s cannot be formatted: %s" % (key, e)) return False message = item.message if method.upper() == 'GET': try: if params is not None: resource = url if not resource.endswith('?'): resource = resource + '?' resource = resource + urllib.parse.urlencode(params) else: resource = url request = urllib.request.Request(resource) request.add_header('User-agent', srv.SCRIPTNAME) if auth is not None: request.add_header("Authorization", "Basic %s" % auth) resp = urllib.request.urlopen(request, timeout=timeout) data = resp.read() except Exception as e: srv.logging.warn("Cannot GET %s: %s" % (resource, e)) return False return True if method.upper() == 'POST': try: request = urllib.request.Request(url) if params is not None: if tojson is not None: encoded_params = json.dumps(params) request.add_header('Content-Type', 'application/json') else: encoded_params = urllib.parse.urlencode(params) else: encoded_params = message request.add_data(encoded_params) request.add_header('User-agent', srv.SCRIPTNAME) if auth is not None: request.add_header("Authorization", "Basic %s" % auth) resp = urllib.request.urlopen(request, timeout=timeout) data = resp.read() # print "POST returns ", data except Exception as e: srv.logging.warn("Cannot POST %s: %s" % (url, e)) return False return True srv.logging.warn("Unsupported HTTP method: %s" % (method)) return False
return '\r\n'.join(flattened) if __name__ == '__main__': # Create the form with simple fields form = MultiPartForm() form.add_field('firstname', 'Doug') form.add_field('lastname', 'Hellmann') # Add a fake file form.add_file('biography', 'bio.txt', fileHandle=StringIO('Python developer and blogger.')) # Build the request request = urllib.request.Request('http://localhost:8080/') request.add_header('User-agent', 'PyMOTW (http://www.doughellmann.com/PyMOTW/)') body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) print() print('OUTGOING DATA:') print(request.get_data()) print() print('SERVER RESPONSE:') print(urllib.request.urlopen(request).read())