def login(username, password): """ 输入自己的账号密码,模拟登录知乎 """ # 检测到11位数字则是手机登录 if re.match(r'\d{11}$', account): url = 'http://www.zhihu.com/login/phone_num' data = {'_xsrf': get_xsrf(), 'password': password, 'remember_me': 'true', 'phone_num': username } else: url = 'https://www.zhihu.com/login/email' data = {'_xsrf': get_xsrf(), 'password': password, 'remember_me': 'true', 'email': username } # 若不用验证码,直接登录 post_data = urlparse.urlencode(data).encode('utf-8') r = opener.open(url, post_data) result = r.read().decode('utf-8') # 打印返回的响应,r = 1代表响应失败,msg里是失败的原因 # 要用验证码,post后登录 if (json.loads(result))["r"] == 1: data['captcha'] = get_captcha() post_data = urlparse.urlencode(data).encode('utf-8') r = opener.open(url, post_data) result = r.read().decode('utf-8') print((json.loads(result))['msg']) # 保存cookie到本地 cookie.save(ignore_discard=True, ignore_expires=True)
def render(self, context): try: start = self.start.resolve(context) current_url = self.current_url.resolve(context) page_arg_names = self.page_arg_names.resolve(context) except Exception: raise template.TemplateSyntaxError( "Invalid variables passed to start_url tag") names = page_arg_names.split(",") params = {names[0]: start} url_parts = list(url_parse.urlparse(current_url)) query = dict(parse_qsl(url_parts[4])) for name in names[1:]: if name not in query: query.update({name: 1}) query.update(params) try: url_parts[4] = urllib.urlencode(query) except AttributeError: url_parts[4] = url_parse.urlencode(query) return url_parse.urlunparse(url_parts)
def to_url(self): """Serialize as a URL for a GET request.""" base_url = urlparse.urlparse(self.url) try: query = base_url.query except AttributeError: # must be python <2.5 query = base_url[4] query = parse_qs(query) for k, v in self.items(): query.setdefault(k, []).append(v) try: scheme = base_url.scheme netloc = base_url.netloc path = base_url.path params = base_url.params fragment = base_url.fragment except AttributeError: # must be python <2.5 scheme = base_url[0] netloc = base_url[1] path = base_url[2] params = base_url[3] fragment = base_url[5] try: url = (scheme, netloc, path, params, urllib.urlencode(query, True), fragment) return urllib.urlunparse(url) except AttributeError: url = (scheme, netloc, path, params, urlparse.urlencode(query, True), fragment) return urlparse.urlunparse(url)
def __str__(self): data = {'oauth_consumer_key': self.key, 'oauth_consumer_secret': self.secret} try: return urllib.urlencode(data) except AttributeError: return urlparse.urlencode(data)
def post(url, data=None, json=None, headers=None, ctx=None): """ .. function:: post(url,data=None,json=None,headers=None,ctx=None) Implements the POST method of the HTTP protocol. A tcp connection is made to the host:port given in the url using the default net driver. If *headers* is given as a dictionary, each pair (key, value) is appropriately sent as a HTTP request header. Mandatory headers are transparently handled: "Host:" is always derived by parsing *url*; other headers are set to defaults if not given: for example "Connection: close" is sent if no value for "Connection" is specified in *headers*. To request a permanent connection, *headers* must contain the pair {"Connection":"Keep-Alive"}. If *data* is provided (always as dictionary), each pair (key, value) will be form-encoded and send in the body of the request with {"content-type":"application/x-www-form-urlencoded"} appended in the headers. If *json* is provided (always as dictionary), json data will send in the body of the request with {"content-type":"application/json"} appended in the headers. .. note:: if both (*data* and *json*) dict are provided, json data are ignored and post request is performed with urlencoded data. *post* returns a :class:`Response` instance. Exceptions can be raised: :exc:`HTTPConnectionError` when the HTTP server can't be contacted; :exc:`IOError` when the source of error lies at the socket level (i.e. closed sockets, invalid sockets, etc..) """ pdata = None if data is not None: pdata = [urlparse.urlencode(data), "application/x-www-form-urlencoded"] elif json is not None: pdata = [json_encoder.dumps(json), "application/json"] return _verb(url, pdata, None, headers, None, "POST", ctx)
def _request(self, **kwargs): # Check for missing variables resource = self.interface def validate_kwarg(key, target): # Always allow list if isinstance(key, list): return all([validate_kwarg(x, target[0]) for x in key]) return isinstance(key, target) for k in resource.get('required', []): if k not in [x.split(':')[0] for x in kwargs.keys()]: raise ValueError('Missing required argument: %s' % k) if isinstance(kwargs.get(k), list) and not isinstance(resource['required'][k], list): raise ValueError('Wrong argument type: %s is not a list' % k) elif not validate_kwarg(kwargs.get(k), resource['required'][k]): if isinstance(resource['required'][k], list): raise ValueError('Wrong arguemnt type: %s is not a list of %ss' % (k, resource['required'][k][0])) raise ValueError('Wrong arguemnt type: %s is not a %s' % (k, resource['required'][k])) conduit = self.api.conduit if conduit: # Already authenticated, add session key to json data kwargs['__conduit__'] = conduit elif self.method == 'conduit' and self.endpoint == 'connect': # Not authenticated, requesting new session key token = str(int(time.time())) kwargs['authToken'] = token kwargs['authSignature'] = self.api.generate_hash(token) else: # Authorization is required, silently auth the user self.api.connect() kwargs['__conduit__'] = self.api.conduit url = urlparse.urlparse(self.api.host) if url.scheme == 'https': conn = httplib.HTTPSConnection(url.netloc, timeout=self.api.timeout) else: conn = httplib.HTTPConnection(url.netloc, timeout=self.api.timeout) path = url.path + '%s.%s' % (self.method, self.endpoint) headers = { 'User-Agent': 'python-phabricator/%s' % str(self.api.clientVersion), 'Content-Type': 'application/x-www-form-urlencoded' } body = urlparse.urlencode({ "params": json.dumps(kwargs), "output": self.api.response_format }) # TODO: Use HTTP "method" from interfaces.json conn.request('POST', path, body, headers) response = conn.getresponse() data = self._parse_response(response.read()) return Result(data['result'])
def _post_ST2(server, payload): data = urlencode(payload) try: content = urlopen(server, data).read() except IOError: return None else: return content
def get_authorization_url(self, nexturl='/'): params = { 'client_id': self.client_id, 'response_type': 'code', 'redirect_uri': self.callback_url + '&next_url=' + nexturl } url = self.AUTH_URL + "?" + urlparse.urlencode(params) return url
def authorize(self, url, state): base_url = "https://www.wunderlist.com/oauth/authorize?" params = { 'client_id': self.client_ID, 'redirect_uri': url, 'state': state } return base_url + urlparse.urlencode(params)
def get(url, params): full_url = url if params != None and len(params) > 0: data = urlparse.urlencode(params) full_url = full_url + "?" + data f = urllib2.urlopen(full_url) content = f.read() return content
def get_authorization_url(self, nexturl='/'): params = { 'client_id': self.client_id, 'response_type': 'code', 'redirect_uri': self.callback_url, 'scope': 'openid email', } # url = self.AUTH_URL + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote) url = self.AUTH_URL + "?" + urlparse.urlencode(params) return url
def replace_query_param(url, key, val): """ Given a URL and a key/val pair, set or replace an item in the query parameters of the URL, and return the new URL. """ (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url) query_dict = urlparse.parse_qs(query) query_dict[key] = [val] query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True) return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def get(url, params): full_url = url if params != None and len(params) > 0: data = urlparse.urlencode(params) full_url = full_url + "?" + data with urllib2.urlopen(full_url) as f: content = f.read() return content
def to_postdata(self): """Serialize as post data for a POST request.""" d = {} for k, v in self.items(): d[k.encode('utf-8')] = to_utf8_optional_iterator(v) # tell urlencode to deal with sequence values and map them correctly # to resulting querystring. for example self["k"] = ["v1", "v2"] will # result in 'k=v1&k=v2' and not k=%5B%27v1%27%2C+%27v2%27%5D try: return urllib.urlencode(d, True).replace('+', '%20') except AttributeError: return urlparse.urlencode(d, True).replace('+', '%20')
def get_pdata(data, json): pdata = None if data is not None: if type(data) == PDICT: pdata = [ urlparse.urlencode(data), "application/x-www-form-urlencoded" ] else: pdata = [data, ""] elif json is not None: pdata = [json_encoder.dumps(json), "application/json"] return pdata
def SynchronizedRequest(connection_policy, method, base_url, path, request_data, query_params, headers): """Performs one synchronized http request according to the parameters. :Parameters: - `connection_policy`: documents.ConnectionPolicy - `method`: str - `base_url`: str - `path`: str - `request_data`: str, unicode, file-like stream object, dict, list or None - `query_params`: dict - `headers`: dict :Returns: tuple of (result, headers), where both result and headers are dicts. """ request_body = None if request_data: request_body = _RequestBodyFromData(request_data) if not request_body: raise errors.UnexpectedDataType( 'parameter data must be a JSON object, string or' + ' readable stream.') request_options = {} parse_result = urlparse.urlparse(base_url) request_options['host'] = parse_result.hostname request_options['port'] = parse_result.port request_options['path'] = path request_options['method'] = method if query_params: request_options['path'] += '?' + urlparse.urlencode(query_params) request_options['headers'] = headers if (request_body and (type(request_body) in [str, bytes] or (unicode is not None and type(request_body) is unicode))): request_options['headers'][http_constants.HttpHeaders.ContentLength] = ( len(request_body)) elif request_body is None: request_options['headers'][http_constants.HttpHeaders.ContentLength] = 0 return _InternalRequest(connection_policy, request_options, request_body)
def post(url, params): if params != None and len(params) > 0: data = urlparse.urlencode(params) else: data = "" data = data.encode('utf-8') request = urllib2.Request(url) request.add_header("Content-Type", "application/x-www-form-urlencoded;charset=utf-8") with urllib2.urlopen(request, data) as f: content = f.read() return content
def get_normalized_parameters(self): """Return a string that contains the parameters that must be signed.""" items = [] for key, value in self.items(): if key == 'oauth_signature': continue # 1.0a/9.1.1 states that kvp must be sorted by key, then by value, # so we unpack sequence values into multiple items for sorting. try: if isinstance(value, basestring): items.append((to_utf8_if_string(key), to_utf8(value))) else: try: value = list(value) except TypeError as e: assert 'is not iterable' in str(e) items.append((to_utf8_if_string(key), to_utf8_if_string(value))) else: items.extend((to_utf8_if_string(key), to_utf8_if_string(item)) for item in value) except NameError: if isinstance(value, str): items.append((to_utf8_if_string(key), to_utf8(value))) else: try: value = list(value) except TypeError as e: assert 'is not iterable' in str(e) items.append((to_utf8_if_string(key), to_utf8_if_string(value))) else: items.extend((to_utf8_if_string(key), to_utf8_if_string(item)) for item in value) # Include any query string parameters from the provided URL query = urlparse.urlparse(self.url)[4] url_items = self._split_url_string(query).items() url_items = [(to_utf8(k), to_utf8(v)) for k, v in url_items if k != 'oauth_signature' ] items.extend(url_items) items.sort() try: encoded_str = urllib.urlencode(items) except AttributeError: encoded_str = urlparse.urlencode(items) # Encode signature parameters per Oauth Core 1.0 protocol # spec draft 7, section 3.6 # (http://tools.ietf.org/html/draft-hammer-oauth-07#section-3.6) # Spaces must be encoded with "%20" instead of "+" return encoded_str.replace('+', '%20').replace('%7E', '~')
def wrapper(self, *args, **kwargs): if not self.get_current_user(): url = self.get_login_url() if "?" not in url: if urlparse.urlsplit(url).scheme: # if login url is absolute, make next absolute too next_url = self.request.full_url() else: next_url = self.request.uri url += "?" + urlparse.urlencode(dict(next=next_url)) if settings.REDIRECT_LOGIN: self.redirect(url) return else: raise HTTPError(403) return method(self, *args, **kwargs)
def generate(uri, key, ttl, policy_name=None): """ .. function:: generate(uri, key, ttl, policy_name=None) Generate a SAS for target :samp:`uri` signed with :samp:`key` valid till :samp:`ttl` (passed as epoch) and with optional :samp:`policy_name`. """ sign_key = urlparse.quote_plus(uri) + '\n' + str(int(ttl)) hh = hmac.HMAC(base64.standard_b64decode(key), sha2.SHA2(hashtype=sha2.SHA256)) hh.update(sign_key) signature = base64.standard_b64encode(hh.digest()) rawtoken = {'sr': uri, 'sig': signature, 'se': str(int(ttl))} if policy_name is not None: rawtoken['skn'] = policy_name return 'SharedAccessSignature ' + urlparse.urlencode(rawtoken)
def to_string(self): """Returns this token as a plain string, suitable for storage. The resulting string includes the token's secret, so you should never send or store this string where a third party can read it. """ data = { 'oauth_token': self.key, 'oauth_token_secret': self.secret, } if self.callback_confirmed is not None: data['oauth_callback_confirmed'] = self.callback_confirmed try: return urllib.urlencode(data) except AttributeError: return urlparse.urlencode(data)
def testProtourlencodeUnrecognizedField(self): """Test that unrecognized fields are saved and can be accessed.""" class MyMessage(messages.Message): number = messages.IntegerField() decoded = to_url.decode_message(MyMessage, self.unexpected_tag_message) self.assertEqual(1, len(decoded.all_unrecognized_fields())) self.assertEqual('unexpected', decoded.all_unrecognized_fields()[0]) # Unknown values set to a list of however many values had that name. self.assertEqual((['whatever'], messages.Variant.STRING), decoded.get_unrecognized_field_info('unexpected')) repeated_unknown = urlencode([('repeated', 400), ('repeated', 'test'), ('repeated', '123.456')]) decoded2 = to_url.decode_message(MyMessage, repeated_unknown) self.assertEqual((['400', 'test', '123.456'], messages.Variant.STRING), decoded2.get_unrecognized_field_info('repeated'))
def request(self, resource, method="get", args=None, body=None, parts=None, headers={}): params = None path = resource headers['User-Agent'] = 'Basic Agent' BOUNDARY = r'00hoYUXOnLD5RQ8SKGYVgLLt64jejnMwtO7q8XE1' CRLF = r'\r\n' if parts: # Attempt to find the Mimetype headers[ 'Content-Type'] = 'multipart/form-data; boundary=' + BOUNDARY encode_string = StringIO() for part in parts: if part.type != 'file': continue encode_string.write(r'--' + BOUNDARY) encode_string.write(CRLF) body = None if part.type == 'file': encode_string.write( r'Content-Disposition: form-data; name="%s"; filename="%s"' % (part.key, os.path.basename(part.text))) encode_string.write(CRLF) content_type = self.get_content_type(part.text) encode_string.write(r'Content-Type: %s' % content_type) encode_string.write(CRLF) encode_string.write(r'Content-Transfer-Encoding: base64') encode_string.write(CRLF) encode_string.write(CRLF) with open(part.text, "rb") as file: body = base64.b64encode(file.read()).decode() #body=str(file.read()) else: encode_string.write( r'Content-Disposition: form-data; name="%s"' % (part.key)) encode_string.write(CRLF) encode_string.write(r'Content-Type: %s' % part.contentType) encode_string.write(CRLF) encode_string.write(CRLF) body = part.text encode_string.write(body) encode_string.write(CRLF) encode_string.write(r'--' + BOUNDARY + r'--' + CRLF) body = encode_string.getvalue() headers['Content-Length'] = str(len(body)) elif body: if not headers.get('Content-Type', None): headers['Content-Type'] = 'text/xml' headers['Content-Length'] = str(len(body)) else: if not headers.get('Content-Type', None): headers['Content-Type'] = 'text/xml' domain = self.base_url.split("//")[1].split("/")[0] if args: if utils.getPythonVersion() == '2': path += r"?" + urllib.urlencode(args) else: path += r"?" + urlparse.urlencode(args) request_path = [] if self.path != "/": if self.path.endswith('/'): request_path.append(self.path[:-1]) else: request_path.append(self.path) if path.startswith('/'): request_path.append(path[1:]) else: request_path.append(path) # if self.host=='211.136.86.203':# kuan chang com IP # url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path)) # else:# normal flow url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path)) url = url.rstrip('/') logger.info('request: %s' % url) resp, content = self.h.request(url, method.upper(), body=body, headers=headers) encoding = 'UTF-8' if 'content-type' in resp and 'charset' in resp['content-type']: items = resp['content-type'].split('=') encoding = items[1] if 'content-type' in resp and 'application' in resp[ 'content-type'] and 'json' not in resp['content-type']: encoding = None if 'content-type' in resp and 'image' in resp['content-type']: encoding = None if encoding is not None: return {r'headers': resp, r'body': content.decode(encoding)} else: # not return binary data return {r'headers': resp, r'body': ''}
def _verb(url, data=None, params=None, headers=None, connection=None, verb=None, ctx=None, stream_callback=None, stream_chunk=512): urlp = urlparse.parse(url) netl = urlparse.parse_netloc(urlp[1]) host = netl[2] # print(verb,urlp,netl) if connection: sock = connection else: sock = _connect(host, netl[3], urlp[0], ctx) # print("CREATED SOCKET",sock.channel) #Generate and send Request Line msg = bytearray(BUFFER_LEN) p = 0 endline = "\r\n" p = add_to_buffer(msg, verb, p) p = add_to_buffer(msg, " ", p) if not urlp[2]: p = add_to_buffer(msg, "/", p) else: p = add_to_buffer(msg, urlp[2], p) if (verb in zverbs) and (urlp[-2] or params): p = add_to_buffer(msg, "?", p) if urlp[-2]: p = add_to_buffer(msg, urlp[-2], p) if params: p = add_to_buffer(msg, "&", p) if params: p = add_to_buffer(msg, urlparse.urlencode(params), p) if msg[p - 1] != __ORD(" "): p = add_to_buffer(msg, " ", p) p = add_to_buffer(msg, "HTTP/1.1", p) p = add_to_buffer(msg, endline, p) __elements_set(msg, p) #clamp buffer #send cmd try: # print(">>",msg) sock.sendall(msg) except Exception as e: sock.close() raise HTTPConnectionError __elements_set(msg, BUFFER_LEN) p = 0 #Generated and send Request headers p = add_to_buffer(msg, "Host: ", p) p = add_to_buffer(msg, host, p) if netl[3]: p = add_to_buffer(msg, ":", p) p = add_to_buffer(msg, netl[3], p) p = add_to_buffer(msg, endline, p) __elements_set(msg, p) # print(">>",msg) sock.sendall(msg) __elements_set(msg, BUFFER_LEN) p = 0 rh = {} if headers: for k in headers: rh[k.lower()] = headers[k] if "connection" not in rh: rh["connection"] = "close" if data is not None: rh["content-length"] = str(len(data[0])) #data[0] is actual data rh["content-type"] = data[1] #data[1] is data type header for k, v in rh.items(): p = add_to_buffer(msg, k, p) p = add_to_buffer(msg, ": ", p) p = add_to_buffer(msg, v, p) p = add_to_buffer(msg, endline, p) __elements_set(msg, p) # print(">>",msg) sock.sendall(msg) __elements_set(msg, BUFFER_LEN) p = 0 # Generate and send Body p = add_to_buffer(msg, endline, p) if data is not None: p = add_to_buffer(msg, data[0], p) __elements_set(msg, p) # print(">>",msg) sock.sendall(msg) __elements_set(msg, BUFFER_LEN) p = 0 #Parse Response rr = Response() ssock = streams.SocketStream(sock) buffer = msg msg = _readline(ssock, buffer, 0, BUFFER_LEN) # print("<<",msg) if msg.startswith("HTTP/1.1"): rr.status = int(msg[9:12]) __elements_set(msg, BUFFER_LEN) msg = _readline(ssock, buffer, 0, BUFFER_LEN) # print("<<",msg) #print(msg) #print(">[",msg,"]",msg=="\n",msg==endline) while not (msg == endline or msg == "\n"): idx_cl = msg.find(__ORD(":")) if idx_cl < 0: sock.close() raise HTTPResponseError rr.headers[str(msg[0:idx_cl].lower())] = str(msg[idx_cl + 1:-2].strip(endline)) __elements_set(msg, BUFFER_LEN) msg = _readline(ssock, buffer, 0, BUFFER_LEN) # print("<<",msg) # print(msg) #print(">[",msg,"]",msg=="\n",msg==endline) #print(rr.headers) rr.connection = sock if "content-length" in rr.headers: bodysize = int(rr.headers["content-length"]) #print("bodysize",bodysize) contentsize = bodysize reset_content = False if stream_callback is not None: reset_content = True contentsize = stream_chunk rr.content = bytearray(contentsize) tmp = 0 rdr = 0 while bodysize > 0: if reset_content: ofs = 0 else: ofs = tmp rdr = sock.recv_into(rr.content, min(bodysize, contentsize), ofs=ofs) if rdr and stream_callback: __elements_set(rr.content, rdr) stream_callback(rr.content) #print(rdr,rr.content[tmp:tmp+rdr]) tmp += rdr bodysize -= rdr if not rdr: break #print("CLOSED SOCKET A",sock.channel,rdr,tmp,bodysize) sock.close() rr.connection = None elif "transfer-encoding" in rr.headers: while True: __elements_set(msg, BUFFER_LEN) msg = _readline(ssock, buffer, 0, BUFFER_LEN) chsize = int(msg, 16) #print("chsize",chsize) if chsize: msg = sock.recv(chsize) #print(msg) if msg: rr.content.extend(msg) else: break __elements_set(buffer, BUFFER_LEN) msg = _readline(ssock, buffer, 0, BUFFER_LEN) else: __elements_set(buffer, BUFFER_LEN) msg = _readline(ssock, buffer, 0, BUFFER_LEN) #terminator break #print("CLOSED SOCKET B",sock.channel) sock.close() else: while True: tmp = sock.recv(32) if tmp: rr.content.extend(tmp) #print(tmp) else: break #print("CLOSED SOCKET C",sock.channel) sock.close() rr.connection = None return rr
class ProtourlencodeConformanceTest(test_util.TestCase, test_util.PytractsConformanceTestBase): PROTOLIB = to_url encoded_partial = urlencode(sorted([('double_value', 1.23), ('int64_value', -100000000000), ('int32_value', 1020), ('string_value', u'a string'), ('enum_value', 'VAL2'), ], key=lambda kv: kv[0])) encoded_full = urlencode(sorted([('double_value', 1.23), ('float_value', -2.5), ('int64_value', -100000000000), ('uint64_value', 102020202020), ('int32_value', 1020), ('bool_value', 'true'), ('string_value', u'a string\u044f'.encode('utf-8')), ('bytes_value', 'YSBieXRlc//+'), ('enum_value', 'VAL2'), ], key=lambda kv: kv[0])) encoded_repeated = urlencode(sorted([('double_value', 1.23), ('double_value', 2.3), ('float_value', -2.5), ('float_value', 0.5), ('int64_value', -100000000000), ('int64_value', 20), ('uint64_value', 102020202020), ('uint64_value', 10), ('int32_value', 1020), ('int32_value', 718), ('bool_value', 'true'), ('bool_value', 'false'), ('string_value', u'a string\u044f'.encode('utf-8')), ('string_value', u'another string'.encode('utf-8')), ('bytes_value', 'YSBieXRlc//+'), ('bytes_value', 'YW5vdGhlciBieXRlcw=='), ('enum_value', 'VAL2'), ('enum_value', 'VAL1'), ], key=lambda kv: kv[0])) encoded_nested = urlencode(sorted([('nested.a_value', 'a string')], key=lambda kv: kv[0])) encoded_repeated_nested = urlencode(sorted([('repeated_nested-0.a_value', 'a string'), ('repeated_nested-1.a_value', 'another string'), ], key=lambda kv: kv[0])) unexpected_tag_message = 'unexpected=whatever' encoded_default_assigned = urlencode(sorted([('a_value', 'a default'),], key=lambda kv: kv[0])) encoded_nested_empty = urlencode(sorted([('nested', '')], key=lambda kv: kv[0])) encoded_repeated_nested_empty = urlencode(sorted([('repeated_nested-0', ''), ('repeated_nested-1', '')], key=lambda kv: kv[0])) encoded_extend_message = urlencode(sorted([('int64_value-0', 400), ('int64_value-1', 50), ('int64_value-2', 6000)], key=lambda kv: kv[0])) encoded_string_types = urlencode( sorted([('string_value', 'Latin')], key=lambda kv: kv[0])) encoded_invalid_enum = urlencode([('enum_value', 'undefined')]) def testParameterPrefix(self): """Test using the 'prefix' parameter to encode_message.""" class MyMessage(messages.Message): number = messages.IntegerField() names = messages.StringField(repeated=True) message = MyMessage() message.number = 10 message.names = [u'Fred', u'Lisa'] encoded_message = to_url.encode_message(message, prefix='prefix-') self.assertEqual({'prefix-number': ['10'], 'prefix-names': ['Fred', 'Lisa']}, parse_qs(encoded_message)) self.assertEqual(message, to_url.decode_message(MyMessage, encoded_message, prefix='prefix-')) def testProtourlencodeUnrecognizedField(self): """Test that unrecognized fields are saved and can be accessed.""" class MyMessage(messages.Message): number = messages.IntegerField() decoded = to_url.decode_message(MyMessage, self.unexpected_tag_message) self.assertEqual(1, len(decoded.all_unrecognized_fields())) self.assertEqual('unexpected', decoded.all_unrecognized_fields()[0]) # Unknown values set to a list of however many values had that name. self.assertEqual((['whatever'], messages.Variant.STRING), decoded.get_unrecognized_field_info('unexpected')) repeated_unknown = urlencode([('repeated', 400), ('repeated', 'test'), ('repeated', '123.456')]) decoded2 = to_url.decode_message(MyMessage, repeated_unknown) self.assertEqual((['400', 'test', '123.456'], messages.Variant.STRING), decoded2.get_unrecognized_field_info('repeated')) def testDecodeUUID(self): from uuid import UUID class Foo(messages.Message): bar = messages.UUIDField() m = to_url.decode_message(Foo, 'bar=06335e84-2872-4914-8c5d-3ed07d2a2f16') self.assertEqual(UUID("06335e84-2872-4914-8c5d-3ed07d2a2f16"), m.bar) def testDecodeUUIDInvalid(self): from uuid import UUID class Foo(messages.Message): bar = messages.UUIDField() with self.assertRaises(messages.DecodeError): to_url.decode_message(Foo, 'bar=bad') def testDecodeDateTimeIso8601(self): class MyMessage(messages.Message): a_datetime = messages.DateTimeISO8601Field() m = to_url.decode_message(MyMessage, 'a_datetime=2012-09-30T15:31:50.262000') self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime) m = to_url.decode_message(MyMessage, 'a_datetime=2012-09-30T15%3A31%3A50.262000') self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime) def testDecodeInvalidDateTimeIso8601(self): class MyMessage(messages.Message): a_datetime = messages.DateTimeISO8601Field() self.assertRaises(messages.DecodeError, to_url.decode_message, MyMessage, 'a_datetime=invalid') def testDecodeDateTimeMsInteger(self): class MyMessage(messages.Message): a_datetime = messages.DateTimeMsIntegerField() m = to_url.decode_message(MyMessage, 'a_datetime=1349019110262') self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime) m = to_url.decode_message(MyMessage, 'a_datetime=1349019110262') self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime) def testDecodeInvalidDateTimeMsInteger(self): class MyMessage(messages.Message): a_datetime = messages.DateTimeISO8601Field() self.assertRaises(messages.DecodeError, to_url.decode_message, MyMessage, 'a_datetime=invalid') def test_decode_message_from_url(self): class AnimalSounds(messages.Message): cow = messages.StringField() dog = messages.StringField() foo = AnimalSounds(cow='moo', dog='woof') self.assertEqual(foo, to_url.decode_message_from_url(AnimalSounds, "http://example.com?cow=moo&dog=woof")) def test_decode_message_from_url_repeated(self): class Animals(messages.Message): animals = messages.StringField(repeated=True) number = messages.IntegerField() tmp = Animals(animals=['dog', 'cat'], number=2) self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?animals=dog&animals=cat&number=2")) def test_decode_message_from_url_repeated_alias(self): class Animals(messages.Message): animals = messages.StringField(name='a', repeated=True) number = messages.IntegerField() tmp = Animals(animals=['dog', 'cat'], number=2) self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?a=dog&a=cat&number=2")) def test_decode_message_from_url_repeated_alias_dashes(self): class Animals(messages.Message): animals = messages.StringField(name='a-m', repeated=True) number = messages.IntegerField() tmp = Animals(animals=['dog', 'cat'], number=2) self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?a-m=dog&a-m=cat&number=2")) def test_encode_message_from_url_repeated_alias(self): class Animals(messages.Message): animals = messages.StringField(name='a', repeated=True) number = messages.IntegerField() tmp = Animals(animals=['dog', 'cat'], number=2) self.assertEqual("a=dog&a=cat&number=2", to_url.encode_message(tmp)) def test_decode_message_from_url_repeated_not_filled_out(self): class Animals(messages.Message): animals = messages.StringField(repeated=True) number = messages.IntegerField() result = to_url.decode_message_from_url(Animals, "http://example.com?number=2") self.assertFalse(Animals.animals.is_set(result)) self.assertEqual([], result.animals) def test_decode_message_from_url_repeated_message(self): class Animal(messages.Message): name = messages.StringField() size = messages.IntegerField() class Animals(messages.Message): animals = messages.MessageField(Animal, repeated=True) number = messages.IntegerField() dog = Animal(name='dog', size=12) cat = Animal(name='cat', size=10) tmp = Animals(animals=[dog, cat], number=2) self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?animals-0.name=dog&animals-0.size=12&animals-1.name=cat&animals-1.size=10&number=2")) def test_encode_message_repeated_message_field(self): class Animal(messages.Message): name = messages.StringField() size = messages.IntegerField() class Animals(messages.Message): animals = messages.MessageField(Animal, repeated=True) number = messages.IntegerField() dog = Animal(name='dog', size=12) cat = Animal(name='cÄt', size=10) tmp = Animals(animals=[dog, cat], number=2) encoded_message = to_url.encode_message(tmp) self.assertEqual({'number': ['2'], 'animals-0.name': ['dog'], 'animals-0.size': ['12'], 'animals-1.name': ['cÄt'], 'animals-1.size': ['10']}, parse_qs(encoded_message)) self.assertEqual(tmp, to_url.decode_message(Animals, encoded_message))
def _on_request(self, _request, session, method, url, data=None, headers=None, **kw): # Create request contract based on incoming params req = Request(method) req.headers = headers or {} req.body = data # Expose extra variadic arguments req.extra = kw # Compose URL if not kw.get('params'): req.url = str(url) else: req.url = str(url) + '?' + urlencode( [(x, y) for x, y in kw['params'].items()]) # Match the request against the registered mocks in pook mock = self.engine.match(req) # If cannot match any mock, run real HTTP request if networking # or silent model are enabled, otherwise this statement won't # be reached (an exception will be raised before). if not mock: return _request(session, method, url, data=data, headers=headers, **kw) # Simulate network delay if mock._delay: yield from asyncio.sleep(mock._delay / 1000) # noqa # Shortcut to mock response res = mock._response # Aggregate headers as list of tuples for interface compatibility headers = [] for key in res._headers: headers.append((key, res._headers[key])) # Create mock equivalent HTTP response _res = HTTPResponse(req.method, self._url(urlunparse(req.url))) # response status _res.version = (1, 1) _res.status = res._status _res.reason = http_reasons.get(res._status) _res._should_close = False # Add response headers _res._raw_headers = tuple(headers) _res._headers = multidict.CIMultiDictProxy( multidict.CIMultiDict(headers)) if res._body: _res.content = SimpleContent( res._body.encode('utf-8', errors='replace'), ) else: # Define `_content` attribute with an empty string to # force do not read from stream (which won't exists) _res.content = EmptyStreamReader() # Return response based on mock definition return _res
def acs(r): try: import urlparse as _urlparse from urllib import unquote except: import urllib.parse as _urlparse from urllib.parse import unquote saml_client = _get_saml_client(get_current_domain(r)) resp = r.POST.get('SAMLResponse', None) next_url = r.session.get('login_next_url', _default_next_url()) # Use RelayState if available, else fall back to next_url. next_url = r.POST.get('RelayState', next_url) if not resp: return HttpResponseRedirect( get_reverse([denied, 'denied', 'django_saml2_auth:denied'])) authn_response = saml_client.parse_authn_request_response( resp, entity.BINDING_HTTP_POST) if authn_response is None: return HttpResponseRedirect( get_reverse([denied, 'denied', 'django_saml2_auth:denied'])) print('return addy') print(authn_response.return_addrs) user_identity = authn_response.get_identity() print('authn_response') print(authn_response) print('user_identity') print(user_identity) if user_identity is None: return HttpResponseRedirect( get_reverse([denied, 'denied', 'django_saml2_auth:denied'])) user_email = user_identity[settings.SAML2_AUTH.get( 'ATTRIBUTES_MAP', {}).get('email', 'Email')][0] user_name = user_identity[settings.SAML2_AUTH.get( 'ATTRIBUTES_MAP', {}).get('username', 'UserName')][0] user_first_name = user_identity[settings.SAML2_AUTH.get( 'ATTRIBUTES_MAP', {}).get('first_name', 'FirstName')][0] user_last_name = user_identity[settings.SAML2_AUTH.get( 'ATTRIBUTES_MAP', {}).get('last_name', 'LastName')][0] target_user = None is_new_user = False try: target_user = User.objects.get(username=user_name) if settings.SAML2_AUTH.get('TRIGGER', {}).get('BEFORE_LOGIN', None): import_string( settings.SAML2_AUTH['TRIGGER']['BEFORE_LOGIN'])(user_identity) except User.DoesNotExist: new_user_should_be_created = settings.SAML2_AUTH.get( 'CREATE_USER', True) if new_user_should_be_created: target_user = _create_new_user(user_name, user_email, user_first_name, user_last_name) if settings.SAML2_AUTH.get('TRIGGER', {}).get('CREATE_USER', None): import_string(settings.SAML2_AUTH['TRIGGER']['CREATE_USER'])( user_identity) is_new_user = True else: return HttpResponseRedirect( get_reverse([denied, 'denied', 'django_saml2_auth:denied'])) r.session.flush() if target_user.is_active: target_user.backend = 'django.contrib.auth.backends.ModelBackend' login(r, target_user) else: return HttpResponseRedirect( get_reverse([denied, 'denied', 'django_saml2_auth:denied'])) if settings.SAML2_AUTH.get('USE_JWT') is True: # We use JWT auth send token to frontend jwt_token = jwt_encode(target_user) params = {"uid": target_user.id, "token": jwt_token} frontend_url = settings.SAML2_AUTH.get('FRONTEND_URL', next_url) if next_url and next_url != _default_next_url(): frontend_url = next_url # Reconstruct URL with added parameters. url_parts = list( _urlparse.urlparse(frontend_url, allow_fragments=False)) query = dict(_urlparse.parse_qsl(url_parts[4])) query.update(params) url_parts[4] = _urlparse.urlencode(query) return HttpResponseRedirect(_urlparse.urlunparse(url_parts)) if is_new_user: try: return render(r, 'django_saml2_auth/welcome.html', {'user': r.user}) except TemplateDoesNotExist: return HttpResponseRedirect(next_url) else: return HttpResponseRedirect(next_url)
def encode_message(message, prefix=''): """Encode Message instance to url-encoded string. Args: message: Message instance to encode in to url-encoded string. prefix: Prefix to append to field names of contained values. Returns: String encoding of Message in URL encoded format. Raises: messages.ValidationError if message is not initialized. """ message.check_initialized() parameters = [] def build_message(parent, prefix): """Recursively build parameter list for URL response. Args: parent: Message to build parameters for. prefix: Prefix to append to field names of contained values. Returns: True if some value of parent was added to the parameters list, else False, meaning the object contained no values. """ has_any_values = False for field in sorted(parent.all_fields(), key=lambda f: f.name): if not parent.has_value_assigned(field.name): continue values = parent.get_assigned_value(field.name) # Found a value. Ultimate return value should be True. has_any_values = True # Normalize all values in to a list. if not field.repeated: values = [values] for index, item in enumerate(values): # Create a name with an index if it is a repeated field. if field.repeated and isinstance(field, messages.MessageField): field_name = '%s%s-%s' % (prefix, field.name, index) else: field_name = prefix + field.name if isinstance(field, messages.DateTimeISO8601Field): # DateTimeField stores its data as a ISO8601 compliant string. parameters.append((field_name, item.isoformat())) elif isinstance(field, messages.DateTimeMsIntegerField): parameters.append((field_name, util.datetime_to_ms(item))) elif isinstance(field, messages.MessageField): # Message fields must be recursed in to in order to construct # their component parameter values. if not build_message(item, field_name + '.'): # The nested message is empty. Append an empty value to # represent it. parameters.append((field_name, '')) elif isinstance(field, messages.BooleanField): parameters.append((field_name, item and 'true' or 'false')) elif isinstance(field, messages.BytesField): parameters.append( (field_name, base64.b64encode(item).decode('utf-8'))) else: #if isinstance(item, unicode): # item = item.encode('utf-8') parameters.append((field_name, str(item))) return has_any_values build_message(message, prefix) # Also add any unrecognized values from the decoded string. for key in message.all_unrecognized_fields(): values, _ = message.get_unrecognized_field_info(key) if not isinstance(values, (list, tuple)): values = (values, ) for value in values: parameters.append((key, value)) return urlencode(parameters)
import urllib import urlparse import re import urlparse ##x = urllib.urlopen('http://www.google.com') ##print x.read() url = 'http://www.pythonprgramming.net' values = {'s':'basic', 'submit':'search'} data = urlparse.urlencode(values) data = data.encode('utf-8') req = urllib.request.Request(url, data) resp = urllib.request.urlopen(req) respData = resp.read() print (respData) ##paragraphs = re.findall(r'<p>(.*?)</p>', str(respData)) for eachP in paragraphs: print(eachP)
def build_query(self, params): params = urlparse.urlencode(params) return params
def _request(self, **kwargs): # Check for missing variables resource = self.interface def validate_kwarg(key, target): # Always allow list if isinstance(key, list): return all([validate_kwarg(x, target[0]) for x in key]) return isinstance(key, target) for k in resource.get('required', []): if k not in [x.split(':')[0] for x in kwargs.keys()]: raise ValueError('Missing required argument: %s' % k) if isinstance(kwargs.get(k), list) and not isinstance( resource['required'][k], list): raise ValueError('Wrong argument type: %s is not a list' % k) elif not validate_kwarg(kwargs.get(k), resource['required'][k]): if isinstance(resource['required'][k], list): raise ValueError( 'Wrong arguemnt type: %s is not a list of %ss' % (k, resource['required'][k][0])) raise ValueError('Wrong arguemnt type: %s is not a %s' % (k, resource['required'][k])) conduit = self.api.conduit if conduit: # Already authenticated, add session key to json data kwargs['__conduit__'] = conduit elif self.method == 'conduit' and self.endpoint == 'connect': # Not authenticated, requesting new session key token = str(int(time.time())) kwargs['authToken'] = token kwargs['authSignature'] = self.api.generate_hash(token) else: # Authorization is required, silently auth the user self.api.connect() kwargs['__conduit__'] = self.api.conduit url = urlparse.urlparse(self.api.host) if url.scheme == 'https': conn = httplib.HTTPSConnection(url.netloc, timeout=self.api.timeout) else: conn = httplib.HTTPConnection(url.netloc, timeout=self.api.timeout) path = url.path + '%s.%s' % (self.method, self.endpoint) headers = { 'User-Agent': 'python-phabricator/%s' % str(self.api.clientVersion), 'Content-Type': 'application/x-www-form-urlencoded' } body = urlparse.urlencode({ "params": json.dumps(kwargs), "output": self.api.response_format }) # TODO: Use HTTP "method" from interfaces.json conn.request('POST', path, body, headers) response = conn.getresponse() data = self._parse_response(response.read()) return Result(data['result'])
def request(self, resource, method = "get", args = None, body = None, parts=None, headers={}): params = None path = resource headers['User-Agent'] = 'Basic Agent' BOUNDARY = r'00hoYUXOnLD5RQ8SKGYVgLLt64jejnMwtO7q8XE1' CRLF = r'\r\n' if parts: # Attempt to find the Mimetype headers['Content-Type']='multipart/form-data; boundary='+BOUNDARY encode_string = StringIO() for part in parts: if part.type != 'file': continue encode_string.write(r'--' + BOUNDARY) encode_string.write(CRLF) body = None if part.type == 'file': encode_string.write(r'Content-Disposition: form-data; name="%s"; filename="%s"' % (part.key, os.path.basename(part.text))) encode_string.write(CRLF) content_type = self.get_content_type(part.text) encode_string.write(r'Content-Type: %s' % content_type) encode_string.write(CRLF) encode_string.write(r'Content-Transfer-Encoding: base64') encode_string.write(CRLF) encode_string.write(CRLF) with open(part.text, "rb") as file: body = base64.b64encode(file.read()).decode() #body=str(file.read()) else: encode_string.write(r'Content-Disposition: form-data; name="%s"' % (part.key)) encode_string.write(CRLF) encode_string.write(r'Content-Type: %s' % part.contentType) encode_string.write(CRLF) encode_string.write(CRLF) body = part.text encode_string.write(body) encode_string.write(CRLF) encode_string.write(r'--' + BOUNDARY + r'--' + CRLF) body = encode_string.getvalue() headers['Content-Length'] = str(len(body)) elif body: if not headers.get('Content-Type', None): headers['Content-Type']='text/xml' headers['Content-Length'] = str(len(body)) else: if not headers.get('Content-Type', None): headers['Content-Type']='text/xml' domain=self.base_url.split("//")[1].split("/")[0] if args: if utils.getPythonVersion() == '2': path += r"?" + urllib.urlencode(args) else: path += r"?" + urlparse.urlencode(args) request_path = [] if self.path != "/": if self.path.endswith('/'): request_path.append(self.path[:-1]) else: request_path.append(self.path) if path.startswith('/'): request_path.append(path[1:]) else: request_path.append(path) # if self.host=='211.136.86.203':# kuan chang com IP # url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path)) # else:# normal flow url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path)) url = url.rstrip('/') logger.info('request: %s' % url) resp, content = self.h.request(url, method.upper(), body=body, headers=headers ) encoding = 'UTF-8' if 'content-type' in resp and 'charset' in resp['content-type']: items = resp['content-type'].split('=') encoding = items[1] if 'content-type' in resp and 'application' in resp['content-type'] and 'json' not in resp['content-type']: encoding = None if 'content-type' in resp and 'image' in resp['content-type']: encoding = None if encoding is not None: return {r'headers':resp, r'body':content.decode(encoding)} else: # not return binary data return {r'headers':resp, r'body':''}
def __init__( self, url, cache_dir, pkg_dir, proxy_addr="", proxy_user="", proxy_pass="", gpg_verify=True, channel_label=None, ): self.url = url parts = url.rsplit('/dists/', 1) self.base_url = [parts[0]] parsed_url = urlparse.urlparse(url) query = urlparse.parse_qsl(parsed_url.query) new_query = [] suite = None component = None arch = None for qi in query: if qi[0] == "uyuni_suite": suite = qi[1] elif qi[0] == "uyuni_component": component = qi[1] elif qi[0] == "uyuni_arch": arch = qi[1] else: new_query.append(qi) if suite: parsed_url = parsed_url._replace( query=urlparse.urlencode(new_query)) base_url = urlparse.urlunparse(parsed_url) path_list = parsed_url.path.split("/") log2(0, 0, "Base URL: {}".format(base_url)) log2(0, 0, "Suite: {}".format(suite)) log2(0, 0, "Component: {}".format(component)) if "/" not in suite: path_list.append("dists") path_list.extend(suite.split("/")) if component: path_list.extend(component.split("/")) if "/" not in suite: if arch is None: rhnSQL.initDB() h = rhnSQL.prepare(""" SELECT ca.label AS arch_label FROM rhnChannel AS c LEFT JOIN rhnChannelArch AS ca ON c.channel_arch_id = ca.id WHERE c.label = :channel_label """) h.execute(channel_label=channel_label) row = h.fetchone_dict() if row and "arch_label" in row: aspl = row["arch_label"].split("-") if len(aspl) == 3 and aspl[0] == "channel" and aspl[ 2] == "deb": arch_trans = { "ia32": "i386", "arm": "armhf", } if aspl[1] in arch_trans: arch = arch_trans[aspl[1]] else: arch = aspl[1] if arch: log2(0, 0, "Channel architecture: {}".format(arch)) path_list.append("binary-{}".format(arch)) while "" in path_list: path_list.remove("") parsed_url = parsed_url._replace(path="/".join(path_list)) self.url = url = urlparse.urlunparse(parsed_url) self.base_url = [base_url] # Make sure baseurl ends with / and urljoin will work correctly if self.base_url[0][-1] != '/': self.base_url[0] += '/' self.urls = self.base_url self.sslclientcert = self.sslclientkey = self.sslcacert = None self.proxy = proxy_addr self.proxy_username = proxy_user self.proxy_password = proxy_pass self.gpg_verify = gpg_verify self.basecachedir = cache_dir if not os.path.isdir(self.basecachedir): fileutils.makedirs(self.basecachedir, user='******', group='www') self.includepkgs = [] self.exclude = [] self.pkgdir = pkg_dir self.http_headers = {}