def _get_encoded_url(self):
"""Convert any UTF-8 char in :obj:`File.file_path` into a url encoded ASCII string."""
sres = urllib_parse.urlsplit(self.file_path)
return urllib_parse.urlunsplit(
urllib_parse.SplitResult(sres.scheme, sres.netloc,
urllib_parse.quote(sres.path), sres.query,
sres.fragment))
def get_auction_rade(topic_id=''):
# ctx.log.info(topic_id)
js = {
"queryId": "itemSaleRoomListsQuery",
"filterMap": {
"saleRoomId": topic_id,
"timeFlag": "all"
},
"pageInfo": {
"currentPage": 1,
"pageCount": 0,
"recordCount": 0,
"pageSize": "200"
}
}
data = 'queryData=' + parse.quote(json.dumps(js))
r = s3.post(
"http://www.testsite.com/ecp/item/input/iteminput/querysriteminfo.action",
data=data)
auction_ids = r.json()['data']['data']
print len(auction_ids)
print data
print auction_ids
return auction_ids
def iri_to_uri(iri):
"""
Convert an Internationalized Resource Identifier (IRI) portion to a URI
portion that is suitable for inclusion in a URL.
This is the algorithm from section 3.1 of RFC 3987, slightly simplified
since the input is assumed to be a string rather than an arbitrary byte
stream.
Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or
b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded
result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/').
"""
# The list of safe characters here is constructed from the "reserved" and
# "unreserved" characters specified in sections 2.2 and 2.3 of RFC 3986:
# reserved = gen-delims / sub-delims
# gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
# sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
# / "*" / "+" / "," / ";" / "="
# unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
# Of the unreserved characters, urllib.parse.quote() already considers all
# but the ~ safe.
# The % character is also added to the list of safe characters here, as the
# end of section 3.1 of RFC 3987 specifically mentions that % must not be
# converted.
if iri is None:
return iri
#WARNING: The following 2 line might come to bite back - have to pay some attentions
#elif isinstance(iri, Promise):
else:
iri = str(iri)
return quote(iri, safe="/#%[]=:;$&()+,!?*@'~")
def find_auction_rade(auction_ids):
urls = []
for a in auction_ids:
aid = str(a['LOT_ID'])
js = {"lotId": aid, "saleRoomId": "", "showRoom": ""}
data = 'dataStr=' + parse.quote(json.dumps(js))
r = s3.post(
"http://www.testsite.com/ecp/catalogueinfo/getcatalogueinfobylotid.action",
data=data)
# print(r.headers)
# print(r.content)
try:
images = []
images.append(r.json()['data']['majorItemPic']['filepath'] +
".jpg")
for i in r.json()['data']['otherItemPicList']:
images.append(i['filepath'] + ".jpg")
auction = [
re.sub(r'[,. -!?:/\\]', '',
r.json()['data']['itemInfo']['itemName']), images
]
urls.append(auction)
except:
print(data)
print(r.headers)
return urls
def build_url_request(self, command, args):
"""
Helper function to construct and return the url that will be requested from the server
@param command - the command name (i.e., prefix.commandName)
@param args - dictionary of the arguments to pass
@returns the url to fetch (i.e, http://servername:8080/remote/prefix.commandName.do?arg=value)
"""
url = '%s/%s.do' % (self.baseurl, command)
query_string = "&".join([
"%s=%s" % (quote(str(k)), quote(unicode(v)))
for k, v in args.items()
])
if not query_string:
return url
return '%s?%s' % (url, query_string)
def safe_path(entity_id, test_id):
s = quote(entity_id)
s = s.replace('/', '%2F')
if not os.path.isdir('log/{}'.format(s)):
os.makedirs('log/{}'.format(s))
return 'log/{}/{}'.format(s, test_id)
def test_request_info_simple_get(self):
uri, body, h_args, cis = self.client.request_info(AuthorizationRequest,
method="GET")
assert url_compare(
uri, '{}?redirect_uri={}&response_type=code&client_id=1'.format(
self.authorization_endpoint, quote(self.redirect_uri,
safe="")))
assert body is None
assert h_args == {}
def test_request_info_simple_get(self):
uri, body, h_args, cis = self.client.request_info(AuthorizationRequest,
method="GET")
assert url_compare(uri,
'{}?redirect_uri={}&response_type=code&client_id=1'.format(
self.authorization_endpoint,
quote(self.redirect_uri, safe="")))
assert body is None
assert h_args == {}
def escape_uri_path(path, encode_percent=True):
"""
Escape the unsafe characters from the path portion of a Uniform Resource
Identifier (URI).
"""
# These are the "reserved" and "unreserved" characters specified in
# sections 2.2 and 2.3 of RFC 2396:
# reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
# unreserved = alphanum | mark
# mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
# The list of safe characters here is constructed subtracting ";", "=",
# and "?" according to section 3.3 of RFC 2396.
# The reason for not subtracting and escaping "/" is that we are escaping
# the entire path, not a path segment.
if encode_percent:
return quote(path, safe="/:@&+$,-_.!~*'()")
else:
return quote(path, safe="/:@&+$,-_.!~*'()%")
def test_request_info_simple_get_with_extra_args(self):
uri, body, h_args, cis = self.client.request_info(
AuthorizationRequest, method="GET", extra_args={"rock": "little"})
assert url_compare(uri,
'{}?redirect_uri={}&response_type=code&client_id=1&rock=little'.format(
self.authorization_endpoint,
quote(self.redirect_uri, safe="")))
assert body is None
assert h_args == {}
assert isinstance(cis, AuthorizationRequest)
def safe_url(url):
if url.startswith('https://'):
url = 's_' + url[8:]
elif url.startswith('http://'):
url = url[7:]
s = quote(url)
s = s.replace('/', '%2F')
s = s.replace('%', '')
s = s.replace('//', '/')
return s
def safe_url(url):
if url.startswith('https://'):
url = 's_' + url[8:]
elif url.startswith('http://'):
url = url[7:]
s = quote(url)
s = s.replace('/', '%2F')
s = s.replace('%', '')
s = s.replace('//', '/')
return s
def test_request_info_simple_get_with_extra_args(self):
uri, body, h_args, cis = self.client.request_info(
AuthorizationRequest, method="GET", extra_args={"rock": "little"})
assert url_compare(
uri,
'{}?redirect_uri={}&response_type=code&client_id=1&rock=little'.
format(self.authorization_endpoint,
quote(self.redirect_uri, safe="")))
assert body is None
assert h_args == {}
assert isinstance(cis, AuthorizationRequest)
def safe_path(eid, *args):
s = quote(eid)
s = s.replace('/', '%2F')
path = 'log/{}'.format(s)
for arg in args[:-1]:
path = '{}/{}'.format(path, arg)
if not os.path.isdir(path):
os.makedirs(path)
return '{}/{}'.format(path, args[-1])
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
displaypath = html.escape(urllib_parse.unquote(self.path))
enc = sys.getfilesystemencoding()
title = 'Directory listing for %s' % displaypath
r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
'"http://www.w3.org/TR/html4/strict.dtd">')
r.append('<html>\n<head>')
r.append('<meta http-equiv="Content-Type" '
'content="text/html; charset=%s">' % enc)
r.append('<title>%s</title>\n</head>' % title)
r.append('<body>\n<h1>%s</h1>' % title)
r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
r.append('<li><a href="%s">%s</a></li>'
% (urllib_parse.quote(linkname), html.escape(displayname)))
# # Use this instead:
# r.append('<li><a href="%s">%s</a></li>'
# % (urllib.quote(linkname), cgi.escape(displayname)))
r.append('</ul>\n<hr>\n</body>\n</html>\n')
encoded = '\n'.join(r).encode(enc)
f = io.BytesIO()
f.write(encoded)
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
r = []
displaypath = html.escape(urllib_parse.unquote(self.path))
enc = sys.getfilesystemencoding()
title = 'Directory listing for %s' % displaypath
r.append('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" '
'"http://www.w3.org/TR/html4/strict.dtd">')
r.append('<html>\n<head>')
r.append('<meta http-equiv="Content-Type" '
'content="text/html; charset=%s">' % enc)
r.append('<title>%s</title>\n</head>' % title)
r.append('<body>\n<h1>%s</h1>' % title)
r.append('<hr>\n<ul>')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
r.append('<li><a href="%s">%s</a></li>' %
(urllib_parse.quote(linkname), html.escape(displayname)))
# # Use this instead:
# r.append('<li><a href="%s">%s</a></li>'
# % (urllib.quote(linkname), cgi.escape(displayname)))
r.append('</ul>\n<hr>\n</body>\n</html>\n')
encoded = '\n'.join(r).encode(enc)
f = io.BytesIO()
f.write(encoded)
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html; charset=%s" % enc)
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
return f
def test_request_info_with_req_and_extra_args(self):
uri, body, h_args, cis = self.client.request_info(
AuthorizationRequest,
method="GET",
request_args={"state": "init"},
extra_args={"rock": "little"})
expected = '{}?state=init&redirect_uri={}&response_type=code&client_id=1&rock=little'
assert url_compare(uri, expected.format(self.authorization_endpoint,
quote(self.redirect_uri,
safe="")))
assert body is None
assert h_args == {}
assert isinstance(cis, AuthorizationRequest)
def test_request_info_simple(self):
req_args = {"state": "hmm", "response_type": "code"}
uri, body, h_args, cis = self.client.request_info(AuthorizationRequest,
request_args=req_args)
# default == "POST"
assert uri == self.authorization_endpoint
body_elts = body.split('&')
expected_body = "state=hmm&redirect_uri={}&response_type=code&client_id=1".format(
quote(self.redirect_uri, safe=""))
expected_body_elts = expected_body.split('&')
assert set(body_elts) == set(expected_body_elts)
assert h_args == {
'headers': {'Content-Type': 'application/x-www-form-urlencoded'}}
assert isinstance(cis, AuthorizationRequest)
def test_request_info_with_req_and_extra_args(self):
uri, body, h_args, cis = self.client.request_info(
AuthorizationRequest,
method="GET",
request_args={"state": "init"},
extra_args={"rock": "little"})
expected = '{}?state=init&redirect_uri={}&response_type=code&client_id=1&rock=little'
assert url_compare(
uri,
expected.format(self.authorization_endpoint,
quote(self.redirect_uri, safe="")))
assert body is None
assert h_args == {}
assert isinstance(cis, AuthorizationRequest)
def test_request_info_simple(self):
req_args = {"state": "hmm", "response_type": "code"}
uri, body, h_args, cis = self.client.request_info(
AuthorizationRequest, request_args=req_args)
assert uri == self.authorization_endpoint
body_elts = body.split('&')
expected_body = "state=hmm&redirect_uri={}&response_type=code&client_id=1".format(
quote(self.redirect_uri, safe=""))
expected_body_elts = expected_body.split('&')
assert set(body_elts) == set(expected_body_elts)
assert h_args == {
'headers': {
'Content-Type': 'application/x-www-form-urlencoded'
}
}
assert isinstance(cis, AuthorizationRequest)
def download(self, custom_path=None, out=None, timeout=None):
"""
Download this file. By default, the file is saved in the current working directory with its
original filename as reported by Telegram. If a :attr:`custom_path` is supplied, it will be
saved to that path instead. If :attr:`out` is defined, the file contents will be saved to
that object using the ``out.write`` method.
Note:
`custom_path` and `out` are mutually exclusive.
Args:
custom_path (:obj:`str`, optional): Custom path.
out (:obj:`object`, optional): A file-like object. Must be opened in binary mode, if
applicable.
timeout (:obj:`int` | :obj:`float`, optional): If this value is specified, use it as
the read timeout from the server (instead of the one specified during creation of
the connection pool).
Raises:
ValueError: If both ``custom_path`` and ``out`` are passed.
"""
if custom_path is not None and out is not None:
raise ValueError('custom_path and out are mutually exclusive')
# Convert any UTF-8 char into a url encoded ASCII string.
sres = urllib_parse.urlsplit(self.file_path)
url = urllib_parse.urlunsplit(
urllib_parse.SplitResult(sres.scheme, sres.netloc,
urllib_parse.quote(sres.path), sres.query,
sres.fragment))
if out:
buf = self.bot.request.retrieve(url)
out.write(buf)
else:
if custom_path:
filename = custom_path
else:
filename = basename(self.file_path)
self.bot.request.download(url, filename, timeout=timeout)
def download(self, custom_path=None, out=None, timeout=None):
"""
Download this file. By default, the file is saved in the current working directory with its
original filename as reported by Telegram. If a :attr:`custom_path` is supplied, it will be
saved to that path instead. If :attr:`out` is defined, the file contents will be saved to
that object using the ``out.write`` method.
Note:
`custom_path` and `out` are mutually exclusive.
Args:
custom_path (:obj:`str`, optional): Custom path.
out (:obj:`object`, optional): A file-like object. Must be opened in binary mode, if
applicable.
timeout (:obj:`int` | :obj:`float`, optional): If this value is specified, use it as
the read timeout from the server (instead of the one specified during creation of
the connection pool).
Raises:
ValueError: If both ``custom_path`` and ``out`` are passed.
"""
if custom_path is not None and out is not None:
raise ValueError('custom_path and out are mutually exclusive')
# Convert any UTF-8 char into a url encoded ASCII string.
sres = urllib_parse.urlsplit(self.file_path)
url = urllib_parse.urlunsplit(urllib_parse.SplitResult(
sres.scheme, sres.netloc, urllib_parse.quote(sres.path), sres.query, sres.fragment))
if out:
buf = self.bot.request.retrieve(url)
out.write(buf)
else:
if custom_path:
filename = custom_path
else:
filename = basename(self.file_path)
self.bot.request.download(url, filename, timeout=timeout)
def test_complex():
js = {
"monkey": "cat",
"aform": parse.urlencode([
("choice", 17),
("choice", 18),
("choice", parse.quote("test+1fsf")),
("choice", "ZnNmc2Q="),
("json", json.dumps({"chained": "json2", "aaa": "bbb"}))
]),
"foo": ["b", "a", {"b": "c", "d": "e"}, "this=is&a=form"],
"bar": False,
"ttt": None,
"key-with.a-dot": "value-with.a-dot",
"中文": "中文"
}
nested = r"""_callback({})""".format(json.dumps(js, ensure_ascii=False))
root = load(nested)
assert isinstance(root, JSONPNode)
assert str(root["monkey"]) \
== "PlainNode<key='monkey' parent='<root>' depth=1 data='cat'>"
assert str(root["bar"]) \
== "PlainNode<key='bar' parent='<root>' depth=1 data=False>"
assert str(root["ttt"]) \
== "PlainNode<key='ttt' parent='<root>' depth=1 data=None>"
assert str(root["key-with.a-dot"]) \
== "PlainNode<key='key-with.a-dot' parent='<root>' depth=1 data='value-with.a-dot'>"
assert str(root["中文"]) == "PlainNode<key='中文' parent='<root>' depth=1 data='中文'>"
assert str(root["aform"]) \
== """FormNode<key='aform' parent='<root>' depth=1 data=QueryDict([('choice', '17'), ('choice', '18'), ('choice', 'test%2B1fsf'), ('choice', 'ZnNmc2Q='), ('json', '{"chained": "json2", "aaa": "bbb"}')])>"""
assert str(root["foo"]) \
== "JSONNode<key='foo' parent='<root>' depth=1 data=['b', 'a', {'b': 'c', 'd': 'e'}, 'this=is&a=form']>"
# 测试fork
new_root = root["aform"]["json"]["aaa"].fork_tree("doge")
_node = new_root["aform"]["json"]["aaa"]
assert _node.data == "doge"
assert _node.parent.data == {'aaa': 'doge', 'chained': 'json2'}
assert str(_node.parent.parent) \
== """FormNode<key='aform' parent='<root>' depth=1 data=QueryDict([('choice', '17'), ('choice', '18'), ('choice', 'test%2B1fsf'), ('choice', 'ZnNmc2Q='), ('json', '{"chained": "json2", "aaa": "doge"}')])>"""
# 测试fork, 同时改变key和value
new_root = root["aform"]["json"]["aaa"].fork_tree("doge2", key="kite")
_node = new_root["aform"]["json"]["kite"]
assert "aaa" not in new_root["aform"]["json"] # key重命名
assert _node.data == "doge2"
assert _node.parent.data == {'kite': 'doge2', 'chained': 'json2'}
# fork后原有的node不发生改变
assert str(root["aform"]["json"]["aaa"]) \
== "PlainNode<key='aaa' parent='json' depth=3 data='bbb'>"
assert root["aform"]["json"]["aaa"].index_in_parent == 1
assert root["aform"]["json"]["aaa"].abskey \
== ('<root>', 'aform#1', 'json#4', 'aaa#1')
assert str(root["foo"]["2"]) \
== "JSONNode<key='2' parent='foo' depth=2 data={'b': 'c', 'd': 'e'}>"
# 遍历叶子
all_leaves = [
"monkey",
"choice", "choice",
"chained", "aaa", "urlencode", "base64",
"0", "1", "this",
"b", "a", "d",
"bar", "ttt", "key-with.a-dot", "中文",
]
_all_leaves = copy.copy(all_leaves)
for leaf in root.iter_all_leaves():
# print(leaf, leaf.abskey)
_all_leaves.remove(leaf.key)
assert not _all_leaves
def filter_query_characters(value):
# Convert to a string, and filter anything that has no business being in a URL
return quote(str(value))
def urlquote(url, safe='/'):
"""
A legacy compatibility wrapper to Python's urllib.parse.quote() function.
(was used for unicode handling on Python 2)
"""
return quote(url, safe)
def getpath(environ):
"""Builds a path."""
return ''.join([quote(environ.get('SCRIPT_NAME', '')),
quote(environ.get('PATH_INFO', ''))])
def getpath(environ):
"""Builds a path."""
return ''.join([
quote(environ.get('SCRIPT_NAME', '')),
quote(environ.get('PATH_INFO', ''))
])
def _get_encoded_url(self):
"""Convert any UTF-8 char in :obj:`File.file_path` into a url encoded ASCII string."""
sres = urllib_parse.urlsplit(self.file_path)
return urllib_parse.urlunsplit(urllib_parse.SplitResult(
sres.scheme, sres.netloc, urllib_parse.quote(sres.path), sres.query, sres.fragment))
