def test_feed_url(self):
"""
Does the `feed_url` property exist and work correctly?
"""
retriever = Retriever()
base_url, query = urllib.splitquery(retriever.feed_url)
params = parse_qs(query)
self.assertTrue(hasattr(retriever, 'feed_url'))
# check default parameters
self.assertFalse(params.get('tagged'))
self.assertEqual('desc', params.get('order')[0])
self.assertEqual('creation', params.get('sort')[0])
self.assertEqual('stackoverflow', params.get('site')[0])
retriever = Retriever(tag='professionalism', site='workplace')
base_url, query = urllib.splitquery(retriever.feed_url)
params = parse_qs(query)
# check custom parameters
self.assertEqual('professionalism', params.get('tagged')[0])
self.assertEqual('desc', params.get('order')[0])
self.assertEqual('creation', params.get('sort')[0])
self.assertEqual('workplace', params.get('site')[0])
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def document_quote(document):
"""Quote given document."""
doc, query = parse.splitquery(document)
doc = url_quote_part(doc, '/=,')
if query:
return "%s?%s" % (doc, query)
return doc
def get_code(client_id, username, password):
MAX_RETRIES = 20 #!Added https://stackoverflow.com/questions/33895739/python-requests-module-error-cant-load-any-url-remote-end-closed-connection
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=MAX_RETRIES)
session.mount('https://', adapter)
session.mount('http://', adapter)
url = "https://api3.stromer-portal.ch/users/login/"
s = requests.session()
res = s.get(url)
s.cookies
qs = urlencode({
"client_id":
client_id,
"response_type":
"code",
"redirect_url":
"stromerauth://auth",
"scope":
"bikeposition bikestatus bikeconfiguration bikelock biketheft bikedata bikepin bikeblink userprofile",
})
data = {
"password": password,
"username": username,
"csrfmiddlewaretoken": s.cookies.get("csrftoken"),
"next": "/o/authorize/?" + qs,
}
res = s.post(url, data=data, headers=dict(Referer=url), allow_redirects=False)
res = s.send(res.next, allow_redirects=False)
_, qs = splitquery(res.headers["Location"])
code = parse_qs(qs)["code"][0]
return code
def processCredentials(self):
'''Entry point for processing the Login page.'''
login = self.request.get('login', '')
if not login:
return
user = self.get_user_from_login(login)
if user:
auditor = LoginAuditor(self.siteInfo, user)
password = self.get_password_from_user(user)
if self.password_matches(login, password):
self.store_password_cookie_for_user(user, password)
uri = self.get_redirect()
u = splitquery(uri)[0].replace('login_redirect', '')
persist = self.request.get('__ac_persistent', '')
auditor.info(LOGIN, persist, u)
self.state = (True, True, True)
self.request.response.redirect(uri)
else: # Password does not match
auditor.info(BADPASSWORD)
self.state = (False, True, True)
else: # There is no user
auditor = AnonLoginAuditor(self.context, self.siteInfo)
auditor.info(BADUSERID, login)
self.state = (False, False, False)
assert(self.state)
def get_code(client_id, username, password):
url = "https://api3.stromer-portal.ch/users/login/"
s = requests.session()
res = s.get(url)
s.cookies
qs = urlencode({
"client_id":
client_id,
"response_type":
"code",
"redirect_url":
"stromerauth://auth",
"scope":
"bikeposition bikestatus bikeconfiguration bikelock biketheft bikedata bikepin bikeblink userprofile",
})
data = {
"password": password,
"username": username,
"csrfmiddlewaretoken": s.cookies.get("csrftoken"),
"next": "/o/authorize/?" + qs,
}
res = s.post(url,
data=data,
headers=dict(Referer=url),
allow_redirects=False)
res = s.send(res.next, allow_redirects=False)
_, qs = splitquery(res.headers["Location"])
code = parse_qs(qs)["code"][0]
return code
def get_access_code(client, client_config, user, extra_scopes,
extra_claims):
client.force_login(user)
resp = client.get(
'/openid/authorize?' +
urlencode({
'redirect_uri': client_config.redirect_uris,
'client_id': 'test',
'scope': '%s %s' % ('openid', ' '.join(extra_scopes)),
'claims': ' '.join(extra_claims),
'response_type': 'code'
}))
assert resp.status_code == 302
redirect_server, redirect_query = splitquery(resp.url)
assert redirect_server == 'http://localhost:8000/complete/test/'
redirect_query = parse_qs(redirect_query)
assert redirect_query['state'] == ['1234']
assert 'code' in redirect_query
code = redirect_query['code'][0]
resp = client.get(
'/openid/token?' + urlencode({
'redirect_uri': client_config.redirect_uris,
'grant_type': 'authorization_code',
'code': code,
}),
HTTP_AUTHORIZATION='Basic ' +
base64.b64encode('test:b'.encode('utf-8')).decode('ascii'))
data = json.loads(resp.content.decode('utf-8'))
return data['access_token']
def __construct_developer_data(base_url, item, html_tag):
img_tag = html_tag.select_one("div > a > img.rounded.avatar-user")
name = html_tag.select_one("div h1.h3 > a")
repo_a_tag = html_tag.select_one("article h1.h4 > a")
repo_description = html_tag.select_one("article div.f6.mt-1")
repo_details = {}
if img_tag:
avatar = img_tag.get("src").strip()
item["avatar"] = splitquery(avatar)[0]
if img_tag.parent and img_tag.parent.name == "a":
href = img_tag.parent.get("href").strip().split("/")
if len(href) == 2:
username = href[-1].strip()
profile = f"{base_url}/{username}"
item["username"] = username
item["profile"] = profile
if name:
name = name.text.strip()
item["name"] = name
if repo_a_tag:
href = repo_a_tag.get("href").strip().split("/")
if len(href) == 3:
repo_details["name"] = href[-1].strip()
url = f'{base_url}{"/".join(href)}'
repo_details["url"] = url
if repo_description:
description = repo_description.text.strip()
repo_details["description"] = description
if repo_details:
item["popular_repository"] = repo_details
def processCredentials(self):
'''Entry point for processing the Login page.'''
login = self.request.get('login', '')
if not login:
return
user = self.get_user_from_login(login)
if user:
auditor = LoginAuditor(self.siteInfo, user)
password = self.get_password_from_user(user)
if self.password_matches(login, password):
self.store_password_cookie_for_user(user, password)
uri = self.get_redirect()
u = splitquery(uri)[0].replace('login_redirect', '')
persist = self.request.get('__ac_persistent', '')
auditor.info(LOGIN, persist, u)
self.state = (True, True, True)
self.request.response.redirect(uri)
else: # Password does not match
auditor.info(BADPASSWORD)
self.state = (False, True, True)
else: # There is no user
auditor = AnonLoginAuditor(self.context, self.siteInfo)
auditor.info(BADUSERID, login)
self.state = (False, False, False)
assert (self.state)
def match(self, path):
spath, qs = urlparse.splitquery(path)
for cre, klass, args in self.handlers:
m = cre.match(spath)
if m is not None:
return functools.partial(klass, **args)
return None
def parse_path(root: str, path: str):
path, query = parse.splitquery(path)
if path == "/":
path = "/index.html"
query = parse.parse_qs(query, keep_blank_values=True)
filename = path[path.find('/') + 1:]
return filename, root + path, get_extention(path), query
def make_query(url, new_query_dict):
path, query_str = splitquery(url)
old_query_dict = extract_query(query_str) if query_str else {}
for i, j in new_query_dict.items():
old_query_dict[i] = j
query_str = urlencode(old_query_dict)
return path + '?' + query_str
def get_this_url_with_params(self, request):
server, query = splitquery(request.build_absolute_uri())
params = {
'authp':
self.request_parameters.pack(
key=OpenIDClient.self_instance().get_key('AES'))
}
return server + '?' + urlencode(params)
def check_response_contains_code(resp):
assert resp.status_code == 302
redirect_server, redirect_query = splitquery(resp.url)
assert redirect_server == 'http://localhost:8000/complete/test/'
redirect_query = parse_qs(redirect_query)
assert redirect_query['state'] == ['1234']
assert 'code' in redirect_query
def extract_image(self, tag):
"""Extract the url from an static or dynamic image"""
return self.to_absolute_link(
parse.splitquery(
tag.attrs.get('src', tag.attrs.get('data-src'))
)[0]
)
def _uris_to_tuples(uris):
tup = []
for uri in uris:
base, query = splitquery(uri)
if query:
tup.append((base, query))
else:
tup.append((base, ""))
return tup
def _uris_to_tuples(uris):
tup = []
for uri in uris:
base, query = splitquery(uri)
if query:
tup.append((base, query))
else:
tup.append((base, ""))
return tup
def _verify_redirect_uri(self, areq):
"""
MUST NOT contain a fragment
MAY contain query component
:return: An error response if the redirect URI is faulty otherwise
None
"""
try:
_redirect_uri = urlparse.unquote(areq["redirect_uri"])
part = urlparse.urlparse(_redirect_uri)
if part.fragment:
raise URIError("Contains fragment")
(_base, _query) = splitquery(_redirect_uri)
if _query:
_query = urlparse.parse_qs(_query)
match = False
for regbase, rquery in self.cdb[str(areq["client_id"])][
"redirect_uris"]:
if _base == regbase or _redirect_uri.startswith(regbase):
# every registered query component must exist in the
# redirect_uri
if rquery:
for key, vals in rquery.items():
assert key in _query
for val in vals:
assert val in _query[key]
# and vice versa, every query component in the redirect_uri
# must be registered
if _query:
if rquery is None:
raise ValueError
for key, vals in _query.items():
assert key in rquery
for val in vals:
assert val in rquery[key]
match = True
break
if not match:
raise RedirectURIError("Doesn't match any registered uris")
# ignore query components that are not registered
return None
except Exception as err:
logger.error("Faulty redirect_uri: %s" % areq["redirect_uri"])
try:
_cinfo = self.cdb[str(areq["client_id"])]
except KeyError:
logger.info("Unknown client: %s" % areq["client_id"])
raise UnknownClient(areq["client_id"])
else:
logger.info("Registered redirect_uris: %s" % _cinfo)
raise RedirectURIError(
"Faulty redirect_uri: %s" % areq["redirect_uri"])
def get_op_identifier_by_cb_uri(url: str):
uri = splitquery(url)[0]
for k, v in current_app.rph.issuer2rp.items():
_cntx = v.get_service_context()
for endpoint in ("redirect_uris",
"post_logout_redirect_uris",
"frontchannel_logout_uri",
"backchannel_logout_uri"):
if uri in _cntx.get(endpoint, []):
return k
def check_is_redirect_to_consent_page(client_config, resp):
assert resp.status_code == 302
# check that this is a url to the login server and there is a ?next=...
consent_server, consent_query = splitquery(resp.url)
assert consent_server == '/openid/consent/%s/' % client_config.id
consent_query = parse_qs(consent_query)
assert 'next' in consent_query
# parse the ?next=... and check that it goes back to the authorize endpoint with ?authp
next_server, next_query = splitquery(consent_query['next'][0])
assert next_server == 'http://testserver/openid/authorize'
next_query = parse_qs(next_query)
assert 'authp' in next_query
# check that ?authp param contains correctly encrypted value
AuthenticationParameters.unpack(
next_query['authp'][0],
key=OpenIDClient.self_instance().get_key('AES'))
def _verify_redirect_uri(self, areq):
"""
MUST NOT contain a fragment
MAY contain query component
:return: An error response if the redirect URI is faulty otherwise
None
"""
try:
_redirect_uri = urlparse.unquote(areq["redirect_uri"])
part = urlparse.urlparse(_redirect_uri)
if part.fragment:
raise URIError("Contains fragment")
(_base, _query) = splitquery(_redirect_uri)
if _query:
_query = urlparse.parse_qs(_query)
match = False
for regbase, rquery in self.cdb[str(areq["client_id"])]["redirect_uris"]:
if _base == regbase or _redirect_uri.startswith(regbase):
# every registered query component must exist in the
# redirect_uri
if rquery:
for key, vals in rquery.items():
assert key in _query
for val in vals:
assert val in _query[key]
# and vice versa, every query component in the redirect_uri
# must be registered
if _query:
if rquery is None:
raise ValueError
for key, vals in _query.items():
assert key in rquery
for val in vals:
assert val in rquery[key]
match = True
break
if not match:
raise RedirectURIError("Doesn't match any registered uris")
# ignore query components that are not registered
return None
except Exception as err:
logger.error("Faulty redirect_uri: %s" % areq["redirect_uri"])
try:
_cinfo = self.cdb[str(areq["client_id"])]
except KeyError:
logger.info("Unknown client: %s" % areq["client_id"])
raise UnknownClient(areq["client_id"])
else:
logger.info("Registered redirect_uris: %s" % _cinfo)
raise RedirectURIError(
"Faulty redirect_uri: %s" % areq["redirect_uri"])
def url_changed(self, change) -> None:
"Callback to be called when the input URL is changed."
value = change['new']
try:
q = splitquery(value)[1]
except AttributeError:
return
params = parse_qs(q)
self.req_pane.get_child_named('Parameters').children = \
[Text(description=key, value=params[key][0]) for key in params]
def _prep_params(self, url, mapping):
mapping.setdefault('params', {})
if self.per_page is not None:
mapping['params'].setdefault('per_page', self.per_page)
# Params in URL will take precedence
if url:
qs = splitquery(url)[-1]
if qs:
for k, _ in parse_qsl(qs):
if k in mapping['params']:
del mapping['params'][k]
def vcr_custom_request_filter(request):
hidden_values = [(os.environ.get("CLIENT_ID")),
(os.environ.get("CLIENT_SECRET")),
(os.environ.get("USER_ID"))]
new_url, _ = parse.splitquery(request.uri)
for value in hidden_values:
new_url = new_url.replace(value, "HIDDEN")
request.uri = new_url
return request
def verify_redirect_uris(registration_request):
verified_redirect_uris = []
try:
client_type = registration_request["application_type"]
except KeyError: # default
client_type = "web"
if client_type == "web":
try:
if registration_request["response_types"] == ["code"]:
must_https = False
else: # one has to be implicit or hybrid
must_https = True
except KeyError:
must_https = True
else:
must_https = False
for uri in registration_request["redirect_uris"]:
_custom = False
p = urlparse(uri)
if client_type == "native":
if p.scheme not in ["http", "https"]: # Custom scheme
_custom = True
elif p.scheme == "http" and p.hostname in ["localhost", "127.0.0.1"]:
pass
else:
logger.error(
"InvalidRedirectURI: scheme:%s, hostname:%s",
p.scheme,
p.hostname,
)
raise InvalidRedirectURIError(
"Redirect_uri must use custom scheme or http and " "localhost"
)
elif must_https and p.scheme != "https":
raise InvalidRedirectURIError("None https redirect_uri not allowed")
elif p.scheme not in ["http", "https"]: # Custom scheme
raise InvalidRedirectURIError(
"Custom redirect_uri not allowed for web client"
)
elif p.fragment:
raise InvalidRedirectURIError("redirect_uri contains fragment")
if _custom is True: # Can not verify a custom scheme
verified_redirect_uris.append((uri, {}))
else:
base, query = splitquery(uri)
if query:
verified_redirect_uris.append((base, parse_qs(query)))
else:
verified_redirect_uris.append((base, {}))
return verified_redirect_uris
def do_GET(self):
print("http GET....")
config_data = get_config()
url, query = url_parser.splitquery(self.path)
if query.split('&')[0].split('=')[0] == 'wrf' and query.split('&')[1].split('=')[0] == 'hec':
wrf_id = query.split('&')[0].split('=')[1]
hec_id = query.split('&')[1].split('=')[1]
print("wrf_id:", wrf_id)
print("hec_id:", hec_id)
try:
print("get_wrf_files...")
client = storage.Client.from_service_account_json(config_data["KEY_FILE_PATH"])
bucket = client.get_bucket(config_data["BUCKET_NAME"])
prefix = config_data["INITIAL_PATH_PREFIX"] + wrf_id + '_'
blobs = bucket.list_blobs(prefix=prefix)
for blob in blobs:
if fnmatch.fnmatch(blob.name, "*" + config_data["WRF_RAINCELL_FILE_ZIP"]):
directory = config_data["WINDOWS_PATH"] + wrf_id.split("_")[1]
if not os.path.exists(directory):
os.makedirs(directory)
download_location = directory + '/' + config_data["WRF_RAINCELL_FILE_ZIP"]
blob.download_to_filename(download_location)
zip_ref = zipfile.ZipFile(download_location, 'r')
zip_ref.extractall(directory)
zip_ref.close()
os.remove(download_location)
src_file = directory + '/' + config_data["WRF_RAIN_CELL_FILE"]
des_file = directory + '/' + config_data["RAIN_CELL_FILE"]
os.rename(src_file, des_file)
else:
print("File prefix didn't match.")
hec_prefix = config_data["INITIAL_PATH_PREFIX"] + hec_id + '_'
hec_blobs = bucket.list_blobs(prefix=hec_prefix)
for blob in hec_blobs:
if fnmatch.fnmatch(blob.name, "*" + "INFLOW.DAT"):
directory = config_data["WINDOWS_PATH"] + hec_id.split("_")[1]
if not os.path.exists(directory):
os.makedirs(directory)
download_location = directory + '/"INFLOW.DAT'
blob.download_to_filename(download_location)
elif fnmatch.fnmatch(blob.name, "*" + "OUTFLOW.DAT"):
directory = config_data["WINDOWS_PATH"] + hec_id.split("_")[1]
if not os.path.exists(directory):
os.makedirs(directory)
download_location = directory + '/OUTFLOW.DAT'
blob.download_to_filename(download_location)
else:
print("File prefix didn't match.")
except Exception as e:
print('Rain cell/Mean-Ref/Rain fall file download failed|Exception:', e.args)
else:
print("Need wrf id and hec id to proceed...")
def urldecode(url):
"""
>>> urldecode('http://google.com/search?q=bar&x=y')
('http://google.com/search', {'q': 'bar', 'x': 'y'})
>>> urldecode('http://google.com/')
('http://google.com/', {})
"""
base, query = splitquery(url)
query = query or ""
items = [item.split('=', 1) for item in query.split('&') if '=' in item]
d = {unquote(k): unquote_plus(v) for (k, v) in items}
return base, d
def latlng_parse(link):
_, query_string = parse.splitquery(link)
query = parse.parse_qs(query_string)
try:
latlng = query["center"]
latitude, longitude = [float(i) for i in latlng[0].split(",")]
except TypeError:
regex_search = re.search(r"png%7C(.*?)%2C(.*?)&", link).groups()
latitude, longitude = [float(i) for i in regex_search]
except:
latitude, longitude = None, None
return latitude, longitude
def pack_redirect_uri(redirect_uris):
ruri = []
for uri in redirect_uris:
if urlparse(uri).fragment:
print("Faulty redirect uri, contains fragment", file=sys.stderr)
base, query = splitquery(uri)
if query:
ruri.append([base, parse_qs(query)])
else:
ruri.append([base, query])
return ruri
def pack_redirect_uri(redirect_uris):
ruri = []
for uri in redirect_uris:
if urlparse(uri).fragment:
print("Faulty redirect uri, contains fragment", file=sys.stderr)
base, query = splitquery(uri)
if query:
ruri.append([base, parse_qs(query)])
else:
ruri.append([base, query])
return ruri
def shop_sign_up(session: Session, link):
logging.info("shop_sign_up : {}".format(link))
path_part, query_part = splitquery(link)
if path_part == "https://lzkj-isv.isvjcloud.com/sign/signActivity":
# 例：https://lzkj-isv.isvjcloud.com/sign/signActivity?activityId=ee6bbff8a06f4a8dbaa67cd9843bc14f&venderId=681974
try:
api = "https://lzkj-isv.isvjcloud.com/sign/wx/signUp"
session.headers.update({
'Host': 'lzkj-isv.isvjcloud.com',
'Origin': 'https://lzkj-isv.isvjcloud.com',
'Accept': 'application/json',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'X-Requested-With': 'XMLHttpRequest',
'Referer': link
})
activity_id = parse_qs(query_part).get('activityId')[0]
data = "actId=" + activity_id + "&pin=" + session.cookies.get(
'pin')
ret_json = json.loads(session.post(api, data=data).text)
logging.info("shop_sign_up: {}".format(str(ret_json)))
except Exception:
logging.exception(sys.exc_info())
elif path_part == "https://gzsl-isv.isvjcloud.com/wuxian/mobileForApp/dist/views/pages/myIntegral.html":
# 例：https: //gzsl-isv.isvjcloud.com/wuxian/mobileForApp/dist/views/pages/myIntegral.html?v=1000133822
try:
api = "https://gzsl-isv.isvjcloud.com/wuxian/user/sign/457"
session.headers.update({
'Host': 'gzsl-isv.isvjcloud.com',
'Origin': 'https://gzsl-isv.isvjcloud.com',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Content-Type': 'application/json;charset=UTF-8',
'X-Requested-With': 'XMLHttpRequest',
'Referer': link
})
data = {"activityId": 457}
ret = session.post(api, data=data).text
logging.info("shop_sign_up: {}".format(str(ret)))
except Exception as e:
logging.exception(sys.exc_info())
else:
logging.error("shop_sign_up: not supported...")
def request(self,
localpart='/',
method='GET',
data=None,
host="0.0.0.0:8080",
headers=None,
https=False,
env=None):
path, maybe_query = splitquery(localpart)
query = maybe_query or ""
env = env or {}
env = dict(env,
HTTP_HOST=host,
REQUEST_METHOD=method,
PATH_INFO=path,
QUERY_STRING=query,
HTTPS=str(https))
headers = headers or {}
for k, v in headers.items():
env['HTTP_' + k.upper().replace('-', '_')] = v
if 'HTTP_CONTENT_LENGTH' in env:
env['CONTENT_LENGTH'] = env.pop('HTTP_CONTENT_LENGTH')
if 'HTTP_CONTENT_TYPE' in env:
env['CONTENT_TYPE'] = env.pop('HTTP_CONTENT_TYPE')
if method not in ["HEAD", "GET", "DELETE"]:
data = data or ''
if isinstance(data, dict):
q = urlencode(data)
else:
q = data
env['wsgi.input'] = BytesIO(q.encode('utf-8'))
if 'CONTENT_LENGTH' not in env:
# if not env.get('CONTENT_TYPE', '').lower().startswith('multipart/') and 'CONTENT_LENGTH' not in env:
env['CONTENT_LENGTH'] = len(q)
response = Storage()
def start_response(status, headers):
response.status = status
response.status_code = int(status.split()[0])
response.headers = dict(headers)
response.header_items = headers
data = self.wsgifunc()(env, start_response)
response.data = b"".join(data)
return response
def parse(self, response):
rel_urls = response.xpath(
'//div[@class="information"]/a/@href').extract()
for rel_url in rel_urls:
yield Request(urljoin(self.base_url, rel_url),
callback=self.detail_parse)
next_page = response.xpath(
'//a[@class="nextPage" and text()="▶"]').extract_first()
if next_page:
url, q = splitquery(response.url)
d = dict(parse_qsl(q))
d['cpage'] = int(d.get('cpage', 0)) + 1
q = urlencode(d)
yield Request(f'{url}?{q}', callback=self.parse)
def jr_sign_mrbyb(self):
"""京东钢镚-每日蹦一蹦"""
job_name = "京东钢镚-每日蹦一蹦"
index_url = "https://red-e.jd.com/resources/pineapple/index.html?merchantCode=4B2CE697A1AEE055"
try:
merchantCode = splitquery(index_url)[1]
merchantCode = parse_qs(merchantCode).get('merchantCode')[0]
self.session.headers.update({
'Host': 'coin.jd.com',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Referer': index_url
})
# https://coin.jd.com/mlgnActivity/coins/checkLottery.html?merchantCode=4B2CE697A1AEE055&callback=jQuery33109727842609730113_1553737619367&_=1553737619368
test_api = "https://coin.jd.com/mlgnActivity/coins/checkLottery.html?merchantCode=" + merchantCode + "&_=" + str(
int(time.time() * 1000)) + "&callback=jQuery"
r = self.session.get(test_api).text
# jQuery({"data":3,"success":true})
if r is None or (not "jQuery" in r):
self.logger.error("{} 获取数据失败...".format(job_name))
return
as_json = json.loads(r[7:-1])
self.logger.info("{} {}".format(job_name, str(as_json)))
if as_json['data']:
sign_data = as_json['data']
else:
self.logger.error('{} 获取数据失败'.format(job_name))
return
if sign_data <= 0:
self.logger.info("{} 抽奖次数已用完...")
return
# 割吊绳 api 未完成；
for i in range(0, sign_data):
award_api = "https://coin.jd.com/mlgnActivity/coins/awardNeedLogin.html?merchantCode=" + merchantCode + "&_=" + str(
int(time.time() * 1000)) + "&callback=jQuery"
self.session.get(award_api)
self.logger.info(job_name + ' 已完成')
except Exception:
self.logger.error("{} 签到发生异常".format(job_name))
self.logger.exception(sys.exc_info())
def gopher_open(self, req):
host = req.get_host()
if not host:
raise GopherError('no host given')
host = unquote(host)
selector = req.get_selector()
type, selector = splitgophertype(selector)
selector, query = splitquery(selector)
selector = unquote(selector)
if query:
query = unquote(query)
fp = gopherlib.send_query(selector, query, host)
else:
fp = gopherlib.send_selector(selector, host)
return addinfourl(fp, noheaders(), req.get_full_url())
def get_authorization_code(client, client_config, user):
client.force_login(user)
resp = client.get('/openid/authorize?' + urlencode({
'redirect_uri': client_config.redirect_uris,
'client_id': 'test',
'scope': 'openid',
'response_type': 'code'
}))
assert resp.status_code == 302
redirect_server, redirect_query = splitquery(resp.url)
assert redirect_server == 'http://localhost:8000/complete/test/'
redirect_query = parse_qs(redirect_query)
assert redirect_query['state'] == ['1234']
assert 'code' in redirect_query
code = redirect_query['code'][0]
return code
def get_next_page(self):
max_page = self.get_max_page()
base_url, query = parse.splitquery(self.current_page_url)
if not query:
query='page=1'
_, count = parse.parse_qsl(query)[0]
count = int(count) + 1
if count > max_page:
self._has_next_page = False
next_query = parse.urlencode({'page': count})
url = base_url+ '?' + next_query
return url, count
def get_redirect(self):
retval = ''
cameFrom = self.request.get('came_from', '')
cacheBuster = seedGenerator()
url, query = splitquery(cameFrom)
if not url:
# Oddly, setting the default came_from to / does not work.
url = '/'
# Put a cache-buster at the end of the query
if query:
query = query + '&nc=%s' % cacheBuster
else:
query = 'nc=%s' % cacheBuster
retval = '?'.join((url, query))
assert retval
return retval
def do_handle(self, message, payload, user):
need_auth = False
spath, qs = urlparse.splitquery(message.path)
if qs is not None:
qs = urlparse.parse_qs(qs, keep_blank_values=True)
if 'need_auth' in qs:
need_auth = True
response = self.start_response(200, message.version)
response.add_header('Content-Type', 'text/html')
response.send_headers()
template = self.kw['tmpl_env'].get_template('intro.jinja2')
response.write(template.render(version=__version__,
user=user,
need_auth=need_auth).encode('utf8'))
return response
def get_redirect(self):
retval = ''
cameFrom = self.request.get('came_from', '')
cacheBuster = seedGenerator()
url, query = splitquery(cameFrom)
if not url:
# Oddly, setting the default came_from to / does not work.
url = '/'
# Put a cache-buster at the end of the query
if query:
query = query + '&nc=%s' % cacheBuster
else:
query = 'nc=%s' % cacheBuster
retval = '?'.join((url, query))
assert retval
return retval
def __parse_obj_table(self, div):
from urllib import parse
from pymal.account_objects.my_manga import MyManga as obj
links_div = div.findAll(name='td', recorsive=False)[1]
link = links_div.find(name='a', attrs={'class': 'animetitle'})
link_id = int(link['href'].split('/')[2])
if self.__account.is_auth:
my_link = links_div.find(name='a', attrs={'class': 'List_LightBox'})
_, query = parse.splitquery(my_link['href'])
my_link_id = int(parse.parse_qs(query)['id'][0])
else:
my_link_id = 0
return obj(link_id, my_link_id, self.__account)
def __parse_obj_table(self, div):
from urllib import parse
from pymal.account_objects.my_anime import MyAnime as obj
links_div = div.findAll(name="td", recorsive=False)[1]
link = links_div.find(name="a", attrs={"class": "animetitle"})
link_id = int(link["href"].split("/")[2])
if self.__account.is_auth:
my_link = links_div.find(name="a", attrs={"class": "List_LightBox"})
_, query = parse.splitquery(my_link["href"])
my_link_id = int(parse.parse_qs(query)["id"][0])
else:
my_link_id = 0
return obj(link_id, my_link_id, self.__account)
def __parse_obj_table(self, div):
from urllib import parse
from pymal.account_objects.my_anime import MyAnime as obj
links_div = div.findAll(name='td', recorsive=False)[1]
link = links_div.find(name='a', attrs={'class': 'animetitle'})
link_id = int(link['href'].split('/')[2])
if self.__account.is_auth:
my_link = links_div.find(name='a', attrs={'class': 'List_LightBox'})
_, query = parse.splitquery(my_link['href'])
my_link_id = int(parse.parse_qs(query)['id'][0])
else:
my_link_id = 0
return obj(link_id, my_link_id, self.__account)
def _url(self, obj_or_id):
if isinstance(obj_or_id, self.model):
id_ = self._id(obj_or_id)
url = getattr(obj_or_id, '_url', self.url)
else:
id_ = obj_or_id
url = getattr(self.model, '_url', self.url)
if callable(url):
return url(id_)
url, query = splitquery(url)
url = '{}/{}'.format(url, id_)
if query is not None:
url = '{}?{}'.format(url, query)
return url
def load_template(self, template_name):
"""
template_name == dotted.path.to.template (without .ext)
@@ 1.1: Let's dump the dotted notation. It was a bad idea, the engine that
inspired it (Kid) is probably dead, and it was always meaningless
to everyone else anyway. Also, let's support an extension as a synonym
for "format", i.e. /blog/feed.rss would be the same as /blog/feed?format=rss
"""
template, args = splitquery(template_name)
if args:
args = dict([a.split('=')
for a in args.split('&')])
else:
args = {}
parts = template.split('.')
template_filename = parts.pop()
template_path = ''
if parts:
template_path = os.path.join(*parts)
return template_path, template_filename, args
def do_handle(self, message, payload, user):
if '?' in message.path:
path, qs = urlparse.splitquery(message.path)
else:
path = message.path
qs = yield from aiohttp_read_all_into_bytearray(payload)
platform = path[len('/oauth2/code_cb/'):]
code_data = urlparse.parse_qs(qs, keep_blank_values=True)
logger.debug('code_data = %r', code_data)
try:
user = yield from oauth2_code_cb(
platform, code_data, self.kw['config'], self.kw['redis'])
exc = None
except OAuth2StepError:
return self.redirect_to('/', message)
except OAuth2BadData:
return self.redirect_to('/', message)
except Exception as e:
user = None
exc = e
logger.debug('user = %r', user)
if user is None:
logger.debug(
'Cannot retrieve user info ' +
'for platform = %r, code_data = %r, exc = %r',
platform, code_data, exc)
response = self.start_response(500, message.version)
response.send_headers()
return response
cookie_content = [
('shinpachi_user_key', '{}:{}'.format(user.platform, user.key))
]
return self.redirect_to('/', message, set_cookies=cookie_content)
def _verify_redirect_uris(self, registration_request):
verified_redirect_uris = []
must_https = False # don't verify https!
for uri in registration_request["redirect_uris"]:
p = urlparse(uri)
if registration_request["application_type"] == "native" and p.scheme == "http":
if p.hostname != "localhost":
raise InvalidRedirectURIError(
"Http redirect_uri must use localhost")
elif must_https and p.scheme != "https":
raise InvalidRedirectURIError(
"None https redirect_uri not allowed")
elif p.fragment:
raise InvalidRedirectURIError(
"redirect_uri contains fragment")
base, query = splitquery(uri)
if query:
verified_redirect_uris.append((base, parse_qs(query)))
else:
verified_redirect_uris.append((base, query))
return verified_redirect_uris
def issue_report_link(args, root_url, f, test_browser=None):
"""Generate a bug reporting link for the current issue."""
# always select the first failed module.
# It might not be the fatal one but better be safe and assume the first
# failed module introduces a problem in the whole job
test_details_page = test_browser.get_soup(f['href'])
current_build_overview = splitquery(test_details_page.find(id='current-build-overview').a['href'])
overview_params = parse_qs(current_build_overview[-1])
group = overview_params['groupid'][0]
build = overview_params['build'][0]
scenario_div = test_details_page.find(class_='previous').div.div
scenario = re.findall('Results for (.*) \(', scenario_div.text)[0]
latest_link = absolute_url(root_url, scenario_div.a)
module, url, details = get_failed_module_details_for_report(f, root_url)
previous_results = test_details_page.find(id='previous_results', class_='overview').find_all('tr')[1:]
previous_results_list = [(i.td['id'], {'status': status(i),
'details': get_test_details(i),
'build': i.find(class_='build').text}) for i in previous_results]
good = re.compile('(?<=_)(passed|softfailed)')
def build_link(v):
return '[%s](%s)' % (v['build'], absolute_url(root_url, v['details']))
first_known_bad = build + ' (current job)'
last_good = '(unknown)'
for k, v in previous_results_list:
if good.search(v['status']):
last_good = build_link(v)
break
first_known_bad = build_link(v)
description = """### Observation
openQA test in scenario %s fails in
%s%s
## Reproducible
Fails since (at least) Build %s
## Expected result
Last good: %s (or more recent)
## Further details
Always latest result in this scenario: [latest](%s)
""" % (scenario, urljoin(root_url, str(url)), details, first_known_bad, last_good, latest_link)
config_section = 'product_issues:%s:product_mapping' % root_url.rstrip('/')
# the test module name itself is often not specific enough, that is why we step upwards from the current module until we find the module folder and
# concatenate the complete module name in format <folder>-<module> and search for that in the config for potential mappings
first_step_url = urljoin(str(url), '1/src')
start_of_current_module = test_details_page.find('a', {'href': first_step_url})
try:
module_folder = start_of_current_module.parent.parent.parent.parent.find(class_='glyphicon-folder-open').parent.text.strip()
except AttributeError: # pragma: no cover
module_folder = ''
log.warn("Could not find module folder on test details page searching for parents of %s" % first_step_url)
complete_module = module_folder + '-' + module
component_config_section = 'product_issues:%s:component_mapping' % root_url.rstrip('/')
try:
components_config_dict = dict(config.items(component_config_section))
component = [v for k, v in iteritems(components_config_dict) if re.match(k, complete_module)][0]
except (NoSectionError, IndexError) as e: # pragma: no cover
log.info("No matching component could be found for the module_folder '%s' and module name '%s' in the config section '%s'" % (module_folder, module, e))
component = ''
try:
product = config.get(config_section, group)
except NoOptionError as e: # pragma: no cover
log.info("%s. Reporting link for product will not work." % e)
product = ''
product_entries = OrderedDict([
('product', product),
('component', component),
('short_desc', '[Build %s] openQA test fails%s' % (build, ' in %s' % module if module else '')),
('bug_file_loc', urljoin(root_url, str(url))),
('comment', description)
])
product_bug = urljoin(config.get('product_issues', 'report_url'), 'enter_bug.cgi') + '?' + urlencode(product_entries)
test_entries = OrderedDict([
('issue[subject]', '[Build %s] test %sfails' % (build, module + ' ' if module else '')),
('issue[description]', description)
])
test_issue = config.get('test_issues', 'report_url') + '?' + urlencode(test_entries)
return ': report [product bug](%s) / [openQA issue](%s)' % (product_bug, test_issue)
def _verify_redirect_uri(self, areq):
"""
MUST NOT contain a fragment
MAY contain query component
:return: An error response if the redirect URI is faulty otherwise
None
"""
try:
_redirect_uri = unquote(areq["redirect_uri"])
part = urlparse(_redirect_uri)
if part.fragment:
raise URIError("Contains fragment")
(_base, _query) = splitquery(_redirect_uri)
if _query:
_query = parse_qs(_query)
match = False
for regbase, rquery in self.cdb[str(areq["client_id"])]["redirect_uris"]:
# The URI MUST exactly match one of the Redirection URI
if _base != regbase:
continue
if not rquery and not _query:
match = True
break
if not rquery or not _query:
continue
# every registered query component must exist in the
# redirect_uri
is_match_query = True
for key, vals in _query.items():
if key not in rquery:
is_match_query = False
break
for val in vals:
if val not in rquery[key]:
is_match_query = False
break
if not is_match_query:
break
if not is_match_query:
continue
match = True
break
if not match:
raise RedirectURIError("Doesn't match any registered uris")
# ignore query components that are not registered
return None
except Exception:
logger.error("Faulty redirect_uri: %s" % areq["redirect_uri"])
try:
_cinfo = self.cdb[str(areq["client_id"])]
except KeyError:
try:
cid = areq["client_id"]
except KeyError:
logger.error('No client id found')
raise UnknownClient('No client_id provided')
else:
logger.info("Unknown client: %s" % cid)
raise UnknownClient(areq["client_id"])
else:
logger.info("Registered redirect_uris: %s" % sanitize(_cinfo))
raise RedirectURIError(
"Faulty redirect_uri: %s" % areq["redirect_uri"])
def found_terminator (self):
if self.current_request:
self.current_request.found_terminator()
else:
header = self.in_buffer
self.in_buffer = ''
lines = string.split (header, '\r\n')
# --------------------------------------------------
# crack the request header
# --------------------------------------------------
while lines and not lines[0]:
# as per the suggestion of http-1.1 section 4.1, (and
# Eric Parker <*****@*****.**>), ignore a leading
# blank lines (buggy browsers tack it onto the end of
# POST requests)
lines = lines[1:]
if not lines:
self.close_when_done()
return
request = lines[0]
command, uri, version = crack_request (request)
header = join_headers (lines[1:])
# unquote path if necessary (thanks to Skip Montanaro for pointing
# out that we must unquote in piecemeal fashion).
rpath, rquery = splitquery(uri)
if '%' in rpath:
if rquery:
uri = unquote (rpath) + '?' + rquery
else:
uri = unquote (rpath)
r = http_request (self, request, command, uri, version, header)
self.request_counter.increment()
self.server.total_requests.increment()
if command is None:
self.server.log_info ('Bad HTTP request: %s' % repr(request), 'error')
r.error (400)
return
# --------------------------------------------------
# handler selection and dispatch
# --------------------------------------------------
for h in self.server.handlers:
if h.match (r):
try:
self.current_request = r
# This isn't used anywhere.
# r.handler = h # CYCLE
h.handle_request (r)
except:
self.server.exceptions.increment()
(file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
self.server.log_info(
'Server Error: %s, %s: file: %s line: %s' % (t,v,file,line),
'error')
try:
r.error (500)
except:
pass
return
# no handlers, so complain
r.error (404)
def request(self, localpart='/', method='GET', data=None,
host="0.0.0.0:8080", headers=None, https=False, **kw):
"""Makes request to this application for the specified path and method.
Response will be a storage object with data, status and headers.
>>> urls = ("/hello", "hello")
>>> app = application(urls, globals())
>>> class hello:
... def GET(self):
... web.header('Content-Type', 'text/plain')
... return "hello"
...
>>> response = app.request("/hello")
>>> response.data
b'hello'
>>> response.status
'200 OK'
>>> response.headers['Content-Type']
'text/plain'
To use https, use https=True.
>>> urls = ("/redirect", "redirect")
>>> app = application(urls, globals())
>>> class redirect:
... def GET(self): raise web.seeother("/foo")
...
>>> response = app.request("/redirect")
>>> response.headers['Location']
'http://0.0.0.0:8080/foo'
>>> response = app.request("/redirect", https=True)
>>> response.headers['Location']
'https://0.0.0.0:8080/foo'
The headers argument specifies HTTP headers as a mapping object
such as a dict.
>>> urls = ('/ua', 'uaprinter')
>>> class uaprinter:
... def GET(self):
... return 'your user-agent is ' + web.ctx.env['HTTP_USER_AGENT']
...
>>> app = application(urls, globals())
>>> app.request('/ua', headers = {
... 'User-Agent': 'a small jumping bean/1.0 (compatible)'
... }).data
b'your user-agent is a small jumping bean/1.0 (compatible)'
"""
path, maybe_query = splitquery(localpart)
query = maybe_query or ""
if 'env' in kw:
env = kw['env']
else:
env = {}
env = dict(env, HTTP_HOST=host, REQUEST_METHOD=method, PATH_INFO=path, QUERY_STRING=query, HTTPS=str(https))
headers = headers or {}
for k, v in headers.items():
env['HTTP_' + k.upper().replace('-', '_')] = v
if 'HTTP_CONTENT_LENGTH' in env:
env['CONTENT_LENGTH'] = env.pop('HTTP_CONTENT_LENGTH')
if 'HTTP_CONTENT_TYPE' in env:
env['CONTENT_TYPE'] = env.pop('HTTP_CONTENT_TYPE')
if method not in ["HEAD", "GET"]:
data = data or ''
if isinstance(data, dict):
q = urlencode(data)
else:
q = data
env['wsgi.input'] = BytesIO(q.encode('utf-8'))
if 'CONTENT_LENGTH' not in env:
#if not env.get('CONTENT_TYPE', '').lower().startswith('multipart/') and 'CONTENT_LENGTH' not in env:
env['CONTENT_LENGTH'] = len(q)
response = web.storage()
def start_response(status, headers):
response.status = status
response.headers = dict(headers)
response.header_items = headers
data = self.wsgifunc()(env, start_response)
response.data = b"".join(data)
return response
def test_command_cgi_url():
req = command._prep_get(command.Operation.list_files, DIR="/DCIM/WOMP",
url="http://192.168.0.1")
print(req.url)
url, _ = parse.splitquery(req.url)
assert url == "http://192.168.0.1/command.cgi"
def do_handle(self, message, payload, user):
url_path_prefix = self.kw['url_path_prefix']
root_path = self.kw['root_path']
path, qs = urlparse.splitquery(message.path[len(url_path_prefix):])
path = root_path + path
if not self.kw['resource_provider'].has_resource(path):
raise aiohttp.HttpErrorException(404)
try:
fstream = self.kw['resource_provider'].get_resource_stream(
self.kw['resource_mgr'], path)
except IsADirectoryError:
raise aiohttp.HttpErrorException(404)
try:
file_size = os.path.getsize(
self.kw['resource_provider'].get_resource_filename(
self.kw['resource_mgr'], path))
content_range = self.get_req_range(
message.headers.getall('RANGE', ()), file_size)
content_length = content_range[1] - content_range[0] + 1
if content_range[0] == 0 and content_length >= file_size:
content_range = None
except FileNotFoundError:
file_size = None
content_length = None
content_range = None
if content_range is None:
response = self.start_response(200, message.version)
else:
response = self.start_response(206, message.version)
response.add_header('Content-Range',
'bytes {}-{}/{}'
.format(content_range[0],
content_range[1],
file_size))
content_type, encoding = mimetypes.guess_type(path)
if content_type is None:
content_type = 'application/octet-stream'
response.add_header('Content-Type', content_type)
if content_length is not None:
response.add_header('Content-Length', str(content_length))
response.send_headers()
async_stream = AsyncFileWrapper(fileobj=fstream)
if content_range is not None and content_range[0] > 0:
async_stream.seek(content_range[0])
self.static_fstream = async_stream
if content_range is not None:
yield from async_stream.copy_to(response, content_range[1] - content_range[0] + 1)
else:
yield from async_stream.copy_to(response)
async_stream.close()
del self.static_fstream
return response
def test_command_cgi_query():
req = command._prep_get(command.Operation.list_files, DIR="/DCIM/WOMP")
_, query = parse.splitquery(req.url)
query_map = parse.parse_qs(query)
assert query_map == {"DIR": ["/DCIM/WOMP"], "op": ["100"]}
def do_GET(self):
# self._set_headers()
#get request params
path = self.path
query = parse.splitquery(path)
self._get_handler(query[1]);
async def request(
self, localpart="/", method="GET", data=None, host="0.0.0.0:8080", headers=None, https=False, **kw
):
"""Makes request to this application for the specified path and method.
Response will be a storage object with data, status and headers.
>>> urls = ("/hello", "hello")
>>> app = application(urls, globals())
>>> class hello:
... def GET(self):
... web.header('Content-Type', 'text/plain')
... return "hello"
...
>>> response = await app.request("/hello")
>>> response.data
b'hello'
>>> response.status
'200 OK'
>>> response.headers['Content-Type']
'text/plain'
To use https, use https=True.
>>> urls = ("/redirect", "redirect")
>>> app = application(urls, globals())
>>> class redirect:
... def GET(self): raise web.seeother("/foo")
...
>>> response = await app.request("/redirect")
>>> response.headers['Location']
'http://0.0.0.0:8080/foo'
>>> response = await app.request("/redirect", https=True)
>>> response.headers['Location']
'https://0.0.0.0:8080/foo'
The headers argument specifies HTTP headers as a mapping object
such as a dict.
>>> urls = ('/ua', 'uaprinter')
>>> class uaprinter:
... def GET(self):
... return 'your user-agent is ' + web.ctx.env['HTTP_USER_AGENT']
...
>>> app = application(urls, globals())
>>> await app.request('/ua', headers = {
... 'User-Agent': 'a small jumping bean/1.0 (compatible)'
... }).data
b'your user-agent is a small jumping bean/1.0 (compatible)'
"""
path, maybe_query = splitquery(localpart)
query = maybe_query or ""
server = host.split(":")
if len(server) == 1:
server.append(80)
scope = dict(
server=server,
method=method,
path=path,
query_string=safebytes(query),
https=str(https),
headers=[],
scheme="http",
http_version="1.1",
root_path="",
)
if "scope" in kw:
scope.update(kw["scope"])
headers = headers or {}
has_content_length = False
has_content_type = False
for k, v in headers.items():
bytes_key = safebytes(k.lower().replace("-", "_"))
if bytes_key == b"content_length":
has_content_length = True
scope["headers"].append((bytes_key, safebytes(v)))
elif bytes_key == b"content_type":
has_content_type = True
scope["headers"].append((bytes_key, safebytes(v)))
else:
scope["headers"].append((b"http_" + bytes_key, safebytes(v)))
q = ""
if method not in ["HEAD", "GET"]:
data = data or ""
if isinstance(data, dict):
q = urlencode(data)
else:
q = data
# env["wsgi.input"] = BytesIO(q.encode("utf-8"))
# if not env.get('CONTENT_TYPE', '').lower().startswith('multipart/') and 'CONTENT_LENGTH' not in env:
if not has_content_length:
scope["headers"].append((b"content_length", len(q)))
if not has_content_type:
scope["headers"].append((b"content_type", "application/x-www-form-urlencoded"))
logger.getChild("application.request").debug("scope(%s)", scope)
response = web.storage()
response_data = []
async def receive():
return {"type": "http.request", "body": q.encode("utf8"), "more_body": False}
async def send_response(message):
if message["type"] == "http.response.start":
response.status = f'{message["status"]}'
response.headers = dict(message["headers"])
response.header_items = message["headers"]
elif message["type"] == "http.response.body":
response_data.append(safebytes(message["body"]))
await self.asgifunc()(scope)(receive, send_response)
logger.getChild("application.request").debug("response(%s)", response_data)
response.data = b"".join(response_data)
return response
