def extend_webapp(session: requests.sessions.Session) -> None:
r = session.get(
"https://www.pythonanywhere.com/user/{}/webapps/".format(LOGIN))
csrfmiddlewaretoken = csrfgetter(r.text).csrfToken
session.post(
WEB_APP_URL,
headers={
"Referer":
"https://www.pythonanywhere.com/user/{}/webapps/".format(LOGIN)
},
data={"csrfmiddlewaretoken": csrfmiddlewaretoken})
def generate_device_auth(self, session: requests.sessions.Session,
client_id: str, launcher_access_token: str,
user_agent: str) -> requests.models.Response:
res = session.post(
f"https://account-public-service-prod.ol.epicgames.com/account/api/public/account/{client_id}/deviceAuth",
headers={
"Authorization": f"Bearer {launcher_access_token}",
"User-Agent": user_agent
})
return res
def exchange_code_auth(self, session: requests.sessions.Session,
email: str) -> requests.models.Response:
res = session.post(
"https://account-public-service-prod.ol.epicgames.com/account/api/oauth/token",
headers={"Authorization": f"basic {launcher_token}"},
data={
"grant_type": "exchange_code",
"exchange_code": input(f"Enter exchange code for {email}: "),
"token_type": "eg1"
})
return res
def device_auth(self, session: requests.sessions.Session, deviceId: str,
accountId: str, secret: str) -> requests.models.Response:
res = session.post(
"https://account-public-service-prod.ol.epicgames.com/account/api/oauth/token",
headers={"Authorization": f"basic {ios_token}"},
data={
"grant_type": "device_auth",
"device_id": deviceId,
"account_id": accountId,
"secret": secret,
"token_type": "eg1"
})
return res
def session_post(
session: requests.sessions.Session, endpoint: str, data: Dict[str, Any]
) -> dict:
"""
Posts JSON `data` to `endpoint` using the `session`.
Handles errors and returns a json response dict.
"""
endpoint = get_full_endpoint_url(endpoint)
response = session.post(
endpoint, json=data, cert=REQUESTS_CERT, headers=REQUESTS_HEADERS
)
json_response = response.json()
handle_code_msg(json_response)
return json_response
def email_and_password_auth(self, session: requests.sessions.Session,
email: str,
password: str) -> requests.models.Response:
session.get(f"{base}/id/api/csrf")
res = session.post(
f"{base}/id/api/login",
headers={"x-xsrf-token": session.cookies.get("XSRF-TOKEN")},
data={
"email": email,
"password": password,
"rememberMe": False,
"captcha": ""
},
cookies=session.cookies)
return res
def submit_snapshot_request(session: requests.sessions.Session, url: str,
headers: dict) -> requests.models.Response:
save_url = url_for_savepage(url)
sub_headers = headers.copy()
sub_headers.update({"Referer": SAVE_ENDPOINT})
resp = session.post(save_url,
headers=sub_headers,
data={
"url": url,
"capture_all": "on"
})
# todo: error out on status code != 200
if resp.status_code != 200:
raise ServerStatusError(
f"""Server status was NOT OK; returned {resp.status_code} for: {save_url}"""
)
else:
return resp
def login(session: requests.sessions.Session):
url="https://passport.shangxueba.com/user/userlogin.aspx?url=https%3A//www.shangxueba.com/"
code_url="https://passport.shangxueba.com/VerifyCode.aspx"
page=session.get(url)
soup=BeautifulSoup(page.text,'lxml')
#登录账号
flag=False
max_login_time=15
while max_login_time > 0 and not flag:
code_image=session.get(code_url).content
with open("code.jpeg","wb") as f:
f.write(code_image)
code = get_code("./code.jpeg")
headers={
"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
"Referer":"https://passport.shangxueba.com/user/userlogin.aspx?url=https%3A//www.shangxueba.com/"
}
data={
"__EVENTTARGET":"",
"__EVENTARGUMENT":"",
"__VIEWSTATE":soup.find('input', id='__VIEWSTATE')["value"],
"__EVENTVALIDATION":soup.find('input', id='__EVENTVALIDATION')["value"],
"txtName": "username",
"txtPassword": "******",
"txtVerifycode":code,
"hidcode":"",
"hidflag":"1",
"Button1":""
}
req=session.post(url,headers=headers,data=data)
if "欢迎您回来" in req.text:
flag=True
break
if "验证码错误" in req.text:
max_login_time -= 1
continue
else:
max_login_time -= 1
continue
if flag == False:
return False
else:
save_cookie(session)
return session
def extend_task(session: requests.sessions.Session) -> None:
r = session.get(
"https://www.pythonanywhere.com/user/{}/tasks_tab/".format(LOGIN))
CSRFToken = csrfgetter(r.text).csrfToken
r = session.post(
TASK_URL,
headers={
"Referer":
"https://www.pythonanywhere.com/user/{}/tasks_tab/".format(LOGIN),
"X-CSRFToken":
CSRFToken
})
if r.headers.get("Content-Type") == "application/json":
r = r.json()
if r.get("status") != "success":
raise Exception(
"[Update task] status != success. Response json: {}".format(r))
else:
raise Exception("[Update task] Server returns non json")
def validate_fhir_resource(
http: requests.sessions.Session,
json_data: str,
resource_name: str,
validation_server_url: str,
) -> None:
"""
Calls the validation server url to validate the given resource
:param http: Http Session to use
:param json_data: json data for resource
:param resource_name: name of resource
:param validation_server_url: url to validation server
"""
# check each resource against the validation server
headers = {"Content-Type": "application/fhir+json"}
full_validation_uri: furl = furl(validation_server_url)
full_validation_uri /= resource_name
full_validation_uri /= "$validate"
validation_response: Response = http.post(
url=full_validation_uri.url,
data=json_data.encode("utf-8"),
headers=headers,
)
if validation_response.ok:
operation_outcome: Dict[str, Any] = validation_response.json()
if operation_outcome["issue"][0]["severity"] == "error":
raise FhirValidationException(
url=full_validation_uri.url,
json_data=json_data,
response_text=validation_response.text,
response_status_code=validation_response.status_code,
message="FhirSender: Validation Failed",
)
else:
raise FhirValidationException(
url=full_validation_uri.url,
json_data=json_data,
response_text=validation_response.text,
response_status_code=validation_response.status_code,
message="FhirSender: Validation Failed",
)
def dump_leaf_page(self, node_id: str, menu_path: str, text: str,
session: requests.sessions.Session) -> str:
custom_page_name, index = re.search('(.+)_([\d]+_[\d]+)',
node_id).groups()
subpage_index, field_index = [int(i) for i in index.split('_')]
leaf_post = {
"method": "ajax.PageElementSettingAjax_rpc_html_kYjvPyv3_",
"params": [
{
"custom_page_name": custom_page_name,
"interface_type": "client",
"menu_path": menu_path,
"field_index": field_index,
"subpage_index": subpage_index,
"element_type": "",
}
],
"id": 1
}
self.jison.load(session.post(self.URL_SOURCE,
json=leaf_post,
headers=self.interface_header).json())
leaf_type = self.jison.get_object('title', value_only=True).lower()
if leaf_type in ['gap', 'title', 'text']:
return leaf_type
if leaf_type == 'xplan':
page_html = self.jison.get_multi_object('html')[2].get('html')
else:
page_html = self.jison.get_multi_object('html')[0].get('html')
page = {}
soup = BeautifulSoup(page_html, 'html5lib')
# basic information for each page
try:
name = soup.find('option', {'selected': True}).getText()
if re.findall('\[(.+?)\] (.+)', name):
name = '--'.join(re.findall('\[(.+?)\] (.+)', name)[0])
except:
name = '(Empty)'
if name == 'no':
name = '(Empty)'
if name == 'Client':
name = text
content = {'entities': []}
if soup.find('input', {'checked': True, 'id': 'entity_types_1'}):
content.get('entities').append('individual')
if soup.find('input', {'checked': True, 'id': 'entity_types_2'}):
content.get('entities').append('superfund')
if soup.find('input', {'checked': True, 'id': 'entity_types_3'}):
content.get('entities').append('partnership')
if soup.find('input', {'checked': True, 'id': 'entity_types_4'}):
content.get('entities').append('trust')
if soup.find('input', {'checked': True, 'id': 'entity_types_5'}):
content.get('entities').append('company')
page['leaf_basic'] = content
# try to acquire table content if an `xplan` page
if leaf_type == 'xplan':
content = {'table1': [], 'table2': []}
if name.lower() in self.subgroup_name_ref:
content['subgroup'] = self.subgroup_name_ref.get(name.lower())
table1_method = "ajax.XplanElementListSettingAjax_rpc_html_WEaBDM8__"
table2_method = "ajax.XplanElementEditSettingAjax_rpc_html_HBm947gH_"
leaf_post_xtable = {
"method": None,
"params": [
{
"has_partner": False,
"domain": "factfind",
"guid": "00000000-0000-0000-0000-000000000000",
"cover_type": "",
"coa_access": False,
"entity_type": 0,
"locale": "AU",
"custom_page_name": "",
"editable": False,
"subpage_index": 0,
"partnerid": 0,
"mode": "edit",
"extra_params": "",
"field_index": 7,
"is_partner": False,
"entityid": 0,
"list_name": "",
"element_name": self.jison.get_object('element_name',
value_only=True),
"render_method": "factfind"
}
],
"id": 1
}
leaf_post_xtable['method'] = table1_method
self.jison.load(
session.post(self.URL_SOURCE, json=leaf_post_xtable,
headers=self.interface_header).json()
)
# if this `xplan` page has list with tabs
# process table content
table_html = self.jison.get_object('tabs_html', value_only=True)
if table_html:
list_view_html = table_html.get('_above-tabs')
full_view_html = table_html.get('_hidden-items')
soup = BeautifulSoup(list_view_html, 'html5lib')
for td in soup.find_all('td',
{'style': 'white-space: nowrap'}):
content.get('table1').append(td.findNext('td').getText())
soup = BeautifulSoup(full_view_html, 'html5lib')
for td in soup.find_all('td',
{'style': 'white-space: nowrap'}):
content.get('table2').append(td.findNext('td').getText())
# if this `xplan` page has list with checkbox or empty page
# process `page_html`
else:
for input in soup.find_all('input', {'checked': 'checked',
'name': 'xstore_listfields'}):
content.get('table1').append(
input.findNext('label').getText())
for input in soup.find_all('input', {'checked': 'checked',
'name': 'xstore_capturefields'}):
content.get('table2').append(
input.findNext('label').getText())
if content.get('table1') or content.get('table2'):
page['leaf_xplan'] = content
# not an `xplan` page
else:
if leaf_type == 'group':
content = {'group': []}
for option in soup.find('select',
{'id': 'select_fields_0'}).find_all(
'option'):
content.get('group').append(option.getText())
page['leaf_group'] = content
elif leaf_type == 'field':
leaf_type = 'variable'
InterfaceLeafPage(node_id, name, leaf_type, menu_path, page).new(
specific_db=self.db)
return leaf_type
def r_dump_interface(self, menu_path: str,
session: requests.sessions.Session,
specific: list = None, debug: bool = False) -> list:
"""
get `children` under current `menu_path`
then acquire `sub_children` for each child in `children`
`_q` is a queue for thread, indicating method is running as a thread
"""
menu_post = {
"method": "ajax.MenuTreeAjax_rpc_load_node_gG7QNYAS_",
"params": [{
"interface_type": "client",
"menu_path": menu_path
}],
"id": 1
}
self.jison.load(session.post(self.URL_SOURCE, json=menu_post,
headers=self.interface_header).json())
local_children = self.jison.get_object('children', value_only=True)
children = []
# loop in `children`, acquire `sub_children` for each child
for child in local_children:
if child.get('hidden'):
continue
text = child.get('title')
child_id = child.get('node_id')
child_path = re.search('client_([0-9_\-]+)', child_id)
specific_flag = False
sub_children = []
if debug:
print(text)
if specific and specific[0].lower() in text.lower():
specific.pop(0)
specific_flag = True
elif specific:
continue
# normal node case (id starts with `client_xxx`)
if child_path:
child_path = child_path.group(1).replace('-', '/')
sub_children = self.r_dump_interface(child_path, session,
specific=specific)
# if current child has no child and with a valid leaf id (a leaf)
if not sub_children and re.search('(.+)_([\d]+_[\d]+)', child_id):
leaf_type = self.dump_leaf_page(child_id, menu_path, text,
session)
if leaf_type in ['gap', 'title', 'text']:
continue
children.append({
'id': child_id,
'text': text,
'type': leaf_type
})
# if current child has no children but with an invalid leaf id
elif not sub_children:
children.append({
'id': child_id,
'text': text,
'type': 'other'
})
# else, append `sub_children` to current child
else:
children.append({
'id': child_id,
'text': text,
'type': 'root',
'children': sub_children
})
if specific_flag:
return children
return children
def authenticate(self, session: requests.sessions.Session, email: str,
password: str, user_agent: str) -> str:
session.get(f"{base}/id/api/csrf")
device_auth_details = AuthUtil.get_device_auth_details(self).get(
email, {})
if os.path.isfile(
"device_auths.json") is False or device_auth_details == {}:
res = AuthUtil.email_and_password_auth(self, session, email,
password)
log.debug(f"\nemail_and_password_auth")
log.debug(res.status_code)
log.debug(res.text)
if res.status_code == 409:
return AuthUtil.authenticate(self, session, email, password,
user_agent)
if res.status_code == 431:
session.get(f"{base}/id/api/csrf")
res = session.post(f"{base}/id/api/login/mfa",
headers={
"x-xsrf-token":
session.cookies.get("XSRF-TOKEN")
},
data={
"code":
input("Please enter the 2fa code: "),
"method": "authenticator",
"rememberDevice": False
},
cookies=session.cookies)
if res.status_code == 400:
raise ValueError("Wrong 2fa code entered.")
if not res.status_code == 400:
res = session.get(f"{base}/id/api/exchange",
headers={
"x-xsrf-token":
session.cookies.get("XSRF-TOKEN")
},
cookies=session.cookies)
exchange_code_ = res.json()["code"]
res = session.post(
f"{base_public_service}/account/api/oauth/token",
headers={"Authorization": f"basic {launcher_token}"},
data={
"grant_type": "exchange_code",
"exchange_code": exchange_code_,
"token_type": "eg1"
})
client_id = res.json()["account_id"]
launcher_access_token = res.json()["access_token"]
res = AuthUtil.generate_device_auth(self, session, client_id,
launcher_access_token,
user_agent)
if res.status_code == 200:
details = {
"deviceId": res.json().get("deviceId"),
"accountId": res.json().get("accountId"),
"secret": res.json().get("secret")
}
AuthUtil.store_device_auth_details(self, email, details)
if res.status_code == 400:
res = AuthUtil.exchange_code_auth(self, session, email)
log.debug(f"\nexchange_code_auth")
log.debug(res.status_code)
log.debug(res.text)
if res.status_code == 400:
raise ValueError("Wrong exchange_code entered.")
if res.status_code == 200:
launcher_access_token = res.json()["access_token"]
client_id = res.json()["account_id"]
res = AuthUtil.generate_device_auth(
self, session, client_id, launcher_access_token,
user_agent)
if res.status_code == 200:
details = {
"deviceId": res.json().get("deviceId"),
"accountId": res.json().get("accountId"),
"secret": res.json().get("secret")
}
AuthUtil.store_device_auth_details(
self, email, details)
else:
res = AuthUtil.device_auth(self, session, **device_auth_details)
log.debug(f"\ndevice_auth")
log.debug(res.status_code)
log.debug(res.text)
if res.status_code == 400:
raise ValueError("Wrong device auth detail entered.")
if res.status_code == 200:
launcher_access_token = res.json()["access_token"]
return launcher_access_token