def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=0&perPage=15&allRows=8084&order='
response = browser.request('post', url_1)
response1 = json.loads(response.content)['pageBean']
page = response1['allPage']
browser.close()
for i in range(0, page):
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=' + str(
i) + '&allPage=539&perPage=15&allRows=8084&order='
res = browser.request('post', url_1)
res1 = json.loads(res.content)['list']
browser.close()
for j in range(len(res1)):
sql = "insert into t_jkftsythzpbaxx_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(sql, [
url_1, res.content,
parse(res1[j]),
str(i + 1) + '_' + str(j + 1)
])
def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
for i in range(1, 330):
browser = Chrome(chrome_options=option)
url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList&on=true&page=' + str(
i) + '&pageSize=15&productName=&conditionType=1&applyname=&applysn'
res = browser.request('post', url)
# print(res.text)
res1 = json.loads(res.content)['list']
# print(res1)
browser.close()
for j in range(len(res1)):
sql = "insert into t_hzpscxkhzqy_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(
sql,
[url, res.content,
parse(res1[j]),
str(i) + '_' + str(j + 1)])
def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
#先调一次post请求，获取一级目录页的页数
url = "http://125.35.6.80:8181/ftban/itownet/fwAction.do?method=getBaNewInfoPage&on=true&page=1&pageSize=15&productName=&conditionType=1&applyname=&applysn="
browser = Chrome(chrome_options=option)
res = browser.request("post", url)
html = res.text
browser.close()
json_return = json.loads(html)
pageCount = json_return["pageCount"]
for i in range(1, pageCount + 1): #遍历目录网页
try:
url_1 = "http://125.35.6.80:8181/ftban/itownet/fwAction.do?method=getBaNewInfoPage&on=true&page=" + str(
i
) + "&pageSize=15&productName=&conditionType=1&applyname=&applysn="
browser = Chrome(chrome_options=option)
res = browser.request("post", url_1)
html = res.text
browser.close()
json_return = json.loads(html)
d = json_return['list'] #得到页面列表信息
for j in range(len(d)):
sql = "insert into t_gcftsythzpbaxx_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(
sql,
[url_1, html,
parse2json(d[j]),
str(i) + '_' + str(j + 1)])
except Exception as e:
print(e)
time.sleep(5)
def selenium_headless(self, WINDOW_SIZE):
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE)
CHROME_PATH = '/Applications/Google Chrome.app' #Change 1
CHROMEDRIVER_PATH = './chromedriver' #Change 2
#Load the sites list
SITES_LIST = pd.read_csv("Top_500_sites.csv")
print("Loaded the sites list file")
SITES = []
for index, row in SITES_LIST.iterrows():
SITES.append(row['Site'])
#Query the sites
FAILED = 0
print("Querying...")
for rank, site in enumerate(SITES):
try:
webdriver = Chrome(executable_path=CHROMEDRIVER_PATH,
chrome_options=chrome_options)
resp = webdriver.request('GET', 'https://' + str(site))
print("Quering site #" + str(rank + 1) + " -> " + str(resp))
if str(resp) == "<Response [403]>":
print("403 received for:" + site)
FAILED += 1
continue
except Exception as ex:
print("Exception occurred for " + str(site))
print("And the exception is:" + str(ex))
FAILED += 1
print("Failed for:" + str(FAILED) + " out of 500")
webdriver.close()
def main():
option = None
mysql_db = DataBase()
# 配置文件中开启是否无头，生产阶段关闭
if if_headless():
option = webdriver.ChromeOptions()
option.add_argument(argument='headless')
option.add_argument('--no-sandbox')
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=0&perPage=15&allRows=8084&order='
response = browser.request('post', url_1)
response1 = json.loads(response.content)['pageBean']
page = response1['allPage']
browser.close()
for i in range(0, page):
browser = Chrome(chrome_options=option)
url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=' + str(
i) + '&allPage=539&perPage=15&allRows=8084&order='
res1 = browser.request('post', url_1)
res1 = json.loads(res1.content)['list']
browser.close()
for j in range(len(res1)):
browser = Chrome(chrome_options=option)
url_2 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?'
params = {'method': 'show', 'id': res1[j]['id']}
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Content-Length': '31',
'Content-Type':
'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie':
'_gscu_515232071=60234697wnhvr115; _gscbrs_515232071=1; JSESSIONID=DA4CEE8CEE0F521678039F251D0A32AD',
'Host': 'cpnp.nmpa.gov.cn',
'Origin': 'http://cpnp.nmpa.gov.cn',
'Referer':
'http://cpnp.nmpa.gov.cn/province/webquery/show.jsp?id=50BF34D2A36759BA',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
res2 = browser.request('post', url_2, data=params, headers=headers)
browser.close()
sql = "insert into t_jkftsythzpbaxx_baxq(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)"
mysql_db.exetcute_sql(sql, [
url_2, res2.text,
parse(json.loads(res2.text)),
str(i + 1) + '_' + str(j + 1)
])
# def test_delete(self):
# print("test delete function")
# ret = delete('5fb088e3c3abc3288dbf35fe')
# print(ret)
# self.assertRegex(ret, 'done')
def test_delete(self):
print("test delete function")
ret = post({
"name": "Su Kayle",
"email": "*****@*****.**",
"password": "******",
"mobile": "042312123",
"gender": "Female"
})
# ret = post({"name":"Su Kayle","email":"*****@*****.**","password":"******"})
print(ret)
self.assertRegex(ret, 'done')
if __name__ == '__main__':
unittest2.main()
# delete('5fb088e3c3abc3288dbf35fe')
# put('5fb08833c3abc3288dbf35fc', {"name":"Dam Smith","password":"******"})
# put('5fb088e3c3abc3288dbf35fe', {"name":"Parv Sing","password":"******"})
# put('5fafe4c7bfd3790e81800940', {"name":"Python Cool","password":"******"})
# post({"name":"Su Kayle","email":"*****@*****.**","password":"******","mobile":"042312123","gender":"Female"})
# listall()
session.close()
driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options)
print("logging into outlook")
driver.get("https://outlook.office.com/owa/?ae=Folder&t=IPF.Appointment")
try:
driver.find_element_by_name("loginfmt").send_keys("*****@*****.**")
driver.find_element_by_id("idSIButton9").click()
print("entered username, waiting for password prompt")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'branding')))
print("password prompt loaded")
except TimeoutException:
print("Loading password prompt took too much time!")
driver.close();
exit(1)
passwd = getpass.getpass()
driver.find_element_by_id("passwordInput").send_keys(passwd)
driver.find_element_by_id("submitButton").click()
print("entered password, waiting for 2FA token")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'idSIButton9')))
driver.find_element_by_id("idSIButton9").click()
print("asking to remember credentials for next time")
except TimeoutException:
print("Loading 2FA page took too much time!")
driver.close();
exit(1)
try:
minted_password = os.environ['minted_password']
except KeyError:
minted_password = input("Enter your minted.com password:")
driver.get(url)
# Selenium deals with lgin form
email_elem = driver.find_element_by_name('email')
email_elem.send_keys(minted_email)
password_elem = driver.find_element_by_name('password')
password_elem.send_keys(minted_password)
login_submit = driver.find_element_by_class_name('loginButton')
login_submit.click()
sleep(5) # to load JS and be nice
# Request address book contents as json
response = driver.request('GET','https://addressbook.minted.com/api/contacts/contacts/?format=json')
listings = response.json()
# Create dataframe to hold addresses
address_book = pd.DataFrame(listings)
# Export to excel and csv
address_book.to_excel('./data/minted-addresses-api.xlsx')
address_book.to_csv('./data/minted-addresses-api.csv', index=False)
# Close selenium webdriver
driver.close()
def issue_request():
# region Variable Declarations
global start_date, end_date, \
instrument_type, property_address, \
authenticated
instrument_types = []
recording_dates = []
dated_dates = []
grantors = []
grantees = []
result_page_links = []
property_addresses = []
property_owner_names = []
property_owner_addresses = []
pdf_text = ""
# endregion Variable Declarations
# Browser Instantiation
browser = Chrome(ChromeDriverManager().install())
# The site to which we will navigate while also handling its session.
browser.get(login_url)
# Locates and populates the elements containing the username and password entry fields.
username_element = browser.find_element_by_id("USERID")
username_element.send_keys(username)
password_element = browser.find_element_by_id("PASSWORD")
password_element.send_keys(password)
# Login
password_element.send_keys(Keys.RETURN)
authenticated = True if "invalid" not in str(
browser.page_source) else False
if authenticated:
# Navigation to and Selection of Advanced Search
browser.get(instrument_type_search_url)
# Ensure Page Elements Have Loaded
time.sleep(1)
# Reveal Additional Search Fields
search_type = browser.find_element_by_id("lbSearchType")
search_type.click()
# Ensure Page Elements Have Loaded
time.sleep(1)
# Issue an Advanced Search Query
start_date_field = browser.find_element_by_id("STARTDATE")
start_date_field.send_keys(start_date)
end_date_field = browser.find_element_by_id("ENDDATE")
end_date_field.send_keys(end_date)
instrument_type_field = browser.find_element_by_id("INSTRUMENT_TYPES")
instrument_type_field.send_keys(instrument_type)
search_button = browser.find_element_by_id("SearchButton")
search_button.click()
# Harvest the Query
document_links = set()
result_page_repeater_ids = set(
link.get_attribute("id")
for link in browser.find_elements_by_xpath("//a[@href]")
if "PageRepeater" in link.get_attribute("href"))
for result_page_repeater_id in result_page_repeater_ids:
result_page_repeater = browser.find_element_by_id(
result_page_repeater_id)
result_page_repeater.click()
current_page_document_links = set(
link.get_attribute("href")
for link in browser.find_elements_by_xpath("//a[@href]")
if "PK" in link.get_attribute("href"))
document_links = document_links | current_page_document_links
for result_page_link in document_links:
# Procedurally Extracting Instrument Types, Recording Dates,
# Dated Dates, Grantors, Grantees, & Property Addresses
malformed_property_address_counter = 0
browser.get(result_page_link)
column_data = browser.find_elements_by_class_name("coldata")
for column_datum in column_data:
parsed_datum = column_datum.text.split('\n')
if malformed_property_address_counter == 0:
instrument_types.append(parsed_datum[0])
elif malformed_property_address_counter == 4:
recording_dates.append(parsed_datum[0])
elif malformed_property_address_counter == 5:
dated_dates.append(parsed_datum[0])
elif malformed_property_address_counter == 9:
grantors.append(parsed_datum)
elif malformed_property_address_counter == 10:
grantees.append(parsed_datum)
malformed_property_address_counter += 1
view_document_button = browser.find_element_by_id(
"BTN_VIEW_DOCUMENT")
on_click = view_document_button.get_attribute("onclick")
pdf_download_link = base_search_url + str(on_click).split('\'')[1]
pdf = browser.request("GET", pdf_download_link).content
pdf_image = pdf2image.convert_from_bytes(pdf)
for page in pdf_image:
page_text = pytesseract.image_to_string(page)
if "Real Estate" in page_text \
or "PROPERTY TO BE DISTRIBUTED" in page_text \
or "Decedent resided at" in page_text \
or "Real Property" in page_text:
pdf_text = page_text
break
property_address = pyap.parse(pdf_text, country="US")
if len(property_address) > 0:
property_address = str(property_address[0])
result_page_links.append(result_page_link)
property_addresses.append(property_address)
update_activity_display(
"\n\n* Found " + str(len(property_addresses)) + " entries for \"" +
instrument_type + "\" cases in the Clay"
" County database. Compare"
" this number to the"
" number of populated fields"
" for property addresses in"
" the generated CSV file to"
" reconcile any"
" discrepancies that may"
" occur in the PDF"
" address-parsing process. *\n", activity_display)
valid_property_address_counter = 0
malformed_property_address_counter = 1
for result_page_link, property_address in zip(result_page_links,
property_addresses):
try:
if len(property_address) == 0:
raise NoSuchElementException
elif len(property_address) > 0:
# Navigation to Second Page
browser.get(gis_url)
# Ensure Page Elements Have Loaded
time.sleep(2)
# Click Agree
agree_field = browser.find_element_by_id(
"dojox_mobile_Button_0")
agree_field.click()
# Navigation to Address Search
address_search_field = browser.find_element_by_id(
"searchButton")
address_search_field.click()
# Ensure Page Elements Have Loaded
time.sleep(2)
address_search_tab = browser.find_element_by_id(
"addressTab")
address_search_tab.click()
# Ensure Page Elements Have Loaded
time.sleep(2)
search_input_field = browser.find_element_by_id(
"search_input")
# Ensure Page Elements Have Loaded
time.sleep(3)
# Enter Address
search_input_field.send_keys(property_address)
# Ensure Page Elements Have Loaded
time.sleep(2)
# Click Submit
search_input_field.send_keys(Keys.RETURN)
# Ensure Search Results Have Loaded
time.sleep(3)
# Harvesting Property Owner Names
gis_results_container = browser.find_element_by_id(
"resultsGeocodeContainer0")
for line in gis_results_container.text.split("\n"):
if "Current Owner" in line:
property_owner_name = line.split(":")[1].strip()
property_owner_names.append(property_owner_name)
break
# Collecting Property Owner Addresses
tabs = browser.window_handles
for link in gis_results_container.find_elements_by_xpath(
"//a[@href]"):
if "parcelid" in link.get_attribute("href"):
link.click()
tabs = browser.window_handles
browser.switch_to.window(tabs[1])
break
property_owner_addresses.append(
browser.find_element_by_xpath(
"/html/body/table[2]/tbody/tr[7]/td").text + ', ' +
browser.find_element_by_xpath(
"/html/body/table[2]/tbody/tr[9]/td").text)
browser.close()
browser.switch_to.window(tabs[0])
valid_property_address_counter += 1
else:
property_owner_names.append([])
property_owner_addresses.append([])
except NoSuchElementException:
malformed_property_address_result_page_link = result_page_link
update_activity_display(
"\nAn invalid address entered into the Clay County GIS address entry field was found.\n"
+ "It was located in the PDF found here:",
activity_display)
activity_display.config(state=NORMAL)
activity_display.pack()
activity_display.insert(
INSERT,
"Link to PDF #" + str(malformed_property_address_counter) +
" for Manual Inspection\n",
hyperlink.add(lambda: webbrowser.open(
malformed_property_address_result_page_link)))
activity_display.config(state=DISABLED)
property_owner_names.append([])
property_owner_addresses.append([])
malformed_property_address_counter += 1
pass
# Exporting Scraped Data to CSV File
df_data = [
instrument_types, recording_dates, dated_dates, grantors, grantees,
result_page_links, property_addresses, property_owner_names,
property_owner_addresses
]
df = pd.DataFrame(data=df_data).T
df.columns = [
"Instrument Type", "Recording Date", "Dated Date", "Grantor(s)",
"Grantee(s)", "Result Page Link", "Property Address",
"Current Property Owner Name", "Owner Mailing Address"
]
df.to_csv("Realtor Estate Data Export (" + str(date.today()) + ").csv")
# Cleaning Up the Browser Instance
browser.close()
return requests.get(instrument_type_search_url).status_code
def main():
global GBF
timestart = time_now()
profile = path.abspath("./" + CFG.profile)
parser = argparse.ArgumentParser(
prog='gbf-scraper.py',
description=
'A simple script for scraping various parts of Granblue Fantasy',
usage=
'gbf-scraper.py [profile] [options]\nexample: python gbf-scraper.py profile2 -i 1 8000',
formatter_class=argparse.MetavarTypeHelpFormatter)
parser.add_argument('profile',
nargs='?',
help='overwrites the default profile path',
type=str)
parser.add_argument(
'--individual',
'-i',
nargs=2,
help=
'scrapes GW individual rankings between the specified start and end',
type=int)
parser.add_argument(
'--guild',
'-g',
nargs=4,
help=
'scrapes GW guild rankings between the specified start and end prelim and seed pages',
type=int)
parser.add_argument(
'--members',
'-m',
help='scrape member data from guilds specified in config.py',
action='store_true')
parser.add_argument('--info',
'-n',
help='scrapes rank info from a guild specified',
type=int)
parser.add_argument(
'--login',
'-l',
help='pauses the script upon starting up to allow logging in',
action='store_true')
args = parser.parse_args()
if len(argv) == 1:
parser.print_help()
quit()
if args.profile is not None:
log('Changing profile path to {}'.format(args.profile))
profile = path.abspath('./' + args.profile)
options = webdriver.ChromeOptions()
log('Using profile at: {}'.format(profile))
options.add_argument('user-data-dir=%s' % profile)
for cargs in CHROME_ARGUMENTS.split():
options.add_argument(cargs)
GBF = Chrome(chrome_options=options)
GBF.get('http://game.granbluefantasy.jp/#mypage')
if args.login:
log('Pausing to login')
input('Press enter to continue...')
if args.individual is not None:
log('GW: {} Scraping individual rankings from page {} to page {}'.
format(GW_NUMBER, *args.individual))
gw_individual(*args.individual)
if args.guild is not None:
log('GW: {}\n Scraping prelim guild rankings from page {} to \
page {} and seed rankings from page {} to page {}'.format(
GW_NUMBER, *args.guild))
gw_guild(*args.guild)
if args.info is not None:
# TODO scrape more than just ranks?
log('Scraping guild info for guild {}'.format(args.info))
guild_ranks(args.info)
if args.members:
# DEFUNCT, used to be for EOP rankings
log('Defunct -m flag')
guild_members()
alert_operator('Task finished. {} seconds elapsed.'.format(time_now() -
timestart),
pause=False)
GBF.close()
quit()
class UmsConnection():
def __init__(self, user_phone, user_password, cookies=None):
self._user_phone = user_phone
self._user_password = user_password
self._is_authorized = False
self._captcha = None # REMOVE
options = Options()
options.add_argument("--headless")
WINDOW_SIZE = "1620,880" # ?
options.add_argument("--window-size=%s" % WINDOW_SIZE)
self.webdriver = Chrome(chrome_options=options)
if cookies:
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
for cookie in cookies:
self.webdriver.add_cookie(cookie)
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
self._is_authorized = self.try_is_authorized()
if not self.is_authorized:
self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do")
phone_input = self.webdriver.find_element_by_id('user12')
phone_input.send_keys(user_phone)
password_input_secinfo = self.webdriver.find_element_by_id('secinfo')
password_input_secinfo.click()
password_input = self.webdriver.find_element_by_id("secinfo2")
password_input.send_keys(user_password)
def get_captcha(self) -> bytes:
captcha = self.webdriver.find_element_by_id('imageC')
location = captcha.location
size = captcha.size
screenshot = self.webdriver.get_screenshot_as_png()
im = Image.open(BytesIO(screenshot))
left = location['x']
top = location['y']
right = location['x'] + size['width']
bottom = location['y'] + size['height']
im = im.crop((left, top, right, bottom))
self._captcha = im # REMOVE
stream = BytesIO()
im.save(stream, format="PNG")
return stream.getvalue()
def send_captcha_key(self, key) -> bool:
captcha_input = self.webdriver.find_element_by_id('imageObj')
captcha_input.send_keys(key)
submit_button = self.webdriver.find_element_by_id('index_login')
submit_button.click()
try:
WebDriverWait(self.webdriver, 2).until(
EC.url_contains(("messages.megafon.ru/onebox/mix.do")))
self._is_authorized = True
self._captcha.save(f'captchas/{key}.png') # REMOVE
return True
except exceptions.TimeoutException:
return False
def try_is_authorized(self) -> bool:
# TODO
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': 10,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.status_code == 200
@property
def is_authorized(self):
return self._is_authorized
def get_chat_list(self, number=10):
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': number,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.text
def get_one_box(self, address, number=10, from_num=1):
url = r"https://messages.megafon.ru/onebox/oneboxList.do"
params = [
('umReq.ctlg', '1,2'),
('umReq.numFlg', '1'),
('umReq.mType', '6149'),
('umReq.srt', '0'),
('umReq.lType', '0'),
('umReq.dFlg', '0'),
('umReq.srtDr', '0'),
('umReq.rdFlg', '0'),
('umReq.bNum', from_num),
('umReq.eNum', number),
('umReq.snd', address),
('umReq.rcv', self._user_phone),
('umReq.bTime', ''),
('umReq.eTime', ''),
('umReq.impt', '-1'),
('umReq.t', ''),
('umReq.threadFlag', '1'),
('rownid', random.random()),
]
response = self.webdriver.request('GET', url, params=params)
return response.text
def close(self):
self.webdriver.close()
def get_cookies_json(self):
cookies = self.webdriver.get_cookies()
return json.dumps(cookies)
def send_outlook():
home_dir = str(Path.home())
chrome_cache_path = f"{home_dir}/.chrome_cache"
print(f"loading chrome, caching to: {chrome_cache_path}")
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--no-startup-window")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-sync-preferences")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-background-networking")
chrome_options.add_argument("--no-first-run")
chrome_options.add_argument("--aggressive-tab-discard")
chrome_options.add_argument("--user-agent=Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.2)")
chrome_options.add_argument(f"--user-data-dir={chrome_cache_path}/user-data")
chrome_options.add_argument(f"--data-path={chrome_cache_path}/data-path")
chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/disk-cache")
chrome_options.add_argument(f"--homedir={chrome_cache_path}")
chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/cache-dir")
prefs={"profile.managed_default_content_settings.images": 2, 'disk-cache-size': 4096 }
chrome_options.add_experimental_option("prefs",prefs)
delay = 60
chrome_options.binary_location = "/usr/bin/chromium-browser"
driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options)
print("logging into outlook")
driver.get("https://outlook.office.com/owa/")
try:
driver.find_element_by_name("loginfmt").send_keys("*****@*****.**")
driver.find_element_by_id("idSIButton9").click()
print("entered username, waiting for password prompt")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'passwordInput')))
print("password prompt loaded")
except TimeoutException:
print("Loading password prompt took too much time!")
print(driver.page_source)
driver.close();
exit(1)
passwd = getpass.getpass()
driver.find_element_by_id("passwordInput").send_keys(passwd)
driver.find_element_by_id("submitButton").click()
print("entered password, waiting for 2FA token")
try:
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'idSIButton9')))
driver.find_element_by_id("idSIButton9").click()
print("asking to remember credentials for next time")
except TimeoutException:
print("Loading 2FA page took too much time!")
print(driver.page_source)
driver.close();
exit(1)
print("2FA accepted, loading office landing page")
except NoSuchElementException:
print("already logged in")
try:
print("waiting for landing page to load")
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'lnkBrwsAllFldrs')))
except TimeoutException:
print(driver.page_source)
print("Loading landing page too much time!")
driver.close();
exit(1)
try:
eink.send_update("Loading Tasks")
print("loading tasks")
driver.find_element_by_id("lnkBrwsAllFldrs").click()
driver.find_element_by_id("selbrfld").click()
Select(driver.find_element_by_id("selbrfld")).select_by_visible_text("Tasks")
driver.find_element_by_id("selbrfld").click()
driver.find_element_by_xpath("(.//*[normalize-space(text()) and normalize-space(.)='Sent Items'])[1]/following::img[1]").click()
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'lvw')))
except TimeoutException:
print(driver.page_source)
print("Loading todo list took too much time!")
driver.close();
exit(1)
elements = driver.find_elements_by_css_selector("td h1 a")
for i, element in enumerate(elements):
eink.send_todo(i, element.text)
try:
eink.send_update("Loading Calendar")
print("loading calendar")
driver.find_element_by_id("lnkNavCal").click()
myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'cntnttp')))
print("calendar loaded, dumping entries")
except TimeoutException:
print(driver.page_source)
print("Loading calendar took too much time!")
driver.close();
exit(1)
elements = driver.find_elements_by_css_selector("td.v a")
for i, element in enumerate(elements):
eink.send_meeting(i,element.get_attribute('title'))
def test_basic_training():
Config.init_config('../app_conf/testing.ini')
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument("--disable-user-media-security")
chrome_options.add_argument("--use-fake-ui-for-media-stream")
chrome_options.add_argument("--use-fake-device-for-media-stream")
chrome_options.add_argument(
'--use-file-for-fake-audio-capture={}/simple_phrases_russian.wav'.
format(os.getcwd()))
chrome_options.add_experimental_option('detach', True)
driver = Chrome(options=chrome_options)
response = driver.request('POST',
'http://127.0.0.1:5000/lti',
data={
'lis_person_name_full':
Config.c.testing.lis_person_name_full,
'ext_user_username':
Config.c.testing.session_id,
'custom_task_id':
Config.c.testing.custom_task_id,
'custom_task_description':
Config.c.testing.custom_task_description,
'custom_attempt_count':
Config.c.testing.custom_attempt_count,
'custom_required_points':
Config.c.testing.custom_required_points,
'custom_criteria_pack_id':
Config.c.testing.custom_criteria_pack_id,
'roles':
Config.c.testing.roles,
'lis_outcome_service_url':
Config.c.testing.lis_outcome_service_url,
'lis_result_sourcedid':
Config.c.testing.lis_result_source_did,
'oauth_consumer_key':
Config.c.testing.oauth_consumer_key,
})
driver.get('http://127.0.0.1:5000/upload_presentation/')
driver.find_element_by_id('upload-presentation-form')
data = open('test_data/test_presentation_file_0.pdf', 'rb')
response = driver.request(
'POST',
'http://127.0.0.1:5000/handle_presentation_upload/',
files=dict(presentation=data))
pos = response.text.find("setupPresentationViewer(\"")
assert pos != -1
training_id = response.text[pos + 25:pos + 49]
driver.get('http://127.0.0.1:5000/trainings/{}/'.format(training_id))
driver.find_element_by_id('record').click()
step = 3
sleep(2 * step)
driver.find_element_by_id('next').click()
sleep(step)
driver.find_element_by_id('done').click()
sleep(step)
total_wait_time = 60
wait_time = 0
while wait_time < total_wait_time:
driver.get('http://127.0.0.1:5000/trainings/statistics/{}/'.format(
training_id))
try:
feedback_element = WebDriverWait(driver, step).until(
EC.presence_of_element_located((By.ID, 'feedback')))
if feedback_element.text.startswith('Оценка за тренировку'):
break
else:
wait_time += step
sleep(step)
except TimeoutException:
wait_time += step
driver.close()
assert wait_time < total_wait_time
