def main(): option = None mysql_db = DataBase() # 配置文件中开启是否无头,生产阶段关闭 if if_headless(): option = webdriver.ChromeOptions() option.add_argument(argument='headless') option.add_argument('--no-sandbox') browser = Chrome(chrome_options=option) url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=0&perPage=15&allRows=8084&order=' response = browser.request('post', url_1) response1 = json.loads(response.content)['pageBean'] page = response1['allPage'] browser.close() for i in range(0, page): browser = Chrome(chrome_options=option) url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=' + str( i) + '&allPage=539&perPage=15&allRows=8084&order=' res = browser.request('post', url_1) res1 = json.loads(res.content)['list'] browser.close() for j in range(len(res1)): sql = "insert into t_jkftsythzpbaxx_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)" mysql_db.exetcute_sql(sql, [ url_1, res.content, parse(res1[j]), str(i + 1) + '_' + str(j + 1) ])
def main(): option = None mysql_db = DataBase() # 配置文件中开启是否无头,生产阶段关闭 if if_headless(): option = webdriver.ChromeOptions() option.add_argument(argument='headless') option.add_argument('--no-sandbox') for i in range(1, 330): browser = Chrome(chrome_options=option) url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList&on=true&page=' + str( i) + '&pageSize=15&productName=&conditionType=1&applyname=&applysn' res = browser.request('post', url) # print(res.text) res1 = json.loads(res.content)['list'] # print(res1) browser.close() for j in range(len(res1)): sql = "insert into t_hzpscxkhzqy_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)" mysql_db.exetcute_sql( sql, [url, res.content, parse(res1[j]), str(i) + '_' + str(j + 1)])
def main(): option = None mysql_db = DataBase() # 配置文件中开启是否无头,生产阶段关闭 if if_headless(): option = webdriver.ChromeOptions() option.add_argument(argument='headless') option.add_argument('--no-sandbox') #先调一次post请求,获取一级目录页的页数 url = "http://125.35.6.80:8181/ftban/itownet/fwAction.do?method=getBaNewInfoPage&on=true&page=1&pageSize=15&productName=&conditionType=1&applyname=&applysn=" browser = Chrome(chrome_options=option) res = browser.request("post", url) html = res.text browser.close() json_return = json.loads(html) pageCount = json_return["pageCount"] for i in range(1, pageCount + 1): #遍历目录网页 try: url_1 = "http://125.35.6.80:8181/ftban/itownet/fwAction.do?method=getBaNewInfoPage&on=true&page=" + str( i ) + "&pageSize=15&productName=&conditionType=1&applyname=&applysn=" browser = Chrome(chrome_options=option) res = browser.request("post", url_1) html = res.text browser.close() json_return = json.loads(html) d = json_return['list'] #得到页面列表信息 for j in range(len(d)): sql = "insert into t_gcftsythzpbaxx_lbsj(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)" mysql_db.exetcute_sql( sql, [url_1, html, parse2json(d[j]), str(i) + '_' + str(j + 1)]) except Exception as e: print(e) time.sleep(5)
def selenium_headless(self, WINDOW_SIZE): chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE) CHROME_PATH = '/Applications/Google Chrome.app' #Change 1 CHROMEDRIVER_PATH = './chromedriver' #Change 2 #Load the sites list SITES_LIST = pd.read_csv("Top_500_sites.csv") print("Loaded the sites list file") SITES = [] for index, row in SITES_LIST.iterrows(): SITES.append(row['Site']) #Query the sites FAILED = 0 print("Querying...") for rank, site in enumerate(SITES): try: webdriver = Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options) resp = webdriver.request('GET', 'https://' + str(site)) print("Quering site #" + str(rank + 1) + " -> " + str(resp)) if str(resp) == "<Response [403]>": print("403 received for:" + site) FAILED += 1 continue except Exception as ex: print("Exception occurred for " + str(site)) print("And the exception is:" + str(ex)) FAILED += 1 print("Failed for:" + str(FAILED) + " out of 500") webdriver.close()
def main(): option = None mysql_db = DataBase() # 配置文件中开启是否无头,生产阶段关闭 if if_headless(): option = webdriver.ChromeOptions() option.add_argument(argument='headless') option.add_argument('--no-sandbox') browser = Chrome(chrome_options=option) url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=0&perPage=15&allRows=8084&order=' response = browser.request('post', url_1) response1 = json.loads(response.content)['pageBean'] page = response1['allPage'] browser.close() for i in range(0, page): browser = Chrome(chrome_options=option) url_1 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?method=query&querytype=productname&pfid=&content=&dataPage=' + str( i) + '&allPage=539&perPage=15&allRows=8084&order=' res1 = browser.request('post', url_1) res1 = json.loads(res1.content)['list'] browser.close() for j in range(len(res1)): browser = Chrome(chrome_options=option) url_2 = 'http://cpnp.nmpa.gov.cn/province/webquery/wq.do?' params = {'method': 'show', 'id': res1[j]['id']} headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Connection': 'keep-alive', 'Content-Length': '31', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': '_gscu_515232071=60234697wnhvr115; _gscbrs_515232071=1; JSESSIONID=DA4CEE8CEE0F521678039F251D0A32AD', 'Host': 'cpnp.nmpa.gov.cn', 'Origin': 'http://cpnp.nmpa.gov.cn', 'Referer': 'http://cpnp.nmpa.gov.cn/province/webquery/show.jsp?id=50BF34D2A36759BA', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36', 'X-Requested-With': 'XMLHttpRequest' } res2 = browser.request('post', url_2, data=params, headers=headers) browser.close() sql = "insert into t_jkftsythzpbaxx_baxq(c_bh, dt_insertTime, c_url, b_content, c_json,c_page) VALUES (REPLACE(UUID(),\"-\",\"\"), sysdate(), %s,%s,%s,%s)" mysql_db.exetcute_sql(sql, [ url_2, res2.text, parse(json.loads(res2.text)), str(i + 1) + '_' + str(j + 1) ])
# def test_delete(self): # print("test delete function") # ret = delete('5fb088e3c3abc3288dbf35fe') # print(ret) # self.assertRegex(ret, 'done') def test_delete(self): print("test delete function") ret = post({ "name": "Su Kayle", "email": "*****@*****.**", "password": "******", "mobile": "042312123", "gender": "Female" }) # ret = post({"name":"Su Kayle","email":"*****@*****.**","password":"******"}) print(ret) self.assertRegex(ret, 'done') if __name__ == '__main__': unittest2.main() # delete('5fb088e3c3abc3288dbf35fe') # put('5fb08833c3abc3288dbf35fc', {"name":"Dam Smith","password":"******"}) # put('5fb088e3c3abc3288dbf35fe', {"name":"Parv Sing","password":"******"}) # put('5fafe4c7bfd3790e81800940', {"name":"Python Cool","password":"******"}) # post({"name":"Su Kayle","email":"*****@*****.**","password":"******","mobile":"042312123","gender":"Female"}) # listall() session.close()
driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options) print("logging into outlook") driver.get("https://outlook.office.com/owa/?ae=Folder&t=IPF.Appointment") try: driver.find_element_by_name("loginfmt").send_keys("*****@*****.**") driver.find_element_by_id("idSIButton9").click() print("entered username, waiting for password prompt") try: myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'branding'))) print("password prompt loaded") except TimeoutException: print("Loading password prompt took too much time!") driver.close(); exit(1) passwd = getpass.getpass() driver.find_element_by_id("passwordInput").send_keys(passwd) driver.find_element_by_id("submitButton").click() print("entered password, waiting for 2FA token") try: myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'idSIButton9'))) driver.find_element_by_id("idSIButton9").click() print("asking to remember credentials for next time") except TimeoutException: print("Loading 2FA page took too much time!") driver.close(); exit(1)
try: minted_password = os.environ['minted_password'] except KeyError: minted_password = input("Enter your minted.com password:") driver.get(url) # Selenium deals with lgin form email_elem = driver.find_element_by_name('email') email_elem.send_keys(minted_email) password_elem = driver.find_element_by_name('password') password_elem.send_keys(minted_password) login_submit = driver.find_element_by_class_name('loginButton') login_submit.click() sleep(5) # to load JS and be nice # Request address book contents as json response = driver.request('GET','https://addressbook.minted.com/api/contacts/contacts/?format=json') listings = response.json() # Create dataframe to hold addresses address_book = pd.DataFrame(listings) # Export to excel and csv address_book.to_excel('./data/minted-addresses-api.xlsx') address_book.to_csv('./data/minted-addresses-api.csv', index=False) # Close selenium webdriver driver.close()
def issue_request(): # region Variable Declarations global start_date, end_date, \ instrument_type, property_address, \ authenticated instrument_types = [] recording_dates = [] dated_dates = [] grantors = [] grantees = [] result_page_links = [] property_addresses = [] property_owner_names = [] property_owner_addresses = [] pdf_text = "" # endregion Variable Declarations # Browser Instantiation browser = Chrome(ChromeDriverManager().install()) # The site to which we will navigate while also handling its session. browser.get(login_url) # Locates and populates the elements containing the username and password entry fields. username_element = browser.find_element_by_id("USERID") username_element.send_keys(username) password_element = browser.find_element_by_id("PASSWORD") password_element.send_keys(password) # Login password_element.send_keys(Keys.RETURN) authenticated = True if "invalid" not in str( browser.page_source) else False if authenticated: # Navigation to and Selection of Advanced Search browser.get(instrument_type_search_url) # Ensure Page Elements Have Loaded time.sleep(1) # Reveal Additional Search Fields search_type = browser.find_element_by_id("lbSearchType") search_type.click() # Ensure Page Elements Have Loaded time.sleep(1) # Issue an Advanced Search Query start_date_field = browser.find_element_by_id("STARTDATE") start_date_field.send_keys(start_date) end_date_field = browser.find_element_by_id("ENDDATE") end_date_field.send_keys(end_date) instrument_type_field = browser.find_element_by_id("INSTRUMENT_TYPES") instrument_type_field.send_keys(instrument_type) search_button = browser.find_element_by_id("SearchButton") search_button.click() # Harvest the Query document_links = set() result_page_repeater_ids = set( link.get_attribute("id") for link in browser.find_elements_by_xpath("//a[@href]") if "PageRepeater" in link.get_attribute("href")) for result_page_repeater_id in result_page_repeater_ids: result_page_repeater = browser.find_element_by_id( result_page_repeater_id) result_page_repeater.click() current_page_document_links = set( link.get_attribute("href") for link in browser.find_elements_by_xpath("//a[@href]") if "PK" in link.get_attribute("href")) document_links = document_links | current_page_document_links for result_page_link in document_links: # Procedurally Extracting Instrument Types, Recording Dates, # Dated Dates, Grantors, Grantees, & Property Addresses malformed_property_address_counter = 0 browser.get(result_page_link) column_data = browser.find_elements_by_class_name("coldata") for column_datum in column_data: parsed_datum = column_datum.text.split('\n') if malformed_property_address_counter == 0: instrument_types.append(parsed_datum[0]) elif malformed_property_address_counter == 4: recording_dates.append(parsed_datum[0]) elif malformed_property_address_counter == 5: dated_dates.append(parsed_datum[0]) elif malformed_property_address_counter == 9: grantors.append(parsed_datum) elif malformed_property_address_counter == 10: grantees.append(parsed_datum) malformed_property_address_counter += 1 view_document_button = browser.find_element_by_id( "BTN_VIEW_DOCUMENT") on_click = view_document_button.get_attribute("onclick") pdf_download_link = base_search_url + str(on_click).split('\'')[1] pdf = browser.request("GET", pdf_download_link).content pdf_image = pdf2image.convert_from_bytes(pdf) for page in pdf_image: page_text = pytesseract.image_to_string(page) if "Real Estate" in page_text \ or "PROPERTY TO BE DISTRIBUTED" in page_text \ or "Decedent resided at" in page_text \ or "Real Property" in page_text: pdf_text = page_text break property_address = pyap.parse(pdf_text, country="US") if len(property_address) > 0: property_address = str(property_address[0]) result_page_links.append(result_page_link) property_addresses.append(property_address) update_activity_display( "\n\n* Found " + str(len(property_addresses)) + " entries for \"" + instrument_type + "\" cases in the Clay" " County database. Compare" " this number to the" " number of populated fields" " for property addresses in" " the generated CSV file to" " reconcile any" " discrepancies that may" " occur in the PDF" " address-parsing process. *\n", activity_display) valid_property_address_counter = 0 malformed_property_address_counter = 1 for result_page_link, property_address in zip(result_page_links, property_addresses): try: if len(property_address) == 0: raise NoSuchElementException elif len(property_address) > 0: # Navigation to Second Page browser.get(gis_url) # Ensure Page Elements Have Loaded time.sleep(2) # Click Agree agree_field = browser.find_element_by_id( "dojox_mobile_Button_0") agree_field.click() # Navigation to Address Search address_search_field = browser.find_element_by_id( "searchButton") address_search_field.click() # Ensure Page Elements Have Loaded time.sleep(2) address_search_tab = browser.find_element_by_id( "addressTab") address_search_tab.click() # Ensure Page Elements Have Loaded time.sleep(2) search_input_field = browser.find_element_by_id( "search_input") # Ensure Page Elements Have Loaded time.sleep(3) # Enter Address search_input_field.send_keys(property_address) # Ensure Page Elements Have Loaded time.sleep(2) # Click Submit search_input_field.send_keys(Keys.RETURN) # Ensure Search Results Have Loaded time.sleep(3) # Harvesting Property Owner Names gis_results_container = browser.find_element_by_id( "resultsGeocodeContainer0") for line in gis_results_container.text.split("\n"): if "Current Owner" in line: property_owner_name = line.split(":")[1].strip() property_owner_names.append(property_owner_name) break # Collecting Property Owner Addresses tabs = browser.window_handles for link in gis_results_container.find_elements_by_xpath( "//a[@href]"): if "parcelid" in link.get_attribute("href"): link.click() tabs = browser.window_handles browser.switch_to.window(tabs[1]) break property_owner_addresses.append( browser.find_element_by_xpath( "/html/body/table[2]/tbody/tr[7]/td").text + ', ' + browser.find_element_by_xpath( "/html/body/table[2]/tbody/tr[9]/td").text) browser.close() browser.switch_to.window(tabs[0]) valid_property_address_counter += 1 else: property_owner_names.append([]) property_owner_addresses.append([]) except NoSuchElementException: malformed_property_address_result_page_link = result_page_link update_activity_display( "\nAn invalid address entered into the Clay County GIS address entry field was found.\n" + "It was located in the PDF found here:", activity_display) activity_display.config(state=NORMAL) activity_display.pack() activity_display.insert( INSERT, "Link to PDF #" + str(malformed_property_address_counter) + " for Manual Inspection\n", hyperlink.add(lambda: webbrowser.open( malformed_property_address_result_page_link))) activity_display.config(state=DISABLED) property_owner_names.append([]) property_owner_addresses.append([]) malformed_property_address_counter += 1 pass # Exporting Scraped Data to CSV File df_data = [ instrument_types, recording_dates, dated_dates, grantors, grantees, result_page_links, property_addresses, property_owner_names, property_owner_addresses ] df = pd.DataFrame(data=df_data).T df.columns = [ "Instrument Type", "Recording Date", "Dated Date", "Grantor(s)", "Grantee(s)", "Result Page Link", "Property Address", "Current Property Owner Name", "Owner Mailing Address" ] df.to_csv("Realtor Estate Data Export (" + str(date.today()) + ").csv") # Cleaning Up the Browser Instance browser.close() return requests.get(instrument_type_search_url).status_code
def main(): global GBF timestart = time_now() profile = path.abspath("./" + CFG.profile) parser = argparse.ArgumentParser( prog='gbf-scraper.py', description= 'A simple script for scraping various parts of Granblue Fantasy', usage= 'gbf-scraper.py [profile] [options]\nexample: python gbf-scraper.py profile2 -i 1 8000', formatter_class=argparse.MetavarTypeHelpFormatter) parser.add_argument('profile', nargs='?', help='overwrites the default profile path', type=str) parser.add_argument( '--individual', '-i', nargs=2, help= 'scrapes GW individual rankings between the specified start and end', type=int) parser.add_argument( '--guild', '-g', nargs=4, help= 'scrapes GW guild rankings between the specified start and end prelim and seed pages', type=int) parser.add_argument( '--members', '-m', help='scrape member data from guilds specified in config.py', action='store_true') parser.add_argument('--info', '-n', help='scrapes rank info from a guild specified', type=int) parser.add_argument( '--login', '-l', help='pauses the script upon starting up to allow logging in', action='store_true') args = parser.parse_args() if len(argv) == 1: parser.print_help() quit() if args.profile is not None: log('Changing profile path to {}'.format(args.profile)) profile = path.abspath('./' + args.profile) options = webdriver.ChromeOptions() log('Using profile at: {}'.format(profile)) options.add_argument('user-data-dir=%s' % profile) for cargs in CHROME_ARGUMENTS.split(): options.add_argument(cargs) GBF = Chrome(chrome_options=options) GBF.get('http://game.granbluefantasy.jp/#mypage') if args.login: log('Pausing to login') input('Press enter to continue...') if args.individual is not None: log('GW: {} Scraping individual rankings from page {} to page {}'. format(GW_NUMBER, *args.individual)) gw_individual(*args.individual) if args.guild is not None: log('GW: {}\n Scraping prelim guild rankings from page {} to \ page {} and seed rankings from page {} to page {}'.format( GW_NUMBER, *args.guild)) gw_guild(*args.guild) if args.info is not None: # TODO scrape more than just ranks? log('Scraping guild info for guild {}'.format(args.info)) guild_ranks(args.info) if args.members: # DEFUNCT, used to be for EOP rankings log('Defunct -m flag') guild_members() alert_operator('Task finished. {} seconds elapsed.'.format(time_now() - timestart), pause=False) GBF.close() quit()
class UmsConnection(): def __init__(self, user_phone, user_password, cookies=None): self._user_phone = user_phone self._user_password = user_password self._is_authorized = False self._captcha = None # REMOVE options = Options() options.add_argument("--headless") WINDOW_SIZE = "1620,880" # ? options.add_argument("--window-size=%s" % WINDOW_SIZE) self.webdriver = Chrome(chrome_options=options) if cookies: self.webdriver.get('https://messages.megafon.ru/onebox/mix.do') for cookie in cookies: self.webdriver.add_cookie(cookie) self.webdriver.get('https://messages.megafon.ru/onebox/mix.do') self._is_authorized = self.try_is_authorized() if not self.is_authorized: self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do") phone_input = self.webdriver.find_element_by_id('user12') phone_input.send_keys(user_phone) password_input_secinfo = self.webdriver.find_element_by_id('secinfo') password_input_secinfo.click() password_input = self.webdriver.find_element_by_id("secinfo2") password_input.send_keys(user_password) def get_captcha(self) -> bytes: captcha = self.webdriver.find_element_by_id('imageC') location = captcha.location size = captcha.size screenshot = self.webdriver.get_screenshot_as_png() im = Image.open(BytesIO(screenshot)) left = location['x'] top = location['y'] right = location['x'] + size['width'] bottom = location['y'] + size['height'] im = im.crop((left, top, right, bottom)) self._captcha = im # REMOVE stream = BytesIO() im.save(stream, format="PNG") return stream.getvalue() def send_captcha_key(self, key) -> bool: captcha_input = self.webdriver.find_element_by_id('imageObj') captcha_input.send_keys(key) submit_button = self.webdriver.find_element_by_id('index_login') submit_button.click() try: WebDriverWait(self.webdriver, 2).until( EC.url_contains(("messages.megafon.ru/onebox/mix.do"))) self._is_authorized = True self._captcha.save(f'captchas/{key}.png') # REMOVE return True except exceptions.TimeoutException: return False def try_is_authorized(self) -> bool: # TODO url = r"https://messages.megafon.ru/onebox/getChatList.do" get_chats_url_params = { 'startNum': '1', 'endNum': 10, 'reFreshFlag': '1', 'operation': '1', 'chatMsgType': '10100000000110000000000000000000', 't': random.random() } response = self.webdriver.request('GET', url, params=get_chats_url_params) return response.status_code == 200 @property def is_authorized(self): return self._is_authorized def get_chat_list(self, number=10): url = r"https://messages.megafon.ru/onebox/getChatList.do" get_chats_url_params = { 'startNum': '1', 'endNum': number, 'reFreshFlag': '1', 'operation': '1', 'chatMsgType': '10100000000110000000000000000000', 't': random.random() } response = self.webdriver.request('GET', url, params=get_chats_url_params) return response.text def get_one_box(self, address, number=10, from_num=1): url = r"https://messages.megafon.ru/onebox/oneboxList.do" params = [ ('umReq.ctlg', '1,2'), ('umReq.numFlg', '1'), ('umReq.mType', '6149'), ('umReq.srt', '0'), ('umReq.lType', '0'), ('umReq.dFlg', '0'), ('umReq.srtDr', '0'), ('umReq.rdFlg', '0'), ('umReq.bNum', from_num), ('umReq.eNum', number), ('umReq.snd', address), ('umReq.rcv', self._user_phone), ('umReq.bTime', ''), ('umReq.eTime', ''), ('umReq.impt', '-1'), ('umReq.t', ''), ('umReq.threadFlag', '1'), ('rownid', random.random()), ] response = self.webdriver.request('GET', url, params=params) return response.text def close(self): self.webdriver.close() def get_cookies_json(self): cookies = self.webdriver.get_cookies() return json.dumps(cookies)
def send_outlook(): home_dir = str(Path.home()) chrome_cache_path = f"{home_dir}/.chrome_cache" print(f"loading chrome, caching to: {chrome_cache_path}") chrome_options = Options() chrome_options.add_argument("--headless") chrome_options.add_argument("--no-startup-window") chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--disable-sync-preferences") chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--disable-background-networking") chrome_options.add_argument("--no-first-run") chrome_options.add_argument("--aggressive-tab-discard") chrome_options.add_argument("--user-agent=Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 5.2)") chrome_options.add_argument(f"--user-data-dir={chrome_cache_path}/user-data") chrome_options.add_argument(f"--data-path={chrome_cache_path}/data-path") chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/disk-cache") chrome_options.add_argument(f"--homedir={chrome_cache_path}") chrome_options.add_argument(f"--disk-cache-dir={chrome_cache_path}/cache-dir") prefs={"profile.managed_default_content_settings.images": 2, 'disk-cache-size': 4096 } chrome_options.add_experimental_option("prefs",prefs) delay = 60 chrome_options.binary_location = "/usr/bin/chromium-browser" driver = Chrome(executable_path=os.path.abspath("/usr/lib/chromium-browser/chromedriver"), chrome_options=chrome_options) print("logging into outlook") driver.get("https://outlook.office.com/owa/") try: driver.find_element_by_name("loginfmt").send_keys("*****@*****.**") driver.find_element_by_id("idSIButton9").click() print("entered username, waiting for password prompt") try: myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'passwordInput'))) print("password prompt loaded") except TimeoutException: print("Loading password prompt took too much time!") print(driver.page_source) driver.close(); exit(1) passwd = getpass.getpass() driver.find_element_by_id("passwordInput").send_keys(passwd) driver.find_element_by_id("submitButton").click() print("entered password, waiting for 2FA token") try: myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'idSIButton9'))) driver.find_element_by_id("idSIButton9").click() print("asking to remember credentials for next time") except TimeoutException: print("Loading 2FA page took too much time!") print(driver.page_source) driver.close(); exit(1) print("2FA accepted, loading office landing page") except NoSuchElementException: print("already logged in") try: print("waiting for landing page to load") myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.ID, 'lnkBrwsAllFldrs'))) except TimeoutException: print(driver.page_source) print("Loading landing page too much time!") driver.close(); exit(1) try: eink.send_update("Loading Tasks") print("loading tasks") driver.find_element_by_id("lnkBrwsAllFldrs").click() driver.find_element_by_id("selbrfld").click() Select(driver.find_element_by_id("selbrfld")).select_by_visible_text("Tasks") driver.find_element_by_id("selbrfld").click() driver.find_element_by_xpath("(.//*[normalize-space(text()) and normalize-space(.)='Sent Items'])[1]/following::img[1]").click() myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'lvw'))) except TimeoutException: print(driver.page_source) print("Loading todo list took too much time!") driver.close(); exit(1) elements = driver.find_elements_by_css_selector("td h1 a") for i, element in enumerate(elements): eink.send_todo(i, element.text) try: eink.send_update("Loading Calendar") print("loading calendar") driver.find_element_by_id("lnkNavCal").click() myElem = WebDriverWait(driver, delay).until(EC.presence_of_element_located((By.CLASS_NAME, 'cntnttp'))) print("calendar loaded, dumping entries") except TimeoutException: print(driver.page_source) print("Loading calendar took too much time!") driver.close(); exit(1) elements = driver.find_elements_by_css_selector("td.v a") for i, element in enumerate(elements): eink.send_meeting(i,element.get_attribute('title'))
def test_basic_training(): Config.init_config('../app_conf/testing.ini') chrome_options = Options() chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--headless') chrome_options.add_argument("--disable-user-media-security") chrome_options.add_argument("--use-fake-ui-for-media-stream") chrome_options.add_argument("--use-fake-device-for-media-stream") chrome_options.add_argument( '--use-file-for-fake-audio-capture={}/simple_phrases_russian.wav'. format(os.getcwd())) chrome_options.add_experimental_option('detach', True) driver = Chrome(options=chrome_options) response = driver.request('POST', 'http://127.0.0.1:5000/lti', data={ 'lis_person_name_full': Config.c.testing.lis_person_name_full, 'ext_user_username': Config.c.testing.session_id, 'custom_task_id': Config.c.testing.custom_task_id, 'custom_task_description': Config.c.testing.custom_task_description, 'custom_attempt_count': Config.c.testing.custom_attempt_count, 'custom_required_points': Config.c.testing.custom_required_points, 'custom_criteria_pack_id': Config.c.testing.custom_criteria_pack_id, 'roles': Config.c.testing.roles, 'lis_outcome_service_url': Config.c.testing.lis_outcome_service_url, 'lis_result_sourcedid': Config.c.testing.lis_result_source_did, 'oauth_consumer_key': Config.c.testing.oauth_consumer_key, }) driver.get('http://127.0.0.1:5000/upload_presentation/') driver.find_element_by_id('upload-presentation-form') data = open('test_data/test_presentation_file_0.pdf', 'rb') response = driver.request( 'POST', 'http://127.0.0.1:5000/handle_presentation_upload/', files=dict(presentation=data)) pos = response.text.find("setupPresentationViewer(\"") assert pos != -1 training_id = response.text[pos + 25:pos + 49] driver.get('http://127.0.0.1:5000/trainings/{}/'.format(training_id)) driver.find_element_by_id('record').click() step = 3 sleep(2 * step) driver.find_element_by_id('next').click() sleep(step) driver.find_element_by_id('done').click() sleep(step) total_wait_time = 60 wait_time = 0 while wait_time < total_wait_time: driver.get('http://127.0.0.1:5000/trainings/statistics/{}/'.format( training_id)) try: feedback_element = WebDriverWait(driver, step).until( EC.presence_of_element_located((By.ID, 'feedback'))) if feedback_element.text.startswith('Оценка за тренировку'): break else: wait_time += step sleep(step) except TimeoutException: wait_time += step driver.close() assert wait_time < total_wait_time