def download_data(browser_object: webdriver, config_object: configparser.ConfigParser) -> BeautifulSoup: browser_object.get(config_object["rekyl_portal"]["url"]) time.sleep(1) username = browser_object.find_element_by_id("username") password = browser_object.find_element_by_css_selector( "input[type=password") username.send_keys(config_object["rekyl_portal"]["username"]) password.send_keys(config_object["rekyl_portal"]["password"]) browser_object.find_element_by_id("button_login_security_low").click() time.sleep(3) show_errands = Select(browser_object.find_element_by_name("maxhits")) show_errands.select_by_index(10) time.sleep(5) browser_object.find_element_by_name("maxhits").send_keys(Keys.RETURN) time.sleep(5) iframe = browser_object.find_element_by_id("iframe_workorder") browser_object.switch_to.frame(iframe) iframe_source = browser_object.page_source soup = BeautifulSoup(iframe_source, "html.parser") browser_object.close() browser_object.quit() logging.info("Raw data downloaded") return soup
def extract_info_from_iframe(browser: webdriver) -> []: try: # open all other pages with comments button = browser.find_element_by_css_selector( "button[data-dot='strankovani/nacist_dalsi']") while button: try: action = ActionChains(browser) action.move_to_element(button).click().perform() except (ElementClickInterceptedException, StaleElementReferenceException): pass sleep(0.5 * SETTINGS["lazy_factor"]) button = browser.find_element_by_css_selector( "button[data-dot='strankovani/nacist_dalsi']") except NoSuchElementException: pass try: # open threads of subcomments for button in browser.find_elements_by_css_selector( "button[data-dot='nacist_nove_podkomentare']"): try: action = ActionChains(browser) action.move_to_element(button).click().perform() except (ElementClickInterceptedException, StaleElementReferenceException): pass sleep(0.3 * SETTINGS["lazy_factor"]) except NoSuchElementException: pass # now we can use familiar beautiful soup soup = BeautifulSoup(browser.page_source, "html.parser") authors = soup.select("a[class='f_bO'] span") texts = soup.select("p[class='d_aJ']") reactions = soup.select("a[class='f_cQ']") comments = [] progress_bar = tqdm(total=len(authors), desc="Comments", position=0) for i in range(len(authors)): author = authors[i].text text = texts[i].text # expect that reaction can be missing in some comments reactions_count = 0 if len(reactions) > i: reactions_count = reactions[i].text comments.append( Comment(author=author, text=text, reactions=reactions_count)) progress_bar.update(1) # browser doesn't need to be open from now on browser.quit() return comments
def setup_proxy_for_driver(driver: webdriver, test_url=None, times=0): if times > 9: logger.warning('setup_proxy_for_driver no available proxy') raise TooManyTimesException('setup_proxy_for_driver') try: try: # clean cookie and close session driver.delete_all_cookies() driver.quit() except: pass proxy_url = get_proxy().get('proxy') logger.info('proxy get {}'.format(proxy_url)) capabilities = get_capabilities(proxy_url) logger.info('start new session') driver.start_session(capabilities=capabilities) logger.info('start testing proxy') ok = test_proxy(driver, test_url, proxy_url) if not ok: logger.warning('proxy checking failed for {} times'.format(times + 1)) return setup_proxy_for_driver(driver, test_url, times=times + 1) logger.info('proxy works') return driver except SessionNotCreatedException: logger.error('Failed to start a new session') return setup_proxy_for_driver(connect_to_driver(), test_url, times=times + 1) except InvalidSessionIdException as e2: logger.error('Session id invalid {}'.format(e2)) return setup_proxy_for_driver(driver, test_url, times=times + 1) except WebDriverException as e3: logger.error('No active session with ID') return setup_proxy_for_driver(driver, test_url, times=times + 1) except NoProxyAvailableException: logger.error('No proxy') mongo._report_error({ 'error_source': 'proxy', 'message': 'no_proxy_available', 'url': None, 'payload': {} }) return setup_proxy_for_driver(driver, test_url, times=times + 1) except Exception as e: logger.error(f'setup_proxy_for_driver {e}') raise e
def navigateLogin(driver: webdriver) -> HtmlChecker: driver.save_screenshot("Post_Login.png") resultBtn = driver.find_element_by_xpath("//a[@data-id='21']") resultBtn.click() time.sleep(10) driver.save_screenshot("Results.png") pageSource = driver.page_source HtmlObj = HtmlChecker(pageSource) driver.quit() return HtmlObj
def track_time(session: webdriver) -> None: try: st = session.find_element_by_id('div-fichaje-action') print(st.text) st.click() print('waiting to save track info') sleep(10) print("Time tracked") except Exception as e: print(f'Unable to submit track info:{e}') finally: session.quit()
def close_firefox(driver: webdriver, restart: bool = False, lastp: bool = False, no_ads_found: bool = False, time_exc: bool = False) -> None: """Close, quit, destroy the webdriver instance of Firefox and safely close the DB connection.""" if restart: print('Restart of Firefox requested.\n' 'Closing Firefox...\n') driver.close() driver.quit() elif time_exc: print('Failed to load the listing URL in time.\n' 'Closing Firefox and retrying...\n') driver.close() driver.quit() elif lastp: print( 'This was the last page that was set or found in the first listing URL.\n' 'Closing Firefox...\n') driver.close() driver.quit() disconnect_db() elif no_ads_found: print('No ads found on the page. Breaking out of the loop.\n' 'Closing Firefox...\n') driver.close() driver.quit() disconnect_db()
def logout(driver: webdriver, manager: str) -> None: ''' This function log you out and gracefully quits everything. :param driver: The web browser driver. :param manager: The Address of the Manager/FMC. :return: None ''' time.sleep(5) if manager.startswith('http'): driver.get('{}/login.cgi?logout=1'.format(manager)) else: driver.get('https://{}/login.cgi?logout=1'.format(manager)) time.sleep(5) driver.quit() # Gracefully quits everything.
def _selenium_operations(driver: webdriver, user_name: str, user_password: str): # 登录界面 Util.info_print('登录界面', 3) driver.get('https://users.nexusmods.com/auth/sign_in') Util.info_print('请在页面中登录N网账户', 3) Util.info_print('如果设置在conf.ini的账户密码正确,这个过程会自动完成。', 3) Util.info_print('如果不正确,请手动输入账户密码', 3) Util.info_print('每一步操作都设置了一定的的可行时间,超过时间程序就会退出', 3) wait = WebDriverWait(driver, 300) username_inputer = wait.until( presence_of_element_located((By.ID, "user_login"))) userpassword_inputer = wait.until( presence_of_element_located((By.ID, "password"))) commit_button = wait.until( presence_of_element_located((By.XPATH, '//input[@type="submit"]'))) username_inputer.send_keys(user_name) userpassword_inputer.send_keys(user_password) commit_button.click() wait.until(EC.url_changes) # 欢迎界面 index_a = wait.until( EC.presence_of_element_located( (By.XPATH, '//div[@class="links"]/div[@class="left-link"]/a[1]'))) index_a.click() Util.info_print('等待进入首页,请勿操作', 3) # 返回首页后 # Util.info_print('等待从首页中获取cookies', 3) wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, ".intro > h1:nth-child(1)"))) nexus_cookies_list = driver.get_cookies() driver.quit() nexus_cookies = dict() for cookie in nexus_cookies_list: nexus_cookies[cookie['name']] = cookie['value'] return nexus_cookies
def _selenium_operations(driver: webdriver, user_name: str, user_password: str): # 登录界面 _info_print('Coo_5') driver.get('https://users.nexusmods.com/auth/sign_in') _info_print('Coo_6') _info_print('Coo_7') _info_print('Coo_8') _info_print('Coo_9') wait = WebDriverWait(driver, 300) username_inputer = wait.until( presence_of_element_located((By.ID, "user_login"))) userpassword_inputer = wait.until( presence_of_element_located((By.ID, "password"))) commit_button = wait.until( presence_of_element_located((By.XPATH, '//input[@type="submit"]'))) username_inputer.send_keys(user_name) userpassword_inputer.send_keys(user_password) commit_button.click() wait.until(EC.url_changes) # 欢迎界面 index_a = wait.until( EC.presence_of_element_located( (By.XPATH, '//div[@class="links"]/div[@class="left-link"]/a[1]'))) index_a.click() _info_print('Coo_10') # 返回首页后 # Util.info_print('等待从首页中获取cookies', 3) wait.until( EC.presence_of_element_located( (By.CSS_SELECTOR, ".intro > h1:nth-child(1)"))) nexus_cookies_list = driver.get_cookies() driver.quit() nexus_cookies = dict() for cookie in nexus_cookies_list: nexus_cookies[cookie['name']] = cookie['value'] return nexus_cookies
def CaptureStats(self, driver: webdriver) -> HubStatsRecord: """ Capture stats from a BT Smart Hub 2 and return a populated HubStatsRecord :param driver: A configured Selenium webdriver object """ print("Capturing modem stats from BT Smart Hub 2 at " + self.URI + "\n") # Capture the data from the Smart Hub # Start the web session driver.get(self.URI) # Select "Advanced Settings" screen WebDriverWait(driver, 30).until( expected_conditions.element_to_be_clickable( (By.ID, 'gotoA'))).click() # Select "Helpdesk" WebDriverWait(driver, 30).until( expected_conditions.element_to_be_clickable( (By.ID, 'advHelpdesk'))).click() #Put the password int he dialog" WebDriverWait(driver, 30).until( expected_conditions.visibility_of_element_located( (By.ID, 'login_password_input_noshow'))).send_keys(self.AdminPassword) #Click the "OK" buttin WebDriverWait(driver, 30).until( expected_conditions.element_to_be_clickable( (By.ID, 'ok_button'))).click() #Grab the infobox DataTableWebElement = WebDriverWait(driver, 10).until( expected_conditions.element_to_be_clickable((By.ID, 'Infobox'))) # Process uptime UptimeStrings = re.findall( r"[-+]?\d*\.\d+|\d+", DataTableWebElement.find_element_by_id('DSLuptime').text) Uptime = int(UptimeStrings[0]) * 24 + int(UptimeStrings[1]) * 1 + int( UptimeStrings[2]) / 60 + int(UptimeStrings[3]) / 3600 # Process Data rate DatarateStrings = re.findall( r"[-+]?\d*\.\d+|\d+", DataTableWebElement.find_element_by_id('Datarate').text) DatarateValues = [float(i) for i in DatarateStrings] # Process Max data rate MDatarateStrings = re.findall( r"[-+]?\d*\.\d+|\d+", DataTableWebElement.find_element_by_id('MDatarate').text) MDatarateValues = [float(i) for i in MDatarateStrings] # Process noise margin NoiseMarginStrings = re.findall( r"[-+]?\d*\.\d+|\d+", DataTableWebElement.find_element_by_id('noise_margin').text) NoiseMarginValues = [float(i) for i in NoiseMarginStrings] # Close Web browser driver.quit() # Populate the return data structure d = HubStatsRecord() d.Uptime = Uptime d.DataRate_Upload = DatarateValues[0] d.DataRate_Down = DatarateValues[1] d.MaxDataRate_Up = MDatarateValues[0] d.MaxDataRate_Down = MDatarateValues[1] d.SNR_Up = NoiseMarginValues[0] d.SNR_Down = NoiseMarginValues[1] return d
def quit_chromium(driver: webdriver) -> None: driver.quit() if operating_system() != "Windows": run(['osascript', '-e', 'quit app \"Chromium\"'])
def cita_selection(driver: webdriver, context: CustomerProfile): resp_text = body_text(driver) if "Por favor, valide el Captcha para poder continuar" in resp_text: success = process_captcha(driver, context, partially=True) if not success: return None return phone_mail(driver, context, retry=True) elif "DISPONE DE 5 MINUTOS" in resp_text: logging.info("Cita attempt -> selection hit! :)") if context.save_artifacts: driver.save_screenshot(f"citas-{datetime.datetime.now()}.png".replace(":", "-")) try: driver.find_elements_by_css_selector("input[type='radio'][name='rdbCita']")[ 0 ].send_keys(Keys.SPACE) except Exception as e: logging.error(e) pass driver.execute_script("envia();") time.sleep(0.5) driver.switch_to.alert.accept() elif "Seleccione una de las siguientes citas disponibles" in resp_text: logging.info("Cita attempt -> selection hit! :)") if context.save_artifacts: driver.save_screenshot(f"citas-{datetime.datetime.now()}.png".replace(":", "-")) try: slots = driver.find_elements_by_css_selector("#CitaMAP_HORAS tbody [id^=HUECO]") slot_ids = sorted([*map(lambda x: x.get_attribute("id"), slots)]) if slot_ids: slot = slot_ids[0] driver.execute_script(f"confirmarHueco({{id: '{slot}'}}, {slot[5:]});") driver.switch_to.alert.accept() except Exception as e: logging.error(e) return None else: logging.info("Cita attempt -> missed selection :(") return None # 8. Confirmation resp_text = body_text(driver) if "Debe confirmar los datos de la cita asignada" in resp_text: logging.info("Cita attempt -> confirmation hit! :)") if context.telegram_token: dispatcher = context.updater.dispatcher def shutdown(): context.updater.stop() context.updater.is_idle = False def code_received(update, ctx): logging.info(f"Received code: {ctx.args[0]}") element = driver.find_element_by_id("txtCodigoVerificacion") element.send_keys(ctx.args[0]) driver.find_element_by_id("chkTotal").send_keys(Keys.SPACE) driver.find_element_by_id("enviarCorreo").send_keys(Keys.SPACE) btn = driver.find_element_by_id("btnConfirmar") btn.send_keys(Keys.ENTER) resp_text = body_text(driver) ctime = datetime.datetime.now() if "CITA CONFIRMADA Y GRABADA" in resp_text: context.bot_result = True code = driver.find_element_by_id("justificanteFinal").text logging.info(f"Justificante cita: {code}") caption = f"Cita confirmed! {code}" if context.save_artifacts: image_name = f"CONFIRMED-CITA-{ctime}.png".replace(":", "-") driver.save_screenshot(image_name) ctx.bot.send_photo( chat_id=update.effective_chat.id, photo=open(os.path.join(os.getcwd(), image_name), "rb"), caption=caption, ) btn = driver.find_element_by_id("btnImprimir") btn.send_keys(Keys.ENTER) else: ctx.bot.send_message(chat_id=update.effective_chat.id, text=caption) threading.Thread(target=shutdown).start() return True elif "Lo sentimos, el código introducido no es correcto" in resp_text: ctx.bot.send_message( chat_id=update.effective_chat.id, text="Incorrect, please try again" ) else: error_name = f"error-{ctime}.png".replace(":", "-") driver.save_screenshot(error_name) ctx.bot.send_photo( chat_id=update.effective_chat.id, photo=open(os.path.join(os.getcwd(), error_name), "rb"), ) ctx.bot.send_message( chat_id=update.effective_chat.id, text="Something went wrong" ) dispatcher.add_handler(CommandHandler("code", code_received, pass_args=True)) context.updater.start_polling(poll_interval=1.0) for i in range(5): speaker.say("ALARM") # Waiting for response 5 minutes time.sleep(360) threading.Thread(target=shutdown).start() if context.save_artifacts: driver.save_screenshot( f"FINAL-SCREEN-{datetime.datetime.now()}.png".replace(":", "-") ) if context.bot_result: driver.quit() os._exit(0) return None else: for i in range(10): speaker.say("ALARM") logging.info("Press Any button to CLOSE browser") input() driver.quit() os._exit(0) else: logging.info("Cita attempt -> missed confirmation :(") if context.save_artifacts: driver.save_screenshot( f"failed-confirmation-{datetime.datetime.now()}.png".replace(":", "-") ) return None
def fetch_image_urls(search_engine: SearchEngine, query: str, max_img_to_fetch: int, driver: webdriver): try: driver.get(search_engine.url) search_box = driver.find_element_by_css_selector( search_engine.css_selector_search_line) search_box.send_keys(query) search_box.submit() except Exception as ex: driver.quit() print(f"Exception {ex} with search box.") return None image_urls = [] image_count = 0 results_start = 0 while image_count < max_img_to_fetch: scroll_to_end(driver, time_sleep=random.randint(1, 3)) # get all image thumbnail results thumbnail_results = driver.find_elements_by_css_selector( search_engine.css_selector_thumbnail_img) number_results = len(thumbnail_results) print( f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}" ) for img in thumbnail_results[results_start:number_results]: # try to click every thumbnail such that we can get the real image behind it try: img.click() time.sleep(random.randint(1, 3)) except Exception: continue # extract image urls actual_images = driver.find_elements_by_css_selector( search_engine.css_selector_image_url) for actual_image in actual_images: if actual_image.get_attribute( 'src') and 'http' in actual_image.get_attribute('src'): image_urls.append(actual_image.get_attribute('src')) break try: close_preview = driver.find_element_by_css_selector( search_engine.close_preview) close_preview.click() except selenium_exceptions.ElementNotVisibleException as ex: print(ex) continue except Exception as ex: print(f'Error with close_preview: {ex}') return image_urls image_count = len(image_urls) if len(image_urls) >= max_img_to_fetch: print(f"Found: {len(image_urls)} image links, done!") break else: print("Found:", len(image_urls), "image links, looking for more ...") time.sleep(random.randint(10, 30)) if search_engine is GOOGLE_IMG: load_more_button = driver.find_element_by_css_selector( ".mye4qd") if load_more_button: driver.execute_script( "document.querySelector('.mye4qd').click();") # move the result startpoint further down results_start = len(thumbnail_results) return image_urls
def quit_session(driver: webdriver): driver.quit()
def close_browser(driver: webdriver): driver.close() driver.quit()
def cita_selection(driver: webdriver, context: CustomerProfile): try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.TAG_NAME, "body"))) except TimeoutException: logging.error("Timed out waiting for body to load") return None resp_text = driver.find_element_by_tag_name("body").text if "DISPONE DE 5 MINUTOS" in resp_text: logging.info("Cita attempt -> selection hit! :)") if context.save_artifacts: driver.save_screenshot( f"citas-{datetime.datetime.now()}.png".replace(":", "-")) try: driver.find_elements_by_css_selector( "input[type='radio'][name='rdbCita']")[0].send_keys(Keys.SPACE) except Exception as e: logging.error(e) pass btn = driver.find_element_by_id("btnSiguiente") btn.send_keys(Keys.ENTER) driver.switch_to.alert.accept() else: logging.info("Cita attempt -> missed selection :(") return None # 9. Confirmation try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.TAG_NAME, "body"))) except TimeoutException: logging.error("Timed out waiting for body to load") return None resp_text = driver.find_element_by_tag_name("body").text if "Debe confirmar los datos de la cita asignada" in resp_text: logging.info("Cita attempt -> confirmation hit! :)") if context.telegram_token: dispatcher = context.updater.dispatcher def shutdown(): context.updater.stop() context.updater.is_idle = False def code_received(update, ctx): logging.info(f"Received code: {ctx.args[0]}") element = driver.find_element_by_id("txtCodigoVerificacion") element.send_keys(ctx.args[0]) driver.find_element_by_id("chkTotal").send_keys(Keys.SPACE) driver.find_element_by_id("enviarCorreo").send_keys(Keys.SPACE) btn = driver.find_element_by_id("btnConfirmar") btn.send_keys(Keys.ENTER) try: WebDriverWait(driver, DELAY).until( EC.presence_of_element_located((By.TAG_NAME, "body"))) except TimeoutException: logging.info("Timed out waiting for body to load") return None resp_text = driver.find_element_by_tag_name("body").text ctime = datetime.datetime.now() if "CITA CONFIRMADA Y GRABADA" in resp_text: context.bot_result = True code = driver.find_element_by_id("justificanteFinal").text logging.info(f"Justificante cita: {code}") caption = f"Cita confirmed! {code}" if context.save_artifacts: image_name = f"CONFIRMED-CITA-{ctime}.png".replace( ":", "-") driver.save_screenshot(image_name) ctx.bot.send_photo( chat_id=update.effective_chat.id, photo=open(os.path.join(os.getcwd(), image_name), "rb"), caption=caption, ) btn = driver.find_element_by_id("btnImprimir") btn.send_keys(Keys.ENTER) else: ctx.bot.send_message(chat_id=update.effective_chat.id, text=caption) threading.Thread(target=shutdown).start() return True elif "Lo sentimos, el código introducido no es correcto" in resp_text: ctx.bot.send_message(chat_id=update.effective_chat.id, text="Incorrect, please try again") else: error_name = f"error-{ctime}.png".replace(":", "-") driver.save_screenshot(error_name) ctx.bot.send_photo( chat_id=update.effective_chat.id, photo=open(os.path.join(os.getcwd(), error_name), "rb"), ) ctx.bot.send_message(chat_id=update.effective_chat.id, text="Something went wrong") dispatcher.add_handler( CommandHandler("code", code_received, pass_args=True)) context.updater.start_polling(poll_interval=1.0) for i in range(5): speaker.say("ALARM") # Waiting for response 5 minutes time.sleep(360) threading.Thread(target=shutdown).start() if context.save_artifacts: driver.save_screenshot( f"FINAL-SCREEN-{datetime.datetime.now()}.png".replace( ":", "-")) if context.bot_result: driver.quit() os._exit(0) return None else: for i in range(5): speaker.say("ALARM") logging.info("Press Any button to CLOSE browser") input() else: logging.info("Cita attempt -> missed confirmation :(") if context.save_artifacts: driver.save_screenshot( f"failed-confirmation-{datetime.datetime.now()}.png".replace( ":", "-")) return None