def wait_for_next(x: WebDriver): # 反复点击【添加成员】,直到点击通过 try: x.find_element(*locator).click() return x.find_element( By.ID, "username") # 这里为什么要 return 这个,上一句不是已经实现了点击操作了么? except: return False
def get_semester_list(driver: WebDriver): dropdown = driver.find_element(By.ID, "DERIVED_SSS_SCL_SSS_MORE_ACADEMICS") dropdown.find_element(By.XPATH, "//option[. = 'Votre horaire cours']").click() driver.find_element(By.ID, "DERIVED_SSS_SCL_SSS_MORE_ACADEMICS").click() driver.find_element(By.NAME, "DERIVED_SSS_SCL_SSS_GO_1$IMG").click() driver.switch_to.default_content() driver.switch_to.frame('ptifrmtgtframe') wait_for_process_to_end(driver) semester_elements = driver.find_elements_by_class_name('PSRADIOBUTTON') return [i.find_element_by_xpath('./../../..').text for i in semester_elements]
def get_job_items_per_tab(driver: WebDriver, main_container: dict): try: list_containers = driver.find_elements(By.CLASS_NAME, 'list-container')[:-1] keys = list(main_container.keys()) print("Fetching data...") for idx in range(len(list_containers)): job_list = [] job_container = list_containers[idx].find_elements( By.TAG_NAME, 'div')[3] job_items = job_container.find_elements(By.TAG_NAME, 'a') for job_item in tqdm(job_items): driver.execute_script( f"window.open('{job_item.get_attribute('href')}', '_blank');" ) windows = driver.window_handles sleep(3) driver.switch_to.window(windows[1]) driver.implicitly_wait(5) # create a Job object from form input fields company = driver.find_element( By.XPATH, "//input[@placeholder='Company']").get_attribute('value') job_title = driver.find_element( By.XPATH, "//input[@placeholder='+ add title']").get_attribute( 'value') location = driver.find_element( By.XPATH, "//input[@placeholder='+ add location']").get_attribute( 'value') description = driver.find_element(By.CLASS_NAME, 'ql-editor').text post_url = driver.find_element( By.XPATH, "//p[@title='Post URL']/following-sibling::div" ).find_element(By.TAG_NAME, 'a').get_attribute('href') a_job = Job(company, job_title, post_url, location, description) job_list.append(a_job.as_dict()) driver.close() driver.switch_to.window(windows[0]) main_container[keys[idx]] = job_list except Exception as err: print(f"Error getting job_container:" + str(err)) sys.exit(2)
def get_archived_board(driver: WebDriver): try: board_title = driver.find_element( By.XPATH, f"//p[@title='{os.getenv('ARCHIVED_BOARD_NAME')}']") board_title.click() except Exception as err: print(f"Error finding your board {os.getenv('ARCHIVED_BOARD_NAME')}:" + str(err)) sys.exit(2)
def login(driver: WebDriver): try: driver.get(os.getenv("HUNTR_BOARD_URL")) driver.implicitly_wait(5) email_input = driver.find_element(By.XPATH, "//input[@type='email']") password_input = driver.find_element(By.XPATH, "//input[@type='password']") email_input.send_keys(os.getenv("HUNTR_USERNAME")) password_input.send_keys(os.getenv("HUNTR_PASSWORD")) driver.implicitly_wait(5) password_input.send_keys(Keys.ENTER) driver.implicitly_wait(5) print("Logged In!") sleep(1 + random.random() * 4) except Exception as err: print(f"Error logging in with supplied credentials for " f"job board {os.getenv('HUNTR_BOARD_URL')} :" + str(err)) sys.exit(2)
class Application: def __init__(self): self.wd = WebDriver() self.wd.implicitly_wait(5) self.session = SessionHelper(self) self.payment = PaymentHelper(self) self.localization = LocalizationHelper(self) self.navigation = NavigationHelper(self) self.login_page = LoginPage(self) self.currency = Currency(self) self.payment_systems = PaymentSystems(self) def experiment(self, a): self.wd.find_element(By.cssSelector('span[class=\"title\"]:contains(' + someText + ')')) self.wd.find_element_by_xpath() self.wd.find_element_by_class_name('forgot') self.wd.find_element_by_link_text('Forgot password') self.wd.find_element_by_css_selector() def check_exists_by_css_selector(self, selector): try: self.wd.find_elements_by_css_selector(selector) except NoSuchElementException: return False return True def check_exists_by_xpath(self, xpath): try: self.wd.find_element_by_xpath(xpath) except NoSuchElementException: return False return True def destroy(self): self.wd.quit()
def is_bad_date(driver: WebDriver, element_id) -> Tuple[bool, str]: """ Entering the date improperly causes dynamic text to appear right below the input box indicating whether the date is invalid (formatting issue) or unavailable. Read this text and return the appropriate message to the caller. NOTE: not currently used, but good to keep here in case the site changes and we need this pattern again. """ date_error_msg = driver.find_element(by=By.ID, value=element_id) invalid_str = "not valid" unavailable_str = "not available" logger.info(date_error_msg.text) if date_error_msg is not None: if unavailable_str in date_error_msg.text: return (True, unavailable_str) if invalid_str in date_error_msg.text: return (True, invalid_str) return (True, "new error") return (False, "all good")
class UntitledTestCase(unittest.TestCase): def setUp(self): self.driver = WebDriver() self.driver.implicitly_wait(30) self.base_url = "https://www.katalon.com/" self.verificationErrors = [] self.accept_next_alert = True def test_untitled_test_case(self): driver = self.driver driver.get("https://t-admin.ibolit.pro/login") driver.find_element_by_id("login").click() driver.find_element_by_id("login").clear() driver.find_element_by_id("login").send_keys("*****@*****.**") driver.find_element_by_id("password").click() driver.find_element_by_id("password").clear() driver.find_element_by_id("password").send_keys("123456") driver.find_element_by_xpath("//button[@type='submit']").click() driver.find_element_by_link_text(u"Пациенты").click() driver.find_element_by_link_text(u"Добавить пациента").click() driver.find_element_by_id("full_name").click() driver.find_element_by_id("full_name").clear() driver.find_element_by_id("full_name").send_keys("Patient Test Two") driver.find_element_by_id("email").click() driver.find_element_by_id("email").clear() driver.find_element_by_id("email").send_keys("*****@*****.**") driver.find_element_by_id("phone").click() driver.find_element_by_id("phone").clear() driver.find_element_by_id("phone").send_keys("89600087485") time.sleep(1) driver.find_element_by_id("password").click() driver.find_element_by_id("password").clear() driver.find_element_by_id("password").send_keys("qwerty") driver.find_element_by_id("password_confirmation").click() driver.find_element_by_id("password_confirmation").clear() driver.find_element_by_id("password_confirmation").send_keys("qwerty") driver.find_element_by_id("select2-doctor-container").click() time.sleep(1) driver.find_element_by_xpath( "/html[1]/body[1]/span[1]/span[1]/span[2]/ul[1]/li[8]").click() driver.find_element_by_id("comment").click() driver.find_element_by_id("comment").clear() driver.find_element_by_id("comment").send_keys( "тестовые комментарии о пациенте ") driver.find_element_by_xpath("//button[@type='submit']").click() driver.find_element_by_xpath("//i").click() driver.find_element_by_link_text(u"Выйти").click() def is_element_present(self, how, what): try: self.driver.find_element(by=how, value=what) except NoSuchElementException as e: return False return True def is_alert_present(self): try: self.driver.switch_to_alert() except NoAlertPresentException as e: return False return True def close_alert_and_get_its_text(self): try: alert = self.driver.switch_to_alert() alert_text = alert.text if self.accept_next_alert: alert.accept() else: alert.dismiss() return alert_text finally: self.accept_next_alert = True
def scrape_campground(driver: WebDriver, campground: Campground, start_date: datetime, num_days: int) -> bool: """ Use Selenium WebDriver to load page, input desired start date, identify availability table for new data, use BeautifulSoup to parse html table, and use pandas DataFrame to identify availability inside the parsed table. Use Selenium's send_keys functionality to enter start date, see below for info: https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.keys https://stackoverflow.com/a/27799120 platform-specific "select-all": https://stackoverflow.com/a/29807390 Note on why we loop through ARROW_LEFT and BACKSPACE: - COMMAND/CTRL + 'a' doesn't work on linux - date_input.clear() doesn't work on any platform - BACKSPACE prior to sending date doesn't work on any platform - seems to be because recreation.gov auto-fills the date field if it is ever empty, which prevents us from clearing it. This way, we put in the date, backtrack to delete the old date, and then manually refresh the table. Works on mac/linux and headless/nonheadless. :param driver: WebDriver object previously instantiated :param campground: Campground object; url field will be loaded with driver :param start_date: datetime object identifying the date user wishes to arrive at campground :param num_days: int representation of number of nights user wishes to stay at campground :returns: True if start_date/num_days are available, False otherwise """ try: logger.debug("\tGetting campground.url (%s) with driver", campground.url) driver.get(campground.url) try: # check for tutorial window and close it if it appears, otherwise table doesn't load correctly tutorial_close_button = driver.find_element(by=By.XPATH, value=TUTORIAL_CLOSE_BUTTON_XPATH) logger.debug("\tClosing tutorial window") tutorial_close_button.click() except NoSuchElementException: logger.debug("\tNo tutorial this time") pass # we don't actually care if tutorial didn't appear, just move on logger.debug("\tFinding input box tag") start_date_input = wait_for_page_element_load(driver, START_DATE_INPUT_TAG_NAME) if start_date_input is None: # if wait for page element load fails -> abandon this check immediately return False logger.debug("\tInputting start/end dates with send_keys") enter_date_input(start_date, start_date_input) end_date = start_date + timedelta(days=num_days) end_date_input = wait_for_page_element_load(driver, END_DATE_INPUT_TAG_NAME) if end_date_input is None: # if wait for page element load fails -> abandon this check immediately return False enter_date_input(end_date, end_date_input) # wait for table refresh/loading spinning wheel to disappear, otherwise table contents are gibberish/NaN # https://stackoverflow.com/a/29084080 -- wait for element to *not* be visible # https://stackoverflow.com/a/51884408 -- wait for element not to be visible even though it # may already be invisible # https://stackoverflow.com/a/45420111 -- temporarily kill implicit waits to make explicit wait work corectly driver.implicitly_wait(0) WebDriverWait(driver, PAGE_LOAD_WAIT).until(EC.invisibility_of_element_located( (By.CLASS_NAME, TABLE_LOADING_TAG_CLASS))) driver.implicitly_wait(PAGE_LOAD_WAIT) logger.debug("\tFinding availability table tag") availability_table = wait_for_page_element_load(driver, AVAILABILITY_TABLE_TAG_NAME) if availability_table is None: # if wait for page element load fails -> abandon this check immediately return False table_html = availability_table.get_attribute('outerHTML') soup = BeautifulSoup(table_html, 'html.parser') df = parse_html_table(soup) dates_available = all_dates_available(df, start_date, num_days) campground.error_count = 0 # if not errored -> reset error count to 0 return dates_available except Exception as e: campground.error_count += 1 # if errored -> inc error count logger.exception("Campground %s (%s) parsing error!\n%s", campground.name, campground.id, e) logger.exception(str(traceback.format_exc())) return False
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By options = Options() options.headless = False driver = WebDriver(chrome_options=options) driver.get('https://www.baidu.com') # html = driver.page_source # 使用sax,dom,etree,lxml,bs4,xpath,css, # crawpy的xpath,css, XMLFeedSpider,CSVFeedSpider ele_input = driver.find_element(by=By.ID, value='kw') # driver.find_element_by_id('') # 等价于上面的函数 print(type(ele_input)) # ele_input.screenshot('ele_input.png') time.sleep(10) ele_input.send_keys('马哥教育') print(driver.page_source) time.sleep(5) ele_btn = driver.find_element_by_id('su') ele_btn.click() time.sleep(10) print(driver.page_source) driver.quit()
if re.match(r'^\t', lineList[-(x+1)]) is None: matched = lineList[-(x+1)] if "Medium" in matched: titleSearch = re.match(r'(^.*)?\sMedium', matched).group(1) if "Dummy_Length" in matched: titleSearch = re.match(r'(^.*)?\sDummy_Length', matched).group(1) while True: time.sleep(1) we = (driver.find_elements(By.XPATH, "//div[contains(@class,'vnbrowse')]/table/tbody/tr//a")) for x in we: if flagTripped: novel_URL.append(x.get_attribute("href")) elif x.text == titleSearch: flagTripped = True try: we = driver.find_element(By.XPATH, "//ul[contains(@class,'browsetabs')]//a[contains(.,'next')]") we.click() except exceptions.NoSuchElementException: break if flagTripped: vn_file = open("scrapedVNList.txt", "a") break else: vn_file = open("scrapedVNList.txt", "w") while True: time.sleep(1) we = (driver.find_elements(By.XPATH, "//div[contains(@class,'vnbrowse')]/table/tbody/tr//a")) for x in we: novel_URL.append(x.get_attribute("href")) try: time.sleep(10)
def wait_for_next(x: WebDriver): try: x.find_element(*locator).click() return x.find_element(By.ID, "username") except: return False
def xpath_click(driver: WebDriver, xpath): return driver.find_element(By.XPATH, xpath).click()