def wait_for_next(x: WebDriver):
# 反复点击【添加成员】，直到点击通过
try:
x.find_element(*locator).click()
return x.find_element(
By.ID, "username") # 这里为什么要 return 这个，上一句不是已经实现了点击操作了么？
except:
return False
def get_semester_list(driver: WebDriver):
dropdown = driver.find_element(By.ID, "DERIVED_SSS_SCL_SSS_MORE_ACADEMICS")
dropdown.find_element(By.XPATH, "//option[. = 'Votre horaire cours']").click()
driver.find_element(By.ID, "DERIVED_SSS_SCL_SSS_MORE_ACADEMICS").click()
driver.find_element(By.NAME, "DERIVED_SSS_SCL_SSS_GO_1$IMG").click()
driver.switch_to.default_content()
driver.switch_to.frame('ptifrmtgtframe')
wait_for_process_to_end(driver)
semester_elements = driver.find_elements_by_class_name('PSRADIOBUTTON')
return [i.find_element_by_xpath('./../../..').text for i in semester_elements]
def get_job_items_per_tab(driver: WebDriver, main_container: dict):
try:
list_containers = driver.find_elements(By.CLASS_NAME,
'list-container')[:-1]
keys = list(main_container.keys())
print("Fetching data...")
for idx in range(len(list_containers)):
job_list = []
job_container = list_containers[idx].find_elements(
By.TAG_NAME, 'div')[3]
job_items = job_container.find_elements(By.TAG_NAME, 'a')
for job_item in tqdm(job_items):
driver.execute_script(
f"window.open('{job_item.get_attribute('href')}', '_blank');"
)
windows = driver.window_handles
sleep(3)
driver.switch_to.window(windows[1])
driver.implicitly_wait(5)
# create a Job object from form input fields
company = driver.find_element(
By.XPATH,
"//input[@placeholder='Company']").get_attribute('value')
job_title = driver.find_element(
By.XPATH,
"//input[@placeholder='+ add title']").get_attribute(
'value')
location = driver.find_element(
By.XPATH,
"//input[@placeholder='+ add location']").get_attribute(
'value')
description = driver.find_element(By.CLASS_NAME,
'ql-editor').text
post_url = driver.find_element(
By.XPATH, "//p[@title='Post URL']/following-sibling::div"
).find_element(By.TAG_NAME, 'a').get_attribute('href')
a_job = Job(company, job_title, post_url, location,
description)
job_list.append(a_job.as_dict())
driver.close()
driver.switch_to.window(windows[0])
main_container[keys[idx]] = job_list
except Exception as err:
print(f"Error getting job_container:" + str(err))
sys.exit(2)
def get_archived_board(driver: WebDriver):
try:
board_title = driver.find_element(
By.XPATH, f"//p[@title='{os.getenv('ARCHIVED_BOARD_NAME')}']")
board_title.click()
except Exception as err:
print(f"Error finding your board {os.getenv('ARCHIVED_BOARD_NAME')}:" +
str(err))
sys.exit(2)
def login(driver: WebDriver):
try:
driver.get(os.getenv("HUNTR_BOARD_URL"))
driver.implicitly_wait(5)
email_input = driver.find_element(By.XPATH, "//input[@type='email']")
password_input = driver.find_element(By.XPATH,
"//input[@type='password']")
email_input.send_keys(os.getenv("HUNTR_USERNAME"))
password_input.send_keys(os.getenv("HUNTR_PASSWORD"))
driver.implicitly_wait(5)
password_input.send_keys(Keys.ENTER)
driver.implicitly_wait(5)
print("Logged In!")
sleep(1 + random.random() * 4)
except Exception as err:
print(f"Error logging in with supplied credentials for "
f"job board {os.getenv('HUNTR_BOARD_URL')} :" + str(err))
sys.exit(2)
class Application:
def __init__(self):
self.wd = WebDriver()
self.wd.implicitly_wait(5)
self.session = SessionHelper(self)
self.payment = PaymentHelper(self)
self.localization = LocalizationHelper(self)
self.navigation = NavigationHelper(self)
self.login_page = LoginPage(self)
self.currency = Currency(self)
self.payment_systems = PaymentSystems(self)
def experiment(self, a):
self.wd.find_element(By.cssSelector('span[class=\"title\"]:contains(' + someText + ')'))
self.wd.find_element_by_xpath()
self.wd.find_element_by_class_name('forgot')
self.wd.find_element_by_link_text('Forgot password')
self.wd.find_element_by_css_selector()
def check_exists_by_css_selector(self, selector):
try:
self.wd.find_elements_by_css_selector(selector)
except NoSuchElementException:
return False
return True
def check_exists_by_xpath(self, xpath):
try:
self.wd.find_element_by_xpath(xpath)
except NoSuchElementException:
return False
return True
def destroy(self):
self.wd.quit()
def is_bad_date(driver: WebDriver, element_id) -> Tuple[bool, str]:
"""
Entering the date improperly causes dynamic text to appear right below the input box
indicating whether the date is invalid (formatting issue) or unavailable. Read this
text and return the appropriate message to the caller.
NOTE: not currently used, but good to keep here in case the site changes and we need
this pattern again.
"""
date_error_msg = driver.find_element(by=By.ID, value=element_id)
invalid_str = "not valid"
unavailable_str = "not available"
logger.info(date_error_msg.text)
if date_error_msg is not None:
if unavailable_str in date_error_msg.text:
return (True, unavailable_str)
if invalid_str in date_error_msg.text:
return (True, invalid_str)
return (True, "new error")
return (False, "all good")
class UntitledTestCase(unittest.TestCase):
def setUp(self):
self.driver = WebDriver()
self.driver.implicitly_wait(30)
self.base_url = "https://www.katalon.com/"
self.verificationErrors = []
self.accept_next_alert = True
def test_untitled_test_case(self):
driver = self.driver
driver.get("https://t-admin.ibolit.pro/login")
driver.find_element_by_id("login").click()
driver.find_element_by_id("login").clear()
driver.find_element_by_id("login").send_keys("*****@*****.**")
driver.find_element_by_id("password").click()
driver.find_element_by_id("password").clear()
driver.find_element_by_id("password").send_keys("123456")
driver.find_element_by_xpath("//button[@type='submit']").click()
driver.find_element_by_link_text(u"Пациенты").click()
driver.find_element_by_link_text(u"Добавить пациента").click()
driver.find_element_by_id("full_name").click()
driver.find_element_by_id("full_name").clear()
driver.find_element_by_id("full_name").send_keys("Patient Test Two")
driver.find_element_by_id("email").click()
driver.find_element_by_id("email").clear()
driver.find_element_by_id("email").send_keys("*****@*****.**")
driver.find_element_by_id("phone").click()
driver.find_element_by_id("phone").clear()
driver.find_element_by_id("phone").send_keys("89600087485")
time.sleep(1)
driver.find_element_by_id("password").click()
driver.find_element_by_id("password").clear()
driver.find_element_by_id("password").send_keys("qwerty")
driver.find_element_by_id("password_confirmation").click()
driver.find_element_by_id("password_confirmation").clear()
driver.find_element_by_id("password_confirmation").send_keys("qwerty")
driver.find_element_by_id("select2-doctor-container").click()
time.sleep(1)
driver.find_element_by_xpath(
"/html[1]/body[1]/span[1]/span[1]/span[2]/ul[1]/li[8]").click()
driver.find_element_by_id("comment").click()
driver.find_element_by_id("comment").clear()
driver.find_element_by_id("comment").send_keys(
"тестовые комментарии о пациенте ")
driver.find_element_by_xpath("//button[@type='submit']").click()
driver.find_element_by_xpath("//i").click()
driver.find_element_by_link_text(u"Выйти").click()
def is_element_present(self, how, what):
try:
self.driver.find_element(by=how, value=what)
except NoSuchElementException as e:
return False
return True
def is_alert_present(self):
try:
self.driver.switch_to_alert()
except NoAlertPresentException as e:
return False
return True
def close_alert_and_get_its_text(self):
try:
alert = self.driver.switch_to_alert()
alert_text = alert.text
if self.accept_next_alert:
alert.accept()
else:
alert.dismiss()
return alert_text
finally:
self.accept_next_alert = True
def scrape_campground(driver: WebDriver, campground: Campground, start_date: datetime, num_days: int) -> bool:
"""
Use Selenium WebDriver to load page, input desired start date, identify availability table
for new data, use BeautifulSoup to parse html table, and use pandas DataFrame to identify
availability inside the parsed table.
Use Selenium's send_keys functionality to enter start date, see below for info:
https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.keys
https://stackoverflow.com/a/27799120
platform-specific "select-all": https://stackoverflow.com/a/29807390
Note on why we loop through ARROW_LEFT and BACKSPACE:
- COMMAND/CTRL + 'a' doesn't work on linux
- date_input.clear() doesn't work on any platform
- BACKSPACE prior to sending date doesn't work on any platform
- seems to be because recreation.gov auto-fills the date field if it is ever empty,
which prevents us from clearing it. This way, we put in the date, backtrack to
delete the old date, and then manually refresh the table. Works on mac/linux
and headless/nonheadless.
:param driver: WebDriver object previously instantiated
:param campground: Campground object; url field will be loaded with driver
:param start_date: datetime object identifying the date user wishes to arrive at campground
:param num_days: int representation of number of nights user wishes to stay at campground
:returns: True if start_date/num_days are available, False otherwise
"""
try:
logger.debug("\tGetting campground.url (%s) with driver", campground.url)
driver.get(campground.url)
try: # check for tutorial window and close it if it appears, otherwise table doesn't load correctly
tutorial_close_button = driver.find_element(by=By.XPATH, value=TUTORIAL_CLOSE_BUTTON_XPATH)
logger.debug("\tClosing tutorial window")
tutorial_close_button.click()
except NoSuchElementException:
logger.debug("\tNo tutorial this time")
pass # we don't actually care if tutorial didn't appear, just move on
logger.debug("\tFinding input box tag")
start_date_input = wait_for_page_element_load(driver, START_DATE_INPUT_TAG_NAME)
if start_date_input is None: # if wait for page element load fails -> abandon this check immediately
return False
logger.debug("\tInputting start/end dates with send_keys")
enter_date_input(start_date, start_date_input)
end_date = start_date + timedelta(days=num_days)
end_date_input = wait_for_page_element_load(driver, END_DATE_INPUT_TAG_NAME)
if end_date_input is None: # if wait for page element load fails -> abandon this check immediately
return False
enter_date_input(end_date, end_date_input)
# wait for table refresh/loading spinning wheel to disappear, otherwise table contents are gibberish/NaN
# https://stackoverflow.com/a/29084080 -- wait for element to *not* be visible
# https://stackoverflow.com/a/51884408 -- wait for element not to be visible even though it
# may already be invisible
# https://stackoverflow.com/a/45420111 -- temporarily kill implicit waits to make explicit wait work corectly
driver.implicitly_wait(0)
WebDriverWait(driver, PAGE_LOAD_WAIT).until(EC.invisibility_of_element_located(
(By.CLASS_NAME, TABLE_LOADING_TAG_CLASS)))
driver.implicitly_wait(PAGE_LOAD_WAIT)
logger.debug("\tFinding availability table tag")
availability_table = wait_for_page_element_load(driver, AVAILABILITY_TABLE_TAG_NAME)
if availability_table is None: # if wait for page element load fails -> abandon this check immediately
return False
table_html = availability_table.get_attribute('outerHTML')
soup = BeautifulSoup(table_html, 'html.parser')
df = parse_html_table(soup)
dates_available = all_dates_available(df, start_date, num_days)
campground.error_count = 0 # if not errored -> reset error count to 0
return dates_available
except Exception as e:
campground.error_count += 1 # if errored -> inc error count
logger.exception("Campground %s (%s) parsing error!\n%s", campground.name, campground.id, e)
logger.exception(str(traceback.format_exc()))
return False
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.webdriver import WebDriver
from selenium.webdriver.common.by import By
options = Options()
options.headless = False
driver = WebDriver(chrome_options=options)
driver.get('https://www.baidu.com')
# html = driver.page_source
# 使用sax，dom，etree，lxml，bs4，xpath,css,
# crawpy的xpath，css, XMLFeedSpider,CSVFeedSpider
ele_input = driver.find_element(by=By.ID, value='kw')
# driver.find_element_by_id('') # 等价于上面的函数
print(type(ele_input))
# ele_input.screenshot('ele_input.png')
time.sleep(10)
ele_input.send_keys('马哥教育')
print(driver.page_source)
time.sleep(5)
ele_btn = driver.find_element_by_id('su')
ele_btn.click()
time.sleep(10)
print(driver.page_source)
driver.quit()
if re.match(r'^\t', lineList[-(x+1)]) is None:
matched = lineList[-(x+1)]
if "Medium" in matched:
titleSearch = re.match(r'(^.*)?\sMedium', matched).group(1)
if "Dummy_Length" in matched:
titleSearch = re.match(r'(^.*)?\sDummy_Length', matched).group(1)
while True:
time.sleep(1)
we = (driver.find_elements(By.XPATH, "//div[contains(@class,'vnbrowse')]/table/tbody/tr//a"))
for x in we:
if flagTripped:
novel_URL.append(x.get_attribute("href"))
elif x.text == titleSearch:
flagTripped = True
try:
we = driver.find_element(By.XPATH, "//ul[contains(@class,'browsetabs')]//a[contains(.,'next')]")
we.click()
except exceptions.NoSuchElementException:
break
if flagTripped:
vn_file = open("scrapedVNList.txt", "a")
break
else:
vn_file = open("scrapedVNList.txt", "w")
while True:
time.sleep(1)
we = (driver.find_elements(By.XPATH, "//div[contains(@class,'vnbrowse')]/table/tbody/tr//a"))
for x in we:
novel_URL.append(x.get_attribute("href"))
try:
time.sleep(10)
def wait_for_next(x: WebDriver):
try:
x.find_element(*locator).click()
return x.find_element(By.ID, "username")
except:
return False
def xpath_click(driver: WebDriver, xpath):
return driver.find_element(By.XPATH, xpath).click()
