def get_vote_dict(driver: webdriver.Chrome,
js_dict: dict,
members: list,
check_interval: int = 10) -> dict:
def get_arr(s):
if '없습니다' in s:
return []
else:
return s.split()
driver.get(BASE_URL)
sleep(1)
vote_dict = {}
for i, (bill_no, js) in enumerate(js_dict.items()):
driver.execute_script(js)
sleep(2)
pro, con, wdr, _ = [
get_arr(x.text) for x in driver.find_elements_by_tag_name('tbody')
]
vote_dict[bill_no] = get_vote_row(members, pro, con, wdr)
driver.back()
sleep(1)
if i % check_interval == 0:
print('Complete: {} of {}'.format(i, inspect.stack()[0][3]))
print('Complete: {}'.format(inspect.stack()[0][3]))
return vote_dict
class Browser:
def __init__(self, site_url, driverfile):
self.driver = Chrome(driverfile)
self.base_url = site_url
random.seed()
def getDriver(self):
return self.driver
def openURL(self, url, delay=0):
self.driver.get(url)
self.wait(delay)
def openSubPath(self, subpath, delay=0):
if not subpath.startswith('/'):
subpath = '/' + subpath
self.openURL(self.base_url + subpath, delay)
def quit(self):
self.driver.close()
def goBack(self):
self.driver.back()
def goForward(self):
self.driver.forward()
def scrollDown(self):
self.driver.execute_script(
"window.scrollTo(0, document.body.scrollHeight);")
self.wait(1)
def wait(self, seconds):
if (seconds > 0):
time.sleep(seconds)
def microdelay(self):
time.sleep(abs(round(random.gauss(0.3, 0.2), 2)))
defaulttimeout = 8
def getElementBy(self, method, query_string, timeout):
try:
return WebDriverWait(self.driver, timeout).until(
EC.presence_of_element_located((method, query_string)))
except:
return None
def getElementByXPath(self, xpath, timeout=defaulttimeout):
return self.getElementBy(By.XPATH, xpath, timeout)
def getElementByTag(self, tag, timeout=defaulttimeout):
return self.getElementBy(By.TAG_NAME, tag, timeout)
def getElementsByTag(self, tag):
return self.driver.find_elements_by_tag_name(tag)
def getBaseURL(self):
return self.base_url
def get_bk_event(conn, cur):
company = "burger king"
driver = Chrome()
driver.get(URL_1)
# access event page
event_page = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[1]/div/div/div[2]/ul/li[3]/button")
event_page.click()
event_page = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[1]/div/div/div[2]/ul/li[3]/ul/li[1]/a/span")
event_page.click()
driver.implicitly_wait(100)
event_1 = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/ul/li[1]/button/span/img"
)
event_1.click()
driver.back()
xpath_front = "//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/ul/li["
xpath_rear = "]/button/span/img"
for i in range(1, 15):
driver.implicitly_wait(100)
xpath = xpath_front + str(i) + xpath_rear
event = driver.find_element_by_xpath(xpath)
image = event.get_attribute("src")
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/div[1]/h4")
title = title_loc.text
date_loc = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/div[1]/p")
date = date_loc.text
content_loc = driver.find_element_by_xpath(
"//*[@id=\"app\"]/div/div[3]/div[2]/div/div[2]/div[2]/div[1]/p")
content = content_loc.text
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
driver.back()
driver.close()
def get_ph_event(conn, cur):
company = "pizza hut"
driver = Chrome()
driver.get(URL_7)
driver.implicitly_wait(100)
event_loc = driver.find_element_by_xpath("//*[@id=\"gnb\"]/ul/li[3]/a")
event_loc.click()
driver.implicitly_wait(100)
xpath_front = "//*[@id=\"event\"]/div/div/div[3]/ul/li["
xpath_rear = "]"
for i in range(1, 9):
time.sleep(2)
xpath = xpath_front + str(i) + xpath_rear
image_loc = driver.find_element_by_xpath(xpath + "/a")
image = image_loc.get_attribute("style")
image = image.split("(")[1]
image = image.split(")")[0]
image = image.replace("\"", "")
image_loc.click()
time.sleep(3)
title_loc = driver.find_element_by_xpath(
"//*[@id=\"event\"]/div/div/div[3]/div[1]")
title = title_loc.text
date_loc = driver.find_element_by_xpath(
"//*[@id=\"event\"]/div/div/div[3]/div[2]")
date = date_loc.text
content_loc = driver.find_element_by_xpath(
"//*[@id=\"event\"]/div/div/div[4]/div/div/span/img")
content = content_loc.get_attribute("src")
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
driver.back()
driver.close()
def get_md_event(conn, cur):
company = "mcdonalds"
driver = Chrome()
driver.get(URL_2)
xpath_front = "//*[@id=\"promotionList\"]/li["
xpath_rear = "]"
for i in range(1, 6):
driver.implicitly_wait(100)
xpath = xpath_front + str(i) + xpath_rear
event = driver.find_element_by_xpath(xpath)
image_loc = driver.find_element_by_xpath(
"//*[@id=\"promotionList\"]/li[" + str(i) + "]/a/div[1]/img")
image = image_loc.get_attribute('src')
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@id=\"container\"]/div[1]/div[1]/div[2]/div/div/div[1]/h2")
title = title_loc.text
title = title.replace('\n', '')
date_loc = driver.find_element_by_xpath(
"//*[@id=\"container\"]/div[1]/div[1]/div[2]/div/div/div[1]/span/em[1]"
)
date = date_loc.text
date = date.replace('\n', '')
date = date.replace('등록일 :', '')
content_loc = driver.find_element_by_xpath(
"//*[@id=\"container\"]/div[1]/div[1]/div[2]/div/div/article/div[1]/img"
)
content = content_loc.get_attribute('src')
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
driver.back()
driver.close()
def get_edy_event(conn, cur):
company = "ediya"
driver = Chrome()
driver.get(URL_6)
driver.implicitly_wait(100)
xpath_front = "//*[@id=\"contentWrap\"]/div[2]/div[2]/div/ul/li["
xpath_rear = "]"
for i in range(1, 11):
time.sleep(2)
xpath = xpath_front + str(i) + xpath_rear
image_loc = driver.find_element_by_xpath(xpath + "/div[1]/a/img")
image = image_loc.get_attribute("src")
title_loc = driver.find_element_by_xpath(xpath + "/dl/dt/a")
title = title_loc.text
date_loc = driver.find_element_by_xpath(xpath + "/dl/dd")
date = date_loc.text
date = date.split(":")[1]
image_loc.click()
time.sleep(2)
content_loc = driver.find_element_by_xpath(
"//*[@id=\"contentWrap\"]/div[2]/div[2]/div/form/div[3]/p[1]/img")
content = content_loc.get_attribute("src")
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
driver.back()
driver.close()
def get_bbq_event(conn, cur):
company = "bbq"
driver = Chrome()
driver.get(URL_4)
driver.implicitly_wait(100)
xpath = "//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/div[1]/a/img"
event = driver.find_element_by_xpath(xpath)
image = event.get_attribute("src")
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/h3")
title = title_loc.text
print(title)
date_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/ul/li[1]")
date = date_loc.text
print(date)
content_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[2]/img")
content = content_loc.get_attribute("src")
print(content)
print(image)
driver.back()
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
time.sleep(2)
driver.implicitly_wait(100)
xpath = "//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[2]/div[1]/a/img"
event = driver.find_element_by_xpath(xpath)
image = event.get_attribute("src")
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/h3")
title = title_loc.text
print(title)
date_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/ul/li[1]")
date = date_loc.text
print(date)
content_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[2]/p/img")
content = content_loc.get_attribute("src")
print(content)
print(image)
driver.back()
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
time.sleep(2)
driver.implicitly_wait(100)
xpath = "//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[3]/div[1]/a/img"
event = driver.find_element_by_xpath(xpath)
image = event.get_attribute("src")
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/h3")
title = title_loc.text
print(title)
date_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/ul/li[1]")
date = date_loc.text
print(date)
content_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[2]/p/a/img")
content = content_loc.get_attribute("src")
print(content)
print(image)
driver.back()
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
time.sleep(2)
driver.implicitly_wait(100)
xpath = "//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[4]/div[1]/a/img"
event = driver.find_element_by_xpath(xpath)
image = event.get_attribute("src")
event.click()
time.sleep(2)
title_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/h3")
title = title_loc.text
print(title)
date_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[1]/ul/li[1]")
date = date_loc.text
print(date)
content_loc = driver.find_element_by_xpath(
"//*[@class=\"wrapper\"]/div[2]/article/div[2]/div[2]/p/img")
content = content_loc.get_attribute("src")
print(content)
print(image)
driver.back()
if insert_event_list(conn, cur, company, date, image, title, content):
message = title + ' inserted in db' + '\n'
print(message)
driver.close()
def nav_to_summary_page(driver: webdriver.Chrome) -> None:
driver.back()
print("Waiting for summary page to load ...")
WebDriverWait(driver, 60).until(
EC.presence_of_element_located(
(By.CSS_SELECTOR, "div.ms-srch-item-body")))
def main():
mail = input("Enter teams username:"******"Enter teams password:"******"-o" or "--show-output":
driver = Chrome()
else:
chromeOptions = Options()
chromeOptions.add_argument('headless')
driver = Chrome(options=chromeOptions)
except:
chromeOptions = Options()
chromeOptions.add_argument('headless')
driver = Chrome(options=chromeOptions)
driver.get("https://teams.microsoft.com")
email = wait("name", "loginfmt", driver)
email.send_keys(mail)
delay(random.randint(1, 3))
wait("id", "idSIButton9", driver).click()
print("Sent mail id to teams")
delay(random.randint(1, 3))
password = wait(
"xpath",
"//input[@class = 'form-control input ext-input text-box ext-text-box']",
driver)
password.send_keys(passw)
delay(random.randint(1, 3))
wait("xpath", "//input[@type='submit']", driver).click()
print("Sent password to teams")
try:
delay(random.randint(1, 3))
wait("xpath",
"//input[@class = 'button ext-button secondary ext-secondary']",
driver).click()
delay(random.randint(1, 3))
print("Just sit back and relax")
except:
print("Just sit back and relax")
try:
teams = waitall("class", "team-card", driver, 20)
print("Got all ur the teams ;)")
except:
print(
"Ur teams are not organized according to this code... usually code is supposed to handle this... but this code was made for a very small set of people... if you want the feature included... let me know"
)
exit()
delay(random.randint(1, 3))
i = 0
while teams:
teams1 = waitall("class", "team-card", driver, 20)
teams1[i].click()
delay(random.randint(1, 3))
try:
meets = waitall(
"xpath",
"//div[@title = 'Click to see details of this meeting']",
driver)
if len(meets) > 1:
print("Too many meetings to handle... but lemme give it a try")
x = 0
while meets:
meets1 = waitall(
"xpath",
"//div[@title = 'Click to see details of this meeting']",
driver)
meets1[x].click()
delay(random.randint(1, 3))
try:
wait("xpath",
"//button[@data-tid='calv2-sf-add-to-calendar']",
driver).click()
print("Meeting added to calender")
except:
print("Meeting already in Calender.")
print(
"Could also be other issue... code is still in beta\n"
)
x += 1
meets.pop(0)
delay(random.randint(1, 3))
driver.back()
else:
meets[0].click()
delay(random.randint(1, 3))
try:
wait("xpath",
"//button[@data-tid='calv2-sf-add-to-calendar']",
driver).click()
print("Meeting added to calender")
except:
print("Meeting already in Calender.")
print(
"Could also be other issue... code is still in beta\n")
delay(random.randint(1, 3))
driver.back()
except:
print("Looks like you dont have any meetings...")
delay(random.randint(1, 3))
try:
wait(
"xpath",
"//button[@class = 'school-app-back-button ts-sym app-icons-fill-hover app-icons-fill-focus button-command focus-round-border']",
driver).click()
except:
wait(
"xpath",
"//button[@id = 'app-bar-66aeee93-507d-479a-a3ef-8f494af43945']",
driver).click()
delay(random.randint(1, 3))
i += 1
teams.pop(0)
wait("xpath",
"//button[@id = 'app-bar-ef56c0de-36fc-4ef8-b417-3d82ba9d073c']",
driver).click()
delay(random.randint(1, 3))
#upcoming = waitall("css","div.node_modules--msteams-bridges-components-calendar-event-card-dist-es-src-renderers-event-card-renderer-event-card-renderer__eventCard--h5y4X",driver,20)
#running = waitall("class","node_modules--msteams-bridges-components-calendar-event-card-dist-es-src-renderers-event-card-renderer-event-card-renderer__eventCard--h5y4X node_modules--msteams-bridges-components-calendar-event-card-dist-es-src-renderers-event-card-renderer-event-card-renderer__activeCall--25Ch-",driver)
#print("upcoming:",len(upcoming))
#print("running:",len(running))
print(
"Thank you for running the code and giving me all ur details.... Just kidding.. the code is open-source it is safe... check for yourself."
)
delay(5)
driver.quit()
class PortalBrowsing(unittest.TestCase):
def setUp(self):
# test with Firefox, without headless()
# self.driver = Firefox(executable_path='/opt/WebDriver/bin/geckodriver')
# test with Firefox, with headless()
# options = FirefoxOptions()
# options.headless = True
# self.driver = Firefox(executable_path='/opt/WebDriver/bin/geckodriver', options=options)
# test with chrome, without headless()
# self.driver = Chrome('/opt/WebDriver/bin/chromedriver')
# test with chrome, with headless()
options = ChromeOptions()
options.headless = True
# options.headless = False
path_local = '/opt/WebDriver/bin/chromedriver'
path_github_actions = '/home/runner/work/slate-portal/slate-portal/chromedriver'
self.driver = Chrome(executable_path=path_github_actions,
options=options)
# portal on minislate
self.driver.get('http://localhost:5000/slate_portal')
# slate portal
# self.driver.get('https://portal.slateci.io/slate_portal')
self.driver.set_window_size(1920, 1080)
def test_iterate_clusters_pages(self):
helpers = Helpers()
clusters_page = helpers.segue_to_page(self.driver, 'clusters')
page_number = 1
click_next = True
while click_next:
clusters_page.wait_until_clusters_table_loaded()
clusters_links = clusters_page.get_clusters_links_on_cur_page()
num_of_links = len(clusters_links)
# print('page number', page_number)
for i in range(num_of_links):
# print('Testing cluster page:', clusters_links[i].text)
clusters_links[i].click()
clusters_profile_page = page.ClusterPublicProfilePage(
self.driver)
assert clusters_profile_page.is_page_valid()
self.driver.back()
clusters_page.wait_until_clusters_table_loaded()
clusters_page.click_cur_page(page_number)
clusters_page.wait_until_clusters_table_loaded()
clusters_links = clusters_page.get_clusters_links_on_cur_page()
next_button = clusters_page.get_next_button()
if next_button.get_attribute('class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
def test_iterate_apps_pages(self):
helpers = Helpers()
apps_page = helpers.segue_to_page(self.driver, 'applications')
for tab_name in ['Stable Applications', 'Incubator Applications']:
page_number = 1
click_next = True
if tab_name == 'Incubator Applications':
apps_page.click_incubator_apps_tab()
self.driver.implicitly_wait(
3
) # implicitly waiting for Incubator Applications tab loaded
while click_next:
apps_page.wait_until_apps_table_loaded(tab_name)
app_links = apps_page.get_app_links_on_cur_page(tab_name)
number_of_links = len(app_links)
# print('page number', page_number)
for i in range(number_of_links):
# print('Testing app page:', app_links[i].text)
app_links[i].click()
app_detail_page = page.AppsDetailPage(self.driver)
assert app_detail_page.is_page_valid()
self.driver.back()
if tab_name == 'Incubator Applications':
apps_page.click_incubator_apps_tab()
apps_page.wait_until_apps_table_loaded(tab_name)
# make sure on the right apps page
apps_page.click_cur_page(tab_name, page_number)
apps_page.wait_until_apps_table_loaded(tab_name)
app_links = apps_page.get_app_links_on_cur_page(tab_name)
next_button = apps_page.get_next_button(tab_name)
if next_button.get_attribute(
'class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
def test_iterate_instances_pages(self):
helpers = Helpers()
# add an new instance
app_name = 'nginx'
app_suffix = 'add-for-test-iterate'
helpers.add_instance(self.driver, app_name, app_suffix=app_suffix)
# start clicking pages
instances_page = helpers.segue_to_page(self.driver, 'instances')
page_number = 1
click_next = True
while click_next:
instances_page.wait_until_instances_table_loaded()
instance_links = instances_page.get_instance_links_on_cur_page()
num_of_links = len(instance_links)
# print('page number', page_number)
for i in range(num_of_links):
# print('Testing instance page:', instance_links[i].text)
instance_links[i].click()
instance_detail_page = page.InstanceProfilePage(self.driver)
assert instance_detail_page.is_page_valid()
self.driver.back()
instances_page.wait_until_instances_table_loaded()
instances_page.click_cur_page(page_number)
instances_page.wait_until_instances_table_loaded()
instance_links = instances_page.get_instance_links_on_cur_page(
)
next_button = instances_page.get_next_button()
if next_button.get_attribute('class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
# delete the new instance
helpers.delete_instance(self.driver, app_name, app_suffix)
def test_iterate_secrets_pages(self):
helpers = Helpers()
# create a group of secrets
added_secrets = helpers.add_mul_secrets(self.driver)
# iterate secrets
secrets_page = helpers.segue_to_page(self.driver, 'secrets')
page_number = 1
click_next = True
while click_next:
secrets_page.wait_until_secrets_table_loaded()
secrets_links = secrets_page.get_secret_links_on_cur_page()
num_of_links = len(secrets_links)
# print(num_of_links)
# print('page number', page_number)
for i in range(num_of_links):
# print('Testing secret page:', secrets_links[i].text)
secrets_links[i].click()
group_profile_page = page.GroupProfilePage(self.driver)
assert group_profile_page.is_page_valid()
self.driver.back()
secrets_page.wait_until_secrets_table_loaded()
secrets_page.click_cur_page(page_number)
secrets_page.wait_until_secrets_table_loaded()
secrets_links = secrets_page.get_secret_links_on_cur_page()
next_button = secrets_page.get_next_button()
if next_button.get_attribute('class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
# delete the added secrets
for secret in added_secrets['secret_names']:
helpers.delete_secret(self.driver, added_secrets['group_name'],
added_secrets['cluster_name'], secret)
def test_iterate_my_groups_pages(self):
helpers = Helpers()
my_groups_page = helpers.segue_to_page(self.driver, 'my_groups')
page_number = 1
click_next = True
while click_next:
my_groups_page.wait_until_groups_table_loaded()
my_groups_links = my_groups_page.get_group_links_on_cur_page()
num_of_links = len(my_groups_links)
# print('page number', page_number)
for i in range(num_of_links):
# print('Testing group page:', my_groups_links[i].text)
my_groups_links[i].click()
my_groups_detail_page = page.GroupProfilePage(self.driver)
assert my_groups_detail_page.is_page_valid()
self.driver.back()
my_groups_page.wait_until_groups_table_loaded()
my_groups_page.click_cur_page(page_number)
my_groups_page.wait_until_groups_table_loaded()
my_groups_links = my_groups_page.get_group_links_on_cur_page()
next_button = my_groups_page.get_next_button()
if next_button.get_attribute('class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
def test_iterate_all_groups_pages(self):
helpers = Helpers()
all_groups_page = helpers.segue_to_page(self.driver, 'all_groups')
page_number = 1
click_next = True
while click_next:
all_groups_page.wait_until_groups_table_loaded()
all_groups_links = all_groups_page.get_group_links_on_cur_page()
num_of_links = len(all_groups_links)
# print('page number', page_number)
for i in range(num_of_links):
# print('Testing group page:', all_groups_links[i].text)
all_groups_links[i].click()
all_groups_detail_page = page.GroupProfilePage(self.driver)
assert all_groups_detail_page.is_page_valid()
self.driver.back()
all_groups_page.wait_until_groups_table_loaded()
all_groups_page.click_cur_page(page_number)
all_groups_page.wait_until_groups_table_loaded()
all_groups_links = all_groups_page.get_group_links_on_cur_page(
)
next_button = all_groups_page.get_next_button()
if next_button.get_attribute('class').split()[-1] == 'disabled':
click_next = False
else:
next_button.click()
page_number += 1
def test_check_cli_access_page(self):
helpers = Helpers()
helpers.segue_to_page(self.driver, 'cli_access')
def tearDown(self):
# self.driver.implicitly_wait(3)
# time.sleep(3)
self.driver.close()
from selenium.webdriver import Chrome
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = Chrome()
driver.get("https://www.baidu.com/")
driver.implicitly_wait(5) # 跟time.sleep类似，比time.sleep更加智能，不会被动的等待时间结束
wait = WebDriverWait(driver, 10) # 等待时间
ele = wait.until(EC.presence_of_element_located((By.ID, "kw")))
ele.send_keys("数学")
wait = WebDriverWait(driver, 10)
ele = wait.until(EC.element_to_be_clickable((By.ID, 'su')))
ele.click()
driver.get('https://docs.python.org/3/library/multiprocessing.html')
driver.get('https://blog.csdn.net/DarrenXf/article/details/82110962')
i = 1
while i <= 5:
time.sleep(1)
driver.back() # 后退
driver.forward() # 前进
i = i + 1
class WebPageBrowsing(unittest.TestCase):
def setUp(self):
# path = '/Users/lizhang/Documents/webdriver/chromedriver'
# path = '/home/runner/work/slateci-unit-tests/slateci-unit-tests/chromedriver'
path = '/home/runner/work/slateci.github.io/slateci.github.io/chromedriver'
options = ChromeOptions()
options.headless = True
self.driver = Chrome(executable_path=path, options=options)
self.driver.get('https://slateci.io/')
self.driver.set_window_size(1920, 1080)
def test_home_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_home_page()
home_page = page.HomePage(self.driver)
assert home_page.is_page_valid()
home_page.wait_for_page_loaded()
links_in_try_slate = home_page.get_links_in_try_slate()
number_of_links = len(links_in_try_slate)
for i in range(number_of_links):
print('Home Page: Try SLATE -> {}'.format(
links_in_try_slate[i].text))
links_in_try_slate[i].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
self.driver.back()
home_page.wait_for_page_loaded()
links_in_try_slate = home_page.get_links_in_try_slate()
# test Edit This Page on GitHub link
# edit_on_github_link = home_page.getEditOnGitHubLink()
# edit_on_github_link.click()
def test_about_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_about_page()
about_page = page.AboutPage(self.driver)
assert about_page.is_page_valid()
about_page.wait_for_page_loaded()
links = about_page.get_all_links()
number_of_links = len(links)
for i in range(number_of_links):
print(links[i].get_attribute('href'))
links[i].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
print(cur_page.get_page_title())
if cur_page.get_page_title() == 'SLATE / About':
continue
self.driver.back()
about_page.wait_for_page_loaded()
links = about_page.get_all_links()
def test_tech_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_tech_page()
tech_page = page.TechPage(self.driver)
assert tech_page.is_page_valid()
tech_page.wait_for_page_loaded()
links = tech_page.get_all_links()
number_of_links = len(links)
for i in range(number_of_links):
# print(links[i].get_attribute('href'))
print('Tech Page: {}'.format(links[i].text))
links[i].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
self.driver.back()
# reload the page and get links
tech_page.wait_for_page_loaded()
links = tech_page.get_all_links()
def test_docs_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_docs_page()
docs_page = page.DocsPage(self.driver)
assert docs_page.is_page_valid()
docs_page.wait_for_page_loaded()
side_menu_btns = docs_page.get_main_items_in_side_menu()
number_of_btns = len(side_menu_btns)
for i in range(number_of_btns):
# print(side_menu_btns[i].get_attribute('href'))
linkL1 = side_menu_btns[i].text
side_menu_btns[i].click()
self.driver.implicitly_wait(5)
assert docs_page.is_page_valid()
active_side_links = docs_page.get_links_in_active_side_item()
if not active_side_links:
docs_page.iterate_links_doc_content()
side_menu_btns = docs_page.get_main_items_in_side_menu()
continue
number_of_active_links = len(active_side_links)
for j in range(number_of_active_links):
# print(active_side_links[j].get_attribute('href'))
linkL2 = active_side_links[j].text
active_side_links[j].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
docs_page.iterate_links_doc_content(linkL1, linkL2)
# get the side links again
active_side_links = docs_page.get_links_in_active_side_item()
side_menu_btns = docs_page.get_main_items_in_side_menu()
def test_blog_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_blog_page()
blog_page = page.BlogPage(self.driver)
assert blog_page.is_page_valid()
blog_page.wait_for_page_loaded()
while True:
# iterate links
links = blog_page.get_links_in_container_blog()
number_of_links = len(links)
for i in range(number_of_links):
# print(links[i].get_attribute('href'))
print('Blog Page: {}'.format(links[i].text))
if links[i].text == 'Older' or links[
i].text == 'Newer' or links[i].get_attribute(
'href') == 'https://slateci.io/feed.xml':
continue
links[i].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
self.driver.back()
blog_page.wait_for_page_loaded()
links = blog_page.get_links_in_container_blog()
# click Order button
older_btn = blog_page.get_older_btn()
if not older_btn:
break
older_btn.click()
blog_page.wait_for_page_loaded()
def test_comm_page(self):
main_page = page.BasePage(self.driver)
main_page.go_to_comm_page()
comm_page = page.CommPage(self.driver)
assert comm_page.is_page_valid()
comm_page.wait_for_page_loaded()
links = comm_page.get_links_in_container_community()
number_of_links = len(links)
for i in range(number_of_links):
if 'mailto:' in links[i].get_attribute('href'):
continue
print('Community Page: {}'.format(links[i].text))
links[i].click()
self.driver.implicitly_wait(5)
cur_page = page.BasePage(self.driver)
assert cur_page.is_page_valid()
self.driver.back()
comm_page.wait_for_page_loaded()
links = comm_page.get_links_in_container_community()
def tearDown(self):
time.sleep(2)
self.driver.close()
class BasePage:
def __init__(self, driver=None):
if driver is None:
self.driver = Chrome()
self.driver.maximize_window()
self.driver.get(
"https://work.weixin.qq.com/wework_admin/frame#contacts")
cookies = {
"wwrtx.vst":
"zenYAe4CxGbueq5ASVGKquiAk5PdPagPGzKHdLCVqT2i-M2L68XlyLV_-2tP7InD4kOpcBm"
"_stcX8b9Y9z6ec1BgEMdhR-FASZD-wSBX7D37_L7OFcsEYXUePdKC8sPqQBza3KieYk7TE9De"
"2a2AaILp3vEZTlaJMLwFDrOFjOBcFLvhY-k-VmX1gl-BGUklaeVgd8MBeY1ky3t4-2M0yiQlnA"
"7VWwRByLyJxlGrHgCrxZhOhs_BhvyJzLmJOoFNQvhrVSvzAXXoFdHs51gdxA",
"wwrtx.d2st":
"a4861364",
"wwrtx.sid":
"iAu-Z4L3xTLbZ5elezl0oXsd6Y-SXiveFjergOybpzZeb_7vPhAIpt8yVlOv0Ki1",
"wwrtx.ltype":
"1",
"wxpay.corpid":
"1688852500754167",
"wxpay.vid":
"1688852500754167",
}
for k, v in cookies.items():
self.driver.add_cookie({"name": k, "value": v})
self.driver.refresh()
else:
self.driver = driver
def get_visible_element(self, locator, eqc=20) -> WebElement:
'''
定位元素，参数locator为元祖类型
:param locator:
:param eqc:
:return:
'''
try:
ele = WebDriverWait(self.driver, timeout=eqc).until(
EC.visibility_of_element_located(locator))
logger.info('获取{}元素成功'.format(locator))
return ele
except:
logger.error("相对时间内没有定位到{}元素".format(locator))
allure.attach(self.get_windows_img())
def get_presence_element(self, locator, eqc=10):
"""
定位一组元素
:param locator:
:param eqc:
:return:
"""
try:
ele = WebDriverWait(self.driver, timeout=eqc).until(
EC.presence_of_element_located(locator))
logger.info('获取{}元素成功'.format(locator))
return ele
except:
logger.error("相对时间内没有定位到{}元素".format(locator))
allure.attach(self.get_windows_img())
def get_clickable_element(self, locator, eqc=20):
try:
ele = WebDriverWait(self.driver, timeout=eqc).until(
EC.element_to_be_clickable(locator))
logger.info('获取{}元素成功'.format(locator))
return ele
except:
logger.error("相对时间内没有定位到{}元素".format(locator))
allure.attach(self.get_windows_img())
def send_keys(self, locator, text):
'''
发送文本，清空后输入
locator = ('id','xxx')
element.send_keys(locator,text)
'''
element = self.get_visible_element(locator)
element.clear()
element.send_keys(text)
logger.info('SendKeys %s in %s success.' % (text, locator))
def is_text_in_element(self, locator, text, timeout=10):
'''
判断文本在元素里，没有元素返回false打印日志，定位到返回判断结果的布尔值
result = driver.text_in_element(locator,text)
'''
try:
result = WebDriverWait(self.driver, timeout, 1).until(
EC.text_to_be_present_in_element(locator, text))
except TimeoutException:
logger.info('No location to the element.')
allure.attach(self.get_windows_img())
return False
else:
return result
def is_text_in_value(self, locator, value, timeout=10):
'''
判断元素的value值，没定位到元素返回false，定位到返回判断结果布尔值
result = dirver.text_to_be_present_in_element_value(locator,text)
'''
try:
result = WebDriverWait(self.driver, timeout, 1).until(
EC.text_to_be_present_in_element_value(locator, value))
except TimeoutException:
logger.info('No location to the element.')
allure.attach(self.get_windows_img())
return False
else:
return result
def is_title(self, title, timeout=10):
'''
判断元素的title是否完全等于
'''
result = WebDriverWait(self.driver, timeout,
1).until(EC.title_is(title))
return result
def is_title_contains(self, title, timeout=10):
'''
判断元素的title是否包含
'''
result = WebDriverWait(self.driver, timeout,
1).until(EC.title_contains(title))
return result
def is_selected(self, locator, timeout=10):
'''
判断元素是否被选中
'''
result = WebDriverWait(self.driver, timeout, 1).until(
EC.element_located_to_be_selected(locator))
return result
def is_selected_be(self, locator, selected=True, timeout=10):
'''
判断元素的状态是不是符合期望的状态，selected是期望的状态
'''
result = WebDriverWait(self.driver, timeout, 1).until(
EC.element_located_selection_state_to_be(locator, selected))
return result
def is_alert_present(self, timeout=10):
'''
判断页面是否有alert,有的话返回alert，没有返回False
'''
result = WebDriverWait(self.driver, timeout,
1).until(EC.alert_is_present())
return result
def is_visibility(self, locator, timeout=10):
'''
元素可见，返回本身，不可见返回False
'''
result = WebDriverWait(self.driver, timeout, 1).until(
EC.visibility_of_element_located(locator))
return result
def is_invisibility(self, locator, timeout=10):
'''
元素可见返回本身，不可见返回Ture,没有找到元素也返回Ture
'''
result = WebDriverWait(self.driver, timeout, 1).until(
EC.invisibility_of_element_located(locator))
return result
def is_clickable(self, locator, timeout=10):
'''
元素可以点击is_enabled返回本身，不可点击返回False
'''
result = WebDriverWait(self.driver, timeout,
1).until(EC.element_to_be_clickable(locator))
return result
def is_located(self, locator, timeout=10):
'''
判断元素有没有被定位到(并不意味着可见),定位到返回element，没有定位到返回False
'''
result = WebDriverWait(self.driver, timeout, 1).until(
EC.presence_of_all_elements_located(locator))
return result
def move_to_element(self, locator):
'''
鼠标悬停操作
locator=('id','xxx')
driver.move_to_element(locator)
'''
element = self.get_visible_element(locator)
ActionChains(self.driver).move_to_element(element).perform()
# logger.info('ActionChins move to %s' % locator)
def back(self):
self.driver.back()
logger.info('back driver!')
def forward(self):
self.driver.forward()
logger.info('forward driver!')
def close(self):
self.driver.close()
logger.info('close driver!')
def refresh(self):
return self.driver.refresh()
def get_title(self):
'''
获取title
'''
logger.info('git dirver title.')
return self.driver.title()
def get_text(self, locator):
'''
获取文本
'''
element = self.get_visible_element(locator)
# logger.info('get text in %s' % locator)
text = element.text
return text
def get_attribute(self, locator, name):
'''
获取属性
'''
element = self.get_visible_element(locator)
logger.info('get attribute in %s' % locator)
return element.get_attribute(name)
def js_execute(self, js):
'''
执行js
'''
try:
logger.info('Execute js.%s' % js)
return self.driver.execute_script(js)
except:
allure.attach(self.get_windows_img())
logger.info('failed to excute js')
def js_focus_element(self, locator):
'''
聚焦元素
'''
target = self.get_visible_element(locator)
self.driver.execute_script("arguments[0].scrollIntoView();", target)
def js_scroll_top(self):
'''
滚动到顶部
'''
js = 'window.scrollTo(0,0)'
self.js_execute(js)
logger.info('Roll to the top!')
def js_scroll_end(self):
'''
滚动到底部
'''
js = "window.scrollTo(0,document.body.scrollHight)"
self.js_execute(js)
logger.info('Roll to the end!')
def get_windows_img(self):
try:
file_name = contants.screenshot_img
self.driver.get_screenshot_as_file(file_name)
logger.info(
'Had take screenshot and save to folder:output/screenshots')
except NameError as e:
logger.info('Failed to take the screenshot!%s' % e)
self.get_windows_img()
return file_name
def switch_window(self, name=None, fqc=20):
"""
切换窗口，有name切换至该name的窗口，没有则切换最新
:param name:
:param fqc:
:return:
"""
if name is None:
current_handle = self.driver.current_window_handle
WebDriverWait(self.driver,
fqc).until(EC.new_window_is_opened(current_handle))
handles = self.driver.window_handles
return self.driver.switch_to.window(handles[-1])
return self.driver.switch_to.window()
def scrape():
driver = Chrome(ChromeDriverManager().install())
bookieDict = {}
try:
driver.get(url)
time.sleep(15)
# decimal = driver.find_element_by_xpath("//i[contains(text(), 'Decimal Odds')]")
decimal = driver.find_element_by_xpath(
'//*[contains(@class, "style_dropdown")]')
decimal.click()
time.sleep(1)
american = driver.find_element_by_xpath(
"//a[contains(text(), 'American Odds')]")
american.click()
time.sleep(1)
ids = len(
driver.find_elements_by_xpath(
'//a[@data-test-id="Event.GameInfo"]'))
for index in range(ids):
allElements = driver.find_elements_by_xpath(
'//a[@data-test-id="Event.GameInfo"]')
try:
driver.execute_script("arguments[0].click();",
allElements[index])
except:
print('error clicking: ' + str(index))
continue
time.sleep(3)
#look for player props
try:
matchup = driver.find_element_by_xpath(
"//span[contains(text(), 'vs.')]")
except NoSuchElementException:
print('couldn\'t find matchup for ' + str(index))
driver.back()
time.sleep(3)
continue
try:
props = driver.find_element_by_xpath(
"//button[contains(text(), 'Player Props')]")
driver.execute_script("arguments[0].click();", props)
time.sleep(3)
# webpage = driver.page_source.encode("utf-8")
# writeToFile(webpage, "props")
except NoSuchElementException:
print('no player props in ' + matchup.text)
driver.back()
time.sleep(3)
continue
fullLines = driver.find_elements_by_xpath(
'//*[contains(@class, "style_marketGroup")]')
# labelAndOdds = driver.find_elements_by_class_name('style_content__20TF7')
now = datetime.now()
dt_string = now.strftime("%d-%m-%Y %H:%M:%S")
file = open("./outputs/" + dt_string + matchup.text + ".csv", "w+")
for idx in range(len(fullLines)):
try:
parts = fullLines[idx].text.split("\n")
# labels = fullLines[idx].find_elements(By.CLASS_NAME, 'label')
# prices = fullLines[idx].find_elements(By.CLASS_NAME, 'price')
# pdb.set_trace()
overidentifier = matchup.text + parts[0] + parts[1]
underidentifier = matchup.text + parts[0] + parts[3]
firstOddsValue = int(parts[2])
secondOddsValue = int(parts[4])
#TODO clear window when we hit > 30?
if (overidentifier in bookieDict):
if (abs(bookieDict[overidentifier][0] - firstOddsValue)
> 30):
_notify('difference for ' + overidentifier +
' has changed from ' +
bookieDict[overidentifier][0] + ' to ' +
firstOddsValue)
elif (abs(bookieDict[overidentifier][1] -
firstOddsValue) > 30):
_notify('difference for ' + overidentifier +
' has changed from ' +
bookieDict[overidentifier][1] + ' to ' +
firstOddsValue)
bookieDict[overidentifier] = (
min(firstOddsValue, bookieDict[overidentifier][0]),
max(parts[2], bookieDict[overidentifier][1]))
else:
bookieDict[overidentifier] = (firstOddsValue,
firstOddsValue)
if (underidentifier in bookieDict):
if (abs(bookieDict[underidentifier][0] -
secondOddsValue) > 30):
_notify('difference for ' + underidentifier +
' has changed from ' +
str(bookieDict[underidentifier][0]) +
' to ' + str(secondOddsValue))
elif (abs(bookieDict[underidentifier][1] -
secondOddsValue) > 30):
_notify('difference for ' + underidentifier +
' has changed from ' +
str(bookieDict[underidentifier][1]) +
' to ' + str(secondOddsValue))
bookieDict[underidentifier] = (
min(secondOddsValue,
bookieDict[underidentifier][0]),
max(secondOddsValue,
bookieDict[underidentifier][1]))
else:
bookieDict[underidentifier] = (secondOddsValue,
secondOddsValue)
file.write(parts[0] + "," + parts[1] + "," + parts[2] +
"," + parts[3] + "," + parts[4] + "\n")
except (NoSuchElementException, IndexError):
print('line ' + str(idx) + ' is locked/missing in ' +
matchup.text)
continue
file.close()
driver.back()
time.sleep(3)
#WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "contentBlock")))
# webpage = driver.page_source.encode("utf-8")
# writeToFile(webpage, "")
finally:
driver.quit()
class TdsystemCrawler:
def __init__(self, headless: bool = True):
self.root = "https://www.tdsystem.co.jp/"
self.driver = Chrome(options=self.__get_options(headless))
self.driver.implicitly_wait(5)
self.driver.set_page_load_timeout(180)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self.driver:
self.driver.quit()
def get_root(self):
self.driver.get(self.root)
def __get_options(self, headless: bool):
options = Options()
if headless:
options.add_argument("--headless")
return options
def __get_wait_secs(self):
max_wait = 5.0
min_wait = 1.0
mean = 3.0
sigma = 1.0
return min(
[max_wait,
max([min_wait, random.normalvariate(mean, sigma)])])
def _get_meet_result_buttons(self, year: str):
self.get_root()
select_years = Select(self.driver.find_element_by_id("category"))
select_years.select_by_value(year)
time.sleep(self.__get_wait_secs())
select_months = Select(self.driver.find_element_by_name("M"))
select_months.select_by_value("0") # 1~12月
time.sleep(self.__get_wait_secs())
return self.driver.find_elements_by_xpath("//button[text() = '結果']")
def crawl_years(self, years: list[str]):
logger = logging.getLogger(__file__)
for y in years:
logger.info(f"crawl_years: {y}")
buttons = self._get_meet_result_buttons(y)
num_of_meets = len(buttons)
logger.info(f"num_of_meets = {num_of_meets}")
# Traverse all meet results in this year
for i in [11, 14]: # TODO test code
buttons = self._get_meet_result_buttons(y)
buttons[i].click()
logger.info(f"Open: {self.driver.current_url}")
time.sleep(self.__get_wait_secs())
self.driver.back()
return
class BasePage:
driver = None
def __init__(self):
self.driver: Chrome = None
self.wait: WebDriverWait = None
if BasePage.driver is None:
self._start_driver()
else:
self.driver = BasePage.driver
self._init_wait()
self._init_page()
def _start_driver(self):
options = ChromeOptions()
options.page_load_strategy = "none"
if config_load.get_options().get("headless"):
options.add_argument("--window-size=1280,940")
options.add_argument("--start-maximized")
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--headless')
options.add_experimental_option(
"mobileEmulation", {
"userAgent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36"
})
if 'linux' in sys.platform:
self.driver = Chrome(options=options)
else:
self.driver = Chrome(
executable_path='../resources/drivers/chromedriver',
options=options)
self.driver.maximize_window()
BasePage.driver = self.driver
self.page_bottom()
self.page_top()
self.driver.implicitly_wait(5)
def _init_page(self):
data = config_load.get_page(self.__module__.split(".")[-1])
class Locator:
def __init__(self, data):
self.by = data['by']
self.locator = data['locator']
if data is not None:
for i in data:
self.__dict__[i] = Locator(data[i])
def __getattr__(
self,
item,
):
raise ValueError("can't find element")
def _init_wait(self):
ignored_exceptions = (
NoSuchElementException,
StaleElementReferenceException,
)
self.wait = WebDriverWait(self.driver,
15,
ignored_exceptions=ignored_exceptions)
def find_element(self, by, locator):
self.wait.until(
expected_conditions.visibility_of_element_located((by, locator)))
return self.driver.find_element(by, locator)
# def click(self, by, locator):
# self.wait.until(expected_conditions.element_to_be_clickable((by, locator)))
# self.driver.find_element(by, locator).click()
def click(self, by, locator):
count = 1
while count <= RETRY_COUNT:
try:
self.wait.until(
expected_conditions.element_to_be_clickable((by, locator)))
self.driver.find_element(by, locator).click()
break
except Exception as e:
time.sleep(3)
count += 1
if count > RETRY_COUNT:
raise e
def move_to_elements(self, by, locator):
self.wait.until(
expected_conditions.visibility_of_element_located((by, locator)))
ActionChains(self.driver).move_to_element(
self.driver.find_element(by, locator)).perform()
def find_elements(self, by, locator):
self.wait.until(
expected_conditions.visibility_of_element_located((by, locator)))
return self.driver.find_elements(by, locator)
def send_keys(self, by, locator, data):
self.wait.until(
expected_conditions.element_to_be_clickable((by, locator)))
# self.driver.find_element(by, locator).clear() # 在发送之前，清理。 在个在某些情况下并不能很好的工作，采用下面这种方式
self.driver.find_element(by, locator).send_keys(Keys.CONTROL,
'a') #工作正常
self.driver.find_element(by, locator).send_keys(data)
def get_property(self, by, locator, name):
return self.find_element(by, locator).get_property(name)
def get_text(self, by, locator):
return self.find_element(by, locator).text
def get_tag_name(self, by, locator):
return self.find_element(by, locator).tag_name
def page_down(self):
self.driver.find_element_by_tag_name('html').send_keys(Keys.PAGE_DOWN)
def page_up(self):
self.driver.find_element_by_tag_name('html').send_keys(Keys.PAGE_UP)
def page_slightly_down(self):
self.driver.find_element_by_tag_name('html').send_keys(Keys.ARROW_DOWN)
def page_slightly_up(self):
self.driver.find_element_by_tag_name('html').send_keys(Keys.ARROW_UP)
def set_page_position(self, x, y):
self.driver.execute_script("window.scrollTo(%s, %s)" % (x, y))
def page_top(self):
self.set_page_position(0, 0)
def page_bottom(self):
self.set_page_position(0, 10000)
def quit(self):
self.driver.quit()
BasePage.driver = None
def isElementExist(self, by, locator):
flag = True
try:
self.find_element(by, locator)
return flag
except:
flag = False
return flag
def page_back(self):
url = self.driver.current_url
self.driver.back()
self.wait.until_not(expected_conditions.url_to_be(url))
self.wait.until(
expected_conditions.presence_of_all_elements_located(
("xpath", "html")))
# self.wait.until(expected_conditions.visibility_of_all_elements_located(("xpath", "html")))
self.driver.get(self.driver.current_url)
def change_window(self):
time.sleep(1)
handles = self.driver.window_handles
index_handle = self.driver.current_window_handle
for handle in handles:
if handle != index_handle:
self.driver.switch_to.window(handle)
else:
continue
def close_window(self):
time.sleep(1)
self.driver.close()
handles = self.driver.window_handles
for handle in handles:
self.driver.switch_to.window(handle)
break
def get_attribute(self, by, locator, name):
return self.find_element(by, locator).get_attribute(name)
def get_url(self):
return self.driver.current_url
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
chrome = Chrome()
chrome.get("https://y.qq.com")
btn = chrome.find_element_by_class_name("search_input__btn")
act = ActionChains(chrome)
ActionChains.move_to_element(act, btn).perform()
search_input = chrome.find_element_by_xpath("/html/body/div[1]/"
"div/div[1]/div[1]/input")
search_input.click()
search_input.send_keys("等你下课")
btn.click()
span = chrome.find_element_by_class_name("songlist__songname")
ActionChains.move_to_element(ActionChains(chrome), span).perform()
chrome.find_element_by_class_name("list_menu__icon_play").click()
chrome.back()
chrome.close()
chrome.forward()
class Drivery: # Don't give me that 'too many public methods' nonsense. pylint: disable=R0904
"""Because Module-Level-State is apparently a terrible idea, have a class singleton.
Wraps a WebDriver instance, and does a bunch of other useful things."""
def __init__(self, globs: dict):
if globs is None: return # Just to handle the CP thing declaration.
# To aid in checking for Page Loaded Status, track the last link clicked.
self.last_link = ''
self.base_url = globs['base_url']
self.locale = globs['locale']
self.locale_url = self.base_url + self.locale
self.auth = globs['auth']
self.cn_mode = globs['cn_mode'] # yeah, but CP needs it apparently.
# A workaround. Firefox gets suspicious when you hide a password in the url.
if globs['browser'] == 'firefox':
p = FirefoxProfile()
p.set_preference('network.http.phishy-userpass-length', 255)
self.driver = Firefox(p)
# Chrome, too, just up and decided to stop supporting this one day.
elif globs['browser'] == 'chrome':
c = ChromeOptions()
c.add_argument(
'--disable-blink-features=BlockCredentialedSubresources')
self.driver = Chrome(chrome_options=c)
else:
self.driver = BROWSERS[globs['browser']]()
self.driver.implicitly_wait(LONG_WAIT)
self.driver.maximize_window()
def set_wait(self, wait: int) -> None:
"""Set the driver's implicit waiting value and the LONG_WAIT value too."""
global LONG_WAIT
LONG_WAIT = wait
self.driver.implicitly_wait(wait)
def close(self) -> None:
"""The testing bit calls this at the end of each test. Clears the session."""
self.driver.quit()
###Navigation Methods.###
def splash_page(self) -> None:
"""Navigates to the Splash Page."""
self.get(self.base_url + '/splash.html')
def open_home_page(self) -> None:
"""Opens the Welcome Page. Shortcut method."""
self.get(self.locale_url)
def current_url(self) -> str:
"""Kind of a technicality. Returns the current url."""
return self.driver.current_url
def back(self) -> None:
"""The rule is, only the drivery module is allowed to invoke self.driver directly."""
self.driver.back()
def refresh(self) -> None:
"""It's The Rules"""
self.driver.refresh()
def page_title(self) -> str:
"""Gets the Page Title. RULES."""
return self.driver.title
def get(self, url: str) -> None:
"""For whatever reason, there is no Basic Authentication that works across all browsers.
This has workarounds for each. Ironically, only IE supports the correct method."""
isie = isinstance(self.driver, Ie)
if not isie and self.auth:
url = re.sub('(https?://)', r'\1{0}:{1}@'.format(*self.auth), url)
self.driver.get(url)
if isie and self.auth:
self.driver.switch_to.alert.authenticate(*self.auth)
def close_window(self) -> None:
"""Closes the currently-focused window or tab. Try not to use this when only one is left."""
self.driver.close()
self.switch_to_window(0)
def close_other_windows(self) -> None:
"""Closes all open tabs and windows except for the original one."""
while len(self.driver.window_handles) != 1:
self.switch_to_window(1)
self.close_window()
def current_scroll(self) -> int:
"""Returns the window's vertical scroll position as stated by javascript's window.scrollY"""
return self.driver.execute_script('return window.scrollY;')
def wait_for_page(self, url=None) -> None:
"""Holds up execution until the current page's url contains the Last Link value (or, if
given, a custom url) and its document.readyState is 'complete'. A decent approximation?"""
script = 'return document.readyState === "complete";'
if url is None:
url = self.last_link
try:
WebDriverWait(self.driver,
LONG_WAIT).until(lambda _: url in self.current_url())
WebDriverWait(
self.driver,
LONG_WAIT).until(lambda _: self.driver.execute_script(script))
except TimeoutException:
raise TimeoutException(
'Timed out waiting for {0} to load.'.format(url)) from None
def wait_until_present(self, selector: str) -> WebElement:
"""Holds up execution until the selectored elment is visibly present.
Use this instead of quietly_find if the target is in the DOM, but hidden."""
try:
return WebDriverWait(self.driver, LONG_WAIT).until(
EC.visibility_of_element_located((By.CSS_SELECTOR, selector)))
except TimeoutException:
raise TimeoutException(
'Timed out waiting for {0} to appear.'.format(
selector)) from None
def wait_until_gone(self, selector: str) -> WebElement:
"""Holds up execution until the selectored element is not visibly present.
EC doesn't seem to support local searches, so be sure the selector is page-unique."""
try:
# The poll_freq value is not the wait-til-fail time. Apparently.
self.driver.implicitly_wait(0.5)
ret = WebDriverWait(self.driver, LONG_WAIT).until(
EC.invisibility_of_element_located(
(By.CSS_SELECTOR, selector)))
self.driver.implicitly_wait(LONG_WAIT)
return ret
except TimeoutException:
raise TimeoutException(
'Timed out waiting for {0} to disappear.'.format(
selector)) from None
def wait_until(self, condition: Callable, desc: str) -> Any:
"""Holds up execution, repeatedly calling the given function until it returns truthy.
The given condition lambda should have no inputs.
desc should explain what the lambda does, as the contents can't really be examined."""
try:
return WebDriverWait(self.driver,
LONG_WAIT).until(lambda _: condition())
except TimeoutException:
raise TimeoutException(
'Timed out waiting for condition: {0}'.format(desc)) from None
def switch_to_window(self, window: int) -> None:
"""Switch WebDriver's focus to the given open tab or window. Zero based indexing."""
self.driver.switch_to.window(self.driver.window_handles[window])
def switch_to_frame(self, selector: str) -> None:
"""Switches WebDriver's focus into the given iframe.
Alternatively, set selector to a None to revert to the main window."""
if selector is None:
self.driver.switch_to.default_content()
else:
self.driver.switch_to.frame(self.flashy_find_element(selector))
def fix_url(self, url: str) -> str:
"""Use this to remove that /content/asp/ stuff from URLs."""
return re.sub(r'((/|^)\w\w)_(\w\w(/|$|.))',
r'\1-\3', (url or '').replace('/content/asp/', '/'),
count=1)
###Some methods to shorten Element Manipulation/Verification.###
def scroll_element(self, elle: WebElement) -> WebElement:
"""Scrolls a single element into view, it wouldn't make sense to do multiple."""
self.driver.execute_script(SCROLL_SCRIPT, elle)
return elle
def blip_element(self, elle: ELEMENT_OR_LIST) -> ELEMENT_OR_LIST:
"""Scrolls (an) element(s) into view, and highlights (i)t(hem).
Returns the found element(s) as well, just for chaining purposes."""
# Kick off the highlight animation, list-wrapped for the sake of only writing one handler.
self.driver.execute_script(BLIP_SCRIPT, to_list(elle))
return elle
def check_visible_quick(self,
selector: str,
within: WebElement = None) -> bool:
"""Check for an element without potentially spending a lot of time polling the DOM.
Ususally used when asserting an element's absence, saves waiting the full timeout."""
within = within or self.driver
self.driver.implicitly_wait(SHORT_WAIT)
elements = within.find_elements_by_css_selector(selector)
self.driver.implicitly_wait(LONG_WAIT)
return len(elements) != 0 and elements[0].is_displayed()
def execute_mouse_over(self, element: WebElement) -> None:
"""Simulates the mouse moving into an element."""
ActionChains(self.driver).move_to_element(element).perform()
def override_click(self, element: WebElement) -> None:
"""Use this if the thing you want to click is ''""'behind''"'" something else."""
ActionChains(self.driver).move_to_element(element).click().perform()
@find_error_improver
def quietly_find_element(self,
selector: str,
within: WebElement = None) -> WebElement:
"""Finds a single element matching a CSS selector, optionally within a given element."""
within = within or self.driver
return within.find_element_by_css_selector(selector)
@find_error_improver
def quietly_find_elements(self,
selector: str,
within: WebElement = None) -> ELEMENT_LIST:
"""Finds multiple elements that match a CSS selector, optionally within a given element."""
within = within or self.driver
return to_list(within.find_elements_by_css_selector(selector))
@find_error_improver
def flashy_find_element(self,
selector: str,
within: WebElement = None) -> WebElement:
"""Finds a single element matching a CSS selector, highlights it as well."""
within = within or self.driver
return self.blip_element(within.find_element_by_css_selector(selector))
@find_error_improver
def flashy_find_elements(self,
selector: str,
within: WebElement = None) -> ELEMENT_LIST:
"""Finds multiple elements that match a CSS selector, highlights them as well.
The browser-provided webdriver self.driver implementations seem to not return a
list when only one element matches, so fixing that here."""
within = within or self.driver
return self.blip_element(
to_list(within.find_elements_by_css_selector(selector)))
@find_error_improver
def find_visible_element(self,
selector: str,
within: WebElement = None) -> WebElement:
"""Given a selector that could match multiple different elements,
return the one that is currently visible, not the first one that matches."""
within = within or self.driver
return self.blip_element([
x for x in within.find_elements_by_css_selector(selector)
if x.is_displayed()
][0])
def get_parent_element(self, element: WebElement) -> WebElement:
"""Gets the immediate parent of the given element."""
return element.find_element_by_xpath('..')
def bring_to_front(self, element: WebElement) -> WebElement:
"""Uses a JS to ensure the selected element is at the front.
Usually in order to properly click() it."""
self.driver.execute_script('arguments[0].style.zIndex = "10000";',
element)
def execute_script(self, script: str, *args) -> Any:
"""Executes a javascript snippet, returning what the script returns."""
return self.driver.execute_script(script, *args)
class Bet365Collector(Collector):
#Input: URL of sport
def __init__(self, url, os):
self.url = url
try:
if os == 'MAC':
self.driver = Chrome()
self.driver.implicitly_wait(10)
elif os == 'WIN':
options = webdriver.ChromeOptions()
options.binary_location = r"C:\Program Files (x86)\Google\Chrome Beta\Application\chrome.exe"
chrome_driver_binary = r"/chromedriver.exe"
self.driver = webdriver.Chrome(chrome_driver_binary,
chrome_options=options)
else:
raise ValueError
except ValueError:
print('Please specify MAC or WIN in the cmd line arguments')
#Returns a list of page_sources. Corresponding to today's games.
def get_games(self):
game_urls = []
self.driver.get(self.url)
time.sleep(1)
#Retrieves all game links
games = len(
self.driver.find_elements_by_class_name(
"sl-CouponParticipantGameLineTwoWay_NameText"))
#Travels to each game
for i in range(0, games, 2):
self.driver.find_elements_by_class_name(
"sl-CouponParticipantGameLineTwoWay_NameText")[i].click()
game_urls.append(self.driver.current_url)
self.driver.back()
self.games = game_urls
def get_page_sources(self):
page_sources = []
for game in self.games:
self.driver.get(game)
time.sleep(1)
try:
#Tries to find the player-market tab
self.driver.find_element_by_xpath(
"//div[contains(text(), 'Player Markets')]").click()
#Tries to open assists, rebounds and all show all dropdowns
try:
self.driver.find_element_by_xpath(
"//div[contains(text(), 'Player Assists')]").click()
time.sleep(1)
self.driver.find_element_by_xpath(
"//div[contains(text(), 'Player Rebounds')]").click()
time.sleep(1)
showMoreDropdowns = self.driver.find_elements_by_class_name(
"msl-ShowMore")
for dropdown in showMoreDropdowns:
dropdown.click()
time.sleep(1)
except NoSuchElementException:
print(
"ERROR: Either assist, rebound was not found. Exporting data anyway"
)
except NoSuchElementException:
print("ERROR: Player Markets Not Found")
self.driver.close()
continue
# appends the page source
page_sources.append(self.driver.page_source)
self.driver.close()
return page_sources
with Chrome() as driver:
# your code inside this indent
Chrome(executable_path='/path/to/chromedriver')
driver.get("https://selenium.dev")
driver.current_url # read the browser url
driver.back()
driver.current_window_handle # WebDriver does not make the distinction between windows and tabs. If your site opens a new tab or window, Selenium will let you work with it using a window handle. Each window has a unique identifier which remains persistent in a single
######## new tab and switch
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# Start the driver api
with webdriver.Firefox() as driver:
# Open URL
driver.get("https://seleniumhq.github.io")
class ACE():
def __init__(self):
options = ChromeOptions()
options.add_argument("--headless")
options.add_argument('--disable-logging')
options.add_argument("--start-maximized")
options.add_argument("--no-sandbox")
self.browser = Chrome(
executable_path='/home/mancunian92/Documents/chromedriver',
chrome_options=options)
self.browser.get(ACE_URL)
self.searchResults = []
self.dropDownOptions = []
self.request = {}
time.sleep(2)
self.getDropDownOptions()
def getDropDownOptions(self):
self.formOptions = self.browser.find_element_by_name(
"formPickupLocation")
self.dropDownOptions = [
o.text for o in Select(
self.browser.find_element_by_name(
"formPickupLocation")).options
]
def pushDateToBrowser(self, date, diffMonths, isDropOff=False):
if isDropOff:
self.browser.find_element_by_id("inline_Dropoff_Date_1").click()
while (diffMonths > 0):
self.browser.find_elements_by_class_name(
"l-form-b__field--datetime")[1].find_element_by_class_name(
"pika-next").click()
diffMonths -= 1
tableElem = self.browser.find_elements_by_class_name(
"l-form-b__field--datetime")[1].find_element_by_class_name(
"pika-lendar").find_element_by_xpath("./table/tbody")
else:
self.browser.find_element_by_id("inline_Pickup_Date_1").click()
while (diffMonths > 0):
self.browser.find_elements_by_class_name(
"l-form-b__field--datetime")[0].find_element_by_class_name(
"pika-next").click()
diffMonths -= 1
tableElem = self.browser.find_elements_by_class_name(
"l-form-b__field--datetime")[0].find_element_by_class_name(
"pika-lendar").find_element_by_xpath("./table/tbody")
expr = "./tr/td[@data-day=" + str(date.day) + "]/button"
tableElem.find_element_by_xpath(expr).click()
def pushTimeToBrowser(self, time, isDropoff=False):
if isDropoff:
Select(self.browser.find_element_by_name(
"formDropoffTime")).select_by_value(time)
else:
Select(self.browser.find_element_by_name(
"formPickupTime")).select_by_value(time)
def selectDates(self, pickupDateInString, dropOffDateInString, pickupTime,
dropOffTime):
currDate = datetime.datetime.now()
pickupDate = datetime.datetime.strptime(pickupDateInString, '%d/%b/%Y')
dropoffDate = datetime.datetime.strptime(dropOffDateInString,
'%d/%b/%Y')
diffMonthsPickup = (pickupDate.year - currDate.year) * 12 + (
pickupDate.month - currDate.month)
diffMonthsDropoff = (dropoffDate.year - pickupDate.year) * 12 + (
dropoffDate.month - pickupDate.month)
self.pushDateToBrowser(pickupDate, diffMonthsPickup)
self.pushDateToBrowser(dropoffDate, diffMonthsDropoff, isDropOff=True)
self.pushTimeToBrowser(pickupTime)
self.pushTimeToBrowser(dropOffTime, isDropoff=True)
def selectLocation(self,
pickupLocation,
dropOffLocation,
isSamePickup=True):
# First, select the pick up location.
pickupIndex = self.dropDownOptions.index(pickupLocation)
Select(self.formOptions).select_by_index(pickupIndex)
if isSamePickup != True:
dropOffElem = Select(
self.browser.find_element_by_name("formDropoffLocation"))
dropOffOptions = [o.text for o in dropOffElem.options]
dropOffIndex = dropOffOptions.index(dropOffLocation)
dropOffElem.select_by_index(dropOffIndex)
def enterPromocode(self, promoCode):
self.browser.find_element_by_name("formPromoCode").send_keys(promoCode)
def search(self, searchRequest):
pickupDate = searchRequest["pickupDate"]
dropoffDate = searchRequest["dropDate"]
pickupLocation = searchRequest["pickupPoint"]
dropoffLocation = searchRequest["dropPoint"]
pickupTime = searchRequest["pickupTime"]
dropoffTime = searchRequest["dropTime"]
self.selectDates(pickupDate, dropoffDate, pickupTime, dropoffTime)
self.selectLocation(pickupLocation,
dropoffLocation,
isSamePickup=False)
self.browser.find_element_by_class_name(
"l-hero__booking-action__submit--btn").click()
def parseCars(self, ace):
try:
parsedCars = []
cars = WebDriverWait(self.browser, MAX_TIMEOUT).until(
lambda x: x.find_element_by_class_name("l-cars__cards"))
cars = cars.find_elements_by_class_name("c-vehicle-card")
totalCars = len(cars)
for i in range(0, totalCars):
parsedCar = {
"carName": "",
"carType": "",
"gearType": "",
"maxSeats": "",
"maxLuggage": "",
"image": "",
"carCost": "",
"totalCost": "",
"currencyCode": "",
"insuranceDetails": [],
"otherOptions": []
}
# Hide the itineray summary
itemSummary = self.browser.find_element_by_class_name(
"l-booking-summary-bar")
self.browser.execute_script(
"arguments[0].style.visibility='hidden'", itemSummary)
action = ActionChains(self.browser)
detailsButton = self.browser.find_elements_by_class_name(
"c-vehicle-card")[i].find_element_by_class_name(
"c-vehicle-card__action")
action.move_to_element(detailsButton)
action.click(detailsButton).perform()
WebDriverWait(self.browser, MAX_TIMEOUT).until(
lambda x: x.find_element_by_class_name("l-booking__step"))
parsedCar = self.parseCarDetail(self.browser, parsedCar)
parsedCars.append(parsedCar)
self.browser.back()
WebDriverWait(self.browser, MAX_TIMEOUT).until(
lambda x: x.find_element_by_class_name("l-cars__cards"))
return parsedCars
except TimeoutException:
print("Am i here in time out exception??")
return parsedCars
def parseCarDetail(self, carDetail, parsedCar):
elements = self.browser.find_elements_by_class_name("l-booking__step")
inner = carDetail.find_element_by_class_name("l-vehicle-panel__inner")
insuranceDetails = elements[0].find_elements_by_class_name(
"c-option-card__main")
otherOptions = elements[1].find_elements_by_class_name(
"x-option-card__main")
parsedCar["carName"] = inner.find_element_by_class_name(
"l-vehicle-panel__subtitle").get_attribute("textContent")
parsedCar["carType"] = inner.find_element_by_class_name(
"l-vehicle-panel__title").get_attribute("textContent")
parsedCar["image"] = inner.find_element_by_class_name(
"l-vehicle-panel__image").find_element_by_xpath(
'./img').get_attribute("src")
specifications = inner.find_element_by_class_name(
"l-vehicle-panel__specifications")
parsedCar["gearType"] = specifications.find_element_by_xpath(
'//img[contains(@src, "transmission")]').get_attribute("alt")
parsedCar["maxSeats"] = specifications.find_element_by_xpath(
'//img[contains(@src, "passengers")]').get_attribute("alt")
parsedCar["maxLuggage"] = specifications.find_element_by_xpath(
'//img[contains(@src, "luggage")]').get_attribute("alt")
cost = WebDriverWait(self.browser, MAX_TIMEOUT).until(
lambda x: x.find_element_by_class_name("l-vehicle-panel__total"))
parsedCar["carCost"] = cost.find_element_by_class_name(
"l-vehicle-panel__total-item-total").get_attribute("textContent")
totalcost = cost.find_element_by_class_name(
"l-vehicle-panel__total-price")
parsedCar["currencyCode"] = totalcost.find_element_by_xpath(
'./span').get_attribute("textContent")
parsedCar["totalCost"] = totalcost.get_attribute("textContent")
for _ins in insuranceDetails:
insur = {
"name":
_ins.find_element_by_class_name(
"c-option-card__title").get_attribute("textContent"),
"price":
_ins.find_element_by_class_name(
"c-option-card__price").get_attribute("textContent")
}
parsedCar["insuranceDetails"].append(insur)
for _opt in otherOptions:
opt = {
"title":
_opt.find_element_by_class_name(
"x-option-card__title").get_attribute("textContent"),
"price":
_opt.find_element_by_class_name(
"x-option-card__price").get_attribute("textContent")
}
parsedCar["otherOptions"].append(opt)
return parsedCar
empty_list = []
downloads.append(empty_list)
pdfs.append(empty_list)
# If there's other type of content
else:
try:
misc.append(tex.text)
except:
empty_list = []
misc.append(empty_list)
# append article info to list
downloaded_data.append(
[news, meta, intro, paras, downloads, pdfs, misc])
browser.back()
except:
excep_inner = 1
print("Exception category 2")
browser.close()
break
flag += 1
if excep_inner == 0 or excep_outer == 0:
browser.close()
# Save data in a pandas dataframe
df = pd.DataFrame(
downloaded_data,
class WebPage(Engine):
def __init__(self, executable_path):
super(WebPage, self).__init__()
self._url = None
self._executable_path = executable_path
self._driver = None
def start(self):
self._driver = Chrome(executable_path=self._executable_path)
self._driver.maximize_window()
if self._url is not None:
self._driver.get(self._url)
Logger.info(self, 'Engine started')
def stop(self):
self._driver.close()
self._driver = None
Logger.info(self, 'Engine stopped')
def _find_element_by_id(self, id):
return self._driver.find_element_by_id(id)
def _find_element_by_xpath(self, xpath):
return self._driver.find_element_by_xpath(xpath)
def _find_element_by_class_name(self, class_name):
return self._driver.find_element_by_class_name(class_name)
def _find_elements_by_class_name(self, class_name):
return self._driver.find_elements_by_class_name(class_name)
def _find_element_by_name(self, name):
return self._driver.find_element_by_name(name)
def _find_element_by_link_text(self, link_text):
return self._driver.find_element_by_link_text(link_text)
def _wait_for_element_by_xpath(self, xpath, timeout=5):
WebDriverWait(self._driver, timeout).until(
EC.presence_of_element_located((By.XPATH, xpath)))
def _wait_for_element_by_class_name(self, class_name, timeout=0.5):
WebDriverWait(self._driver, timeout).until(
EC.presence_of_element_located((By.CLASS_NAME, class_name)))
return self._driver.find_element_by_class_name(class_name)
def _wait_for_visibility_by_xpath(self, xpath, timeout=5):
WebDriverWait(self._driver, timeout).until(
EC.visibility_of_element_located((By.XPATH, xpath)))
def _wait_for_clickability_by_xpath(self, xpath, timeout=5):
WebDriverWait(self._driver, timeout).until(
EC.element_to_be_clickable((By.XPATH, xpath)))
def _wait_for_frame_availability_by_xpath(self, xpath, timeout=5):
WebDriverWait(self._driver, timeout).until(
EC.frame_to_be_available_and_switch_to_it((By.XPATH, xpath)))
def _switch_to_frame(self, frame_web_element):
self._driver.switch_to.frame(frame_web_element)
@except_to_bool(exc=(NoAlertPresentException, TimeoutException))
def _is_alert_present(self, timeout=0.1):
WebDriverWait(self._driver, timeout).until(EC.alert_is_present())
def _back(self):
self._driver.back()
sleep(1)
