def get_games_id(comp):
dates = [d for d in date_range(START_DATE, END_DATE)]
games_id = []
chrome_options = Options()
chrome_options.add_argument('--dns-prefetch-disable')
driver = Chrome(chrome_options=chrome_options)
for day in dates:
driver.get(
'http://www.espn.com.ar/futbol/resultados/_/liga/{}/fecha/{}'.
format(comp, day)
)
game_link_driver = driver.find_elements_by_class_name(
'mobileScoreboardLink '
)
for game_driver in game_link_driver:
game_id = game_driver.get_attribute('href')[46:53]
games_id.append((game_id, day))
driver.quit
# print(games_id)
return games_id
def checkin(info, keepOpen=False, emailBoardingPass=True):
url = MAIN_SW_URL + '&confirmationNumber=' + info['confNum'] + \
'&lastName=' + info['lastName'] + '&firstName=' + info['firstName']
try:
browser = Chrome(CHROME_DRIVER_PATH)
browser.get(url)
time.sleep(0.5)
# Click the check in button
buttonCheckIn = browser.find_element_by_xpath(CHECKIN_BUTTON_XPATH)
buttonCheckIn.click()
time.sleep(0.5)
# Click the email boarding pass button
if emailBoardingPass:
radioButtonEmail = browser.find_element_by_xpath(EMAIL_RADIO_BUTTON_XPATH)
radioButtonEmail.click()
time.sleep(0.1)
buttonFinal = browser.find_element_by_xpath(FINAL_BUTTON_XPATH)
buttonFinal.click()
time.sleep(0.5)
if not keepOpen:
browser.quit()
print('successful checkin!')
report = messages.successfulCheckIn()
status = 1
except:
if not keepOpen:
browser.quit()
report = messages.failedCheckIn()
status = -1
return [status, report]
def test_we_can_launch_multiple_chrome_instances(self):
driver1 = Chrome()
driver2 = Chrome()
driver3 = Chrome()
driver1.quit()
driver2.quit()
driver3.quit()
def test_network_conditions_emulation():
driver = Chrome()
driver.set_network_conditions(
offline=False,
latency=56, # additional latency (ms)
throughput=789)
conditions = driver.get_network_conditions()
assert conditions['offline'] is False
assert conditions['latency'] == 56
assert conditions['download_throughput'] == 789
assert conditions['upload_throughput'] == 789
def selenium_driver():
if os.environ.get('SAUCE_USERNAME'):
driver = make_sauce_driver()
elif os.environ.get('JUPYTER_TEST_BROWSER') == 'chrome':
driver = Chrome()
else:
driver = Firefox()
yield driver
# Teardown
driver.quit()
def get_cookie(username, password, proxy):
"""
Launch a chrome to get cookies
"""
chromeopts = ChromeOptions()
if proxy:
chromeopts.add_argument('--proxy-server=%s' % proxy)
web = Chrome(chrome_options=chromeopts)
try:
return signin(web, username, password)
finally:
web.quit()
def setUp(self):
""" Start a new browser instance for each test """
self._screenshot_number = 1
self.browser = os.getenv('SELENIUM_BROWSER',
settings.SELENIUM_DEFAULT_BROWSER)
if os.getenv('SELENIUM_HOST'):
self.sel = self.sauce_labs_driver()
elif self.browser == 'firefox':
self.sel = Firefox()
elif self.browser == 'htmlunit':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.HTMLUNITWITHJS)
elif self.browser in ['ios', 'ipad', 'ipod', 'iphone']:
capabilities = {
'app': 'safari',
'browserName': '',
'device': 'iPhone Simulator',
'os': 'iOS 6.1'
}
self.sel = RemoteWebDriver(command_executor=self.appium_command_executor(),
desired_capabilities=capabilities)
elif self.browser == 'opera':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.OPERA)
elif self.browser == 'iexplore':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.INTERNETEXPLORER)
elif self.browser == 'phantomjs':
self.sel = PhantomJS(service_args=['--debug=true',
'--webdriver-loglevel=DEBUG'])
elif self.browser == 'safari':
# requires a Safari extension to be built from source and installed
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.SAFARI)
else:
self.sel = Chrome()
# Give the browser a little time; Firefox throws random errors if you
# hit it too soon
time.sleep(1)
class AltomTest(unittest.TestCase):
driver = None
home_page = None
def setUp(self, browser):
self.browser = browser
if "firefox" in self.browser:
profile = FirefoxProfile()
# profile.set_preference("plugin.state.silverlight", 2)
# profile.set_preference("browser.download.folderList", 1)
# profile.set_preference("pdfjs.disabled", False);
# profile.set_preference("pdfjs.firstRun", True);
self.driver = Firefox(profile) # get a new firefox session
if "chrome" in self.browser:
chromedriver = "/usr/local/bin/chromedriver"
options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['disable-component-update'])
options.add_argument("--user-data-dir=./browser_resources/chrome_data_dir/")
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = Chrome(executable_path=chromedriver, chrome_options=options)
self.home_page = home.Home(self.driver)
def tearDown(self):
self.driver.quit()
def screenshot(self, name='.'):
if name == '.':
name = self.browser
now = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
name = '%s-%s.png' % (name, now)
output_path = "../test_results/screenshots/"
if not os.path.exists(output_path):
os.makedirs(output_path)
retries = 5
for r in xrange(retries):
try:
self.driver.get_screenshot_as_file(output_path + name)
break
except Exception as e:
print "retry to take screenshot: %d out of %d" % (r + 1, retries)
print e.message
if r == retries - 1:
print "ERROR: failed to take screenshot"
print "screenshot saved with file name: %s" %name
return name
def __init__(self):
self.driver = Chrome('/usr/local/bin/chromedriver')
self._max_window()
self.driver.implicitly_wait(10)
self.homepage = Homepage(self.driver)
self.profile_page = ProfilePage(self.driver)
def setUp(self):
""" Start a new browser instance for each test """
self._screenshot_number = 1
self.browser = os.getenv("SELENIUM_BROWSER", settings.SELENIUM_DEFAULT_BROWSER)
command_executor = os.getenv("SELENIUM_COMMAND_EXECUTOR", "")
executor_was_set_explicitly = command_executor != ""
command_executor = command_executor or "http://127.0.0.1:4444/wd/hub"
if os.getenv("SELENIUM_HOST"):
self.sel = self.sauce_labs_driver()
elif executor_was_set_explicitly and self.browser in ("chrome", "firefox"):
if self.browser == "chrome":
caps = DesiredCapabilities.CHROME
browser_profile = None
else:
caps = DesiredCapabilities.FIREFOX
browser_profile = self.get_firefox_profile()
self.sel = RemoteWebDriver(
command_executor=command_executor, desired_capabilities=caps, browser_profile=browser_profile
)
elif self.browser == "firefox":
self.sel = Firefox(self.get_firefox_profile())
elif self.browser == "htmlunit":
self.sel = RemoteWebDriver(
command_executor=command_executor, desired_capabilities=DesiredCapabilities.HTMLUNITWITHJS
)
elif self.browser in ["ios", "ipad", "ipod", "iphone"]:
capabilities = {"app": "safari", "browserName": "", "device": "iPhone Simulator", "os": "iOS 6.1"}
self.sel = RemoteWebDriver(
command_executor=self.appium_command_executor(), desired_capabilities=capabilities
)
elif self.browser == "opera":
self.sel = RemoteWebDriver(
command_executor=command_executor, desired_capabilities=DesiredCapabilities.OPERA
)
elif self.browser == "iexplore":
self.sel = RemoteWebDriver(
command_executor=command_executor, desired_capabilities=DesiredCapabilities.INTERNETEXPLORER
)
elif self.browser == "phantomjs":
self.sel = PhantomJS(service_args=["--debug=true", "--webdriver-loglevel=DEBUG"])
elif self.browser == "safari":
# requires a Safari extension to be built from source and installed
self.sel = RemoteWebDriver(
command_executor=command_executor, desired_capabilities=DesiredCapabilities.SAFARI
)
else:
self.sel = Chrome()
self.sel.set_page_load_timeout(settings.SELENIUM_PAGE_LOAD_TIMEOUT)
# Give the browser a little time; Firefox throws random errors if you
# hit it too soon
time.sleep(1)
def run_test(self,browser_name):
"""
"""
# print "Browser name = " + TestSuite().get_browser_name()
if settings.global_browser is not None:
browser_name = settings.global_browser
print "BROWSER NAME = " + browser_name
if browser_name == "firefox":
self.browser = Firefox()
elif browser_name == "ie":
self.browser = Ie()
elif browser_name == "opera":
self.browser = Opera()
elif browser_name == "chrome":
self.browser = Chrome()
self.logger = logger.Logger(self.get_log_path())
self.logger.log_name = self.get_log_name()
self.logger.log("====================\r\n")
self.logger.log(self.get_log_name()+"\r\n")
self.logger.log("====================\r\n")
# IF true => test succeeded, else => test failed
status = True
# while not self.test_is_running:
try:
self.test_is_running = True
# Method is overridden in child object
self.run(self.browser,self.logger)
self.logger.log("\r\n<<<< SUCCESS >>>>\r\n")
self.browser.quit()
except TestFailedException as e:
status = False
self.browser.get_screenshot_as_file(self.get_screen_path())
self.logger.log("\r\n ERROR: " + e.value + "\r\n Stacktrace: "
+ traceback.format_exc() + "<<<< TEST FAILED >>>>\r\n")
try:
self.browser.quit()
except NoSuchWindowException as e:
print "ERROR: Cannot close window after Test Failed: " + e.message
self.logger.log("ERROR: Cannot close window after Test Failed: " + e.message)
finally:
self.test_is_running = False
self.logger.dump_to_filesystem()
return {self.logger.log_name:[self.logger.get_logs(),status]}
def open_cds(cfg_path: str ='', pass_path: str='') -> WebDriver:
config_path = cfg_path if cfg_path else 'config.json'
with open(config_path) as f:
config = json.load(f)
password_path = pass_path if pass_path else '.pass'
with open(password_path) as f:
credential = json.load(f)
chrome_option = webdriver.ChromeOptions()
chrome_option.add_experimental_option("prefs",
{"download.default_directory": os.path.join(os.getcwd(), 'template')})
if 'driver_executable_path' in config and os.path.exists(config['driver_executable_path']):
browser_driver = Chrome(executable_path=config['driver_executable_path'], chrome_options=chrome_option)
else:
browser_driver = Chrome(chrome_options=chrome_option)
browser_driver.get(config['url_link_cds_eit100'])
input_email = browser_driver.find_element_by_css_selector('input[name= "USER"]')
input_email.send_keys(credential['user_name'])
input_password = browser_driver.find_element_by_css_selector('input[name = "PASSWORD"]')
input_password.send_keys(decrypt(credential['password'], str(uuid.getnode())))
btn_login = browser_driver.find_element_by_css_selector('input[value = "Log on"]')
btn_login.click()
return browser_driver
def setUp(self, browser):
self.browser = browser
if "firefox" in self.browser:
profile = FirefoxProfile()
# profile.set_preference("plugin.state.silverlight", 2)
# profile.set_preference("browser.download.folderList", 1)
# profile.set_preference("pdfjs.disabled", False);
# profile.set_preference("pdfjs.firstRun", True);
self.driver = Firefox(profile) # get a new firefox session
if "chrome" in self.browser:
chromedriver = "/usr/local/bin/chromedriver"
options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['disable-component-update'])
options.add_argument("--user-data-dir=./browser_resources/chrome_data_dir/")
os.environ["webdriver.chrome.driver"] = chromedriver
self.driver = Chrome(executable_path=chromedriver, chrome_options=options)
self.home_page = home.Home(self.driver)
def browser_work(arg):
# 打开浏览器
browser = Chrome()
# 访问用户登录url
browser.get(const.USER_LOGIN_URL)
# 等待网页加载
time.sleep(2)
while 1:
if browser.current_url.split('#')[0] == const.USER_LOGIN_URL:
try:
user_login(browser)
except Exception, e:
pass
elif browser.current_url == const.LOGIN_SUCCESS_URL:
if arg:
try:
user_logout(browser)
except Exception, e:
pass
def extract_citation_for_publication(link):
"""
this function craws the list of articles from a given link. If it has next page, it will continue to it until there is none
@param[in] profile_url the link of google scholar profile you want to crawl
@return the list of articles as a list where each entry is dictionary
"""
browser=Browser('chromedriver.exe')
citation={}
# go the citation view
# as the page is written is javascript, we are not able to get its content via urllib2
# intead we will use Selenium to simulate a web browser to render the page
# req=urllib2.Request(publication[k]['link'], headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'})
# p=urllib2.urlopen(req)
# sub_soup=BeautifulSoup(p.readlines()[0], 'html.parser')
# s=sub_soup.find(id='gs_ccl')
browser.get(link)
while True:
citation_root=browser.find_element_by_id('gs_ccl')
citation_list=citation_root.find_elements_by_class_name('gs_r')
for citation_item in citation_list:
# title
title=citation_item.find_element_by_class_name('gs_rt').text
# try to get the downloading link, if there is one
try:
link=citation_item.find_element_by_id('gs_ggsW2')
link=link.find_element_by_link_text(link.text).get_attribute('href')
except:
link=None
# author
author_line=citation_item.find_element_by_class_name('gs_a')
author_name=author_line.text.split(', ')
author={}
# for each of the author, find its link if its exits
for a in author_name:
try:
print '.',
# there is a google scholar profile with author
item=author_line.find_element_by_link_text(a)
author[a]=item.get_attribute('href')
except:
# there is not such profile
author[a]=None
# we can also press the cite button to get the detailed citation information, skipped here
citation[title]={'link':link, 'author': author}
# go to next page, if there is one
if not next_page(browser):
break
# close
browser.close()
return citation
def extract_publication(profile_url, verbose=verbose_citation_list):
"""
this function crawl the publication list from the google scholar profile
@param[in] profile_url the link of google scholar profile you want to crawl
@param[in] verbose the level of information you want to scrawl. By default, we will scraw the detailed citation list for each of your publicaiton
@return the list of pulication as a list, where each entry is a dictionary
"""
# scholar's artical list
browser=Browser()
browser.get(profile_url)
publication={}
while True:
publication_list=browser.find_elements_by_class_name('gsc_a_tr')
for publication_item in publication_list:
title=publication_item.find_element_by_class_name('gsc_a_at').text
print title
author=publication_item.find_elements_by_class_name('gs_gray')[0].text.split(', ')
vendor=publication_item.find_elements_by_class_name('gs_gray')[1].text
try:
citation=int(publication_item.find_element_by_class_name('gsc_a_ac').text)
link=publication_item.find_element_by_class_name('gsc_a_ac').get_attribute('href')
except:
citation=0
link=None
try:
year=int(publication_item.find_element_by_class_name('gsc_a_h').text)
except:
year=None
"""
# to get citation for every paper, but will be detected as robot
if citation>0 and verbose>=verbose_citation_list:
print 'and its citation list',
# to solve anti-crawl, but not work
# time.sleep(2)
cited_by=extract_citation_for_publication(link)
else:
cited_by=None
print 'finished'
publication[title]={'link':link,'author':author,'vendor':vendor,'citation':citation, 'cited by': cited_by, 'year':year}
"""
publication[title]={'link':link,'author':author,'vendor':vendor,'citation':citation, 'cited by': citation, 'year':year}
if not next_page_new(browser):
break
browser.close()
return publication
def extract_movies(max_page_num=5):
browser = Browser()
browser.get(URL)
movies = {}
while True:
movie_list = browser.find_elements_by_class_name("item")
for movie in movie_list:
title = movie.find_element_by_tag_name("p").text.strip()
rating = movie.find_element_by_tag_name("strong").text.strip()
movies[title] = rating
if max_page_num > 0:
max_page_num -= 1
if not have_more(browser):
break
else:
break
browser.close()
return movies
def weibo_login():
"""
show login page and ask user to enter psd and usr to login
:return: utils
"""
driver = Chrome(executable_path=CHROME_DRIVER_PATH)
driver.get("http://weibo.com")
try:
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "W_nologin_logo"))
)
except Exception as e:
print e
else:
print u"请在登陆页面上进行登陆操作, 之后再文件夹下寻找cookies文件"
retcode = 1 if raw_input(u"登陆成功?(Y/N)") == 'Y' else 0
return retcode, driver.get_cookies()
finally:
driver.quit()
def scrap_nasdaq():
try:
import selenium
except ImportError:
log.error('\nThere is no such module installed')
import sys
sys.exit(-1)
from selenium.webdriver import Chrome
dr = Chrome()
dr.get("http://eoddata.com/stocklist/NASDAQ.htm")
e = [i.text for i in [i for i in dr.find_elements_by_xpath('//*[@id="ctl00_cph1_divLetters"]/table/tbody/tr/td')]]
abec = {}
for znm in e:
dr.get("http://eoddata.com/stocklist/NASDAQ/{}.html".format(znm))
abec[znm] = [symb.text for symb in dr.find_elements_by_xpath('//*[@id="ctl00_cph1_divSymbols"]/table/tbody/tr/td[1]/a')]
for key, val in abec.items():
with open("nasdaq.txt", "a") as ofile:
for rec in val:
ofile.write(rec + "\n")
def extract_hongren(max_page_num=5):
suffix = "hongren"
# 正常情况下，把driver文件所在路径加到Path环境变量里就可以了
# 但是我这里不知道怎么回事就是不行，干脆放在代码所在目录下面了
browser = Browser('chromedriver.exe')
browser.get(BASE_URL + suffix)
items = {}
while True:
item_list = browser.find_elements_by_class_name('wall_item')
for item in item_list:
href = item.find_element(By.CSS_SELECTOR, ".pic_box.pic").get_attribute("href")
desc = item.find_elements_by_class_name("desc")[0].text.strip()
items[href] = desc
if max_page_num > 0:
max_page_num -= 1
if not scroll_to_next(browser):
break
else:
break
browser.close()
return items
class TestCase(TestSuite):
"""
pydoc
"""
__log_name = ""
__log_path = ""
__screen_path = ""
#noinspection PyMissingConstructor
def __init__(self):
self.browser = None
self.wait = None
self.logger = None
self.test_is_running = False
#noinspection PyArgumentList
def run_test(self,browser_name):
"""
"""
# print "Browser name = " + TestSuite().get_browser_name()
if settings.global_browser is not None:
browser_name = settings.global_browser
print "BROWSER NAME = " + browser_name
if browser_name == "firefox":
self.browser = Firefox()
elif browser_name == "ie":
self.browser = Ie()
elif browser_name == "opera":
self.browser = Opera()
elif browser_name == "chrome":
self.browser = Chrome()
self.logger = logger.Logger(self.get_log_path())
self.logger.log_name = self.get_log_name()
self.logger.log("====================\r\n")
self.logger.log(self.get_log_name()+"\r\n")
self.logger.log("====================\r\n")
# IF true => test succeeded, else => test failed
status = True
# while not self.test_is_running:
try:
self.test_is_running = True
# Method is overridden in child object
self.run(self.browser,self.logger)
self.logger.log("\r\n<<<< SUCCESS >>>>\r\n")
self.browser.quit()
except TestFailedException as e:
status = False
self.browser.get_screenshot_as_file(self.get_screen_path())
self.logger.log("\r\n ERROR: " + e.value + "\r\n Stacktrace: "
+ traceback.format_exc() + "<<<< TEST FAILED >>>>\r\n")
try:
self.browser.quit()
except NoSuchWindowException as e:
print "ERROR: Cannot close window after Test Failed: " + e.message
self.logger.log("ERROR: Cannot close window after Test Failed: " + e.message)
finally:
self.test_is_running = False
self.logger.dump_to_filesystem()
return {self.logger.log_name:[self.logger.get_logs(),status]}
def run(self):
pass
def get_browser(self):
pass
# dump test case results to file system
# return TestCase log
def get_log_name(self):
return self.__log_name
def get_log_path(self):
return self.__log_path
def get_screen_path(self):
return self.__screen_path
def set_log_name(self,name):
self.__log_path = settings.get_topface_reports_path() + name + ".log"
self.__log_name = name
self.__screen_path = settings.get_topface_reports_path() + name + ".png"
def do_method(self,method,element_type=None,*args):
dao = DataAccessObject()
t0 = time.time()
self.logger.log("TIME0 = " + str(t0))
method(*args)
result_time = time.time() - t0
self.logger.log("EXECUTION TIME = " + str(result_time) + " seconds")
try:
if element_type == profiling_events.login_event:
self.logger.log("INSERT INTO LOGIN_TIMELINE_TABLE")
dao.insert_into_login_timeline_table(result_time)
elif element_type == profiling_events.message_sent_event:
self.logger.log("INSERT INTO PROFILE_EVENTS_TABLE")
dao.insert_into_send_message_timeline_table(result_time)
elif element_type == profiling_events.questionary_edited_event:
self.logger.log("INSERT INTO QUESTIONARY_TIMELINE_TABLE")
dao.insert_into_questionary_timeline_table(result_time)
elif element_type == profiling_events.user_marked_event:
self.logger.log("INSERT INTO MARKS_TIMELINE_TABLE")
dao.insert_into_mark_user_timeline_table(result_time)
elif element_type == profiling_events.user_navigated_event:
self.logger.log("INSERT INTO NAVIGATION_TIMELINE_TABLE")
dao.insert_into_user_navigation_timeline_table(result_time)
except TestFailedException as e:
raise TestFailedException("Failed to put in log time metrics " + e.message)
raise
started_at = datetime.now()
print("now it is %s" % started_at)
print("newest ver = %d" % CDDA_version_max)
CDDA_version = CDDA_version_first
# keep fetching until we get to CDDA_version_max. TODO: before ending, check again (if we had very long fetching)
if CDDA_version == CDDA_version_max + 1:
input('No new CDDA versions to make hmtl out off!')
exit()
pull_url = 'https://github.com/CleverRaven/Cataclysm-DDA/pull/{}'
commit_url = 'https://github.com/CleverRaven/Cataclysm-DDA/commit/{}'
driver = Chrome(r'C:\Program Files\chromedriver.exe'
) # if that throws error, fix this line
while CDDA_version <= CDDA_version_max:
url = "http://gorgon.narc.ro:8080/job/Cataclysm-Matrix/{}/".format(
CDDA_version)
req = Request(url)
try:
webpage_read = urlopen(req).read().decode("utf-8")
except: # except HTTPError throws NameError
if CDDA_version <= CDDA_version_max:
print("\t%d doesn't exist" % CDDA_version)
CDDA_version += 1
continue
break
class AutomatorMixin(object):
class UnexpectedSituation(Exception):
pass
data_property_class = None
def __init__(self, steps, data_args=[]):
self.steps = steps
self.data = self.data_property_class(*data_args)
def run(self):
options = ChromeOptions()
options.add_argument('--test-type')
self.driver = Chrome(chrome_options=options)
self.perform_steps()
self.driver.close()
def find_element(self, selector):
LOG.info('finding selector "%s"' % selector)
return self.driver.find_element_by_css_selector(selector)
@property
def action_method_lookup(self):
return self.get_action_method__lookup()
def get_action_method__lookup(self):
return {
'click': self.perform_click,
'fill_form': self.perform_fill_form,
'select_drop_down': self.perform_select_drop_down,
}
def get_css_selector(self, action):
return action.get('css_selector')
def get_action_value(self, action):
if 'value' in action:
value = action['value']
elif 'property' in action:
property_name = action['property']
value = getattr(self.data, property_name)
else:
raise AutomatorMixin.UnexpectedSituation('Cannot find key "property" or "value"')
return value
def perform_steps(self):
for step in self.steps:
if 'url' in step:
self.driver.get(step['url'])
if 'actions' in step:
self.perform_actions(step['actions'])
def perform_actions(self, actions):
for action in actions:
action_method = self.action_method_lookup[action['type']]
action_method(action)
def perform_click(self, action):
selector = self.get_css_selector(action)
if selector:
self.find_element(selector).click()
return
# Find by id. This will be needed when people use "." in their id names. Such as kfc's survey
css_id = action['id_selector']
LOG.info(css_id)
self.driver.find_element_by_id(css_id).click()
def perform_fill_form(self, action):
selector = self.get_css_selector(action)
value = self.get_action_value(action)
self.find_element(selector).send_keys(value)
def perform_select_drop_down(self, action):
selector = self.get_css_selector(action)
value = self.get_action_value(action)
Select(self.find_element(selector)).select_by_value(value)
def get_browser(headless: bool = True, browser_class: int = 1) -> Firefox:
"""
获取一个浏览器
:param headless:
:param browser_class: 浏览器种类,0是谷歌, 1 是火狐, 服务器端不能使用谷歌
:return:
"""
"""
selenium安装方法: pip3 install selenium
firefox的headless浏览器
因为headless的浏览器的语言跟随操作系统,为了保证爬回来的数据是正确的语言,
这里必须设置浏览器的初始化参数,
注意，使用headless必须先安装对应浏览器正常的版本,然后再安装headless版本
比如火狐的headless
下载火狐的geckodriver驱动。(当前文件夹下已经有一个了)地址是：
https://github.com/mozilla/geckodriver/releases
下载后解压是一个geckodriver 文件。拷贝到/usr/local/bin目录下，然后加上可执行的权限
sudo chmod +x /usr/local/bin/geckodriver
chrome的headless浏览器
https://chromedriver.storage.googleapis.com/index.html?path=2.35/
你也可以自行搜索chromedriver的下载地址,解压是个可执行文件,放到chrome的目录即可.
一般ubuntu下面,chrome的目录是/opt/google/chrome/
据说使用root权限运行的话,chrome的headless浏览器会报异常.而firefox的headless浏览器不会!
"""
if browser_class == 1:
profile = FirefoxProfile()
profile.set_preference("intl.accept_languages", "zh-cn")
options = FirefoxOptions()
options.add_argument("--headless")
if headless:
try:
browser = Firefox(firefox_profile=profile, executable_path=firefox_driver, firefox_options=options)
except Exception as e:
title = "{} Firefox headless浏览器打开失败".format(datetime.datetime.now())
content = "错误原因是：{}".format(e)
send_mail(title=title, content=content)
logger.exception(e)
raise e
else:
try:
browser = Firefox(firefox_profile=profile, executable_path=firefox_driver,)
except Exception as e:
title = "{} Firefox headless浏览器打开失败".format(datetime.datetime.now())
content = "错误原因是：{}".format(e)
send_mail(title=title, content=content)
logger.exception(e)
raise e
else:
options = ChromeOptions()
options.add_experimental_option("excludeSwitches", ["ignore-certificate-errors"])
if headless:
options.add_argument("--headless")
try:
browser = Chrome(executable_path=chrome_driver, chrome_options=options)
except Exception as e:
title = "{} Chrome headless浏览器打开失败".format(datetime.datetime.now())
content = "错误原因是：{}".format(e)
send_mail(title=title, content=content)
logger.exception(e)
raise e
else:
try:
browser = Chrome(executable_path=chrome_driver, chrome_options=options)
except Exception as e:
title = "{} Chrome headless浏览器打开失败".format(datetime.datetime.now())
content = "错误原因是：{}".format(e)
send_mail(title=title, content=content) # 这是我自定义的方法
logger.exception(e)
raise e
return browser
class CodeForces:
def __init__(self):
self.opts = Options()
self.opts.set_headless()
self.browser = Chrome(options=self.opts)
def close(self):
self.browser.close()
def getTopRated(self):
try:
self.browser.get('https://www.codeforces.com')
section = self.browser.find_elements_by_tag_name('table')
newsec = section[0].text
newsec = newsec[14:]
newsec = newsec.split('\n')
l = []
for i in range(len(newsec)):
n = newsec[i].split(' ')
l.append(n)
with open('top_rated.csv', 'w') as fil:
f = csv.writer(fil)
f.writerow(['RANK', 'USERNAME', 'RATING'])
for i in range(len(l)):
f.writerow(l[i])
print('Successfully created/updated "top_rated.csv"')
except Exception as e:
print('Error occured')
print(e)
def getTodaysProblems(self):
try:
self.browser.get('https://www.codeforces.com/problemset')
section = self.browser.find_elements_by_class_name('problems')
newsec = section[0].text
newsec = newsec[9:]
newsec = newsec.split('\n')
l = []
n = []
ids = []
newl = [None for i in range(2 * len(l))]
final = [None for i in range(len(l))]
d = []
for i in range(len(newsec)):
if (i % 4) == 1:
n.append(newsec[i])
for i in range(len(newsec)):
if (i % 4) == 0:
ids.append(newsec[i])
for i in range(len(newsec)):
if (i % 4) == 3:
l.append(newsec[i])
for i in range(len(l)):
newl.append(l[i].split(' '))
for i in range(len(newl)):
if newl[i][0] == '':
d.append('None')
continue
val = int(newl[i][0])
if val > 0 and val <= 1700:
d.append('Easy')
elif val > 1700 and val <= 2600:
d.append('Medium')
else:
d.append('Hard')
with open('todays_problem_set.csv', 'w') as fil:
f = csv.writer(fil)
f.writerow(['ID', 'PROBLEM', 'LEVEL', 'DIFFICULTY'])
for i in range(len(n)):
f.writerow([ids[i], n[i], newl[i][0], d[i]])
print('Successfully created/updated "todays_problem_set.csv"')
print('Note: Problem sets change from time to time')
except Exception as e:
print('Error occured')
print(e)
def getLatestContest(self):
try:
self.browser.get('https://codeforces.com/contests')
section = self.browser.find_element_by_class_name('sidebox')
newsec = section.text.split('\n')
name = newsec[2]
section2 = self.browser.find_elements_by_class_name('dark')
d = section2[2].text
d = d[:len(d) - 7]
l = section2[3].text
with open('upcoming_contest.txt', 'w') as fil:
fil.write('UPCOMING CONTEST:\n\n')
fil.write('Name of the contest: ' + name + '\n')
fil.write('Begin date: ' + d + ' UTC+5.5\n')
fil.write('Duration: ' + l + 'hrs\n')
print('Successfully created/updated "upcoming_contest.txt"')
except Exception as e:
print('Error occured')
print(e)
# row.find_elements_by_tag_name('td')
if cells[3].text == "Errors" or cells[3].text == "Completed":
downloadldss = row.find_element_by_css_selector("td.context-menu-ldss")
downloadldss.click()
else:
time.sleep(10)
i = i - 1
print("Will wait 10x" + i + "second(s)")
if i < 1:
break
print("you can check the validation result now!")
if __name__ == "__main__":
browser = Chrome()
tran_name = "Test for CDS48 - 100 users contacts - 16"
with open(".pass") as f:
credential = json.load(f)
browser.get("https://eit1-i.svcs.hp.com/cds/")
email = browser.find_element_by_css_selector('input[name= "USER"]')
email.send_keys(credential["user_name"])
password = browser.find_element_by_css_selector('input[name = "PASSWORD"]')
password.send_keys(credential["password"])
Login = browser.find_element_by_css_selector('input[value = "Log on"]')
Login.click()
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
import sys # ----copied above part-----------
import time
from selenium.webdriver import Chrome
from selenium.webdriver.firefox.options import Options
opts = Options()
#opts.set_headless()
#assert opts.headless # Operating in headless mode ----> remove this line if assertion error is thrown
browser = Chrome()
browser.get('https://quora.com')
#login part
email = browser.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[2]/div[1]/div/div[2]/div/div/div/form/div[2]/div[1]/input')
email.clear()
email.send_keys('*****@*****.**')
password = browser.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[2]/div[1]/div/div[2]/div/div/div/form/div[2]/div[2]/input')
password.clear()
password.send_keys('2Aug@Sd0')
time.sleep(1)
login = browser.find_element_by_xpath('/html/body/div[3]/div/div[2]/div[2]/div[1]/div/div[2]/div/div/div/form/div[2]/div[3]/input')
login.click()
time.sleep(3)
#asking question
def init_driver():
username = '******'
# 测试环境平台端
url = r'https://cs1.jsbooks.com.cn/customer/#/customer/user/login'
password = "******"
#云环境
# url = r'http://www.jssz365.com/user/login'
# password = '******'
driver = Chrome()
driver.implicitly_wait(10)
driver.get(url=url)
driver.maximize_window()
driver.find_element_by_id('userName').send_keys(username)
driver.find_element_by_id('password').send_keys(password)
driver.find_element_by_id('verifyCode').send_keys('1234')
driver.find_element_by_class_name('ant-btn').click()
time.sleep(1)
driver.find_element_by_class_name('ant-btn').click()
# driver.execute_script()
# driver.find_element_by_xpath().__setattr__()
# driver.find_element_by_css_selector()
time.sleep(1)
yield driver
driver.quit()
def reset_route(driver: webdriver.Chrome):
driver.find_element_by_css_selector(
"""body > div.metro-body > div > div.metro-app__sidebar-container > div.metro-sidebar-view > div > div > div.scroll__content > div.metro-input-form > div.metro-input-form__buttons > div"""
).click()
time.sleep(0.1)
def get_travel_time(driver: webdriver.Chrome):
return driver.execute_script("""
return Number(document.getElementsByClassName("route-masstransit-step-view__details-info-left-content")[0].textContent.split(" ")[0]);
""")
def get_travel_interval(driver: webdriver.Chrome):
return driver.execute_script("""
return Number(document.getElementsByClassName("route-metro-details-step-view__metro-minutes")[0].textContent.split(" ")[0]);
""")
def get_line(driver: webdriver.Chrome):
bg_color = driver.execute_script("""
return document.getElementsByClassName("metro-input-form__stop-icon")[0].style.backgroundColor;
""")
return LINE_COLORS.index(bg_color) + 1
def main():
# Set up logging
logging.basicConfig(filename='grade-checker.log',
format='%(asctime)s - %(levelname)s ==> %(message)s',
level=logging.INFO)
logging.info("Grade Checker started")
# setup
primuss_username, primuss_password = init()
# Start browser
chrome_options = Options()
chrome_options.add_argument("--headless")
browser = Chrome(options=chrome_options)
browser.implicitly_wait(10)
browser.get('https://www3.primuss.de/cgi-bin/login/index.pl?FH=fhin')
logging.info("Browser created")
try:
# Login in
username = browser.find_element_by_id("username")
username.click()
username.clear()
username.send_keys(primuss_username)
password = browser.find_element_by_id("password")
username.click()
password.clear()
password.send_keys(primuss_password)
button = browser.find_element_by_xpath(
'/html/body/div/div[5]/form/div[4]/button')
button.click()
logging.info("Successfully logged into primuss")
# Get to grad announcement page
open_menu = browser.find_element_by_xpath(
'//*[@id="main"]/div[1]/div/div[1]/button')
open_menu.click()
my_exams = browser.find_element_by_xpath(
'//*[@id="nav-prim"]/div/ul/li[4]/a')
my_exams.click()
my_grades = browser.find_element_by_xpath(
'//*[@id="main"]/div[2]/div[1]/div[2]/form/input[6]')
my_grades.click()
# Get the current grades
new_grades = browser.find_element_by_xpath(
'//*[@id="content-body"]/table[2]/tbody[2]').get_attribute(
'innerHTML')
logging.info("Retrieved grades from primuss")
# Parse grades from table
results = parse(new_grades)
# check for updates
update = check(results)
# If grades were updated, send email
if update:
logging.info("New grades are available")
notify()
else:
logging.info("No new grades available")
except Exception as e:
logging.error("Exception while retrieving grades from primuss: " +
str(e))
finally:
browser.close()
logging.info("Browser closed. Exiting grade Checker.\n\n")
def __init__(self):
self.driver = Chrome()
def __init__(self):
self.opts = Options()
self.opts.set_headless()
self.browser = Chrome(options=self.opts)
from selenium.webdriver import Chrome, ChromeOptions
import sys
import os
import csv
import urllib
if not os.path.isdir('../data/'):
os.makedirs('../data/')
options = ChromeOptions()
options.add_argument('--headless')
driver = Chrome('../chromedriver', options=options)
driver.get('https://vdata.nikkei.com/economicdashboard/macro/')
# Get rid of 's'
str_list = [chr(i) for i in range(97, 97 + 18)]
str_list.remove('n')
str_list.remove('r')
content_list = driver.find_elements_by_class_name('content__section')
section_lists = [
content.find_elements_by_class_name("a-section")
for content in content_list
]
element_index_list = [
list(range(1,
len(section_list) + 1)) for section_list in section_lists
]
title_list = []
i = 0
def main():
lines = read()
USERNAME = lines[0]
PASSWORD = lines[1]
EVENT = lines[2]
opts = Options()
opts.add_argument('--no-sandbox')
opts.add_argument('--disable-dev-shm-usage')
opts.add_argument('--log-level=3')
opts.add_argument('--headless')
opts.add_argument('--disable-gpu')
browser = Chrome(options=opts)
browser.get(
'http://info.vit.ac.in/gravitas18/gravitas/gravitas_coordinator_login.asp'
)
wait = WebDriverWait(browser, 100)
src = browser.page_source
myElem = wait.until(EC.visibility_of_element_located((By.ID, 'loginid')))
loginfield = browser.find_element_by_id('loginid')
loginfield.send_keys(USERNAME)
pwfield = browser.find_element_by_id('logpassword')
pwfield.send_keys(PASSWORD)
soup = BeautifulSoup(src, 'html.parser')
captcha = soup.find_all("input", type='hidden')
captcha = str(captcha[0])
captcha = captcha[-9:-3]
#print(captcha)
cfield = browser.find_element_by_id('captchacode')
cfield.send_keys(captcha)
loginbtn = browser.find_element_by_name('frmSubmit')
loginbtn.click()
#print("waiting for page load")
#myElem = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'glyphicon glyphicon-menu-down')))
browser.get(
'http://info.vit.ac.in/gravitas18/gravitas/coord_event_participants.asp'
)
#http://info.vit.ac.in/gravitas18/gravitas/coord_event_participants.asp
#http://info.vit.ac.in/gravitas18/gravitas/coord_event_participant_list.asp
myElem = wait.until(EC.visibility_of_element_located((By.NAME, 'form1')))
viewbtn = browser.find_element_by_name('form1')
viewbtn.click()
src = browser.page_source
scrapeRegs(src, EVENT)
browser.quit()
def report_day(browser: webdriver.Chrome, ShouJHM, ShiFSH, ShiFZX, ddlSheng,
ddlShi, ddlXian, XiangXDZ, ShiFZJ, t: dt.datetime):
print(f'正在补报{t.year}-{t.month}-{t.day}')
browser.get(
f'https://selfreport.shu.edu.cn/DayReport.aspx?day={t.year}-{t.month}-{t.day}'
)
time.sleep(1)
print('承诺')
browser.find_element(By.ID, 'p1_ChengNuo-inputEl-icon').click()
checkboxes = browser.find_elements(
By.CSS_SELECTOR, '#p1_pnlDangSZS .f-field-checkbox-icon')
if len(checkboxes) > 0: # 有的人没有答题
print('答题')
checkboxes[0].click()
print('是否在上海', ShiFSH)
# 在上海（校内），在上海（不在校内），不在上海
checkboxes = browser.find_elements(By.CSS_SELECTOR,
'#p1_ShiFSH .f-field-checkbox-icon')
if ShiFSH == '在上海（不在校内）':
checkboxes[1].click()
elif ShiFSH == '不在上海':
checkboxes[2].click()
else:
checkboxes[0].click()
time.sleep(1)
print('是否住校', ShiFZX)
try:
checkboxes = browser.find_elements(
By.CSS_SELECTOR, '#p1_ShiFZX .f-field-checkbox-icon')
checkboxes[0 if ShiFZX else 1].click()
except Exception as e:
print('是否住校提交失败')
print('省市县详细地址', ddlSheng, ddlShi, ddlXian, XiangXDZ[:2])
elem = browser.find_element(
By.CSS_SELECTOR, "#p1_ddlSheng input[name='p1$ddlSheng$Value']")
browser.execute_script(
'''
var elem = arguments[0];
var value = arguments[1];
elem.value = value;
''', elem, ddlSheng)
elem = browser.find_element(By.CSS_SELECTOR,
"#p1_ddlShi input[name='p1$ddlShi$Value']")
browser.execute_script(
'''
var elem = arguments[0];
var value = arguments[1];
elem.value = value;
''', elem, ddlShi)
elem = browser.find_element(By.CSS_SELECTOR,
"#p1_ddlXian input[name='p1$ddlXian$Value']")
browser.execute_script(
'''
var elem = arguments[0];
var value = arguments[1];
elem.value = value;
''', elem, ddlXian)
elem = browser.find_element(By.CSS_SELECTOR,
"#p1_XiangXDZ #p1_XiangXDZ-inputEl")
browser.execute_script(
'''
var elem = arguments[0];
var value = arguments[1];
elem.value = value;
''', elem, XiangXDZ)
print('是否家庭地址', ShiFZJ)
checkboxes = browser.find_elements(By.CSS_SELECTOR,
'#p1_ShiFZJ .f-field-checkbox-icon')
checkboxes[0 if ShiFZJ else 1].click()
time.sleep(0.5)
# 随申码
try:
SuiSM = browser.find_element(By.ID, 'p1_pImages_HFimgSuiSM-inputEl')
if SuiSM.get_attribute('value') == '':
print('未检测到已提交随申码')
upload = browser.find_element(By.NAME, 'p1$pImages$fileSuiSM')
upload.send_keys(draw_XingCM(ShouJHM, t))
WebDriverWait(browser, 10).until(
element_has_no_value((By.NAME, 'p1$pImages$fileSuiSM')))
browser.find_element(By.CSS_SELECTOR,
'#p1_pImages_fileSuiSM a.f-btn').click()
WebDriverWait(browser, 10).until(
element_has_value((By.ID, 'p1_pImages_HFimgSuiSM-inputEl')))
print(SuiSM.get_attribute('value'))
else:
print(f'已提交随申码')
except Exception as e:
print(e)
print('随申码提交失败')
# 行程码
try:
XingCM = browser.find_element(By.ID, 'p1_pImages_HFimgXingCM-inputEl')
if XingCM.get_attribute('value') == '':
print('未检测到已提交行程码')
upload = browser.find_element(By.NAME, 'p1$pImages$fileXingCM')
upload.send_keys(draw_XingCM(ShouJHM, t))
WebDriverWait(browser, 10).until(
element_has_no_value((By.NAME, 'p1$pImages$fileXingCM')))
browser.find_element(By.CSS_SELECTOR,
'#p1_pImages_fileXingCM a').click()
WebDriverWait(browser, 10).until(
element_has_value((By.ID, 'p1_pImages_HFimgXingCM-inputEl')))
print(XingCM.get_attribute('value'))
else:
print(f'已提交行程码')
except Exception as e:
print(e)
print('行程码提交失败')
# 确认提交
browser.find_element(By.ID, 'p1_ctl02_btnSubmit').click()
time.sleep(1)
messagebox = browser.find_element(By.CLASS_NAME, 'f-messagebox')
if '确定' in messagebox.text:
for a in messagebox.find_elements(By.TAG_NAME, 'a'):
if a.text == '确定':
a.click()
break
messagebox = browser.find_element(By.CLASS_NAME, 'f-messagebox')
if '提交成功' in messagebox.text:
return True
else:
print(messagebox.text)
return False
else:
print(messagebox.text)
return False
# This will open a web browser, sign into venmo, and pay someone some money
from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
import cPickle as pickle
import SendKeys
import time
import venmoInfo
import datetime
import os
CHROME_DRIVER_PATH = 'C:\ChromeDriver\chromedriver.exe'
VENMO_URL = 'https://venmo.com/'
browser = Chrome(CHROME_DRIVER_PATH)
browser.get(VENMO_URL)
if os.path.isfile('cookies.pkl'):
# there is a cookie file
cookies = pickle.load(open("cookies.pkl", "rb"))
for cookie in cookies:
browser.add_cookie(cookie)
# click on the sign in link
signin_link = browser.find_element_by_link_text("Sign in")
signin_link.click()
# enter the email and password and send it
username_box = browser.find_element_by_class_name("email-username-phone")
username_box.send_keys(venmoInfo.my_u)
def set_chrome(self,**kwargs):
"""
----------------------------------------------------------------------------
Configuration Method:
1. EXECUTABLE,SERVICE ARGS and PORT
2. DESIRED CAPABILITIES
3. CHROME OPTIONS
Command Line Switches:
http://peter.sh/experiments/chromium-command-line-switches/
Capabilities:
https://sites.google.com/a/chromium.org/chromedriver/capabilities
----------------------------------------------------------------------------
This function should be a class for webdriver.
For now, just setting up Chrome.
driver_browsers = ['android',
'chrome',
'firefox',
'htmlunit',
'internet explorer',
'iPhone',
'iPad',
'opera',
'safari']
"""
def set_defaults(self):
default_settings = {'bin_path' : '/usr/local/bin/chromedriver',
'port' : 15010,
'log_path' : os_environ['BD'] + '/html/logs/chromedriver.log',
'user-agent' : "Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1",
# 1 in 1788 per panopticlick !!
'no_java' : True,
'no_plugins' : True,
'net-log-capture-mode' : 'IncludeCookiesAndCredentials',
'log-level' : 0,
'cookie_content' : {},
'capabilities' :
{ 'acceptSslCerts' : True,
'databaseEnabled' : False,
'unexpectedAlertBehaviour' : "accept",
'applicationCacheEnabled' : False,
'webStorageEnabled' : False,
'browserConnectionEnabled' : False,
'locationContextEnabled' : True,
},
'loggingPrefs' :
{ "driver" : "ALL",
"server" : "ALL",
"browser" : "ALL"},
'true_opts' :
[
'disable-core-animation-plugins',
'disable-plugins',
'disable-extensions',
'disable-plugins-discovery',
'disable-site-engagement-service',
'disable-text-input-focus-manager',
'enable-account-consistency',
'enable-devtools-experiments',
'enable-logging',
'enable-network-information',
'enable-net-benchmarking',
'enable-network-portal-notification',
'enable-strict-site-isolation',
'incognito', # if incognito, extensions must be disabled
'log-net-log',
'scripts-require-action',
'system-developer-mode',
# 'use-mobile-user-agent',
],
'false_opts' :
[
'enable-profiling',
],
}
excluded = [] if not (hasattr(self,'T') and hasattr(self.T,'excluded_defaults')) else self.T.excluded_defaults
for k,v in default_settings.iteritems():
if excluded.count(k):
if T.has_key(k):
del T[k]
else:
T.update( {k : v})
return T
def set_desired_capabilities(self):
from selenium.webdriver import DesiredCapabilities
dc = DesiredCapabilities.CHROME.copy()
platforms = ['WINDOWS', 'XP', 'VISTA', 'MAC', 'LINUX', 'UNIX', 'ANDROID', 'ANY']
# -PROXY OBJECT
# from selenium.webdriver import Proxy
# -READ-WRITE CAPABILITIES
rw_capabilities = [
'acceptSslCerts', # boolean unless specified
'javascriptEnabled',
'databaseEnabled',
'proxy', # Proxy Object
'unexpectedAlertBehaviour', # string {"accept", "dismiss", "ignore"}
'applicationCacheEnabled',
'webStorageEnabled',
'rotatable',
'browserConnectionEnabled',
'locationContextEnabled',
'elementScrollBehavior', # int (align with the top (0) or bottom (1) of the viewport)
'nativeEvents'
]
assert T.has_key('capabilities')
for it in rw_capabilities:
if T['capabilities'].has_key(it):
dc[it] = str(T['capabilities'][it])
# -loggingPrefs OBJECT (dict)
# "OFF", "SEVERE", "WARNING",
# "INFO", "CONFIG", "FINE",
# "FINER","FINEST", "ALL"
if T.has_key('loggingPrefs'):
dc[it] = T['loggingPrefs']
return dc
def set_profile(self):
profile = {#"download.default_directory" : "C:\\SeleniumTests\\PDF",
"download.prompt_for_download" : False,
"download.directory_upgrade" : True,
"plugins.plugins_disabled" : ["Chromoting Viewer",
"Chromium PDF Viewer"],
}
opts.add_experimental_option( "prefs", profile)
def set_performance_logging(self):
perfLogging = {
"enableNetwork" : True,
"enablePage" : True,
"enableTimeline" : True,
#"tracingCategories":<string>,
"bufferUsageReportingInterval" : 1000
}
opts.add_experimental_option( "perfLoggingPrefs",perfLogging)
def set_chrome_options(self):
from selenium.webdriver import ChromeOptions
opts = ChromeOptions()
### Add Boolean Arguments
if T.has_key('true_opts'):
for it in T['true_opts']:
opts.add_argument( '%s=1' % it )
if T.has_key('false_opts'):
for it in T['false_opts']:
opts.add_argument( '%s=0' % it )
value_opts = [
'profile-directory',
'log-level', # 0 to 3: INFO = 0, WARNING = 1, LOG_ERROR = 2, LOG_FATAL = 3
'net-log-capture-mode', # "Default" "IncludeCookiesAndCredentials" "IncludeSocketBytes"'
'register-font-files', # might be windows only
'remote-debugging-port',
'user-agent',
'user-data-dir', # don't use b/c it negates no-extension options
]
### Add Value Arguments
for it in value_opts:
if T.has_key(it):
opts.add_argument( '%s=%s' % (it,T[it]) )
### OTHER CHROME OPTIONS NOT YET FULLY CONFIGURED
# -extensions list str
# -localState dict
# -prefs dict
# set_profile()
# -detach bool
# -debuggerAddress str
# -excludeSwitches list str
# -minidumpPath str
# -mobileEmulation dict
# -perfLoggingPrefs OBJECT (dict)
# set_performance_logging()
return opts
from selenium.webdriver import Chrome
T = {}
if kwargs:
T.update( kwargs)
if (hasattr(self,'T') and hasattr(self.T,'kwargs')):
T.update( self.T.kwargs)
# Cycle Through kwargs and Extract Configs
if hasattr(self.T,'id'):
T.update( self.T.id.__dict__)
if hasattr(self.T.id,'details'):
for k,v in self.T.id.details.__dict__.iteritems():
T.update( { k.strip('_') : v})
if hasattr(self.T.id,'cookie'):
if hasattr(self.T.id.cookie,'content'):
T.update( {'cookie_content' : self.T.id.cookie.content})
# Set Defaults if not provided
if not T.has_key('defaults'):
T = set_defaults(self)
# Config Data Storage if Possible
if T.has_key('SAVE_DIR'):
T['user-data-dir'] = T['SAVE_DIR']
T['profile-directory'] = 'Profile'
if T.has_key('guid'):
T['log_path'] = '%s/%s.log' % (T['SAVE_DIR'],T['guid'])
# Configure with Special Profiles if Requested
special_profiles = os_environ['BD'] + '/html/webdrivers/chrome/profiles'
if T.has_key('no_java') and T['no_java']:
if T.has_key('no_plugins') and T['no_plugins']:
T['user-data-dir'] = special_profiles + '/no_java_no_plugins/'
del T['profile-directory']
else:
T['user-data-dir'] = special_profiles + '/no_java/'
del T['profile-directory']
elif T.has_key('no_plugins') and T['no_plugins']:
T['user-data-dir'] = special_profiles + '/no_plugins/'
del T['profile-directory']
# SERVICE ARGS # ( somewhat documented in executable help, i.e., chromedriver --help )
service_args = ["--verbose",
"--log-path=%(log_path)s" % T]
dc = set_desired_capabilities(self)
opts = set_chrome_options(self)
d = Chrome( executable_path = T['bin_path'],
port = T['port'],
service_args = service_args,
desired_capabilities = dc,
chrome_options = opts)
d.set_window_size( 1280,720)
if T['cookie_content']:
d.add_cookie( T['cookie_content'])
self.config_browser( d,kwargs)
return d,T
class SelemimumCrawler:
__PAGE_URL = "https://www.facebook.com/"
__EMAIL_INPUT_XPATH = "//input[@name='email']"
__PASSWORD_INPUT_XPATH = "//input[@name='pass']"
__ACCOUNT_LINK_XPATH = "//*[@id='userNav']/ul/li[1]/a"
__FRIENDS_AMOUNT_XPATH = "//*[@data-tab-key='friends']/span[1]"
__FRIENDS_LINK_XPATH = "//*[@data-tab-key='friends']"
__FRIENDS_XPATH = "//*[@class='fsl fwb fcb']/a"
__ACCOUNT_NAME = "//*[@id='fb-timeline-cover-name']/a"
def __init__(self, driver_path="chromedriver", options=None):
Logger.info(__file__, "SeleniumCrawler init")
try:
self.driver = Chrome(executable_path=driver_path, options=options)
Logger.info(__file__, "Driver initiated")
except WebDriverException as err:
Logger.error(__file__, err.args)
self.configs = Configuration()
def execute(self):
data = dict()
Logger.info(__file__, f"Opening {self.__PAGE_URL} page")
self.driver.get(self.__PAGE_URL)
Logger.info(__file__, "Finding login input web element")
login_web_element = self.get_element(
by=By.XPATH,
value=self.__EMAIL_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send email to login input web element}")
login_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_LOGIN])
Logger.info(__file__, "Finding password input web element")
password_web_element = self.get_element(
by=By.XPATH,
value=self.__PASSWORD_INPUT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Send password to password input web element}")
password_web_element.send_keys(
self.configs.settings[Values.SETTINGS][Values.FACEBOOK_PASSWORD])
Logger.info(__file__,
"Send Enter key pressing to password input web element}")
password_web_element.send_keys(Keys.ENTER)
Logger.info(__file__, "Finding account link")
account_button_web_element = self.get_element(
by=By.XPATH,
value=self.__ACCOUNT_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on account link")
account_button_web_element.click()
Logger.info(__file__, "Extracting total friends amount value")
total_friends_amount = int(
self.get_element(by=By.XPATH,
value=self.__FRIENDS_AMOUNT_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text)
Logger.debug(__file__, f"Extracted {total_friends_amount} value")
Logger.info(__file__, "Finding friends page link")
friends = self.get_element(by=By.XPATH,
value=self.__FRIENDS_LINK_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.info(__file__, "Click on friends page link")
friends.click()
Logger.info(__file__, "Extracting account name")
name = self.get_element(by=By.XPATH,
value=self.__ACCOUNT_NAME,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME]).text
friends = self.__get_friends_list()
previous_friend_amount = 0
current_friends_amount = len(friends)
while previous_friend_amount != current_friends_amount:
Logger.info(__file__, "Friends list can be scroll")
previous_friend_amount = current_friends_amount
Logger.info(__file__, "Scroll to last friends list element")
ActionChains(self.driver).move_to_element(friends[-1]).perform()
time.sleep(1)
friends = self.__get_friends_list()
current_friends_amount = len(friends)
Logger.info(__file__, "Full friends list already loaded")
Logger.info(__file__, "Extract data from friends web elements list")
data["Account name: "] = name
for friend in friends:
link = friend.get_attribute("href")
data[link[:friend.get_attribute('href').find('fref') -
1]] = friend.text
Logger.info(__file__, "Closing web driver")
Logger.info(__file__, "Closing selenium web driver")
self.driver.quit()
data["Total friends amount"] = total_friends_amount
data["Scanned friends amount"] = current_friends_amount
return data
def __get_friends_list(self):
Logger.info(__file__, "Load friends list web elements")
list = self.get_elements(by=By.XPATH,
value=self.__FRIENDS_XPATH,
wait=self.configs.settings[Values.SETTINGS][
Values.ELEMENT_WAIT_TIME])
Logger.debug(__file__, f"Size of loaded friends list: {len(list)}")
return list
def get_element(self, by, value, wait=0):
try:
return WebDriverWait(self.driver, wait).until(
EC.presence_of_element_located((by, value)))
except TimeoutException as err:
Logger.error(__file__, err.args)
def get_elements(self, by, value, wait=0):
try:
return WebDriverWait(self.driver, wait).until(
EC.presence_of_all_elements_located((by, value)))
except TimeoutException as err:
Logger.error(__file__, err.args)
def processing(url, name, ticker, path_):
options = webdriver.ChromeOptions()
options.add_argument('headless')
browser = Chrome(executable_path = getcwd() + '/Driver/chromedriver_Windows.exe', chrome_options = options)
browser.get(url)
search_form = browser.find_element_by_xpath('''/html/body/div[5]/header/div[1]/div/div[3]/div[1]/input''')
search_form.send_keys(name)
search_form.send_keys(Keys.ENTER)
search_form = browser.find_element_by_xpath('''//*[@id="fullColumn"]/div/div[2]/div[2]/div[1]/a[1]''')
search_form.click()
search_form = browser.find_element_by_xpath('''//*[@id="pairSublinksLevel1"]/li[3]/a''')
search_form.click()
news = []
dates = []
pages_table = browser.find_element_by_xpath('''//*[@id="paginationWrap"]/div[2]''')
for i in range(2):
pages_table.find_element_by_link_text(str(i+1)).click()
table = browser.find_element_by_xpath('''//*[@id="leftColumn"]/div[8]''')
articles = table.find_elements_by_class_name('articleItem')
for article in articles:
text = article.find_element_by_class_name('textDiv').find_element_by_tag_name('a').text
date = article.find_element_by_class_name('articleDetails').find_element_by_class_name('date').text
news.append(text)
dates.append(date)
data = []
for i in range(len(news)):
data.append((replace_shit(change_date(dates[i])), lemmatizator(news[i])))
with open(path_ + 'News' + ticker + '.csv','w') as f:
writer = csv.writer(f)
writer.writerow(('Date', 'New'))
for i in data:
writer.writerow(i)
browser.close()
print(name + ' - done')
def run(self):
options = ChromeOptions()
options.add_argument('--test-type')
self.driver = Chrome(chrome_options=options)
self.perform_steps()
self.driver.close()
from selenium.webdriver import Chrome
from time import sleep
url = 'https://selenium.dunossauro.live/exercicio_06.html'
browser = Chrome()
browser.get(url)
def preencher_form(browser, form, nome, senha):
sleep(2)
context = browser.find_element_by_css_selector(f'.form-{form}')
inputs_form = {
'nome': context.find_element_by_name('nome'),
'senha': context.find_element_by_name('senha'),
'enviar': context.find_element_by_css_selector('[type="submit"]')
}
inputs_form['nome'].send_keys(nome)
inputs_form['senha'].send_keys(senha)
inputs_form['enviar'].click()
for n in range(7):
sleep(2)
form = browser.find_element_by_css_selector('span').text
preencher_form(browser, form, f'elza{n}', f'123{n}')
def test_invalid_data1():
path = "C:\\Users\\chirag\\Downloads\\chromedriver_win32\\chromedriver.exe"
driver = Chrome(executable_path=path)
driver.get("http://www.thetestingworld.com/testings")
driver.maximize_window()
def setUp(self):
options = ChromeOptions()
options.add_argument("--no-sandbox")
self.browser = Chrome(options=options)
import os
from dotenv import load_dotenv
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
load_dotenv()
opts = ChromeOptions()
opts.binary_location = os.environ.get("GOOGLE_CHROME_BIN")
opts.add_argument("--headless")
opts.add_argument("--no-sandbox")
opts.add_argument("--disable-dev-sh-usage")
driver = Chrome(executable_path=os.environ.get("CHROMEDRIVER_PATH"),
options=opts)
#Sub of the Week
sotw = ''
def get_sub_of_the_week():
global sotw
i = 0
while i < 3:
print('Fetching Sub of the Week...')
try:
set_publix_store()
driver.get('https://www.publix.com/savings/weekly-ad')
driver.find_element_by_xpath("//a[@id='deli']").click()
sotw = driver.find_element_by_xpath(
def Driver(url):
driver = Chrome()
driver.get(url)
current_windows = driver.window_handles
print(current_windows)
class FilterKeyError(KeyError):
def __init__(self, key, filters):
# out = 'Potential filters: ' + ' '.join(filters.keys()) + ' doesn\'t contain ' + key
flist = ' '.join(filters.keys())
out = 'Desired filter %s not found in available filters (%s)' %(key, flist)
super(FilterKeyError, self).__init__(out)
class DetailsPage(BasePage):
@property
def foodName(self):
return self.driver.find_element_by_class_name('scout-spot-name')
@property
def foodCuisines(self):
return self.driver.find_element_by_class_name('scout-spot-cuisine')
@property
def foodType(self):
return self.driver.find_element_by_class_name('scout-spot-type')
@property
def openStatus(self):
return self.driver.find_element_by_class_name('scout-spot-status')
if __name__ == '__main__':
from selenium.webdriver import Firefox, Chrome
d = Chrome()
d.get('http://localhost:8001/')
page = HomePage(d)
class WebBot():
def __init__(self):
self.driver = Chrome()
def login(self):
self.driver.get(
"https://adfs.inacap.cl/adfs/ls/?wtrealm=https://siga.inacap.cl/sts/&wa=wsignin1.0&wreply=https://siga.inacap.cl/sts/&wctx=https%3a%2f%2fadfs.inacap.cl%2fadfs%2fls%2f%3fwreply%3dhttps%3a%2f%2fwww.inacap.cl%2ftportalvp%2fintranet-alumno%26wtrealm%3dhttps%3a%2f%2fwww.inacap.cl%2f"
)
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_element_by_xpath('//*[@id="userNameInput"]'))
inputUsr = self.driver.find_element_by_xpath(
'//*[@id="userNameInput"]')
inputUsr.send_keys(username)
inputPass = self.driver.find_element_by_xpath(
'//*[@id="passwordInput"]')
inputPass.send_keys(password)
inputLogin = self.driver.find_element_by_xpath(
'//*[@id="submitButton"]')
inputLogin.click()
sleep(10)
def getSede(self):
el = WebDriverWait(
self.driver, timeout=30).until(lambda d: d.find_element_by_xpath(
'/html/body/div[1]/div/section[3]/div[1]/ul/li[3]/a'))
self.driver.find_element_by_xpath(
'/html/body/div[1]/div/section[3]/div[1]/ul/li[3]/a').click()
sleep(3)
self.driver.switch_to.window(self.driver.window_handles[1])
el = WebDriverWait(
self.driver, timeout=30
).until(lambda d: d.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div/section/div/div/aside/aside[2]/div/div/div[1]/ul/li[2]/a'
))
self.driver.find_element_by_xpath(
'/html/body/div[2]/div[2]/div/div/section/div/div/aside/aside[2]/div/div/div[1]/ul/li[2]/a'
).click()
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_elements_by_css_selector('div.media'))
clases = self.driver.find_elements_by_css_selector('div.media')
for x in clases:
sede = x.find_element_by_tag_name('a').text
match = re.search('Santiago Centro', sede)
if match:
print('Sede: ' + match.group())
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
return
print('La sede no es la especificad')
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
self.driver.close()
input('Pulse una tecla para finalizar')
raise SystemExit
def getCarrera(self):
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_element_by_xpath('//*[@id="t1_contenido"]'))
cont = self.driver.find_element_by_xpath('//*[@id="t1_contenido"]')
carrera = cont.find_element_by_tag_name('h3')
print('La carrera es: ' + carrera.text)
if carrera.text == 'Ingeniería en Informática' or carrera.text == 'Analista Programador':
print('Usted es de la carrera c:')
return
print('Usted no es de la carrera :c')
self.driver.close()
input('Pulse una tecla para finalizar')
raise SystemExit
def getRamos(self):
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_element_by_link_text('Mi Malla'))
self.driver.find_element_by_link_text('Mi Malla').click()
sleep(3)
self.driver.switch_to.window(self.driver.window_handles[1])
pattern = re.compile('^TI[a-zA-Z0-9]*')
ti = []
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_elements_by_css_selector('div.card'))
clases = self.driver.find_elements_by_css_selector('div.card')
for x in clases:
match = pattern.match(x.find_element_by_tag_name('p').text)
if match:
name = x.find_element_by_tag_name('h5').text
print(name)
ti.append(name)
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
return ti
def goToEmail(self, ti):
lista = {}
el = WebDriverWait(
self.driver, timeout=30).until(lambda d: d.find_element_by_xpath(
'/html/body/div/div/section[3]/div[1]/div[1]/div[1]/h5'))
self.driver.find_element_by_xpath(
'/html/body/div/div/section[3]/div[1]/div[1]/div[1]/h5').click()
sleep(1)
el = WebDriverWait(
self.driver, timeout=30).until(lambda d: d.find_element_by_xpath(
'/html/body/div/div/section[3]/div[1]/div[1]/div[1]/ul/li[4]/a'
))
self.driver.find_element_by_xpath(
'/html/body/div/div/section[3]/div[1]/div[1]/div[1]/ul/li[4]/a'
).click()
sleep(3)
self.driver.switch_to.window(self.driver.window_handles[1])
# entrar a curso
el = WebDriverWait(
self.driver,
timeout=30).until(lambda d: d.find_elements_by_class_name('curso'))
cursos = self.driver.find_elements_by_class_name('curso')
for curso in cursos:
if curso.text in ti:
print('\n' + curso.text)
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: curso.find_element_by_tag_name('a'))
curso.find_element_by_tag_name('a').click()
el = WebDriverWait(
self.driver, timeout=30
).until(lambda d: d.find_element_by_xpath(
'/html/body/form/div[4]/div[3]/div/div/div/div[2]/div/div/div[1]/div[2]/label/select'
))
cb = Select(
self.driver.find_element_by_xpath(
'/html/body/form/div[4]/div[3]/div/div/div/div[2]/div/div/div[1]/div[2]/label/select'
))
cb.select_by_visible_text('100')
# sacar lista de alumnos
sleep(2)
el = WebDriverWait(self.driver, timeout=30).until(
lambda d: d.find_elements_by_css_selector('td.sorting_1'))
nom = self.driver.find_elements_by_css_selector('td.sorting_1')
mails = self.driver.find_elements_by_css_selector('td.curso')
# limpiar basura
aux = mails.copy()
for x in aux:
if len(x.text) < 1: mails.remove(x)
# Llenar lista de estudiantes a mandar
for x in range(len(nom)):
lista[x] = {'name': nom[x].text, 'email': mails[x].text}
print('nombre: ' + nom[x].text + '\tCorreo: ' +
mails[x].text + '\n') #cambiar por output de correos
self.driver.find_element_by_xpath(
'//*[@id="btnVolver"]').click()
sleep(1)
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
return lista
class NewVisitorTest(LiveServerTestCase):
def setUp(self):
options = ChromeOptions()
options.add_argument("--no-sandbox")
self.browser = Chrome(options=options)
def TearDown(self):
self.browser.quit()
def wait_for_row_in_list_table(self, row_text):
start_time = time.time()
while True:
try:
table = self.browser.find_element_by_id('id_list_table')
rows = table.find_elements_by_tag_name('tr')
self.assertIn(row_text, [row.text for row in rows])
return
except (AssertionError, WebDriverException) as e:
if time.time() - start_time > MAX_WAIT:
raise e
time.sleep(0.5)
def test_can_start_a_list_for_one_user(self):
# Edith has heard about a cool new online to-do app. She goes
# to check out its homepage
self.browser.get(self.live_server_url)
# She notices the page title and header mention to-do lists
self.assertIn('To-Do', self.browser.title)
header_text = self.browser.find_element_by_tag_name('h1').text
self.assertIn('To-Do', header_text)
# She is invited to enter a to-do item straight away
inputbox = self.browser.find_element_by_id('id_new_item')
self.assertEqual(inputbox.get_attribute('placeholder'),
'Enter a to-do item')
# She types "Buy peacock feathers" into a text box (Edith's hobby
# is tying fly-fishing lures)
inputbox.send_keys('Buy peacock feathers')
# When she hits enter, the page updates, and now the page lists
# "1: Buy peacock feathers" as an item in a to-do list table
inputbox.send_keys(Keys.ENTER)
self.wait_for_row_in_list_table('1: Buy peacock feathers')
# There is still a text box inviting her to add another item. She
# enters "Use peacock feathers to make a fly" (Edith is very
# methodical)
inputbox = self.browser.find_element_by_id('id_new_item')
inputbox.send_keys('Use peacock feathers to make a fly')
inputbox.send_keys(Keys.ENTER)
# The page updates again, and now shows both items on her list
self.wait_for_row_in_list_table(
'2: Use peacock feathers to make a fly')
self.wait_for_row_in_list_table('1: Buy peacock feathers')
# Satisfied, she goes back to sleep
def test_multiple_users_can_start_lists_at_different_urls(self):
# Edith starts a new to-do list
self.browser.get(self.live_server_url)
inputbox = self.browser.find_element_by_id('id_new_item')
inputbox.send_keys('Buy peacock feathers')
inputbox.send_keys(Keys.ENTER)
self.wait_for_row_in_list_table('1: Buy peacock feathers')
# She notices that her list has a unique URL
edith_list_url = self.browser.current_url
self.assertRegex(edith_list_url, '/lists/.+')
# Now a new user, Francis, comes along to the site.
## We use a new browser session to make sure that no information
## of Edith's is coming through from cookies etc
self.browser.quit()
options = ChromeOptions()
options.add_argument("--no-sandbox")
self.browser = Chrome(options=options)
# Francis visits the home page. There is no sign of Edith's
# list
self.browser.get(self.live_server_url)
page_text = self.browser.find_element_by_tag_name('body').text
self.assertNotIn('Buy peacock feathers', page_text)
self.assertNotIn('make a fly', page_text)
# Francis starts a new list by entering a new item. He
# is less interesting than Edith...
inputbox = self.browser.find_element_by_id('id_new_item')
inputbox.send_keys('Buy milk')
inputbox.send_keys(Keys.ENTER)
self.wait_for_row_in_list_table('1: Buy milk')
# Francis gets his own unique URL
francis_list_url = self.browser.current_url
self.assertRegex(francis_list_url, '/lists/.+')
self.assertNotEqual(francis_list_url, edith_list_url)
# Again, there is no trace of Edith's list
page_text = self.browser.find_element_by_tag_name('body').text
self.assertNotIn('Buy peacock feathers', page_text)
self.assertIn('Buy milk', page_text)
# Satisfied, they both go back to sleep
def test_layout_and_styling(self):
# Edith goes to the home page
self.browser.get(self.live_server_url)
self.browser.set_window_size(1024, 768)
# She notices the input box is nicely centered
inputbox = self.browser.find_element_by_id('id_new_item')
self.assertAlmostEqual(inputbox.location['x'] +
inputbox.size['width'] / 2,
512,
delta=10)
class SeleniumTestCase(LiveServerTestCase):
"""
Base class for Selenium tests. Allows tests to be written independently
of which browser they're going to be run in.
"""
@classmethod
def appium_command_executor(cls):
""" Get the command executor URL for iOS simulator testing """
if hasattr(cls, '_appium_executor'):
return cls._appium_executor
# Get the address iWebDriver will connect to
address = None
try:
address = socket.gethostbyname(socket.gethostname())
except:
# Use default address defined below
pass
# If we don't have an address we should use localhost
if not address:
address = '127.0.0.1'
port = 4723
cls._appium_executor = "".join(["http://", address, ":", str(port),
'/wd/hub'])
return cls._appium_executor
@classmethod
def setUpClass(cls):
# Create the screenshots directory if it doesn't exist yet
screenshot_dir = settings.SELENIUM_SCREENSHOT_DIR
if screenshot_dir and not os.path.exists(screenshot_dir):
os.makedirs(screenshot_dir)
super(SeleniumTestCase, cls).setUpClass()
@classmethod
def tearDownClass(cls):
super(SeleniumTestCase, cls).tearDownClass()
def setUp(self):
""" Start a new browser instance for each test """
self._screenshot_number = 1
self.browser = os.getenv('SELENIUM_BROWSER',
settings.SELENIUM_DEFAULT_BROWSER)
if os.getenv('SELENIUM_HOST'):
self.sel = self.sauce_labs_driver()
elif self.browser == 'firefox':
self.sel = Firefox()
elif self.browser == 'htmlunit':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.HTMLUNITWITHJS)
elif self.browser in ['ios', 'ipad', 'ipod', 'iphone']:
capabilities = {
'app': 'safari',
'browserName': '',
'device': 'iPhone Simulator',
'os': 'iOS 6.1'
}
self.sel = RemoteWebDriver(command_executor=self.appium_command_executor(),
desired_capabilities=capabilities)
elif self.browser == 'opera':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.OPERA)
elif self.browser == 'iexplore':
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.INTERNETEXPLORER)
elif self.browser == 'phantomjs':
self.sel = PhantomJS(service_args=['--debug=true',
'--webdriver-loglevel=DEBUG'])
elif self.browser == 'safari':
# requires a Safari extension to be built from source and installed
self.sel = RemoteWebDriver(desired_capabilities=DesiredCapabilities.SAFARI)
else:
self.sel = Chrome()
self.sel.set_page_load_timeout(settings.SELENIUM_PAGE_LOAD_TIMEOUT)
# Give the browser a little time; Firefox throws random errors if you
# hit it too soon
time.sleep(1)
def tearDown(self):
# Check to see if an exception was raised during the test
info = sys.exc_info()
passed = info[0] is None
if not passed:
# Want to see what went wrong
self.screenshot()
self.report_status(passed)
if hasattr(self, 'sel'):
self.sel.quit()
super(SeleniumTestCase, self).tearDown()
# ~~~~~~~~~~~~~~~~~~~~~~~~~ Selenium operations ~~~~~~~~~~~~~~~~~~~~~~~~~~
def assert_hidden(self, selector):
element = self.wait_for_element(selector)
msg = "'%s' should not be visible" % selector
assert not element.is_displayed(), msg
def assert_not_present(self, selector):
assert_raises(NoSuchElementException,
self.sel.find_element_by_css_selector, selector)
def assert_not_visible(self, selector):
""" Ok if it's either missing or hidden """
try:
element = self.sel.find_element_by_css_selector(selector)
except NoSuchElementException:
return
msg = "'%s' should not be visible" % selector
assert not element.is_displayed(), msg
def assert_text_not_in_element(self, selector, text):
""" Verify that the specified element does not contain certain text """
msg = "'%s' should not contain the text '%s'" % (selector, text)
content = self.sel.find_element_by_css_selector(selector).text
assert text not in content, msg
def assert_visible(self, selector):
element = self.wait_for_element(selector)
msg = "'%s' should be visible" % selector
assert element.is_displayed(), msg
def audit_accessibility(self):
""" Check for accessibility violations using the JavaScript library
from Chrome's Developer Tools. """
# First add the library to the page
script = ''
for line in ADD_ACCESSIBILITY_SCRIPT.splitlines():
script += line.strip()
self.sel.execute_script(script)
# Wait for the script to finish loading
self.wait_for_condition('return axs.AuditRule.specs.videoWithoutCaptions !== "undefined";')
# Now run the audit and inspect the results
self.sel.execute_script('axs_audit_results = axs.Audit.run();')
failed = self.sel.execute_script('return axs_audit_results.some(function (element, index, array) { return element.result === "FAIL" });')
if failed:
report = self.sel.execute_script('return axs.Audit.createReport(axs_audit_results);')
raise self.failureException(report)
def click(self, selector):
""" Click the element matching the selector (and retry if it isn't
visible or clickable yet) """
element = self.wait_for_element(selector)
element_was_clicked = lambda driver: lambda_click(element)
msg = "The element matching '%s' should be clickable" % selector
Wait(self.sel).until(element_was_clicked, msg)
return element
def click_link_with_text(self, text):
link_is_present = lambda driver: driver.find_element_by_link_text(text)
msg = "A link with text '%s' should be present" % text
link = Wait(self.sel).until(link_is_present, msg)
link.click()
return link
def click_link_with_xpath(self, xpath):
link_is_present = lambda driver: driver.find_element_by_xpath(xpath)
msg = "A link with xpath '%s' should be present" % xpath
link = Wait(self.sel).until(link_is_present, msg)
link.click()
return link
def enter_text(self, selector, value):
field = self.wait_for_element(selector)
field.send_keys(value)
self.screenshot()
return field
def enter_text_via_xpath(self, xpath, value):
field = self.wait_for_xpath(xpath)
field.send_keys(value)
self.screenshot()
return field
def get(self, relative_url):
self.sel.get('%s%s' % (self.live_server_url, relative_url))
self.screenshot()
def screenshot(self):
if hasattr(self, 'sauce_user_name'):
# Sauce Labs is taking screenshots for us
return
if not hasattr(self, 'browser') or self.browser == 'htmlunit':
# Can't take screenshots
return
screenshot_dir = settings.SELENIUM_SCREENSHOT_DIR
if not screenshot_dir:
return
name = "%s_%d.png" % (self._testMethodName, self._screenshot_number)
path = os.path.join(screenshot_dir, name)
self.sel.get_screenshot_as_file(path)
self._screenshot_number += 1
def select_by_text(self, selector, text):
select = Select(self.wait_for_element(selector))
select.select_by_visible_text(text)
self.screenshot()
return select
def select_by_value(self, selector, value):
select = Select(self.wait_for_element(selector))
select.select_by_value(value)
self.screenshot()
return select
def select_text(self, selector, start=0, end=-1):
""" Selects the specified text range of the element matching the
provided selector by simulating a mouse down, programmatically
selecting the text, and then simulating a mouse up. Doesn't yet work
on IE < 9 or iOS. Doesn't support nested markup either. """
if not hasattr(self, 'select_text_template'):
template = ''
for line in SELECT_TEXT_SOURCE.splitlines():
template += line.strip()
self.select_text_template = template
script = self.select_text_template % (selector, start, end)
self.sel.execute_script(script)
self.screenshot()
def wait_for_background_color(self, selector, color_string):
color = Color.from_string(color_string)
correct_color = lambda driver: Color.from_string(driver.find_element_by_css_selector(selector).value_of_css_property("background-color")) == color
msg = "The color of '%s' should be %s" % (selector, color_string)
Wait(self.sel).until(correct_color, msg)
self.screenshot()
def wait_for_condition(self, return_statement, msg=None):
"""Wait until the provided JavaScript expression returns true.
Note: for this to work, the expression must include the "return"
keyword, not just the expression to be evaluated."""
condition_is_true = lambda driver: driver.execute_script(return_statement)
if not msg:
msg = '"{}" never became true'.format(return_statement)
Wait(self.sel).until(condition_is_true, msg)
def wait_for_element(self, selector):
element_is_present = lambda driver: driver.find_element_by_css_selector(selector)
msg = "An element matching '%s' should be on the page" % selector
element = Wait(self.sel).until(element_is_present, msg)
self.screenshot()
return element
def wait_for_text(self, text):
text_is_present = lambda driver: text in driver.page_source
msg = "The text '%s' should be present on the page" % text
Wait(self.sel).until(text_is_present, msg)
self.screenshot()
def wait_for_xpath(self, xpath):
element_is_present = lambda driver: driver.find_element_by_xpath(xpath)
msg = "An element matching '%s' should be on the page" % xpath
element = Wait(self.sel).until(element_is_present, msg)
self.screenshot()
return element
def wait_until_element_contains(self, selector, text):
""" Wait until the specified element contains certain text """
text_contained = lambda driver: text in driver.find_element_by_css_selector(selector).text
msg = "'%s' should contain the text '%s'" % (selector, text)
Wait(self.sel).until(text_contained, msg)
self.screenshot()
def wait_until_hidden(self, selector):
""" Wait until the element matching the selector is hidden """
element = self.wait_for_element(selector)
element_is_hidden = lambda driver: not element.is_displayed()
msg = "The element matching '%s' should not be visible" % selector
Wait(self.sel).until(element_is_hidden, msg)
self.screenshot()
return element
def wait_until_not_present(self, selector):
""" Wait until the element matching the selector is gone from page """
element_is_present = lambda driver: driver.find_element_by_css_selector(selector)
msg = "There should not be an element matching '%s'" % selector
Wait(self.sel).until_not(element_is_present, msg)
self.screenshot()
def wait_until_not_visible(self, selector):
""" Wait until the element matching the selector is either hidden or
removed from the page """
element_is_visible = lambda driver: driver.find_element_by_css_selector(selector).is_displayed()
msg = "The element matching '%s' should not be visible" % selector
Wait(self.sel).until_not(element_is_visible, msg)
self.screenshot()
def wait_until_option_added(self, selector, option_text):
""" Wait until the specified select option appears; the entire
select widget may be replaced in the process """
end_time = time.time() + settings.SELENIUM_TIMEOUT
while True:
try:
select = Select(self.sel.find_element_by_css_selector(selector))
for option in select.options:
if option.text == option_text:
return option
except (NoSuchElementException, StaleElementReferenceException):
pass
time.sleep(settings.SELENIUM_POLL_FREQUENCY)
if time.time() > end_time:
break
raise TimeoutException("Select option should have been added")
def wait_until_option_disabled(self, selector, option_text):
""" Wait until the specified select option is disabled; the entire
select widget may be replaced in the process """
end_time = time.time() + settings.SELENIUM_TIMEOUT
while True:
try:
select = Select(self.sel.find_element_by_css_selector(selector))
for option in select.options:
if option.text == option_text and not option.is_enabled():
return option
except (NoSuchElementException, StaleElementReferenceException):
pass
time.sleep(settings.SELENIUM_POLL_FREQUENCY)
if time.time() > end_time:
break
raise TimeoutException("Select option should have been disabled")
def wait_until_property_equals(self, selector, name, value):
""" Wait until the specified CSS property of the element matching the
provided selector matches the expected value """
value_is_correct = lambda driver: driver.find_element_by_css_selector(selector).value_of_css_property(name) == value
msg = "The %s CSS property of '%s' should be %s" % (name, selector,
value)
Wait(self.sel).until(value_is_correct, msg)
self.screenshot()
def wait_until_offscreen(self, selector):
""" Wait until the element matching the provided selector has been
moved offscreen (deliberately, not just scrolled out of view) """
end_time = time.time() + settings.SELENIUM_TIMEOUT
while True:
try:
element = self.sel.find_element_by_css_selector(selector)
location = element.location
size = element.size
if location["y"] + size["height"] <= 0:
self.screenshot()
return True
if location["x"] + size["width"] <= 0:
self.screenshot()
return True
except (NoSuchElementException, StaleElementReferenceException):
pass
time.sleep(settings.SELENIUM_POLL_FREQUENCY)
if time.time() > end_time:
break
raise TimeoutException("'%s' should be offscreen" % selector)
def wait_until_onscreen(self, selector):
""" Wait until the element matching the provided selector has been
moved into the viewable page """
end_time = time.time() + settings.SELENIUM_TIMEOUT
while True:
try:
element = self.sel.find_element_by_css_selector(selector)
location = element.location
if location["x"] >= 0 and location["y"] >= 0:
self.screenshot()
return True
except (NoSuchElementException, StaleElementReferenceException):
pass
time.sleep(settings.SELENIUM_POLL_FREQUENCY)
if time.time() > end_time:
break
raise TimeoutException("'%s' should be offscreen" % selector)
def wait_until_property_less_than(self, selector, name, value):
""" Wait until the specified CSS property of the element matching the
provided selector is less than a certain value. Ignores any
non-integer suffixes like 'px'. """
value_is_correct = lambda driver: int(re.match(r'([\d-]+)', driver.find_element_by_css_selector(selector).value_of_css_property(name)).group(1)) < value
msg = "The %s CSS property of '%s' should be less than %s" % (name, selector, value)
Wait(self.sel).until(value_is_correct, msg)
self.screenshot()
def wait_until_visible(self, selector):
""" Wait until the element matching the selector is visible """
element = self.wait_for_element(selector)
element_is_visible = lambda driver: element.is_displayed()
msg = "The element matching '%s' should be visible" % selector
Wait(self.sel).until(element_is_visible, msg)
return element
# ~~~~~~~~~~~~~~~~~~~~~~~~~ Sauce Labs support ~~~~~~~~~~~~~~~~~~~~~~~~~~
def sauce_labs_driver(self):
""" Configure the Selenium driver to use Sauce Labs """
host = os.getenv("SELENIUM_HOST", "ondemand.saucelabs.com")
port = os.getenv("SELENIUM_PORT", "80")
executor = "".join(["http://", host, ":", port, '/wd/hub'])
platform = os.getenv("SELENIUM_PLATFORM", "Windows 7")
version = os.getenv("SELENIUM_VERSION", "")
self.sauce_user_name = os.getenv("SAUCE_USER_NAME")
self.sauce_api_key = os.getenv("SAUCE_API_KEY")
tunnel_id = os.getenv("SAUCE_TUNNEL_ID", "")
build_number = os.getenv('BUILD_NUMBER')
job_name = os.getenv('JOB_NAME')
# http://code.google.com/p/selenium/wiki/DesiredCapabilities
# https://saucelabs.com/docs/additional-config#desired-capabilities
caps = {
'accessKey': self.sauce_api_key,
'capture-html': True,
'browserName': self.browser,
'javascriptEnabled': True,
'name': self.id(),
'platform': platform,
'username': self.sauce_user_name,
'version': version,
}
if build_number and job_name:
caps['build'] = '{} #{}'.format(job_name, build_number)
if tunnel_id:
caps['tunnel-identifier'] = tunnel_id
if settings.SELENIUM_SAUCE_VERSION:
caps['selenium-version'] = settings.SELENIUM_SAUCE_VERSION
remote = webdriver.Remote(command_executor=executor,
desired_capabilities=caps)
# Store the Sauce session ID to output later for Jenkins integration
# See https://saucelabs.com/jenkins/5 for details
sauce_sessions.append('SauceOnDemandSessionID={} job-name={}'.format(remote.session_id, self.id()))
return remote
def report_status(self, passed):
"""Report to Sauce Labs whether or not the test passed, so that can be
reflected in their UI."""
if not hasattr(self, 'sauce_user_name'):
# Not using Sauce Labs for this test
return
url_pattern = 'http://{}:{}@saucelabs.com/rest/v1/{}/jobs/{}'
url = url_pattern.format(self.sauce_user_name,
self.sauce_api_key,
self.sauce_user_name,
self.sel.session_id)
body_content = json.dumps({"passed": passed})
headers = {
'Content-Type': 'application/json',
}
response = requests.put(url, body_content, headers=headers)
return response.status_code == 200
from src.main.web_pypom.controls.base_control import BaseControl
from selenium.webdriver.common.by import By
from src.main.web_pypom.controls.wait import wait
from time import sleep
class Button(BaseControl):
def __init__(self, driver, locator='//*[@id="tsf"]/div[2]/div[3]/center/input[2]'):
self.locator = (By.XPATH, locator)
self.driver = driver
@property
def button(self):
return self.driver.find_element(*self.locator)
@property
@wait
def click(self):
return self.button.click()
if __name__ == '__main__':
from selenium.webdriver import Chrome
driver = Chrome()
driver.get('https://www.google.com')
bc = Button(driver)
bc.click
sleep(3)
driver.quit()
kbo_link.append(link.replace('/watch?v=', '/embed/'))
kbo_title.append(title)
# print(kbo_link)
# print(kbo_commentlink)
driver.close()
# with open('./kbo_title.json','wt') as f:
# json.dump(kbo_title,f)
# with open('./kbo_link.json','wt') as f:
# json.dump(kbo_link,f)
### 유투브 댓글 크롤링 ###
kbo_bonki = []
for li in kbo_commentlink:
delay = 2
browser = Chrome()
browser.implicitly_wait(delay)
start_url = "https://www.youtube.com" + li
browser.get(start_url)
browser.maximize_window()
print(start_url)
time.sleep(3)
body = browser.find_element_by_tag_name('body')
pagedowns = 2 #2번 밑으로
while pagedowns:
body.send_keys(Keys.PAGE_DOWN)
time.sleep(2)
pagedowns -= 1
time.sleep(3)
from selenium.webdriver.common.keys import Keys #all the modes and methods i'm importing from selenium to python tool
from selenium.webdriver import ActionChains #performing actions
import time
from contextlib import closing
from selenium.webdriver import Chrome
Browser = Chrome('C:\Users\skashem\Desktop\chromedriver\chromedriver')
url = 'http://staging.totsy.com/'
def Input_Field(Field_name,Field_value):
Browser.find_element_by_xpath(Field_name).send_keys(Field_value)
def Link(Link_Name):
Browser.find_element_by_xpath(Link_Name).click()
Browser.get(url)
#Log-in Credential
Input_Field("//*[@id='email']","*****@*****.**")
Input_Field("//*[@id='pass']",'skashem3422')
Link("//*[@id='submit-button']")
#Mouse hover to SHOP BY CATEGORY
shop_by_category = Browser.find_element_by_xpath("//*[@id='navByCat']/a/em")
hover = ActionChains(Browser).move_to_element(shop_by_category)
hover.perform()
#Click on boys apparel
Link("//*[@id='navByCat']/ul/li[2]/a")
Link("//*[@id='mainContent']/div[2]/div/div[1]/ul/li[1]/div/a/img")
Link("//*[@id='attribute169']")
m.update(string_id.encode('utf8'))
identifier = m.hexdigest()
print("hash-------------------->", identifier)
create_time = str(datetime.datetime.now())
update_time = ""
insertdb = InsertDB()
data_base.append(
(property_address, street_add, city, state, zipcode,
status_text, phone_number, bathrooms, bedrooms,
tax_assessed_value, zestimate, rent_zestimate, home_type,
parking, year_built, hoa, heating, lot, cooling,
price_sqft, identifier, create_time, update_time))
insertdb.insert_document(data_base, table_name)
# return
if __name__ == "__main__":
print("-----------------start---------------")
options = Options()
options.binary_location = "C:\Program Files\Google\Chrome\Application\chrome.exe"
path = "driver\\chromedriver.exe"
driver = Chrome(executable_path=path, chrome_options=options)
driver.get("https://www.zillow.com/")
time.sleep(2)
driver.maximize_window()
main(driver.page_source, driver)
from selenium.webdriver import Chrome
import getpass
from selenium.webdriver.remote.webdriver import WebDriver
import gzip
import time
import cx_Oracle
browser = Chrome()
def get_job(browser: WebDriver) -> str:
rows = browser.find_elements_by_css_selector('tr.table_row')
for row in rows:
cells = row.find_elements_by_tag_name('td')
for i, c in enumerate(cells):
if c.text == 'AUTOMATION':
tran_run_id= cells[i+10].text
return tran_run_id
browser.get('https://eit1-i.svcs.hp.com/cds/') # Site-Minder url
USER1=input("Please enter the User Name:")
PASSWORD1 = getpass.getpass("Password: "******"USER"]')
email.send_keys(USER1)
password = browser.find_element_by_css_selector('input[name="PASSWORD"]')
password.send_keys(PASSWORD1)
logon = browser.find_element_by_css_selector('input.btn.btn-primary')
logon.click()
filter_btn.click()
first_shoe_detail = web_driver.find_element_by_css_selector('[alt="Appli Air x Night"]')
if first_shoe_detail.is_displayed():
first_shoe_detail.click()
else:
print("Shoe details link is not displayed in order to click")
eyes.check("", Target.window().fully())
eyes.close_async()
except Exception as e:
eyes.abort_async()
print(e)
def tear_down(web_driver, runner):
web_driver.quit()
all_test_results = runner.get_all_test_results(False)
print(all_test_results)
web_driver = Chrome(ChromeDriverManager().install())
consecutive_runs = 10
runner = VisualGridRunner(consecutive_runs)
eyes = Eyes(runner)
set_up(eyes)
try:
ultra_fast_test(web_driver, eyes)
finally:
tear_down(web_driver, runner)
def get_last_report(browser: webdriver.Chrome, t):
print('正在获取手机号...')
browser.get('https://selfreport.shu.edu.cn/PersonInfo.aspx')
time.sleep(1)
# 手机号
ShouJHM = browser.find_element(
By.ID, 'persinfo_ctl00_ShouJHM-inputEl').get_attribute('value')
if NEED_BEFORE:
print('开始补报，正在获取补报日期前一天的填报信息...')
t = START_DT - dt.timedelta(days=1)
else:
print('正在获取前一天的填报信息...')
t = t - dt.timedelta(days=1)
browser.get(
f'https://selfreport.shu.edu.cn/ViewDayReport.aspx?day={t.year}-{t.month}-{t.day}'
)
time.sleep(1)
# 是否在上海，在上海（校内），在上海（不在校内），不在上海
ShiFSH = browser.find_element(By.CSS_SELECTOR,
'#ctl03_ShiFSH #ctl03_ShiFSH-inputEl').text
# 是否住校
ShiFZX = 'f-checked' in browser.find_element(
By.CSS_SELECTOR,
'#ctl03_ShiFZX .f-field-checkbox-icon').get_attribute('class')
# 省
ddlSheng = browser.find_element(
By.CSS_SELECTOR,
'#ctl03_ddlSheng #ctl03_ddlSheng-inputEl').get_attribute('value')
# 市
ddlShi = browser.find_element(
By.CSS_SELECTOR,
'#ctl03_ddlShi #ctl03_ddlShi-inputEl').get_attribute('value')
# 县
ddlXian = browser.find_element(
By.CSS_SELECTOR,
'#ctl03_ddlXian #ctl03_ddlXian-inputEl').get_attribute('value')
# 详细地址
XiangXDZ = browser.find_element(
By.CSS_SELECTOR,
'#ctl03_XiangXDZ #ctl03_XiangXDZ-inputEl').get_attribute('value')
# 是否家庭地址
ShiFZJ = 'f-checked' in browser.find_element(
By.CSS_SELECTOR,
'#ctl03_ShiFZJ .f-field-checkbox-icon').get_attribute('class')
return ShouJHM, ShiFSH, ShiFZX, ddlSheng, ddlShi, ddlXian, XiangXDZ, ShiFZJ
import time
from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
url = "https://demos.devexpress.com/aspxeditorsdemos/ListEditors/MultiSelect.aspx"
chrome_path = '/home/xuananh/Downloads/chromedriver_linux64/chromedriver'
# chrome driver download from link: https://chromedriver.storage.googleapis.com/index.html?path=2.44/
chrome_option = ChromeOptions()
driver = Chrome(executable_path=chrome_path,
options=chrome_option)
driver.maximize_window()
driver.get(url)
# set select mode to Multiple
driver.find_element_by_id("ControlOptionsTopHolder_lbSelectionMode_B-1").click()
time.sleep(0.5)
driver.find_element_by_id("ControlOptionsTopHolder_lbSelectionMode_DDD_L_LBI1T0").click()
time.sleep(1.5)
# perform Multiple Select
select = driver.find_element_by_id("ContentHolder_lbFeatures_LBT")
options = select.find_elements_by_tag_name("td")
action = ActionChains(driver)
# multi_action = action \
# .key_down(Keys.CONTROL) \
