def click_element(self,
locator_type: By = None,
locator_value: str = None,
element: WebElement = None,
from_element: WebElement = None,
time_to_wait: int = TIME_TO_WAIT):
if not element:
element = self.find_element_by(locator_type, locator_value,
from_element, time_to_wait)
element.click()
def click_on_an_element(self, clickable_element, driver):
try:
clickable_element.send_keys(Keys.ENTER)
except:
try:
#clickable_element.location_once_scrolled_into_view
WebElement.click(clickable_element)
except:
actions = ActionChains(driver)
actions.move_to_element(clickable_element).click().perform()
def clickElem(elem:WebElement, timeOut:float=29, pollFreq:float=0.7):
timeElapsed=0.0
while timeElapsed < timeOut:
beginTime=time.time()
try:
elem.click()
return
except StaleElementReferenceException:
pass
time.sleep(pollFreq)
timeElapsed+=(time.time()- beginTime)
raise TimeoutException('[getElemText] Time out elem='+str(elem))
def clickElem(elem: WebElement, timeOut: float = 29, pollFreq: float = 0.7):
timeElapsed = 0.0
while timeElapsed < timeOut:
beginTime = time.time()
try:
elem.click()
return
except StaleElementReferenceException:
pass
time.sleep(pollFreq)
timeElapsed += (time.time() - beginTime)
raise TimeoutException('[getElemText] Time out elem=' + str(elem))
def click(browser: XuexiChrome, element: WebElement):
browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
'source': '''
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
window.alert = function() {
return;
}
'''
})
element.click()
def nf_click_center_of_element(
self,
element: WebElement,
desired_link: str = None,
disable_navigation: bool = False,
skip_action_chain: bool = False,
try_n: int = 1
):
"""Moves pointer to center of element and then clicks"""
if not skip_action_chain:
try:
(
ActionChains(self.browser)
.move_to_element(element)
.move_by_offset(
element.size['width'] // 2,
element.size['height'] // 2,
)
.click()
.perform()
)
self.quota_supervisor.add_server_call()
except MoveTargetOutOfBoundsException:
pass
except StaleElementReferenceException:
self.logger.debug("Stale Element")
if try_n <= 3:
self.browser.execute_script(JS.RELOAD)
self.quota_supervisor.add_server_call()
nf_click_center_of_element(self, element, desired_link, disable_navigation, skip_action_chain, try_n + 1)
if desired_link or skip_action_chain:
try:
explicit_wait(self, "PFL", [], 7, False)
if skip_action_chain or not check_if_in_correct_page(self, desired_link):
try:
element.click()
self.quota_supervisor.add_server_call()
explicit_wait(self, "PFL", [], 7, False)
except ElementClickInterceptedException:
pass
if not check_if_in_correct_page(self, desired_link):
self.browser.execute_script("arguments[0].click();", element)
self.quota_supervisor.add_server_call()
explicit_wait(self, "PFL", [], 7, False)
if not check_if_in_correct_page(self, desired_link):
self.logger.debug("Failed to press element{}".format(
", navigating to desired link" if not disable_navigation else ""))
if not disable_navigation:
web_address_navigator(self, desired_link)
except StaleElementReferenceException:
pass
else:
sleep(2)
def retry_find_and_click(element: WebElement):
result = False
attempts = 1
while attempts < 3 and result is False:
try:
element.click()
result = True
except StaleElementReferenceException:
pass
attempts += 1
time.sleep(0.5)
return result
def click_surely(ele: WebElement, inframe: bool = True) -> None:
"""When clicking on an element, move it onscreen first. BECAUSE IE.
If that doesn't work, manual override, it was probably just behind a blank textbox."""
try:
sc = 'wp=window.top;wp.scrollTo(0,arguments[0].getBoundingClientRect().top-wp.innerHeight/2'
if inframe:
sc += '+wp.$("iframe[src*=\'/content/\']").offset().top'
sc += ')'
# if not isinstance(DRIVER, Firefox):
DRIVER.execute_script(sc, ele)
ele.click()
except WebDriverException:
ActionChains(DRIVER).move_to_element(ele).click().perform()
def click_element(ctx: Context, element: WebElement) -> None:
"""Click the element.
Args:
ctx: The behave context object
element: element to click
Returns:
None
"""
LOGGER.debug("Attempting to click the element")
element.click()
LOGGER.debug("Successfully clicked the element")
def clear_form(
self,
ku_select: Select,
ku_critical: WebElement,
eku_select: Select,
eku_critical: WebElement,
tf_select: Select,
tf_critical: WebElement,
subject: typing.Dict[str, WebElement],
cn_in_san: WebElement,
) -> None:
"""Clear the form."""
ku_select.deselect_all()
eku_select.deselect_all()
tf_select.deselect_all()
if ku_critical.is_selected():
ku_critical.click()
if eku_critical.is_selected():
eku_critical.click()
if tf_critical.is_selected():
tf_critical.click()
if cn_in_san.is_selected():
cn_in_san.click()
for field in subject.values():
field.clear()
def click(self, target: WebElement) -> WebElement:
try:
if type(target) == WebElement:
target.click()
self.__DEBUG(target=target, value="")
except ElementNotInteractableException:
return target
except ElementClickInterceptedException:
return target
except StaleElementReferenceException:
return target
except WebDriverException:
return target
return target
def click(element: WebElement = None,
element_parent: Union[None, WebDriver, WebElement] = None,
locator: Locator = None,
delay: Union[int, float] = None):
""" Clicks on element. Repeats when it fails
It is possible to provide element object directly or its parent object and locator.
:param element: WebElement - if present, directly click on this WebElement
:param element_parent: WebElement - if present first find the element to click on under this element/WebDriver
object using locator parameter
:param locator: locator tuple i.e. (By.CSS_SELECTOR, 'div.my-class')
:param delay: optional delay between attempts
"""
for i in range(3, -1, -1):
if locator and element_parent:
try:
element = element_parent.find_element(*locator)
except WebDriverException as e:
if i == 0:
raise
print("ERROR: cannot find element", e.args)
sleep(0.5)
continue
if not element.is_displayed():
if i == 0:
raise Exception('Element is not displayed')
else:
print('ERROR: element is not displayed')
sleep(0.5)
continue
if not element.is_enabled():
if i == 0:
raise Exception('Element is not enabled')
else:
print('ERROR: element is not enabled')
sleep(0.5)
continue
try:
sleep(0.5)
element.click()
if delay is not None:
sleep(delay)
break
except WebDriverException as e:
if i == 0:
raise
print("ERROR: cannot click on element", e.args)
def __edu_work(self, nav_edu_work: WebElement, info_dict: dict):
# nav_edu_work = _5pwrs[1]
nav_edu_work.click()
sleep.random_sleep(self.__sleep)
pagelet_timeline_medley_about = self.__browser.find_element_by_id(
'pagelet_timeline_medley_about')
lines = pagelet_timeline_medley_about.text.split('\n')
context = lines[11:len(lines)]
info_dict['nav_edu_work'] = '\n'.join(context)
try:
pagelet_eduwork = self.__browser.find_element_by_id(
'pagelet_eduwork')
info_dict['pagelet_eduwork'] = pagelet_eduwork.text
except NoSuchElementException:
self.__log.debug('exception: pagelet_eduwork not find')
def _click_and_nav_away(self, element: WebElement) -> None:
"""Clicks on provided element and navigates away from Paylocity.
The only reason to navigate away from Paylocity is to simulate logging
out of the service. It is unnecessary to close and re-open the web
browser regularly. Further, it does not make sense to stay on the
Paylocity web page, just to have it log use out due to expired session.
Parameters
----------
element : WebElement, required
The web element that should be clicked on.
"""
element.click()
self._driver.get('https://www.google.com')
def __all_relationships(self, nav_all_relationships: WebElement,
info_dict: dict):
# nav_all_relationships = _5pwrs[4]
nav_all_relationships.click()
sleep.random_sleep(self.__sleep)
pagelet_timeline_medley_about = self.__browser.find_element_by_id(
'pagelet_timeline_medley_about')
lines = pagelet_timeline_medley_about.text.split('\n')
context = lines[11:len(lines)]
info_dict['nav_all_relationships'] = '\n'.join(context)
try:
pagelet_relationships = self.__browser.find_element_by_id(
'pagelet_relationships')
info_dict['pagelet_relationships'] = pagelet_relationships.text
except NoSuchElementException:
self.__log.debug('exception: pagelet_relationships not find')
def __year_overviews(self, nav_year_overviews: WebElement,
info_dict: dict):
# nav_year_overviews = _5pwrs[6]
nav_year_overviews.click()
sleep.random_sleep(self.__sleep)
pagelet_timeline_medley_about = self.__browser.find_element_by_id(
'pagelet_timeline_medley_about')
lines = pagelet_timeline_medley_about.text.split('\n')
context = lines[11:len(lines)]
info_dict['nav_year_overviews'] = '\n'.join(context)
fb_profile_edit_experiences = self.__browser.find_element_by_class_name(
'fbProfileEditExperiences')
try:
info_dict[
'fbProfileEditExperiences'] = fb_profile_edit_experiences.text
except NoSuchElementException:
self.__log.debug('exception: fbProfileEditExperiences not find')
def dismiss_overlay_click(driver,
locator: Tuple[By, str] = None,
elem: WebElement = None,
retries: int = 5):
"""If element is blocked by a modal, attempt to close it."""
if elem:
pass
elif locator:
elem = WebDriverWait(driver,
10).until(EC.presence_of_element_located(locator))
else:
raise ValueError("either element, or locator must be provided")
for i in range(retries):
try:
elem.click()
break
except ElementClickInterceptedException as exc:
log.info(
"blocked by overlay, attempting to close before clicking again (%s/%s): %s",
i,
retries,
exc,
)
if len(glob(f"{config.WORKING_DIRECTORY}/*overlay.png")
) >= retries * 2:
log.exception("giving up on closing overlay")
driver.screenshot(
BaseWebScraper.screenshot_path("max_close_overlay"))
raise exc
if detect_and_send_escape_to_close_survey(driver):
pass
elif detect_and_close_survey(driver):
pass
elif detect_and_close_modal(driver):
pass
else:
log.exception("unable to close overlay, raising")
raise exc
driver.sleep(2)
except StaleElementReferenceException:
return
driver.screenshot(BaseWebScraper.screenshot_path("close_overlay"))
def _handle_main_category(self, main_cat_id: int,
main_cat_elem: WebElement):
text_elem = main_cat_elem.find_element_by_class_name("MainTag_Name")
main_cat = MainCategory(text_elem.text, main_cat_id)
self.categories[main_cat_id] = main_cat
main_cat_elem.click() # Open sub categories
inner_tags_elem = self.driver.find_element_by_id("InnerTags")
inner_main_elem = inner_tags_elem.find_element_by_id(
"innerMainTag_{}".format(main_cat_id))
a_elems = inner_main_elem.find_elements_by_tag_name("a")
for a_elem in a_elems:
sub_cat_id = int(a_elem.get_attribute("data-option-value"))
text_elem = a_elem.find_element_by_class_name("MainTag_Name")
sub_cat = SubCategory(text_elem.text, sub_cat_id)
main_cat.sub_categories[sub_cat_id] = sub_cat
img_elem = a_elem.find_element_by_tag_name("img")
img_elem.click()
self._handle_sub_category(sub_cat)
def _test_image_filters(
self,
driver: WebDriver,
expand_settings_elem: WebElement,
image_view_port_elem: WebElement,
) -> None:
self._test_invert(driver, image_view_port_elem)
tone_modal = driver.find_element_by_id('id_tone_modal')
self.assertFalse(tone_modal.is_displayed())
tone_button = driver.find_element_by_id('id_tone_button')
tone_button.click()
self.assertTrue(tone_modal.is_displayed())
brightness_slider = driver.find_element_by_id('id_brightness_slider')
contrast_slider = driver.find_element_by_id('id_contrast_slider')
self._test_apply_modal(
brightness_slider,
contrast_slider,
driver,
image_view_port_elem,
)
tone_button.click()
self._test_cancel_modal(
brightness_slider,
contrast_slider,
driver,
image_view_port_elem,
)
expand_settings_elem.click()
wait = WebDriverWait(driver, timeout=5)
def extra_image_settings_not_displayed(new_driver: WebDriver) -> bool:
extra_image_settings = new_driver.find_element_by_id(
'id_extra_image_settings', )
return not extra_image_settings.is_displayed()
wait.until(extra_image_settings_not_displayed)
def __contact_basic(self, nav_contact_basic: WebElement, info_dict: dict):
# nav_contact_basic = _5pwrs[3]
nav_contact_basic.click()
sleep.random_sleep(self.__sleep)
pagelet_timeline_medley_about = self.__browser.find_element_by_id(
'pagelet_timeline_medley_about')
lines = pagelet_timeline_medley_about.text.split('\n')
context = lines[11:len(lines)]
info_dict['nav_contact_basic'] = '\n'.join(context)
try:
pagelet_contact = self.__browser.find_element_by_id(
'pagelet_contact')
info_dict['pagelet_contact'] = pagelet_contact.text
except NoSuchElementException:
self.__log.debug('exception: pagelet_contact not find')
try:
pagelet_basic = self.__browser.find_element_by_id('pagelet_basic')
info_dict['pagelet_basic'] = pagelet_basic.text
except NoSuchElementException:
self.__log.debug('exception: pagelet_basic not find')
def action_on_elem(self,
web_elem: WebElement,
action: str,
content: str = ""):
log_client = LogsClient(output_file=self.log_output_file,
project_dir=project_dir,
file_name=file_name,
log_run_uuid=self.log_run_uuid)
log_client.set_msg(
log_type="info",
log_msg=f"action: {action} on web element: {web_elem}")
try:
if action == "click":
web_elem.click()
elif action == "send_keys":
web_elem.send_keys(content)
sleep(2)
except NoSuchElementException:
log_client.set_msg(
log_type="error",
log_msg=
f"error while trying to perform action: {action} at web element: {web_elem}"
)
except Exception as e:
log_client.set_msg(
log_type="error",
log_msg=f"the following error occurred with args: {e.args}")
def _do_script_step(self, step: list, parent: WebElement = None) -> list:
"""
单步执行操作
@param {list} step - 要执行操作的步骤参数
['find', op, para1, ...] # 忽略此前步骤查找到的元素，重新全局搜索元素并设置为当前元素, 数组后面为查找指令和对应的参数，参考find_elements
['find_child', op, para1, ...] # 搜索此前元素的子元素并设置为当前元素，数组后面为查找指令和对应的参数，参考find_elements
['click'] # 点击元素
['send_str', '要输入的文本'] # 元素输入文本
['send_keys', 'Keys.CONTROL', 'Keys.SHIFT', 'a', ...] # 输入按键，传入多个代表同时按下
# 特殊按键定义参考 selenium.webdriver.common.keys.Keys
@param {WebElement} parent=None - 父节点，如果不传代表全局搜索
@returns {list} - 如果是执行操作，直接返回包含parent的数组；如果是查询操作，返回查询到的元素清单
"""
_elements = [
parent,
]
_op = step[0]
if _op == 'find':
# 忽略此前步骤查找到的元素，重新全局搜索元素并设置为当前元素
return self.find_elements([step[1:]], parent=None)
elif _op == 'find_child':
return self.find_elements([step[1:]], parent=parent)
elif _op == 'click':
parent.click()
elif _op == 'send_str':
parent.send_keys(step[1])
elif _op == 'send_keys':
# 输入快捷键
_keys = [
eval(_key) if _key.startswith('Keys.') else _key
for _key in step[1:]
]
parent.send_keys(*_keys)
return _elements
def perform_mp(mp_name: str, item: WebElement, main_window_handle=None):
item.click()
time.sleep(3)
if main_window_handle is not None:
window_handles = driver.window_handles
for handle in window_handles:
if handle != main_window_handle:
driver.switch_to.window(handle)
if driver.title.strip() == mp_name:
mp_html = driver.page_source
write_file(mp_name, mp_html, '_mp_html.txt')
# 正则提取
parse_mp_articles(mp_html)
else:
"""
被拦截的情况，需要输入验证码，尝试进行验证码解析（未验证此处理）
"""
image_code = driver.find_element_by_id('seccodeImage')
# 移动到该元素
action = ActionChains(driver).move_to_element(image_code)
# 右键点击该元素
action.context_click(image_code)
# 点击键盘向下箭头
action.send_keys(Keys.ARROW_DOWN)
# 键盘输入V保存图
action.send_keys('v')
time.sleep(3)
# 执行保存
action.perform()
driver.close()
if main_window_handle is not None:
driver.switch_to.window(main_window_handle)
def click(
driver,
css_selector: str = None,
xpath: str = None,
elem: WebElement = None,
scroll: bool = True,
):
"""helper method to click an element, if it is blocked by blockOverlay, waits for the overlay to disappear"""
if elem:
pass
elif css_selector:
elem = driver.find_element_by_css_selector(css_selector)
elif xpath:
elem = driver.find_element_by_xpath(xpath)
else:
raise ValueError("one of css_selector, xpath or elem must be provided")
retries_left = 5
while retries_left > 0:
try:
scroll_to(driver, elem) if scroll is True else None
elem.click()
break
except ElementClickInterceptedException as e:
if "blockUI blockOverlay" in e.msg:
log.info(
"blocked by overlay, waiting for it to go before clicking again"
)
wait_for_block_overlay(driver)
continue
else:
if not close_modal(driver):
raise
finally:
retries_left -= 1
def click_elem(self, elem: WebElement) -> bool:
if elem != None:
elem.click()
return True
return False
def safe_click(driver: WebDriver, element: WebElement) -> None:
try:
element.click()
except ElementClickInterceptedException:
ActionChains(driver).move_to_element(element).perform()
element.click()
browser = webdriver.Firefox()
browser.get("https://webassign.net/login.html")
print("Please login to start the fetching process:")
browser.find_element_by_id("WebAssignUsername").send_keys(
raw_input("Username:"******"WebAssignInstitution").send_keys(
raw_input("Institution:"))
browser.find_element_by_id("WebAssignPassword").send_keys(
raw_input("Password:"******"loginbtn").click()
wait("student.pl")
print("Starting collection of assignments...")
pastclick = browser.find_element_by_xpath(
"//*[@id=\"webAssign\"]/div[2]/div[4]/div[1]/div[1]/div[2]/div[2]/p/a")
WebElement.click(pastclick)
homehtml = browser.page_source
if not os.path.exists(r"Webassign Data"):
os.makedirs(r"Webassign Data")
groupnumber = 0
for i in range(7,
homehtml.count("row container")): #TODO change back after debug
try:
wait("student.pl")
entry = browser.find_element_by_xpath(
"//*[@id=\"webAssign\"]/div[2]/div[3]/div[2]/div[" + str(i + 2) +
"]/div[1]/p[1]/a")
title = entry.text
print(title)
entry.click()
def click_human_like(element: WebElement):
""" Click on button human like """
element.click()
sleep(random.uniform(0.4, 0.9))
def click_element(self, element: WebElement) -> None:
if element.is_displayed():
element.click()
def click_elem(self, button: WebElement, expect_id: str = None) -> bool:
self.browser.execute_script("window.scrollTo(0, %s)" % button.location['y'])
button.click()
if expect_id:
self.wait.until(EC.presence_of_element_located((By.ID, expect_id)))
return True
def get_puppy_id(self, view_details_btn: WebElement):
view_details_btn.click()
url = self.browser.current_url
url = re.sub(pattern='[^0-9]', repl='', string=url)
self.browser.back()
return int(url)
