def get_results(drv):
try:
table_element = WebDriverWait(drv, 60).until(
EC.presence_of_element_located((By.TAG_NAME, "tbody")))
except TimeoutException as e:
raise TimeoutException('Timeout exception')
except UnexpectedAlertPresentException as e:
raise UnexpectedAlertPresentException(
"Возможная причина - исчерпание лимиты у Арсенкина"
) # Это исключение мы нигде более не ловим. Просто сообщаем причину.
sleep(
2
) # Следующий блок вызывал исключение. Т.е. таблица, вроде, есть, а элент th в ней отсутствует. Поэтому добавлен слип.
try:
drv.find_element_by_xpath(
"//th[text()='Релевантный URL в выдаче по запросу']"
) # Проверяем, что данные, действительно, выведены. Одной проверки наличия таблицы недостаточно - будут пропуски ключей.
except NoSuchElementException as e:
raise NoSuchElementException(str(e))
try:
table_html = table_element.get_attribute("innerHTML")
except StaleElementReferenceException:
raise StaleElementReferenceException("StaleElementReferenceException")
return table_html
def test_wait_until_element_is_enabled(waiting):
locator = "//div"
element = mock()
when(waiting).find_element(locator, None).thenReturn(element)
when(element).is_enabled().thenRaise(
StaleElementReferenceException()).thenReturn(True)
waiting.wait_until_element_is_enabled(locator, TIMEOUT)
def test_wait_until_element_is_not_visible(waiting):
locator = "//div"
element = mock()
when(waiting).find_element(locator, required=False).thenReturn(element)
when(element).is_displayed().thenRaise(
StaleElementReferenceException()).thenReturn(False)
waiting.wait_until_element_is_not_visible(locator, TIMEOUT)
def test_wait_until_element_is_enabled_get_attribute_readonly(waiting):
locator = '//div'
element = mock()
when(waiting).find_element(locator, None).thenReturn(element)
when(element).is_enabled().thenReturn(True)
when(element).get_attribute('readonly').thenRaise(
StaleElementReferenceException()).thenReturn(None)
waiting.wait_until_element_is_enabled(locator, TIMEOUT)
def _check_error(payload):
# type: (dict) -> None
error = payload.get("error")
if error:
if error["message"].startswith("stale element reference"):
raise StaleElementReferenceException(error["message"])
else:
raise USDKFailure(error["message"], error["stack"])
def find_elements_by_xpath(self, xpath, retry=10):
try:
return self.web_driver.find_elements_by_xpath(xpath)
except StaleElementReferenceException as e:
if retry <= 0:
raise StaleElementReferenceException()
return self.find_elements_by_xpath(xpath, retry - 1)
return None
def test_wait_until_element_is_enabled_fails(waiting):
locator = "//div"
element = mock()
when(waiting).find_element(locator, None).thenReturn(element)
when(element).is_enabled().thenRaise(StaleElementReferenceException("foo"))
with pytest.raises(AssertionError) as error:
waiting.wait_until_element_is_enabled(locator, TIMEOUT)
assert "Message: foo" in str(error.value)
def wrap(*args, **kwargs):
attempts = 10
for _ in range(attempts):
try:
return method(*args, **kwargs)
except StaleElementReferenceException:
time.sleep(0.5)
else:
raise StaleElementReferenceException("Couldn't handle it")
def test_wait_until_element_is_visible_fails(waiting):
locator = '//div'
element = mock()
when(waiting).find_element(locator, required=False).thenReturn(element)
when(element).is_displayed().thenRaise(
StaleElementReferenceException('foo'))
with pytest.raises(AssertionError) as error:
waiting.wait_until_element_is_visible(locator, TIMEOUT)
assert 'Message: foo' in str(error.value)
def select_noble_metal(self):
self.driver.switch_to_default_content()
WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.ID, "quanbu")))
quanbu_element = self.driver.find_element_by_id("quanbu")
print("quanbu element was found")
quanbu_element.click()
#quanbu_element.click()
#script = 'perbankAtomLocationTW' + \
# '("PBL200204","",dse_sessionId)'
#self.driver.execute_script(script)
self.switch_to_content_frame()
self.logger.debug("before find noble metal link")
noble_metal_xpath = "html/body/div/div/div[6]" + \
"/div/div/div/ul/li[1]/a"
WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.XPATH, noble_metal_xpath)))
self.logger.debug("after find noble metal link")
self.logger.debug("before script")
script = 'AtomSerivceSubmit("PBL201311","","","")'
self.driver.execute_script(script)
self.logger.debug("after script")
counter = 0
while counter < 3:
found_exception = False
try:
self.switch_to_content_frame()
WebDriverWait(self.driver, 10).until(
EC.frame_to_be_available_and_switch_to_it(
(By.NAME, "_market")))
WebDriverWait(self.driver, 10).until(
EC.frame_to_be_available_and_switch_to_it(
(By.NAME, "_left")))
except StaleElementReferenceException as stale_exception:
error_msg = "select noble metal: StaleException {0}"
error_msg = error_msg.format(stale_exception)
self.logger.error(error_msg)
print(error_msg)
found_exception = True
if not found_exception:
break
time.sleep(10)
counter = counter + 1
if counter == 3:
raise StaleElementReferenceException(
"select noble metal: tried 3 times")
return
def f_decorated(*args, **kwargs):
try:
return f(*args, **kwargs)
except URLError as url_error:
if _is_caused_by_server_shutdown(url_error):
raise StaleElementReferenceException(
'The Selenium server this element belonged to is no longer '
'available.')
else:
raise
def test_relocates_if_element_goes_stale(self, mocker, browser_mock,
driver_mock):
browser_mock.wd = driver_mock
el = wd_element(mocker)
locator = {'xpath': './/div', 'id': 'foo'}
mtcher = matcher(mocker, browser_mock, locator)
expect_all(driver_mock, [el])
mtcher.match.side_effect = [StaleElementReferenceException(), el]
assert locate_one(mtcher, locator) == el
def error(self):
if self.changed:
raise StaleElementReferenceException()
errors = {
1: "MEDIA_ERR_ABORTED",
2: "MEDIA_ERR_NETWORK",
3: "MEDIA_ERR_DECODE",
4: "MEDIA_ERR_SRC_NOT_SUPPORTED"
}
return errors[self._el._parent.execute_script("return arguments[0].error", self._el)]
def raise_exception(type_of_exception):
"""
Raises an exception according to type of exception.
"""
if type_of_exception == STALE_ELEMENT_REFERENCE_EXCEPTION:
raise StaleElementReferenceException()
elif type_of_exception == NO_SUCH_ELEMENT_EXCEPTION:
raise NoSuchElementException()
elif type_of_exception == TIMEOUT_EXCEPTION:
raise TimeoutException()
elif type_of_exception == EXCEPTION:
raise Exception()
def find_elements_anti_stale(self, selector):
attempts = 0
while attempts < 5:
try:
elements = self.driver.find_elements_by_css_selector(selector)
time.sleep(1)
[element.is_displayed() for element in elements]
except StaleElementReferenceException:
attempts += 1
else:
return elements
raise StaleElementReferenceException(
f"Exceeded max attempts limit trying to get element {selector}")
def coarse_filter(self, potential_divs: list, result_list: list) -> None:
"""Populate the result set with coarse-grain filtered result for further evaluation.
Linkedin occasionally returns irrelevant search results for unknown reason
Args:
potential_divs (list):
result_list (list):
"""
log_phase = 'Coarse-Filter'
logger.debug('{}: Starting filter...'.format(log_phase))
local_row_index = self.row_index
for div in potential_divs: # web-element
logger.debug('{}: Finding web element(s)...'.format(log_phase))
try:
inner_anchor = div.find_element(By.TAG_NAME, "a")
profile_link = inner_anchor.get_attribute("href")
inner_h3 = inner_anchor.find_element(By.TAG_NAME, "h3")
inner_h3_id = inner_h3.get_attribute("id")
inner_span = inner_anchor.find_element(
By.XPATH, "//h3[@id=\"" + inner_h3_id + "\"]/span[1]/span")
inner_span_text = inner_span.text.lower().replace(" ", "")
# logger.debug('{}: \nrow_first_name: {}\nrow_last_name: {}\ninner_span_text: {}'.format(
# log_phase, self.row_first_name, self.row_last_name, inner_span_text))
if self.row_first_name in inner_span_text and self.row_last_name in inner_span_text:
# TODO 1, mark full name on this profile link to output's FULL_NAME_ON_LINKEDIN column
self.output_data.at[
local_row_index,
"FULL_NAME_ON_LINKEDIN"] = inner_span.text
# TODO 2, mark this profile link to output's PROFILE_LINK column
self.output_data.at[local_row_index,
"PROFILE_LINK"] = profile_link
# TODO 3, increment the row index(local)
local_row_index += 1
result_list.append(profile_link)
except NoSuchElementException:
msg = '{}: Web element could not be found.'.format(log_phase)
logger.exception(msg)
raise NoSuchElementException(msg)
except StaleElementReferenceException:
msg = '{}: Web element lost.'.format(log_phase)
logger.exception(msg)
raise StaleElementReferenceException(msg)
log_result = str(len(result_list))
logger.debug(
'{}: \"{}\" candidates survived from coarse-grain filter.'.format(
log_phase, log_result))
def wait_for_load(a_driver):
element = a_driver.find_element_by_tag_name(u'html')
count = 0
while True:
count += 1
# 超过5s，直接返回,看情况设置
if count > 5:
print(u'Timing out after 5s and returning')
return
print u'睡眠1s'
time.sleep(
0.5) # 检查还是不是同一个element，如果不是，说明这个html标签已经不再DOM中了。如果不是抛出异常
new = a_driver.find_element_by_tag_name(u'html')
if element != new:
raise StaleElementReferenceException(u'刚才重定向了！')
def test_raises_locator_exception_if_element_continues_to_go_stale(
self, mocker, browser_mock, driver_mock):
browser_mock.wd = driver_mock
el = wd_element(mocker)
locator = {'xpath': './/div', 'id': 'foo'}
mtcher = matcher(mocker, browser_mock, locator)
expect_all(driver_mock, [el])
mtcher.match.side_effect = [StaleElementReferenceException()] * 3
message_parts = [
'Unable to locate element collection from', "'xpath': './/div'",
"'id': 'foo'", 'due to changing page'
]
with pytest.raises(LocatorException) as e:
locate_all(mtcher, locator)
assert all(part in e.value.args[0] for part in message_parts)
def _waitFor(self, assertion, *args, **kw):
start = time.time()
while True:
try:
with no_screenshot(self):
assertion(*args, **kw)
except self.failureExceptionClass as e:
if time.time() - start > self.timeout:
raise self.failureException(
f'Timed out after {self.timeout} s. {e.args[0]}')
except StaleElementReferenceException as e:
if time.time() - start > self.timeout:
raise StaleElementReferenceException(
f'Timed out after {self.timeout} s. {e.msg}')
except NoSuchElementException as e:
if time.time() - start > self.timeout:
raise NoSuchElementException(
f'Timed out after {self.timeout} s. {e.msg}')
else:
break
time.sleep(0.1)
def get_noble_metal_price_list(self):
noble_metal_price_list = None
counter = 0
while counter < 3:
found_exception = False
try:
noble_metal_price_list = self.\
_unsafe_get_noble_metal_price_list()
except StaleElementReferenceException as stale_exception:
error_msg = "get_noble_metal_price_list: Stale Exception"
error_msg += "\n{0}".format(stale_exception)
self.logger.error(error_msg)
print(error_msg)
found_exception = True
if not found_exception:
break
counter = counter + 1
time.sleep(10)
if counter == 3:
raise StaleElementReferenceException(
"get_noble_metal_price_list: Tried 3 times.")
return noble_metal_price_list
def find_element_and_verify_visible(locator, max_number_of_tries=1):
s2l = get_s2l()
for i in range(max_number_of_tries):
try:
element = s2l._element_find(locator, True, False)
if element is None:
raise NoSuchElementException(
"Unable to find element with locator: %s" % locator)
if s2l.is_element_visible(element):
logger.info("Element %s verified as visible" % locator)
else:
fail_test("Element %s NOT visible" % locator)
except StaleElementReferenceException:
continue
else:
break
else:
raise StaleElementReferenceException(
"Element %s not attached to DOM after %s try(s)" %
(locator, max_number_of_tries))
def _get_web_element(self, element_type, locator_string, wait_state,
throw_exception, timeout):
"""
Returns web element or web elements based on the element type
:param element_type: can be ElementType.SINGLE or ElementType.MULTIPLE
:param locator_string: unique string to identify web element in DOM
:param wait_state: ElementWaitState
:param throw_exception: by default it is set to true, can be set to false
:param timeout: wait time before throwing any exception
:return: web element or web elements based on elementType
"""
by_locator = None
web_element = None
if timeout is None:
timeout = self.default_wait
try:
# retrieve the element locator in the form of tuple ex:(id, test)
by_locator = self._get_locator(locator_string)
# Check if the element_type has a valid value
if not isinstance(element_type, _ElementType):
self.context.logger.error(
f'{repr(element_type)} must be an instance of _ElementType enum class'
)
raise TypeError(
f'{repr(element_type)} must be an instance of _ElementType enum class'
)
# Retrieves single web element
if element_type is _ElementType.SINGLE:
web_element = self._get_single_web_element(
wait_state, by_locator, timeout)
# Retrieves multiple web element
if element_type is _ElementType.MULTIPLE:
web_element = self._get_multiple_web_elements(
wait_state, by_locator, timeout)
except StaleElementReferenceException as stale_ex:
if throw_exception:
self.context.logger.error(
f'The element {str(by_locator)} is no longer attached to the DOM '
f'i.e. it has been removed from the document or the document has changed'
)
self.context.logger.exception(stale_ex)
raise StaleElementReferenceException(
f'The element {str(by_locator)} is no longer attached to the DOM '
f'i.e. it has been removed from the document or the document has changed.'
f' Error {stale_ex}')
except TimeoutException as timeout_ex:
if throw_exception:
self.context.logger.error(
f'Timed out after {str(timeout)} seconds waiting for the '
f'element {str(by_locator)} state {str(wait_state)}.')
self.context.logger.exception(timeout_ex)
raise TimeoutException(
f'Timed out after {str(timeout)} seconds waiting for the '
f'element {str(by_locator)} state {str(wait_state)}. Error {timeout_ex}'
)
except WebDriverException as ex:
if throw_exception:
self.context.logger.error(
f'An error occurred while identifying the element '
f'{str(by_locator)} on the web page.')
self.context.logger.exception(ex)
raise WebDriverException(
f'An error occurred while identifying the element '
f'{str(by_locator)} on the web page. Error {ex}')
return web_element
def _raise(*a):
raise StaleElementReferenceException("Darn")
def wrapper(self, *args):
if self.changed:
raise StaleElementReferenceException()
return fn(self, *args)
def throwSERE(driver):
raise StaleElementReferenceException("test")
def __init__(self, msg, *args, **kwargs):
WebDriverException(msg, args, kwargs) or StaleElementReferenceException(msg, args, kwargs)
def assertion(*args, **kw):
raise StaleElementReferenceException(
'Element is no longer attached to the DOM')