def clickElem(elem:WebElement, timeOut:float=29, pollFreq:float=0.7):
timeElapsed=0.0
while timeElapsed < timeOut:
beginTime=time.time()
try:
elem.click()
return
except StaleElementReferenceException:
pass
time.sleep(pollFreq)
timeElapsed+=(time.time()- beginTime)
raise TimeoutException('[getElemText] Time out elem='+str(elem))
def enterElem(e:WebElement, func):
prev_url = driver.current_url
uprint('prev_url='+prev_url)
next_url = e.get_attribute('href')
uprint('next_url='+next_url)
driver.get(next_url)
func(prev_url)
def getElemAttrib(e:WebElement,attName:str, trialCount=20,pollInterval=3,default=None) -> str:
for trial in range(trialCount):
try:
return e.get_attribute(attName)
except (StaleElementReferenceException, NoSuchElementException, TimeoutException):
sleep(pollInterval)
return default
def get_el(self, locator: tuple, base: WebElement=None) -> WebElement:
"""
Shortcut to get an element.
"""
if base is None:
base = self.browser
return base.find_element(*locator)
def get_elements_static(target_element: WebElement, verifier: SelecElement) ->[]:
try:
elements = None
if verifier.element_type == NavEleType.IsId:
elements = target_element.find_elements_by_id(verifier.target)
elif verifier.element_type == NavEleType.IsClass:
elements = target_element.find_elements_by_class_name(verifier.target)
elif verifier.element_type == NavEleType.IsCssSelector:
elements = target_element.find_elements_by_css_selector(verifier.target)
elif verifier.element_type == NavEleType.IsName:
elements = target_element.find_elements_by_name(verifier.target)
else:
raise ValueError("Selector not Supported")
return elements
except Exception as inst:
print(type(inst))
print(inst.args)
return None
def get_menu_item(ele: WebElement) -> MenuItem:
"""
:param ele:
:return:
"""
url = ele.find_element_by_tag_name('a')
menu_title = url.text
menu_url = url.get_attribute('href')
menu_item = MenuItem(menu_title, menu_url)
nodes = ele.find_elements_by_css_selector("ul > li")
for node in nodes:
sub_menu = get_menu_item(node)
menu_item.nodes.append(sub_menu)
return menu_item
def _more_text_condition(self, post: WebElement):
try:
big_post = post.find_element_by_class_name("pi_text_more")
big_post_mark = "pi_text_more" in big_post.get_attribute("class")
if big_post_mark:
big_post.click()
sleep(random.uniform(3.2, 6.6))
except NoSuchElementException:
pass
def find_nested_element_by_locator_and_value(self, parent: WebElement, locator: str, value: str) -> WebElement:
try:
element = parent.find_element(locator, value)
self.driver.execute_script("return arguments[0].scrollIntoView();", element)
return element
except TimeoutException:
raise Exception("Element not found")
except NoSuchElementException:
raise Exception("Element not found")
def findVidSrc(vidEl: WebElement):
postersrc = vidEl.get_attribute('data-postersrc')
if not postersrc:
return ['', '']
vid_ids = re.findall(r'^.*\/(.*?)\..*?$', postersrc)
return [
f'https://i.kinja-img.com/gawker-media/image/upload/{vid_ids[0]}.mp4',
f'{vid_ids[0]}.mp4'
]
def clear_nested_element(self, parent: WebElement, locator: Tuple[str, str]):
try:
element = parent.find_element(*locator)
self.driver.execute_script("return arguments[0].scrollIntoView();", element)
element.clear()
except TimeoutException:
raise Exception("Element not found")
except NoSuchElementException:
raise Exception("Element not found")
def find_element(self, by: By, element: WebElement, value: str):
_element: WebElement = None
try:
_element = element.find_element(by, value)
except Exception as e:
_element = None
return _element
return _element
def waitForDissappear( webelmnt: WebElement, counter:int=10 ):
timer=0
while webelmnt.is_displayed()>-1:
if timer>=counter:
Log.write('Ошибка Ожидания исчезновения элемента '+webelmnt)
return False
time.sleep(1)
timer+=1
return True
def dismiss_overlay_click(driver,
locator: Tuple[By, str] = None,
elem: WebElement = None,
retries: int = 5):
"""If element is blocked by a modal, attempt to close it."""
if elem:
pass
elif locator:
elem = WebDriverWait(driver,
10).until(EC.presence_of_element_located(locator))
else:
raise ValueError("either element, or locator must be provided")
for i in range(retries):
try:
elem.click()
break
except ElementClickInterceptedException as exc:
log.info(
"blocked by overlay, attempting to close before clicking again (%s/%s): %s",
i,
retries,
exc,
)
if len(glob(f"{config.WORKING_DIRECTORY}/*overlay.png")
) >= retries * 2:
log.exception("giving up on closing overlay")
driver.screenshot(
BaseWebScraper.screenshot_path("max_close_overlay"))
raise exc
if detect_and_send_escape_to_close_survey(driver):
pass
elif detect_and_close_survey(driver):
pass
elif detect_and_close_modal(driver):
pass
else:
log.exception("unable to close overlay, raising")
raise exc
driver.sleep(2)
except StaleElementReferenceException:
return
driver.screenshot(BaseWebScraper.screenshot_path("close_overlay"))
def __extract(self, service: WebElement) -> dict:
'''Extracts the services from each section element.
Contains URI, title and description for each service.
Parameters
----------
service : WebElement
a services WebElement to scrape information
Returns
-------
dict
information on dict-like basis
'''
return {
"URI" : service.find_element(By.TAG_NAME, "a").get_attribute("href"),
"Title" : service.find_element(By.TAG_NAME, "a").text,
"Description" : service.find_element(By.TAG_NAME, "p").text,
}
def analyze_element(self, el: WebElement):
data_product_promo = el.get_attribute("data-product-promo")
if data_product_promo not in self.plans_promo_blacklist:
term_element = el.find_element_by_css_selector(
"div.card-body div ul")
term = term_element.text
match = re.search(r"(\d+)\s+Months", term)
if match:
term = match.groups()[0]
else:
raise Exception("Term could not match. (%s)" % term)
else:
term = "1"
price_element = el.find_element_by_css_selector(
"div.card-body div.modal-body h1" +
",div.card-body div.product2.cards_div2 h2")
price = re.search(r"(\d+(\.\d+)?)",
price_element.text.split("¢")[0]).groups()[0]
product_name = el.find_element_by_css_selector(
"div.card-body div.modal-body h2" +
",div.card-body div.product2.cards_div2 h4").text
efl_download_link_element = el.find_element_by_css_selector(
"div.modal-footer a")
efl_download_link_element.click()
self.client.switch_to_window(self.client.window_handles[1])
pdf_url = self.client.find_element_by_tag_name("iframe").get_attribute(
"src")
self.client.get(pdf_url)
self.client.close()
self.client.switch_to_window(self.client.window_handles[0])
self.wait_for()
return {
"term": term,
"price": price,
"product_name": product_name,
}
def _find_by_css_selector(root, css_selector, raise_exception):
element = None
try:
# Chrome 96+ returns the element as a dict that includes the id
if isinstance(root, dict):
key = list(root.keys())[0]
shadow_root_el = WebElement(_get_driver(), root[key])
element = shadow_root_el.find_element_by_css_selector(css_selector)
else:
element = root.find_element_by_css_selector(css_selector)
except NoSuchElementException as nse:
if raise_exception:
if root.tag_name == "slot":
raise TaurusException(
"Shadow DOM Slots are currently not supported in Taurus execution."
)
raise nse
pass
return element
def _table_rows(table_element: WebElement,
header=True) -> List[WebElement]:
""" Return table rows
:param table_element: table element
:param header: skips the header when true if table has a header
:return: table rows elements
"""
first_row = 1 if header else 0
return table_element.find_elements_by_css_selector('tr')[first_row:]
def get_emoji(element: WebElement) -> Optional[str]:
"""
get emoji from emoji span
"""
try:
return element.find_element_by_tag_name(
'span').find_element_by_tag_name('span').get_attribute(
'innerHTML')
except NoSuchElementException:
return None
def GetETimeFrameFromBlock(expBlock: WebElement) -> ETimeFrame or None:
"""Returns an [ETimeFrame] from an experience block
Args:
expBlock (WebElement): The experience block
Returns:
ETimeFrame or None: The timeframe
"""
try:
dateRange: list = expBlock.find_element(
By.CSS_SELECTOR, selectors.DATE_RANGE).text.split('–')
start: str = dateRange[0]
end: str = dateRange[1]
length: str = expBlock.find_element(By.CSS_SELECTOR,
selectors.EXP_LENGTH).text
return ETimeFrame(start, end, length)
except:
return None
def _handle_main_category(self, main_cat_id: int,
main_cat_elem: WebElement):
text_elem = main_cat_elem.find_element_by_class_name("MainTag_Name")
main_cat = MainCategory(text_elem.text, main_cat_id)
self.categories[main_cat_id] = main_cat
main_cat_elem.click() # Open sub categories
inner_tags_elem = self.driver.find_element_by_id("InnerTags")
inner_main_elem = inner_tags_elem.find_element_by_id(
"innerMainTag_{}".format(main_cat_id))
a_elems = inner_main_elem.find_elements_by_tag_name("a")
for a_elem in a_elems:
sub_cat_id = int(a_elem.get_attribute("data-option-value"))
text_elem = a_elem.find_element_by_class_name("MainTag_Name")
sub_cat = SubCategory(text_elem.text, sub_cat_id)
main_cat.sub_categories[sub_cat_id] = sub_cat
img_elem = a_elem.find_element_by_tag_name("img")
img_elem.click()
self._handle_sub_category(sub_cat)
def select_option_value_on_nested_element(self, parent: WebElement, locator: Tuple[str, str], value: str):
try:
element = parent.find_element(*locator)
self.driver.execute_script("return arguments[0].scrollIntoView();", element)
select = Select(element)
select.select_by_value(value)
except TimeoutException:
raise Exception("Element not found")
except NoSuchElementException:
raise Exception("Element not found")
def find_visible_element(self,
selector: str,
within: WebElement = None) -> WebElement:
"""Given a selector that could match multiple different elements,
return the one that is currently visible, not the first one that matches."""
within = within or self.driver
return self.blip_element([
x for x in within.find_elements_by_css_selector(selector)
if x.is_displayed()
][0])
def check_visible_quick(self,
selector: str,
within: WebElement = None) -> bool:
"""Check for an element without potentially spending a lot of time polling the DOM.
Ususally used when asserting an element's absence, saves waiting the full timeout."""
within = within or self.driver
self.driver.implicitly_wait(SHORT_WAIT)
elements = within.find_elements_by_css_selector(selector)
self.driver.implicitly_wait(LONG_WAIT)
return len(elements) != 0 and elements[0].is_displayed()
def flashy_find_elements(self,
selector: str,
within: WebElement = None) -> ELEMENT_LIST:
"""Finds multiple elements that match a CSS selector, highlights them as well.
The browser-provided webdriver self.driver implementations seem to not return a
list when only one element matches, so fixing that here."""
within = within or self.driver
return self.blip_element(
to_list(within.find_elements_by_css_selector(selector)))
def _process_contact(self, conversation: WebElement):
# Fetch contact name/number element - if contact saved, would appear as name
contact_name_number = self.wait.until(lambda _: conversation.find_element_by_xpath(
".//span[@class='_1wjpf']")).get_attribute('title')
if contact_name_number not in self.processed_contacts:
cleaned = selenium_methods.clean_contact_number(contact_name_number)
if cleaned:
self.processed_contacts.append(cleaned)
return contact_name_number
def parse_location(experience_row: WebElement) -> str:
try:
return experience_row.find_element_by_xpath(
selectors['profile_position_location']).text
except NoSuchElementException as e:
logging.debug(f"Can't find profile_position_location {e}")
return ''
except Exception as e:
logging.debug(f"Unknown Exception {e}")
return ''
def send_keys_delay_random(self,
element: WebElement,
keys: Union[str, int, float, List[str],
List[int], List[float]],
min_delay: float = 0.025,
max_delay: float = 0.25) -> bool:
_keys = copy.deepcopy(keys)
if type(_keys) in [int, float]:
_keys = str(_keys)
for key in _keys:
if type(key) in [int, float]:
key = str(key)
element.send_keys(key)
time.sleep(random.uniform(min_delay, max_delay))
return True
def bulk_individual_option_toggles(self,
row: WebElement,
i: int = 0) -> List[WebElement]:
""" Return list of toggles in bulk edit individual Visibility
:param row: bulk row
:param i: property index: 0 - first property, 1 - second property
:return: list of toggle elements on the left or right pane
"""
selector = self.BULK_EDIT_OPTION_TOGGLES_LEFT if i == 0 else self.BULK_EDIT_OPTION_TOGGLES_RIGHT
return row.find_elements(*selector)
def get_value_of_elem(elem: WebElement):
class_name = elem.get_attribute('class')
if 'blank' in class_name:
return 'o'
elif 'bombflagged' in class_name:
return 'x'
elif 'bombdeath' in class_name:
raise Exception('It\'s a boy!')
else:
return int(class_name[-1])
def parse_datetime(story_element: WebElement) -> Optional[datetime]:
try:
story_datetime = story_element.find_element_by_class_name("story__datetime")
str_datetime = story_datetime.get_attribute('datetime')
story_timestampz = datetime.fromisoformat(str_datetime)
story_timestamp = story_timestampz.replace(tzinfo=None)
return story_timestamp
except NoSuchElementException:
return None
def global_sku_input(self, row: WebElement = None) -> WebElement:
""" Finds and returns input element for global sku
:param row: listing row element - if None, bulk edit used instead
:return: sku input element
"""
if row:
return row.find_element(*self.GLOBAL_SKU_INPUT)
else:
return self.driver.find_element(*self.BULK_EDIT_GLOBAL_SKU_INPUT)
def _get_feed_post_media(post: WebElement):
"""
Get link to post from post web-element from feed
:param post: WebElement
:return: str
"""
try:
image = post.find_element_by_css_selector('div:nth-child(2) img')
return image.get_attribute('src')
except excp.NoSuchElementException:
pass
try:
video = post.find_element_by_tag_name('video')
return video.get_attribute('src')
except excp.NoSuchElementException:
pass
return False
def bulk_individual_option_inputs(self,
row: WebElement,
i: int = 0) -> List[WebElement]:
""" Return list of input boxes in bulk edit individual Price/Q/SKU
:param row: bulk row
:param i: property index: 0 - first property, 1 - second property
:return: list of input elements on the left or right pane
"""
selector = self.BULK_EDIT_OPTION_INPUTS_LEFT if i == 0 else self.BULK_EDIT_OPTION_INPUTS_RIGHT
return row.find_elements(*selector)
def commenter_action_element(trigger_type: TRIGGER, commenter: WebElement):
trigger_link = None
try:
if trigger_type == TRIGGER.LIKE:
trigger_link = commenter.find_element_by_xpath(
'//div[text()="Like"]')
except NoSuchElementException as e:
print("Like button not found by the FebuBot")
return trigger_link
def get_card_text_img_audio_data(card: WebElement, card_side: str) -> str:
"""
Get the card text
"""
card_img_tag = get_images_from_cards(card, card_side)
card_audio_tag = get_audio_from_cards(card, card_side)
card_info = card.find_element_by_class_name(card_side)
card_text = card_audio_tag + "\n" + card_info.text.replace(
"Play", "") + "\n" + card_img_tag
return card_text
def is_trash_story(cls, story_container: WebElement) -> bool:
"""False for adv and other not normal containers"""
try:
if story_container.find_elements_by_class_name(
"story__placeholder"):
return True
except StaleElementReferenceException:
return True
return False
def getElemAttr(elem:WebElement, attr:str, timeOut:float=30, pollFreq:float=1) -> str:
timeElapsed=0.0
while timeElapsed < timeOut:
beginTime=time.time()
try:
return elem.get_attribute(attr)
except StaleElementReferenceException:
pass
time.sleep(pollFreq)
timeElapsed+=(time.time()- beginTime)
raise TimeoutException('[getElemAttr] Time out elem='+str(elem))
def parse_results_for_bulk_expired_domains_search(self, element: WebElement) ->[]: #return a list of DomainType
if element is not None:
products = element.find_elements_by_class_name(BulkCheckElements.get_element_by_class(BulkCheckElements.class_product).target)
filtered = []
for product in products:
domain = product.find_element_by_class_name(BulkCheckElements.get_element_by_class(BulkCheckElements.class_domain_name).target).text
available = GoDaddy.checkAvailability(product.find_element_by_class_name(
BulkCheckElements.get_element_by_class(BulkCheckElements.class_msg).target).text)
filtered.append(DomainType(domain, available))
return filtered
else:
return None
def clickDownloadableElem(elem:WebElement)->str:
pollFreq=1
filesOld=os.listdir(dlDir)
fileName=getElemText(elem)
href = elem.get_attribute('href')
assert href is not None
uprint('fileName="%s" href="%s"'%(fileName,href))
sql("UPDATE dlink SET href=:href WHERE"
" file_name=:fileName",locals())
uprint("UPDATE dlink SET href=%(href)s WHERE"
" file_name=%(fileName)s"%locals())
return
def versionWalker(tr:WebElement):
global driver, prevTrail
try:
btn=tr.find_element_by_css_selector('button')
btn.click()
time.sleep(0.1)
waitUnti(lambda:all(_.is_displayed() for _ in tr/csss/'ul li a'))
versions = tr/csss/'ul li a'
numVersion = len(versions)
startIdx = getStartIdx()
for idx in range(startIdx, numVersions):
ulog('version idx=%s'%idx)
fwVer = versions[idx].text.strip()
versions[idx].click()
upsert(tr, fwVer)
if idx < numVersions-1:
btn.click()
time.sleep(0.1)
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def elem_has_class(elem: WebElement, className: str):
classes = elem.get_attribute('class')
classes = classes.split()
return className in classes
def __is_valid_element(element: WebElement):
assert (element is not None)
assert element.is_displayed() is True, "element is not displayed on the page"
assert element.is_enabled() is True, "element is not enabled"
def get_value(element: WebElement) -> str:
return element.get_attribute('value')
def getTextA(elm:WebElement)->str:
waitUntil(lambda: elm.is_displayed() and elm.text,30,1)
return elm.text
def is_expanded(self, node: WebElement):
return 'dynatree-expanded' in node.get_attribute('class')
