def from_webelement(cls, element: WebElement):
id = element.find_element_by_class_name("navigation-module-index").text
name = element.find_element_by_class_name(
"navigation-module-title").text
module_name = "".join([id, "-", name])
return cls(module_name, element, [])
def iterate_react_select_options(self,
select: WebElement) -> t.List[WebElement]:
"""Given a .Select rendered by react-select, return an iterable
of .Select-option that can be .click()-ed to select them.
"""
# related: https://github.com/JedWatson/react-select/issues/603#issuecomment-157888562 # noqa
select.find_element_by_class_name('Select-arrow').click()
self.wait(EC.element_to_be_clickable((By.CLASS_NAME, 'Select-option')))
return select.find_elements_by_class_name('Select-option')
def has_button(element: WebElement) -> bool:
"""Return true iff this element has a button."""
try:
element.find_element_by_class_name('pb-buttonList__container')
except NoSuchElementException:
return False
return True
def get_xzw_height(c_main: WebElement) -> int:
"""
URL https://www.xzw.com/fortune/cancer/
Returns: int
"""
top_height = c_main.find_element_by_class_name("top").size["height"]
dl_height = c_main.find_element_by_tag_name("dl").size["height"]
cont_height = c_main.find_element_by_class_name(
"c_cont").size["height"]
return top_height + dl_height + cont_height
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_xpath(
'.//div[@class="term-description"]')
match = re.search(r'\b\d+\b', term_element.text)
if match:
term = match.group()
else:
term = 1
self.wait_for()
plan_element = el.find_element_by_class_name('description-title ')
product_name = plan_element.text
price_element = el.find_element_by_xpath(
'.//div[@class="rate-price"]/div[2]')
price = re.search(r'(\d+(\.\d+)?)', price_element.text).group()
# Popping up learn more details
detail_link = el.find_element_by_xpath('.//a[@class="detailsButton"]')
self.client.execute_script("arguments[0].click();", detail_link)
efl_element = el.find_element_by_xpath(
'.//div[@class="planInfoHolder"]//a[1]')
efl_url = efl_element.get_attribute('href')
self.__download_pdf(efl_url, '//embed')
return {
'term': term,
'price': price,
'product_name': product_name,
}
def parse_author(story_element: WebElement):
try:
author = story_element.find_element_by_class_name("user__nick").text
except NoSuchElementException:
author = None
return author
def text_manager(self, bet_container: WebElement, condition_dict: Dict[str, str], pending_bet: List[str], outer_counter: int, inner_counter: List[str], already_reported: WebElements):
"""pending_bet is the bet (list) that is being written
outer_counter keeps track (int) for self.pending_bets
inner_counter keeps track (list) of results in the same strategy"""
result = bet_container.find_element_by_class_name("myb-SettledBetItem_BetStateContainer ").text #result
#defines res_partial
if 'perdida' in result: #sets res_partial
res_partial = 'red'
elif 'retorno' in result:
res_partial = 'green'
#defines res_final, pending_bet[6] is strategy
if pending_bet[6] != 'e3' and pending_bet[6] != 'e5':
res_final = condition_dict[res_partial]
#if there are more bets of this type and team in sequence
elif (pending_bet[0] == self.pending_bets[outer_counter+1][0]) and (pending_bet[12] == self.pending_bets[outer_counter+1][12]):
res_final = '-'
inner_counter.append(res_partial)
#if this is the last bet of this type in sequence
else:
inner_counter.append(res_partial)
res_final = 'win' if inner_counter.count('green')/len(inner_counter) > 0.5 else 'loss'
inner_counter = []
with io.open('report.txt', 'a+', encoding='utf-8') as report:
text = ", ".join(pending_bet[2:])
text = f"{text}, {text}, {res_partial}, {res_final}, {self.username}"
report.write('\n')
report.write(text)
self.pending_bets.remove(pending_bet)
outer_counter -= 1
already_reported.append(bet_container)
return outer_counter, inner_counter, already_reported
def __init__(self, element: WebElement) -> None:
super().__init__(element)
self.__summary_element: WebElement = element.find_element_by_class_name(
"text-content")
self.summary: str = self.__summary_element.text
def assert_search_header(we: WebElement, window_width: int) -> None:
"""Assert that the search header part of the page is as expected.
Args:
we: WebElement containing the desired content to assert.
window_width: Width of the viewport in pixels for the web driver used
to access the page.
"""
assert_element_by_classes(we, 'img', 'myaku-logo')
assert_element_by_classes(we, 'button', 'search-clear')
assert_element_by_classes(we, 'button', 'search-submit')
assert_element_by_classes(we, 'button', 'search-options-toggle',
'Show search options')
assert_element_by_classes(we, 'legend', 'search-options-legend',
'Romaji Conversion', False)
assert_element_by_classes(we, 'label', 'check-input-label', True, False, 3)
assert_element_by_classes(we, 'input', 'search-options-check-input', None,
False, 3)
nav_link_list = we.find_element_by_class_name('nav-link-list')
assert_element_by_tag(nav_link_list,
'a',
'Github',
attrs={'href': GITHUB_LINK})
if window_width < SM_MIN_WIDTH:
placeholder = SHORT_SEARCH_PLACEHOLDER
else:
placeholder = FULL_SEARCH_PLACEHOLDER
assert_element_by_id(we,
'input',
'search-input',
attrs={'placeholder': placeholder})
def _getBetslipPrediction(self, webElement: WebElement) -> str:
try:
WebDriverWait(self._driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "CSegno")))
finally:
prediction = webElement.find_element_by_class_name(
"CSegno").text.strip().split()[0]
return prediction
def is_option_combobox_open(self, element: WebElement) -> bool:
""" Checks whether option combobox is open
:param element: option combo element (div.rw-input must be its child)
:return: True or False
"""
return 'rw-open' in element.find_element_by_class_name(
'rw-combobox').get_attribute('class')
def get_xzw_top_offset(c_main: WebElement) -> int:
"""
URL https://www.xzw.com/fortune/cancer/
Returns: int
"""
top_height = c_main.find_element_by_class_name("top").size["height"]
return top_height
def _more_text_condition(self, post: WebElement):
try:
big_post = post.find_element_by_class_name("pi_text_more")
big_post_mark = "pi_text_more" in big_post.get_attribute("class")
if big_post_mark:
big_post.click()
sleep(random.uniform(3.2, 6.6))
except NoSuchElementException:
pass
def parse_datetime(story_element: WebElement) -> Optional[datetime]:
try:
story_datetime = story_element.find_element_by_class_name("story__datetime")
str_datetime = story_datetime.get_attribute('datetime')
story_timestampz = datetime.fromisoformat(str_datetime)
story_timestamp = story_timestampz.replace(tzinfo=None)
return story_timestamp
except NoSuchElementException:
return None
def parse_link(story_element: WebElement) -> Tuple:
with open('./parser/story.html', 'w') as file:
file.write(story_element.get_attribute('outerHTML'))
# if not story_element.find_elements_by_class_name("story__title-link"):
# return
link = story_element.find_element_by_class_name("story__title-link")
href = link.get_attribute('href')
return href, link.text
def get_card_text_img_audio_data(card: WebElement, card_side: str) -> str:
"""
Get the card text
"""
card_img_tag = get_images_from_cards(card, card_side)
card_audio_tag = get_audio_from_cards(card, card_side)
card_info = card.find_element_by_class_name(card_side)
card_text = card_audio_tag + "\n" + card_info.text.replace(
"Play", "") + "\n" + card_img_tag
return card_text
def get_images_from_cards(card: WebElement, card_side: str) -> str:
"""
If a card element has an image this function will get this img tag
"""
try:
parent_element = card.find_element_by_class_name(card_side)
img_element = parent_element.find_element_by_tag_name("img")
src_img = img_element.get_attribute("src")
return f'\n<br><img src="{src_img}" >'
except NoSuchElementException:
return ""
def get_audio_from_cards(card: WebElement, card_side: str) -> str:
"""
If a card element has audio with it this function will get this audio tag
"""
try:
parent_element = card.find_element_by_class_name(card_side)
audio_element = parent_element.find_element_by_tag_name("audio")
audio_source_element = audio_element.find_element_by_tag_name("source")
src_audio = audio_source_element.get_attribute("src")
return f'\n<audio controls><source src="{src_audio} type="audio/mpeg"></audio><br>'
except NoSuchElementException:
return ""
def from_web_element(self, web_element: WebElement):
session_text = web_element.text
session_tokens = session_text.split(' ')
self.status = session_tokens[0]
self.session_id = session_tokens[1]
self.session_name = web_element.find_element_by_class_name('title').text
self.url = web_element.get_attribute('data-vote-url').replace('/vote', '')
speaker_text = " ".join(session_tokens[2:-3])
speaker_text = speaker_text.replace(self.session_name, '')
speaker_text = speaker_text.replace('about', '').strip()
self.speaker_text = speaker_text
def get_data_from_element(cls, element: WebElement) -> Dict[str, Any]:
data = {}
data["link"] = element.find_element_by_class_name(
"search-item__link").get_attribute("href")
data["title"] = element.find_element_by_class_name(
"search-item__title").text
data["preview"] = element.find_element_by_class_name(
"search-item__text").text
more = element.find_element_by_class_name("search-item__category").text
more_list = more.split(", ")
# # category, date, time format
# if len(more) == 3: # category, date, time format
# data["category"], date, time = more_list
# else if len(more) == 2: # date, time format
# date, time = more_list
# else:
# data["datetime"] = cls
return data
def _like_condition(self, post: WebElement):
try:
like_on_post = post.find_element_by_class_name("item_like")
liked = "item_sel" in like_on_post.get_attribute("class")
if liked:
self.scroll_to_element_page(post)
else:
like_count = self.get_post_params(like_on_post)
if like_count > random.randint(4500, 10000):
# scroll to like on post
self.scroll_to_element_page(like_on_post)
like_on_post.click()
sleep(random.uniform(0.5, 1.5))
except NoSuchElementException:
pass
def _wrapper_to_info(wrapper: WebElement):
departure_time = _get_departure_time(wrapper)
wrapper.find_element_by_css_selector(".price button.yellow").click()
while True:
try:
sections_wrap = wrapper.find_element_by_class_name("sections-wrap")
break
except NoSuchElementException:
time.sleep(0.1)
standard_class_seats = sections_wrap.find_elements_by_css_selector(".seats")[1].text
return AvailableTrainInfo(departure_time, standard_class_seats)
def _get_item_title(item_element: WebElement) -> Tuple[str, str]:
"""
:param item_element: the item div
:return: returns the title and link of an item
"""
item_elements = item_element.find_element_by_class_name('a-col-right') \
.find_elements_by_class_name('a-row')
item_title_element = item_elements[0]
title = item_title_element.text
try:
link = item_title_element.find_element_by_class_name(
'a-link-normal').get_attribute('href')
except NoSuchElementException:
link = 'not available'
return title, link
def _get_item_price(self, item_element: WebElement, item_index: int,
order_element: WebElement) -> float:
"""
:param item_element: the item div
:param item_index: the index of the item in the order
:param order_element: the order div
:return: returns the price of an item
"""
try:
item_price_str = item_element.find_element_by_class_name(
'a-color-price').text
item_price = self._price_str_to_float(item_price_str)
except (NoSuchElementException, ValueError):
item_price = self._get_item_price_through_details_page(
order_element, item_index)
return item_price
def get_element_static(target_element: WebElement, verifier: SelecElement) -> WebElement:
try:
element = None
if verifier.element_type == NavEleType.IsId:
element = target_element.find_element_by_id(verifier.target)
elif verifier.element_type == NavEleType.IsClass:
element = target_element.find_element_by_class_name(verifier.target)
elif verifier.element_type == NavEleType.IsCssSelector:
element = target_element.find_element_by_css_selector(verifier.target)
elif verifier.element_type == NavEleType.IsName:
element = target_element.find_element_by_name(verifier.target)
else:
raise ValueError("Selector not Supported")
return element
except Exception as inst:
print(type(inst))
print(inst.args)
return None
def _handle_main_category(self, main_cat_id: int,
main_cat_elem: WebElement):
text_elem = main_cat_elem.find_element_by_class_name("MainTag_Name")
main_cat = MainCategory(text_elem.text, main_cat_id)
self.categories[main_cat_id] = main_cat
main_cat_elem.click() # Open sub categories
inner_tags_elem = self.driver.find_element_by_id("InnerTags")
inner_main_elem = inner_tags_elem.find_element_by_id(
"innerMainTag_{}".format(main_cat_id))
a_elems = inner_main_elem.find_elements_by_tag_name("a")
for a_elem in a_elems:
sub_cat_id = int(a_elem.get_attribute("data-option-value"))
text_elem = a_elem.find_element_by_class_name("MainTag_Name")
sub_cat = SubCategory(text_elem.text, sub_cat_id)
main_cat.sub_categories[sub_cat_id] = sub_cat
img_elem = a_elem.find_element_by_tag_name("img")
img_elem.click()
self._handle_sub_category(sub_cat)
def scrape_multi_position_job(self, job: Job, job_element: WebElement):
summary_element = self.find_summary_element(
job_element, 'pv-entity__company-summary-info')
# <h3 class="t-16 t-black t-bold">
# <span class="visually-hidden">Company Name</span>
# <span>University of Colorado Boulder</span>
# </h3>
# company_element = summary_element.find_element_by_class_name('t-16')
company_element = summary_element.find_element_by_tag_name('h3')
company_spans = company_element.find_elements_by_tag_name('span')
job.company.name = self.canonize_company_name(company_spans[1].text)
duration_element = summary_element.find_element_by_tag_name('h4')
duration_spans = duration_element.find_elements_by_tag_name('span')
job.total_duration = duration_spans[1].text.strip()
# <ul class="pv-entity__position-group mt2">
positions_element = job_element.find_element_by_class_name(
'pv-entity__position-group')
# <li class="pv-entity__position-group-role-item">
positions_items = positions_element.find_elements_by_class_name(
'pv-entity__position-group-role-item')
for position_item in positions_items:
position = Position()
# <div class="pv-entity__summary-info-v2 pv-entity__summary-info--background-section pv-entity__summary-info-margin-top mb2">
position_element = position_item.find_element_by_class_name(
'pv-entity__summary-info--background-section')
# <h3 class="t-14"> <span>Title</span> <span>Web Designer</span>
title_element = position_element.find_element_by_tag_name('h3')
title_spans = title_element.find_elements_by_tag_name('span')
position.title = title_spans[1].text.strip()
position.date_range = self.scrape_date_range(position_element)
position.location = self.scrape_location(position_element)
position.duration = self.scrape_duration(position_element)
job.add_position(position)
def get_element_static(target_element: WebElement,
verifier: SelecElement) -> WebElement:
try:
element = None
if verifier.element_type == NavEleType.IsId:
element = target_element.find_element_by_id(verifier.target)
elif verifier.element_type == NavEleType.IsClass:
element = target_element.find_element_by_class_name(
verifier.target)
elif verifier.element_type == NavEleType.IsCssSelector:
element = target_element.find_element_by_css_selector(
verifier.target)
elif verifier.element_type == NavEleType.IsName:
element = target_element.find_element_by_name(verifier.target)
else:
raise ValueError("Selector not Supported")
return element
except Exception as inst:
print(type(inst))
print(inst.args)
return None
def _get_item_price_through_details_page(self, order_element: WebElement,
item_index: int) -> float:
"""
:param order_element: the order div
:param item_index: the index of the item in the order
:returns: the item price found on the order details page
"""
item_price: float = 0
try:
order_details_link = order_element.find_element_by_class_name(
'a-link-normal').get_attribute('href')
self.browser.execute_script(
f'''window.open("{order_details_link}","_blank");''')
self.browser.switch_to.window(self.browser.window_handles[1])
if not ut.wait_for_element_by_class_name(self.browser,
'od-shipments'):
return item_price
od_shipments_element = self.browser.find_element_by_class_name(
'od-shipments')
price_fields: List[
WebElement] = od_shipments_element.find_elements_by_class_name(
'a-color-price')
item_price = self._price_str_to_float(
price_fields[item_index].text)
except (NoSuchElementException, ValueError):
item_price = 0
self.logger.warning(
colored(
f'Could not parse price for order:\n{order_element.text}',
'yellow'))
finally:
self.browser.close()
self.browser.switch_to.window(self.browser.window_handles[0])
return item_price
def get_job_info(self, job: WebElement, category: str):
"""Extract Job Info from a given element and return it in a list
Args:
job(WebElement): A webelement reference to a tag consisting of job info.
category(str): The category the job fits into.
Returns:
info(list): A list containing the following info: company_name, title, job_type, region and category.
"""
company_and_job_type = job.find_elements_by_class_name('company')
company_name = company_and_job_type[0].get_attribute('textContent')
job_type = company_and_job_type[1].get_attribute('textContent')
title = job.find_element_by_class_name('title').get_attribute(
'textContent')
try:
region = job.find_element_by_css_selector(
'section.jobs article ul li span.region.company'
).get_attribute('textContent')
except NoSuchElementException:
region = ''
return [company_name, title, job_type, region, category]
def get_cards_info_of_deck(chrome_webdriver: WebDriver, deck: WebElement):
"""
Get the cards info of each deck
"""
try:
glasses_icon = deck.find_element_by_class_name(
"ion-ios-glasses-outline")
glasses_icon.click()
sleep(2)
cards_window_selection = chrome_webdriver.find_element_by_class_name(
"preview-card-table")
cards_list = cards_window_selection.find_elements_by_class_name(
"preview-card")
csv_file_name = get_file_name_for_csv_files(chrome_webdriver).replace(
"Preview", "").strip()
path = os.getcwd()
path_csv = f"{path}/csv_sem_utf8_delimiter_virgula/"
try:
os.mkdir(path_csv)
except OSError:
pass
with open(f"{path_csv}{csv_file_name}.csv", "w",
newline="") as csv_file:
writer = csv.writer(csv_file, delimiter=",")
for card in cards_list:
front_info = get_card_text_img_audio_data(card, "front")
back_info = get_card_text_img_audio_data(card, "back")
writer.writerow([front_info, back_info])
color_options = [
Fore.GREEN, Fore.MAGENTA, Fore.CYAN, Fore.BLUE, Fore.YELLOW
]
print(choice(color_options) + f"{csv_file_name}.csv salvo!")
chrome_webdriver.find_element_by_class_name("close-button").click()
except NoSuchElementException:
pass
