def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector(
'ul.options > li:last-child')
term = term_element.text.replace('Terms: ', '')
match = re.search(r'(\d+)\s+Months', term)
if match:
term = match.groups()[0]
elif term == 'End-of-Year':
term = '12'
elif term == 'Month-to-month':
term = '1'
else:
raise Exception("Term could not match. (%s)" % term)
price_element = el.find_element_by_css_selector('div.price')
price = price_element.text.split("¢")[0]
plan_element = el.find_element_by_css_selector('div.title')
product_name = plan_element.text
# efl_download_link_element = el.find_element_by_css_selector(
# 'div.card-body.plan-box div.gridPlanLinks span.efl_link a')
# efl_download_link_element.click()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def get_track(article: WebElement) -> Track:
track_id, artwork, url = [None] * 3
try:
track_id_container = article.find_element_by_css_selector(
'.mbs._6m6._2cnj._5s6c')
emoji = Crawler.get_emoji(
track_id_container.find_element_by_tag_name('a'))
track_id = track_id_container.find_element_by_tag_name(
'a').get_attribute('innerHTML')
track_id = re.sub('<span .+</span>', emoji,
track_id) if emoji else track_id
artwork = article.find_element_by_css_selector(
'img.scaledImageFitWidth').get_attribute('src')
url_map = urllib.parse.parse_qs(
article.find_element_by_css_selector('a._52c6').get_attribute(
'href'))
if URL not in url_map.keys():
url = urllib.parse.unquote(
article.find_element_by_css_selector(
'a._52c6').get_attribute('href'))
url = Crawler.parse_track_url(url)
else:
url = url_map[URL][0]
url = Crawler.parse_track_url(url)
return Track(track_id, artwork, url)
except NoSuchElementException:
if track_id is None:
url = article.find_element_by_css_selector('._5_jv._58jw') \
.find_element_by_tag_name('a').get_attribute('innerHTML')
return Track(track_id, artwork, url)
def analyze_element(self, el: WebElement):
if "featured-plan" in el.get_attribute("class").split(" "):
term, price, product_name = self.__featured_analyze_element(el)
else:
term_element = el.find_element_by_xpath(".//ul/li[1]")
match = re.search(r"\b\d+\b", term_element.text)
if match:
term = match.group()
else:
term = 1
price_element = el.find_element_by_css_selector("div.product2 h2")
price = re.search(r"(\d+(\.\d+)?)", price_element.text).group()
plan_element = el.find_element_by_css_selector("div.product2 h4")
product_name = plan_element.text
efl_download_link_element = el.find_element_by_xpath(".//a[1]")
efl_url = efl_download_link_element.get_attribute("href")
self.__download_pdf(efl_url)
return {
"term": term,
"price": price,
"product_name": product_name,
}
def analyze_element(self, el: WebElement):
# term
term = el.find_element_by_css_selector("span.offerSubTitle").text
try:
term = int(term)
except ValueError:
term = 1
# price
price = el.find_element_by_css_selector("div.rate span").text
# product
plan_element = el.find_element_by_xpath(
'.//form/div[@class="offerTitle"]/table/tbody/tr/td/h4')
plan_full_text = plan_element.text
price_text = el.find_element_by_xpath(
'.//form/div[@class="offerTitle"]/table/tbody/tr/td/h4/span').text
plan_offset = len(plan_full_text) - len(price_text)
product_name = plan_full_text[:plan_offset].rstrip()
# download
decline_button = self.client.find_element_by_id("mcx_decline")
if decline_button and decline_button.is_displayed():
decline_button.click()
link_element = el.find_element_by_xpath(
'.//form/div[@class="offerSpecs"]/ul/li/a')
link_element.click()
return {
"term": term,
"price": price,
"product_name": product_name,
}
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector('td:nth-child(5)')
term = term_element.text
match = re.search(r'(\d+)\s+Months', term)
if match:
term = match.groups()[0]
else:
raise Exception("Term could not match. (%s)" % term)
price_element = el.find_element_by_css_selector('td:nth-child(2)')
price = price_element.text.split('$')[1]
plan_element = el.find_element_by_css_selector('td:nth-child(1)')
product_name = plan_element.text
efl_download_link_element = el.find_element_by_css_selector(
'td:nth-child(7) > a:nth-child(1)')
pdf_url = efl_download_link_element.get_attribute('href')
self.client.get(pdf_url)
return {
'term': term,
'price': price,
'product_name': product_name,
}
def analyze_element(self, el: WebElement, el_detail: WebElement):
term_element = el.find_element_by_css_selector('div.term-css div span')
term = term_element.text
price_element = el.find_element_by_css_selector(
'div.type-css div span')
price = price_element.text.split('¢')[0]
# Expand detail page
expand_btn_element = el.find_element_by_css_selector(
'div.hideButton-css div.buttonCss')
expand_btn_element.click()
plan_element = el_detail.find_element_by_css_selector(
'div:nth-child(1) div h4 b')
product_name = plan_element.text
efl_pdf_url = el_detail.find_element_by_css_selector(
'div.text-center span:nth-child(1) a').get_attribute("href")
self.client.get(efl_pdf_url)
expand_btn_element.click()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector(
"div.individual-option-main ul.individual-options-short-list " +
"span.li-text")
term = re.search(r"\b\d+\b", term_element.text).group()
price_element = el.find_element_by_css_selector(
"div.individual-option-main div.individual-cost ")
price = re.search(r"(\d+(\.\d+)?)", price_element.text).group()
plan_element = el.find_element_by_css_selector(
"div.individual-option-main span.name-above-plan")
product_name = plan_element.text
efl_download_link_element = el.find_element_by_xpath(
'.//div[@class="individual-plan-doc"]/a[1]')
main_client = self.client.current_window_handle
self.client.get(efl_download_link_element.get_attribute("href"))
self.client.switch_to.window(main_client)
return {
"term": term,
"price": price,
"product_name": product_name,
}
def analyze_element(self, el: WebElement):
detail_btn = el.find_element_by_css_selector('input.colorButton')
self.client.execute_script("arguments[0].click();", detail_btn)
self.wait_for()
term_element = el.find_element_by_css_selector('div.term')
term = re.search(r'\b\d+\b', term_element.text).group()
price_element = el.find_element_by_css_selector('div.productPrice ')
price = re.search(r'(\d+(\.\d+)?)', price_element.text).group()
try:
plan_element = el.find_element_by_css_selector('div.productName ')
except Exception:
plan_element = el.find_element_by_css_selector(
'div.greenProductName ')
product_name = plan_element.text
efl_download_link_element = el.find_element_by_xpath(
'.//table[2]//tbody//tr[5]//td//a[1]')
efl_download_link_element.click()
main_client = self.client.current_window_handle
self.client.switch_to_window(self.client.window_handles[-1])
self.client.execute_script('window.print();')
self.client.close()
self.client.switch_to.window(main_client)
self.wait_for()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def func_item_wrap(element: WebElement) -> Dict:
a: WebElement = element.find_element_by_css_selector(
css_selector="div.content div h3 a")
img = element.find_element_by_css_selector(
css_selector="div.pic a img").get_attribute("src")
name = a.text
sid = re.search(".*sid: (\\d+),.*",
a.get_attribute("onclick")).group(1)
year = re.search(
".*/ (\\d+)$",
element.find_element_by_css_selector(
css_selector="div.content div div span.subject-cast").text
).group(1)
rating = "0"
try:
element.find_element_by_css_selector(
css_selector="div.content div div span.rating_nums").text
except:
pass
return {
"sid": sid,
"name": name.strip(),
"rating": rating.strip(),
"img": img.strip(),
"year": year
}
def analyze_element(self, el: WebElement, el_detail: WebElement):
el.click()
self.wait_for(2)
term_element = el_detail.find_element_by_css_selector(
"div.row > div.col-sm-4 > div.list-group"
+ " > span.list-group-item:nth-child(1) > span:nth-child(2)"
)
term = term_element.text
price_element = el.find_element_by_css_selector(
"div.row > div.price-col span.price"
)
price = price_element.text.split("¢")[0]
plan_element = el.find_element_by_css_selector(
'div.row > div.offer-col:not([style*="display: none"])' +
" span.offer-name"
)
product_name = plan_element.text
efl_download_link_element = el_detail.find_element_by_css_selector(
"div.row > div.col-sm-8 > div.list-group"
+ " > span.list-group-item:nth-child(2) > span.pull-right"
+ " > a"
)
efl_download_link_element.click()
return {
"term": term,
"price": price,
"product_name": product_name,
}
def click_to_mode(self,
sidebar: WebElement,
mode: FitMode,
timeout: int = 14):
"""
Click the button~
To the beat~
Returns either the final button or None if nothing is found; can be an
easy way to check if the mode set is correct.
"""
fit_button: WebElement = sidebar.find_element_by_css_selector(
"button[data-bind='fit_button']")
timeout_count = 0
while timeout_count < timeout:
fit_button.click()
fit_button = sidebar.find_element_by_css_selector(
"button[data-bind='fit_button']")
observed_mode = self.FitMode(
fit_button.get_attribute("data-lyt.fit"))
if observed_mode and observed_mode == mode:
return fit_button
timeout_count += 1
time.sleep(0.015)
return None
def click_search_result(search_result: RemoteWebElement) -> None:
"""
Clicks on the passed in Amazon search result
:param search_result: RemoteWebElement
:return: None
"""
_ = search_result.location_once_scrolled_into_view
search_result.find_element_by_css_selector("img").click()
def _get_feed_post_link(post: WebElement):
"""
Get link to post from post web-element from feed
:param post: WebElement
:return:
"""
try:
post_link = post.find_element_by_css_selector('div:nth-child(3) div:nth-child(4) a')
except excp.NoSuchElementException:
post_link = post.find_element_by_css_selector('div:nth-child(3) div:nth-child(3) a')
return post_link.get_attribute('href')
def get_user(self, article: WebElement):
name, user_link = [None] * 2
try:
name = article.find_element_by_css_selector('._5pb8.m_1djjlbjav._8o._8s.lfloat._ohe').get_attribute('title')
user_link = self.parse_url(
article.find_element_by_css_selector('._5pb8.m_1djjlbjav._8o._8s.lfloat._ohe').get_attribute('href'))
b = article.find_element_by_css_selector('.igjjae4c.glosn74e.f2jdl7fy.cxfqmxzd').get_attribute("innerHTML")
return User(user_link, name, b)
except NoSuchElementException:
return User(user_link, name)
def __assert_posting_panel(self, web_element: webelement.WebElement,
posting_data: dict) -> None:
title = web_element.find_element_by_css_selector(
'div.panel-heading > h3.panel-title')
if 'name' in posting_data.keys():
self.assertTrue(posting_data['name'] in title.text)
created_at = web_element.find_element_by_css_selector(
'div.panel-heading > h3.panel-title > label.small')
if 'created_at' in posting_data.keys():
self.assertTrue(posting_data['created_at'] in created_at.text)
message = web_element.find_element_by_css_selector('div.panel-body')
if 'messsage' in posting_data.keys():
self.assertEqual(posting_data['message'], message.text.strip())
def analyze_element(self, el: WebElement):
# term
term_element = el.find_element_by_css_selector(
'div.pricing-table-value-small.ng-binding')
term_full_text = term_element.text
unit_text = term_element.find_element_by_xpath('.//span').text
term_text_index = len(term_full_text) - len(unit_text)
term = term_full_text[:term_text_index].rstrip()
# price
price_element = el.find_element_by_css_selector(
'div.pricing-table-value.ng-binding')
price_full_text = price_element.text
currency_text = price_element.find_element_by_xpath('.//sup').text
price_text_index = len(price_full_text) - len(currency_text)
price = price_full_text[:price_text_index].rstrip()
# product
product_name = el.find_element_by_css_selector(
'h1.pricing-table-title').text
# download
# close the survey dialog box if it is open
decline_button = self.client.find_element_by_id("mcx_decline")
if decline_button and decline_button.is_displayed():
decline_button.click()
# open modal for getting download link
signup_form = el.find_element_by_id('signupForm')
modal_link_element = signup_form.find_element_by_xpath('.//a')
modal_link_element.click()
dialog_element = self.client.find_element_by_css_selector(
'div.modal.active')
link_element = dialog_element.find_elements_by_xpath(
'.//div//div[@class="modal-body"]/'
'p[@class="modal-link-container"]/a')[-1]
link_element.click()
# close the modal
button = dialog_element.find_element_by_css_selector(
'a.plan-modal-link-close')
button.click()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def _wrapper_to_info(wrapper: WebElement):
departure_time = _get_departure_time(wrapper)
wrapper.find_element_by_css_selector(".price button.yellow").click()
while True:
try:
sections_wrap = wrapper.find_element_by_class_name("sections-wrap")
break
except NoSuchElementException:
time.sleep(0.1)
standard_class_seats = sections_wrap.find_elements_by_css_selector(".seats")[1].text
return AvailableTrainInfo(departure_time, standard_class_seats)
def analyze_element(self, el: WebElement):
main_client = self.client.current_window_handle
# term
term = el.find_element_by_xpath(
'.//ul[contains(@class, "plancard-attributes")]/li'
).text
term = 0
# price
price_element = el.find_element_by_xpath(
'.//span[@class="average-price"]'
)
price_full_text = price_element.text
currency = el.find_element_by_xpath(
'.//span[@class="average-price"]/span'
).text
price_offset = len(price_full_text) - len(currency)
price = price_full_text[:price_offset].rstrip()
# product
plan_element = el.find_element_by_css_selector('div.plan-price')
product_name = plan_element.text
# download
link_element = el.find_element_by_css_selector(
'div.mt-auto a.edit-btn'
)
link_element.click()
target_modal = link_element.get_attribute("data-target")[1:]
modal = self.wait_until(target_modal)
links = modal.find_elements_by_css_selector("ul.bullet-ul")[-1]
link = links.find_elements_by_xpath(".//li/a")[-1]
link.click()
self.wait_for()
if main_client != self.client.window_handles[-1]:
self.client.switch_to_window(main_client)
button_element = modal.find_element_by_xpath(
'.//div/div/div[@class="modal-header"]/button'
)
button_element.click()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def get_reactions(article: WebElement) -> Optional[str]:
try:
reactions = article.find_element_by_css_selector(
'._81hb').get_attribute('innerHTML')
return reactions
except NoSuchElementException:
return None
def _parse_compact_render_elem(self, video_element: WebElement):
self.title = video_element.find_element_by_css_selector("span#video-title").get_attribute("innerText")
self.url = video_element.find_element_by_tag_name("a").get_attribute("href")
self.channel_name = video_element.find_element_by_xpath(
".//ytd-channel-name//yt-formatted-string"
).get_attribute("innerText")
self.channel_url = None # Channel is not clickable from this element
def analyze_element(self, el: WebElement):
# product
product_name = el.find_element_by_css_selector(
'h6.tcenter.boldtext.ng-binding').text
# download & term
# open the modal
detail_element = el.find_element_by_xpath(".//a")
detail_element.click()
self.wait_for()
dialog_element = self.wait_until('ngdialog-content', by=By.CLASS_NAME)
dd_elements = dialog_element.find_elements_by_xpath('.//dd')
price_text = dd_elements[0].text
price = price_text[:price_text.find('.') + 2]
term = dd_elements[-2].text
term = term.rstrip('Months')
try:
term = int(term)
except ValueError:
term = 1
links = dialog_element.find_elements_by_xpath('.//ul/li')
links[1].click()
button_element = dialog_element.find_element_by_xpath('.//input')
button_element.click()
return {
'term': term,
'price': price,
'product_name': product_name,
}
def get_user(article: WebElement) -> User:
name, user_url = [None] * 2
user_class = '._5pb8.m_1djjlbjav._8o._8s.lfloat._ohe'
badge_class = '.igjjae4c.glosn74e.f2jdl7fy.cxfqmxzd'
try:
name = article.find_element_by_css_selector(
user_class).get_attribute('title')
user_url = Crawler.parse_user_url(
article.find_element_by_css_selector(user_class).get_attribute(
'href'))
badge = article.find_element_by_css_selector(
badge_class).get_attribute("innerHTML")
return User(user_url, name, badge)
except NoSuchElementException:
return User(user_url, name)
def parse_element(self, el: WebElement):
try:
term_element = el.find_element_by_css_selector('div.term h3 span')
except Exception:
term_element = el.find_element_by_css_selector('div.term h2 span')
term = term_element.text
match = re.search(r'(\d+)\s+Months', term)
if match:
term = match.groups()[0]
elif term_element.text == 'Month\nto Month':
term = '1'
else:
if not self.check_aband_popup_visible():
return self.parse_element(el)
else:
return self.analyze_element(self.current_plan)
# raise Exception("Term could not match. (%s)" % term)
try:
price_element = el.find_element_by_css_selector('div.rate h3 span')
except Exception:
price_element = el.find_element_by_css_selector(
'div.rate h2 strong')
price = price_element.text.split('¢')[0]
try:
plan_element = el.find_element_by_css_selector(
'h4.plan-title ~ h2')
except Exception:
plan_element = el.find_element_by_css_selector(
'div.plan-details-area h2 span')
product_name = plan_element.text
# time.sleep(3)
try:
collapse_details = el.find_element_by_css_selector(
'div.plan-details-description a.details')
except Exception:
collapse_details = el.find_element_by_css_selector(
'div.col-sm-12 a.details.confirm-ignore')
collapse_details.click()
collapse_id = collapse_details.get_attribute('data-target').split(
'#')[1]
efl_download_link_element = el.find_element_by_xpath(
'//div[@id="{}"]//a[text()="Electricity Facts Label"]'.format(
collapse_id))
pdf_url = efl_download_link_element.get_attribute("href")
self.client.get(pdf_url)
return {
'term': term,
'price': price,
'product_name': product_name,
}
def _parse_vid_render_elem(self, video_element: WebElement):
title_label = video_element.find_element_by_css_selector("a#video-title")
self.title = title_label.get_attribute("title")
self.url = title_label.get_attribute("href")
channel_label = video_element.find_element_by_xpath(".//ytd-channel-name//a")
self.channel_name = channel_label.get_attribute("innerText")
self.channel_url = channel_label.get_attribute("href")
def analyze_element(self, el: WebElement):
data_product_promo = el.get_attribute("data-product-promo")
if data_product_promo not in self.plans_promo_blacklist:
term_element = el.find_element_by_css_selector(
"div.card-body div ul")
term = term_element.text
match = re.search(r"(\d+)\s+Months", term)
if match:
term = match.groups()[0]
else:
raise Exception("Term could not match. (%s)" % term)
else:
term = "1"
price_element = el.find_element_by_css_selector(
"div.card-body div.modal-body h1" +
",div.card-body div.product2.cards_div2 h2")
price = re.search(r"(\d+(\.\d+)?)",
price_element.text.split("¢")[0]).groups()[0]
product_name = el.find_element_by_css_selector(
"div.card-body div.modal-body h2" +
",div.card-body div.product2.cards_div2 h4").text
efl_download_link_element = el.find_element_by_css_selector(
"div.modal-footer a")
efl_download_link_element.click()
self.client.switch_to_window(self.client.window_handles[1])
pdf_url = self.client.find_element_by_tag_name("iframe").get_attribute(
"src")
self.client.get(pdf_url)
self.client.close()
self.client.switch_to_window(self.client.window_handles[0])
self.wait_for()
return {
"term": term,
"price": price,
"product_name": product_name,
}
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector(
"div.ribbon-rate-section div.ribbon-rate-wrapper " +
"div.row:nth-child(2) div.ribbon-label-data")
term = term_element.text
match = re.search(r"(\d+)\s+Months", term)
if match:
term = match.groups()[0]
else:
if term == "Monthly":
term = "1"
else:
raise Exception("Term could not match. (%s)" % term)
price_element = el.find_element_by_css_selector(
"div.ribbon-rate-section div.ribbon-rate-wrapper " +
"div.ribbon-rate")
price = price_element.text.split("¢")[0]
plan_element = el.find_element_by_css_selector(
"div.ribbon-heading-section div.ribbon-heading " + "span")
product_name = plan_element.text
# el_class = el.get_attribute("class")
# el_class_name = re.search(
# r"(ribbon-gas-text|ribbon-electric-text)", el_class
# ).group()
# if el_class_name == "ribbon-gas-text":
# commodity = COMMODITY.natural_gas
# elif el_class_name == "ribbon-electric-text":
# commodity = COMMODITY.electricity
# else:
# commodity = COMMODITY.electricity
return {
"term": term,
"price": price,
"product_name": product_name,
"commodity": self.get_commodity(),
}
def from_element(cls, filter_elem: WebElement):
"""Initialize a Filter instance from a `WebElement`.
Parameters:
filter_elem: A `WebElement` instance
"""
# get the table property that the filter is applied to
property_elem = filter_elem.find_element_by_css_selector(
cls._property_select_sel
)
filtered_property = property_elem.get_attribute('title')
if filtered_property is None or filtered_property == "Selecione":
# the row is reserved for adding new a filter; skip it
return None
# get whether the operation is to be negated
negate_elem = filter_elem.find_element_by_css_selector(
cls._property_select_sel
)
if negate_elem.get_attribute('checked'):
negate = True
else:
negate = False
# get the filter operation
operation_elem = filter_elem.find_element_by_css_selector(
cls._operation_select_sel
)
operation = operation_elem.get_attribute('title')
# get filter value
value_elem = filter_elem.find_element_by_css_selector(
", ".join([cls._value_select_sel, cls._value_input_sel])
)
value = value_elem.get_attribute('title')
if not value:
value = value_elem.get_attribute('value')
# instantiate filter
return Filter(filtered_property, operation, value, negate)
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector(
'div.h-term-description')
term = term_element.text
match = re.search(r'(\d+)\s+MONTHS', term)
if match:
term = match.groups()[0]
else:
raise Exception("Term could not match. (%s)" % term)
price_element = el.find_element_by_css_selector('div.h-rate-value')
price = price_element.text
plan_element = el.find_element_by_css_selector('div.h-terms-title')
product_name = plan_element.text
return {
'term': term,
'price': price,
'product_name': product_name,
}
def analyze_element(self, el: WebElement):
term_element = el.find_element_by_css_selector(
'div.grid-card-header div.col.grid-month')
term = term_element.text
match = re.search(r'(\d+)\s+Month', term)
if match:
term = match.groups()[0]
else:
raise Exception("Term could not match. (%s)" % term)
price_element = el.find_element_by_css_selector(
'div.rate_container span.rate_amount')
price = price_element.text
plan_element = el.find_element_by_css_selector(
'div.card-body.plan-box div.grid-planName')
product_name = plan_element.text
result = {
'term': term,
'price': price,
'product_name': product_name,
'commodity': self.get_commodity(),
}
try:
efl_download_link_element = el.find_element_by_css_selector(
'div.card-body.plan-box div.gridPlanLinks span.efl_link a, ' +
'div.card-body.plan-box div.gridPlanLinks ' +
'span.rate_plan_link a, ' +
'div.card-body.plan-box div.gridPlanLinks ' +
'span.contract_summary_link a, ' +
'div.card-body.plan-box div.gridPlanLinks ' +
'span.disclosure_statement_link a')
efl_download_link_element.click()
except NoSuchElementException:
self.log("Failed to find EFL Link.", level=logging.WARNING)
result['skip_download'] = True
return result
def scrape_post(self, e: WebElement) -> dict:
date = e.find_element_by_css_selector("abbr._5ptz")
date = date.get_attribute("title")
locale.setlocale(locale.LC_ALL, 'nl_NL.UTF-8')
date = datetime.strptime(date, "%A %d %B %Y om %H:%M")
try:
headline = e.find_element_by_css_selector(
".mbs._6m6._2cnj._5s6c").text
except NoSuchElementException:
logging.debug(f"No headline by: {e}")
headline = "-"
try:
msg = e.find_element_by_css_selector(".userContent").text
except NoSuchElementException:
logging.debug(f"No message by: {e}")
msg = "-"
if msg.strip() == "":
logging.debug(f"No message by: {e}")
msg = "-"
url = e.find_element_by_css_selector(".fsm > ._5pcq")
url = fbposturl(url.get_attribute("href"))
article = dict(title=headline,
date=date,
text=msg,
url=url,
medium="Langstraatmedia")
print(f"artikel is {headline},{date}, {msg}")
try:
reaction = e.find_element_by_css_selector("._81hb").text
article["reactions"] = fbnumber(reaction)
except NoSuchElementException:
logging.debug(f"No reaction found: {e}")
try:
nremark = e.find_element_by_css_selector("._3hg-").text
article["nremarks"] = fbnumber(nremark)
except NoSuchElementException:
logging.debug(f"No remarks by: {e}")
try:
share = e.find_element_by_css_selector("._3rwx").text
article["shares"] = fbnumber(share)
except NoSuchElementException:
logging.debug(f"No shares by: {e}")
try:
link = e.find_element_by_css_selector("._52c6")
link = link.get_attribute("href")
article["article_url"] = fburl(link)
except NoSuchElementException:
logging.debug(f"No link by: {e}")
return article
def get_element_static(target_element: WebElement, verifier: SelecElement) -> WebElement:
try:
element = None
if verifier.element_type == NavEleType.IsId:
element = target_element.find_element_by_id(verifier.target)
elif verifier.element_type == NavEleType.IsClass:
element = target_element.find_element_by_class_name(verifier.target)
elif verifier.element_type == NavEleType.IsCssSelector:
element = target_element.find_element_by_css_selector(verifier.target)
elif verifier.element_type == NavEleType.IsName:
element = target_element.find_element_by_name(verifier.target)
else:
raise ValueError("Selector not Supported")
return element
except Exception as inst:
print(type(inst))
print(inst.args)
return None
def versionWalker(tr:WebElement):
global driver, prevTrail
try:
btn=tr.find_element_by_css_selector('button')
btn.click()
time.sleep(0.1)
waitUnti(lambda:all(_.is_displayed() for _ in tr/csss/'ul li a'))
versions = tr/csss/'ul li a'
numVersion = len(versions)
startIdx = getStartIdx()
for idx in range(startIdx, numVersions):
ulog('version idx=%s'%idx)
fwVer = versions[idx].text.strip()
versions[idx].click()
upsert(tr, fwVer)
if idx < numVersions-1:
btn.click()
time.sleep(0.1)
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
