def analyze_element(self, el: WebElement): # term term = el.find_element_by_css_selector("span.offerSubTitle").text try: term = int(term) except ValueError: term = 1 # price price = el.find_element_by_css_selector("div.rate span").text # product plan_element = el.find_element_by_xpath( './/form/div[@class="offerTitle"]/table/tbody/tr/td/h4') plan_full_text = plan_element.text price_text = el.find_element_by_xpath( './/form/div[@class="offerTitle"]/table/tbody/tr/td/h4/span').text plan_offset = len(plan_full_text) - len(price_text) product_name = plan_full_text[:plan_offset].rstrip() # download decline_button = self.client.find_element_by_id("mcx_decline") if decline_button and decline_button.is_displayed(): decline_button.click() link_element = el.find_element_by_xpath( './/form/div[@class="offerSpecs"]/ul/li/a') link_element.click() return { "term": term, "price": price, "product_name": product_name, }
def analyze_element(self, el: WebElement): # term term_full_text = el.find_element_by_xpath( ".//ul[@class='price-bullet']/li").text try: term = re.search(r'\b\d+\b', term_full_text).group() except Exception: term = 1 # price price = el.find_element_by_xpath( ".//div[@class='price-container']/h4/span").text # product product_name = el.find_element_by_xpath('.//h3').text # download # close the survey dialog box if it is open link = el.find_elements_by_xpath( './/div[@class="gridPlanLinks"]/a')[-1] link.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): # Popping up learn more details detail_link = el.find_element_by_xpath( './/a[@class="utility-details-trigger"]/button') self.client.execute_script("arguments[0].click();", detail_link) price_element = el.find_element_by_xpath( './/div[@class="utility-choice-price"]').text price = re.search(r"(\d+(\.\d+)?)", price_element).group() product_name = el.find_element_by_xpath( './/div[@class="utility-choice-title"]').text match = re.search(r"\b\d+\b", product_name) if match: term = match.group() else: term = 1 self.wait_for(2) efl_element = el.find_element_by_xpath( './/ul[@class="utility-documents"]/li[6]/a') efl_url = efl_element.get_attribute('href') self.client.get(efl_url) return { "term": term, "price": price, "product_name": product_name, }
def analyze_element(self, el: WebElement): if "featured-plan" in el.get_attribute("class").split(" "): term, price, product_name = self.__featured_analyze_element(el) else: term_element = el.find_element_by_xpath(".//ul/li[1]") match = re.search(r"\b\d+\b", term_element.text) if match: term = match.group() else: term = 1 price_element = el.find_element_by_css_selector("div.product2 h2") price = re.search(r"(\d+(\.\d+)?)", price_element.text).group() plan_element = el.find_element_by_css_selector("div.product2 h4") product_name = plan_element.text efl_download_link_element = el.find_element_by_xpath(".//a[1]") efl_url = efl_download_link_element.get_attribute("href") self.__download_pdf(efl_url) return { "term": term, "price": price, "product_name": product_name, }
def parse_user_info(wait: WebDriverWait, row: WebElement): """Parse user info.""" title = row.find_element_by_xpath(".//*[contains(@class, ' title ')]") sub_title = row.find_element_by_xpath( ".//*[contains(@class, 'sub-title')]") description = row.find_element_by_xpath(".//*[contains(@class, 'desc')]") return f'{title.text:20} | {sub_title.text:20} | {description.text:20}'
def parse_description(experience_row: WebElement) -> str: try: description_show_more = experience_row.find_element_by_xpath( selectors['profile_position_description_show_more']) scroll_to_element(description_show_more, 'profile_position_description_show_more') description_show_more.click() except NoSuchElementException as e: logging.debug( f"Can't find profile_position_description_show_more (it's normal if description is short) {e}" ) except Exception as e: logging.debug(f"Unknown Exception {e}") try: description_element = experience_row.find_element_by_xpath( selectors['profile_position_description']) description_text = description_element.text if description_text[-10:] in ['.\nСвернуть', '.\nsee less']: description_text = description_text[:-10] return description_text except NoSuchElementException as e: logging.debug( f"Can't find profile_position_description (it's normal) {e}") return '' except Exception as e: logging.debug(f"Unknown Exception {e}") return ''
def _process_conversation(self, conversation: WebElement): print('\nProcessing conversation...') uid = None try: # Assuming the user is not saved as a contact, 'contact_id' will return the number contact_id = self.wait.until(lambda _: conversation.find_element_by_xpath(".//span[@class='_1wjpf']")) \ .get_attribute('title') contact_id = selenium_methods.clean_contact_number(contact_id) if not contact_id: print('Invalid contact ID') return False # Try get uid from local database, otherwise perform network call, # if this fails then user needs to be created first uid = self.get_uid_from_number_db(contact_id) if not uid: uid = self._get_uid_from_number(contact_id) if not uid: print('User needs to be created') last_message_content = self.wait.until(lambda _: conversation.find_element_by_xpath(".//span[@class='_2_LEW']")) \ .get_attribute('title') last_message = WhatsAppMessage(uid=uid, timestamp=None, sender_name=None, sender_number=contact_id, content=last_message_content) del last_message.timestamp del last_message.sender_name if self.messages_in_sync(last_message): print('Messages in sync') return True print('Processing conversation %s: ID - %s' % (conversation.id, contact_id)) except NoSuchElementException: print('No such element') return False messages_panel = self.load_conversation_messages_panel(conversation) self.previous_conversation_content = messages_panel if uid: # messages = self._extract_and_save_messages(messages_panel) # print('Saving messages to database') # for message in messages: # message.uid = uid # self.save_message(message) # print('Message: %s' % message.__dict__) return True else: username = self._process_new_user(contact_id, messages_panel) if username: print('New user %s ~ %s' % (username, contact_id)) new_uid = self._create_new_user(contact_number=contact_id, username=username) if new_uid: print('Successfully created user: %s' % new_uid) else: print('Failed to create user: %s' % contact_id) else: print('No valid launch sequence for %s' % contact_id) return False
def analyze_element(self, el: WebElement): term_element = el.find_element_by_xpath( './/ul[@class="price-bullet"]/li[2]') match = re.search(r"\b\d+\b", term_element.text) if match: term = match.group() else: term = 1 price_element = el.find_element_by_xpath( './/div[@class="price-container"]//span') price = re.search(r"(\d+(\.\d+)?)", price_element.text).group() plan_element = el.find_element_by_xpath( './/div[@class="featuredPlanDescription"]//span') product_name = plan_element.text efl_element = el.find_element_by_xpath( './/div[@class="gridPlanLinks"]/a[3]') self.client.execute_script("arguments[0].click();", efl_element) self.wait_for(2) return { "term": term, "price": price, "product_name": product_name, }
def analyze_element(self, el: WebElement): price_element = el.find_element_by_xpath( './/div[@class="average-rate"]//span').text price = re.search(r"(\d+(\.\d+)?)", price_element).group() product_name = el.find_element_by_xpath( './/div[@class="panel-body"]//h4').text term = re.search(r"\b\d+\b", product_name).group() # Popping up learn more details detail_link = el.find_element_by_xpath("//article//input") self.client.execute_script("arguments[0].click();", detail_link) modal = self.wait_until('.//div[@id="dnn_ContentPane"]', by=By.XPATH) detail_btn = modal.find_element_by_xpath(".//li[2]/a") self.client.execute_script("arguments[0].click();", detail_btn) efl_element = modal.find_element_by_xpath( '//div[@id="dnn_ContentPane"]//fieldset[4]//span//div[2]/a') self.client.execute_script("arguments[0].click();", efl_element) close_btn = modal.find_element_by_xpath( '//div[@class="modal-footer"]//button') self.client.execute_script("arguments[0].click();", close_btn) return { "term": term, "price": price, "product_name": product_name, }
def analyze_element(self, el: WebElement): term_element = el.find_element_by_xpath( './/div[@class="term-description"]') match = re.search(r'\b\d+\b', term_element.text) if match: term = match.group() else: term = 1 self.wait_for() plan_element = el.find_element_by_class_name('description-title ') product_name = plan_element.text price_element = el.find_element_by_xpath( './/div[@class="rate-price"]/div[2]') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() # Popping up learn more details detail_link = el.find_element_by_xpath('.//a[@class="detailsButton"]') self.client.execute_script("arguments[0].click();", detail_link) efl_element = el.find_element_by_xpath( './/div[@class="planInfoHolder"]//a[1]') efl_url = efl_element.get_attribute('href') self.__download_pdf(efl_url, '//embed') return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): term_element = el.find_element_by_xpath( './/div[@class="row"]/div[1]/span') match = re.search(r'\b\d+\b', term_element.text) if match: term = match.group() else: term = 1 self.wait_for() plan_element = el.find_element_by_xpath( './/div[@id="product-left"]/h2') product_name = plan_element.text # Popping up learn more details detail_link = el.find_element_by_xpath( './/div[@class="row"]/div[3]/a') self.client.execute_script("arguments[0].click();", detail_link) price_element = el.find_element_by_xpath( './/div[@id="product-right"]//h1') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() efl_element = el.find_element_by_xpath( './/div[@id="product-details-right"]/ul/li[1]/a') self.client.execute_script("arguments[0].click();", efl_element) return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): plan_element = el.find_element_by_xpath('.//div[@class="price"]/div') product_name = plan_element.text match = re.search(r'\b\d+\b', product_name) if match: term = match.group() else: term = 1 price_element = el.find_element_by_xpath( './/div[@class="price"]/div[2]') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() # Popping up learn more details detail_btm = el.find_element_by_xpath( './/div[@class="price"]//span/button') self.client.execute_script("arguments[0].click();", detail_btm) modal = self.wait_until('//div[@class="modal-content"]', by=By.XPATH) efl_element = modal.find_element_by_xpath( '//ul[@class="free-night-ul"]/li[1]/a[1]') self.client.execute_script("arguments[0].click();", efl_element) self.wait_for(2) close_btn = modal.find_element_by_xpath( '//div[@class="modal-footer"]/button') self.client.execute_script("arguments[0].click();", close_btn) return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): # product product_name = el.find_element_by_xpath( './/div/span[@class="plan-name"]').text # price price_element = el.find_element_by_xpath( './/div/div[@class="plan-wrapper"]/span') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() # term term = re.search(r'(\d+(\.\d+)?)', product_name).group() # download link = el.find_elements_by_css_selector("a.plan-link") self.client.execute_script("arguments[0].click();", link[1]) self.wait_for() dialog_element = self.wait_until("pdf-modal-dialog", by=By.CLASS_NAME) pdf_iframe_element = dialog_element.find_element_by_id("pdfframe") self.client.switch_to.frame(pdf_iframe_element) download_button = self.wait_until("download") download_button.click() self.client.switch_to.default_content() close_button_element = dialog_element.find_element_by_xpath( './/button') close_button_element.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): term_element = el.find_element_by_xpath('.//h6//small[2]') match = re.search(r'\b\d+\b', term_element.text) if match: term = match.group() else: term = 1 self.wait_for(3) price_element = el.find_element_by_xpath( './/div[@class="price-container"]//span') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() plan_element = el.find_element_by_xpath('.//div[@class="front"]//h3') product_name = plan_element.text # Popping up learn more details detail_link = el.find_element_by_xpath( './/div[@class="front"]//div[@class="txt-center"]/a') self.client.execute_script("arguments[0].click();", detail_link) modal = self.wait_until( '//div[@id="divModelPlanDetails"]/div[@class="modal-dialog"]', by=By.XPATH) efl_element = modal.find_element_by_xpath('.//a[1]') self.client.execute_script("arguments[0].click();", efl_element) close_btn = modal.find_elements_by_id('lblShowPlnDtlsPopupClose') close_btn[0].click() return { 'term': term, 'price': price, 'product_name': product_name, }
def __get_efl_link(self, el: WebElement, i: int): try: el.find_element_by_xpath( './/div[@class="planContractDoc"]/div/div[{}]/'.format(i) + 'a[contains(text(), "Electricity")]') return True except Exception: return False
def parse_result(result: WebElement, xpath: str, attr: str, args: str = None) -> str: out = None try: if args is None: out = str(getattr(result.find_element_by_xpath(xpath), attr)) else: out = str(getattr(result.find_element_by_xpath(xpath), attr)(args)) except NoSuchElementException: pass return out
def analyze_element(self, el: WebElement): main_client = self.client.current_window_handle # term term = el.find_element_by_xpath( './/ul[contains(@class, "plancard-attributes")]/li' ).text term = 0 # price price_element = el.find_element_by_xpath( './/span[@class="average-price"]' ) price_full_text = price_element.text currency = el.find_element_by_xpath( './/span[@class="average-price"]/span' ).text price_offset = len(price_full_text) - len(currency) price = price_full_text[:price_offset].rstrip() # product plan_element = el.find_element_by_css_selector('div.plan-price') product_name = plan_element.text # download link_element = el.find_element_by_css_selector( 'div.mt-auto a.edit-btn' ) link_element.click() target_modal = link_element.get_attribute("data-target")[1:] modal = self.wait_until(target_modal) links = modal.find_elements_by_css_selector("ul.bullet-ul")[-1] link = links.find_elements_by_xpath(".//li/a")[-1] link.click() self.wait_for() if main_client != self.client.window_handles[-1]: self.client.switch_to_window(main_client) button_element = modal.find_element_by_xpath( './/div/div/div[@class="modal-header"]/button' ) button_element.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): detail_btn = el.find_element_by_css_selector('input.colorButton') self.client.execute_script("arguments[0].click();", detail_btn) self.wait_for() term_element = el.find_element_by_css_selector('div.term') term = re.search(r'\b\d+\b', term_element.text).group() price_element = el.find_element_by_css_selector('div.productPrice ') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() try: plan_element = el.find_element_by_css_selector('div.productName ') except Exception: plan_element = el.find_element_by_css_selector( 'div.greenProductName ') product_name = plan_element.text efl_download_link_element = el.find_element_by_xpath( './/table[2]//tbody//tr[5]//td//a[1]') efl_download_link_element.click() main_client = self.client.current_window_handle self.client.switch_to_window(self.client.window_handles[-1]) self.client.execute_script('window.print();') self.client.close() self.client.switch_to.window(main_client) self.wait_for() return { 'term': term, 'price': price, 'product_name': product_name, }
def _getMonkeyPaths(self, element: WebElement, data: dict, jsonList: list): elemPathsTemp = [element.find_element_by_xpath(self.xPathMonkeysNextH3)] elemPaths = [] for _ in range(2): elemPathsTemp.append( elemPathsTemp[len(elemPathsTemp) - 1].find_element_by_xpath( self.xPathMonkeysNextH3Rest ) ) for elemPath in elemPathsTemp: if elemPath.tag_name == "div": elemPaths.append(elemPath.find_element_by_xpath(self.xPathMonkeysInDiv)) else: # if elemPath.tag_name == "table": elemPaths.append( elemPath.find_element_by_xpath(self.xPathMonkeysInTable) ) for idx, elemPath in enumerate(elemPaths): elems = elemPath.find_elements_by_xpath("./tr/td[1]/p/b/a") elemsPrice = elemPath.find_elements_by_xpath("./tr/td[4]") for count, elem, elemPrice in zip(range(1, 6), elems, elemsPrice): jsonList.append( { "Type": "Monkey", "SubType": data["SubType"], "Name": data["Name"], "UpgradedName": elem.get_attribute("title"), "Path": str(10 ** (2 - idx) * count).zfill(3), "Price (Hard)": elemPrice.text, "Link": elem.get_attribute("href"), } )
def _parse_compact_render_elem(self, video_element: WebElement): self.title = video_element.find_element_by_css_selector("span#video-title").get_attribute("innerText") self.url = video_element.find_element_by_tag_name("a").get_attribute("href") self.channel_name = video_element.find_element_by_xpath( ".//ytd-channel-name//yt-formatted-string" ).get_attribute("innerText") self.channel_url = None # Channel is not clickable from this element
def get_modules_for_folder(self, node: WebElement): node.click() title = node.find_element_by_xpath( './/span[@style="font-size: small;"]').text.strip() self.wait.until( EC.presence_of_element_located( (By.XPATH, "//h4[contains(text(), '" + title + "')]"))) self.wait.until( EC.invisibility_of_element_located( (By.XPATH, 'span[@class="fa fa-refresh fa-spin loading-symbol"]'))) heading = self.driver.find_element_by_xpath( '//div[@class="panel-heading"]/strong[contains(text(), "Module")]') table = heading.find_element_by_xpath('../..//table/tbody') data = {} for row in table.find_elements_by_xpath('tr'): fields = row.find_elements_by_xpath('td') if len(fields) > 0: name = fields[0].text.strip() nr = fields[1].text.strip() version = fields[2].text.strip() data[nr] = (name, version) return data
def analyze_element(self, el: WebElement): term_element = el.find_element_by_css_selector( "div.individual-option-main ul.individual-options-short-list " + "span.li-text") term = re.search(r"\b\d+\b", term_element.text).group() price_element = el.find_element_by_css_selector( "div.individual-option-main div.individual-cost ") price = re.search(r"(\d+(\.\d+)?)", price_element.text).group() plan_element = el.find_element_by_css_selector( "div.individual-option-main span.name-above-plan") product_name = plan_element.text efl_download_link_element = el.find_element_by_xpath( './/div[@class="individual-plan-doc"]/a[1]') main_client = self.client.current_window_handle self.client.get(efl_download_link_element.get_attribute("href")) self.client.switch_to.window(main_client) return { "term": term, "price": price, "product_name": product_name, }
def analyze_element(self, el: WebElement): # Popping up learn more details detail_btm = el.find_element_by_xpath('.//a') self.client.execute_script("arguments[0].click();", detail_btm) modal = self.wait_until('//div[@class="modal-content"]', by=By.XPATH) plan_element = modal.find_element_by_xpath( './/h2[@class="custom-header"]') product_name = plan_element.text term_element = modal.find_element_by_xpath( './/div[@class="container-fluid"]/div[4]/div/p') match = re.search(r'\b\d+\b', term_element.text) if match: term = match.group() else: term = 1 price_element = modal.find_element_by_xpath( './/div[@class="container-fluid"]/div[3]/div/p') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() efl_element = modal.find_element_by_xpath( './/div[@class="container-fluid"]/div[4]/div[3]/a[3]') self.client.execute_script("arguments[0].click();", efl_element) self.wait_for(3) cls_btn = modal.find_element_by_xpath( './/button[@class="close text-right"]') cls_btn.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): # product product_name = el.find_element_by_css_selector( 'h6.tcenter.boldtext.ng-binding').text # download & term # open the modal detail_element = el.find_element_by_xpath(".//a") detail_element.click() self.wait_for() dialog_element = self.wait_until('ngdialog-content', by=By.CLASS_NAME) dd_elements = dialog_element.find_elements_by_xpath('.//dd') price_text = dd_elements[0].text price = price_text[:price_text.find('.') + 2] term = dd_elements[-2].text term = term.rstrip('Months') try: term = int(term) except ValueError: term = 1 links = dialog_element.find_elements_by_xpath('.//ul/li') links[1].click() button_element = dialog_element.find_element_by_xpath('.//input') button_element.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def analyze_element(self, el: WebElement): plan_id = el.find_element_by_id('SelectedPlanId').get_attribute( 'value') price_element = el.find_element_by_id('averageRate-{}'.format(plan_id)) price = price_element.text plan_element = el.find_element_by_xpath( './div[@class="row"]/div[1]/div[2]/strong') product_name = plan_element.text collapse_btn = self.wait_until( '//button[@aria-controls="collapse-{}"]'.format(plan_id), By.XPATH) self.client.execute_script("arguments[0].click();", collapse_btn) term_element = self.wait_until( '//div[@id="collapse-{}"]/div[1]/div[1]/div[3]'.format(plan_id), By.XPATH) term = term_element.text match = re.search(r'(\d+)\s+Months', term) if match: term = match.groups()[0] else: raise Exception("Term could not match. (%s)" % term) efl_download_link_element = self.wait_until( '//div[@id="collapse-{}"]/div[2]/div[1]/a'.format(plan_id), By.XPATH) efl_download_link_element.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def find_element(self, xpath: str, src_path="", element: WebElement = None) -> WebElement: target_element: WebElement = None # src_path should only be specified if no element if src_path and not element: self._http_client.get(src_path) # Find element in page or in specified WebElement try: if element: target_element = element.find_element_by_xpath(xpath) else: target_element = self._http_client.find_element_by_xpath(xpath) except (InvalidSelectorException, NoSuchAttributeException) as e: print( "Invalid XPath: Specified XPath was invalid. Please check XPath and try again." ) except NoSuchElementException: print( "Element Not Found: Specified XPath failed to locate element on webpage." ) return target_element
def analyze_element(self, el: WebElement): main_client = self.client.current_window_handle # product product_name = el.find_element_by_xpath('.//div/h2/strong').text # term term = re.search(r'(\d+(\.\d+)?)', product_name).group() # price price_element = el.find_element_by_css_selector( 'h3.Heading_black_k70K9.Heading_center_338zS.Heading_h3_2FwVg') price = re.search(r'(\d+(\.\d+)?)', price_element.text).group() # download # open the modal button_element = el.find_element_by_xpath( ".//button[contains(@class, 'Link_primary_3KooB')]") self.client.execute_script("arguments[0].click();", button_element) dialog_element = self.wait_until('Modal_modalBackdrop_2tUY1', by=By.CLASS_NAME) link = dialog_element.find_element_by_xpath('.//a') link.click() self.wait_for(5) self.client.switch_to_window(self.client.window_handles[-1]) download_button_element = self.wait_until("downloadbutton", by=By.CLASS_NAME, timeout=30) self.client.execute_script("arguments[0].click();", download_button_element) self.wait_for(10) self.client.close() self.client.switch_to.window(main_client) self.wait_for(10) close_button = dialog_element.find_element_by_xpath('.//button') close_button.click() return { 'term': term, 'price': price, 'product_name': product_name, }
def extract_parent_data(self, veranstaltung: WebElement): parent = veranstaltung.find_element_by_xpath('./../..') meta = parent.find_element_by_xpath('following-sibling::div') dates = parent.find_element_by_xpath('following-sibling::table') active = True if 'Inaktiv' in parent.text: active = False return (veranstaltung, active, meta, dates)
def is_livestream(video_element: WebElement) -> bool: """Checks if the given video_element is a livestream instead of a regular video""" try: badge = video_element.find_element_by_xpath( "div[1]/div/ytd-badge-supported-renderer/div[1]/span") return badge.get_attribute("innerText") == "LIVE NOW" except: return False
def _parse_vid_render_elem(self, video_element: WebElement): title_label = video_element.find_element_by_css_selector("a#video-title") self.title = title_label.get_attribute("title") self.url = title_label.get_attribute("href") channel_label = video_element.find_element_by_xpath(".//ytd-channel-name//a") self.channel_name = channel_label.get_attribute("innerText") self.channel_url = channel_label.get_attribute("href")