def getDriver(self, executable_path=None, run_headless=False, load_images=True, proxy_string=None, **kwargs): firefox_profile = self.__getFireFoxOptions( self._validate_proxy_string(proxy_string)) if run_headless: display = Display(visible=0, size=(1024, 768)) display.start() else: display = None if not load_images: firefox_profile.add_extension(os.path.dirname( os.path.realpath(__file__)) + '/disableimageload/quickjava-2.1.2-fx.xpi') firefox_profile.set_preference( 'thatoneguydotnet.QuickJava.curVersion', '2.1.2.1') firefox_profile.set_preference( 'thatoneguydotnet.QuickJava.startupStatus.Images', 2) firefox_profile.set_preference( 'thatoneguydotnet.QuickJava.startupStatus.AnimatedImage', 2) if executable_path: driver = webdriver.Firefox( firefox_profile, executable_path=executable_path) else: driver = webdriver.Firefox(firefox_profile) driver.display = display return driver
def test_deadlock(): d = Display(visible=VISIBLE, size=(600, 400)) d.start() p = EasyProcess([python, '-c', 'import Image;Image.new("RGB",(99, 99)).show()']) p.start() p.sleep(1) # hangs with pipes p.stop() d.stop()
def test_deadlock(): d = Display(visible=VISIBLE, size=(600, 400)) d.start() p = EasyProcess( [python, '-c', 'import Image;Image.new("RGB",(99, 99)).show()']) p.start() p.sleep(1) # hangs with pipes p.stop() d.stop()
def test_deadlock(): # skip these tests for Windows/Mac if not sys.platform.startswith("linux"): return d = Display(visible=VISIBLE, size=(600, 400)) d.start() p = EasyProcess( [python, "-c", 'import Image;Image.new("RGB",(99, 99)).show()']) p.start() p.sleep(1) # hangs with pipes p.stop() d.stop()
def run(self, result_uri, url, retries=1, tests=None, **params): logger.info('Starting getting data ({0} runs) with selenium for url: {1}'.format(retries, url)) self.position = params.get('task_position', self.MIDDLE) self.on_start(params['task_uri']) results = {} if tests is None: tests = self.get_test_list(url) display = None try: if settings.SELENIUM_USE_VIRTUAL_DISPLAY: display = Display() display.start() def run_test(): return self.run_test(test) for test in tests: test_results = zip(range(retries), collect_results(run_test, retries)) results[test['test_name']] = [ { 'run': i + 1, 'result': test_result } for i, test_result in test_results ] finally: if display is not None: display.stop() logger.info('Sending results from Selenium for url: {0}'.format(url)) # posting raw results ApiClient.post(params['raw_result_uri'], { 'result': result_uri, 'generator': 'selenium', 'context': {'url': url, 'origin': 'selenium', }, 'data': results, }) return result_uri
def test_xauth(): """ Test that a Xauthority file is created. """ if not xauth.is_installed(): raise SkipTest("This test needs xauth installed") old_xauth = os.getenv("XAUTHORITY") display = Display(visible=0, use_xauth=True) display.start() new_xauth = os.getenv("XAUTHORITY") ok_(new_xauth is not None) ok_(os.path.isfile(new_xauth)) filename = os.path.basename(new_xauth) ok_(filename.startswith("PyVirtualDisplay.")) ok_(filename.endswith("Xauthority")) display.stop() eq_(old_xauth, os.getenv("XAUTHORITY")) ok_(not os.path.isfile(new_xauth))
def test_xauth(): ''' Test that a Xauthority file is created. ''' if not xauth.is_installed(): raise SkipTest('This test needs xauth installed') old_xauth = os.getenv('XAUTHORITY') display = Display(visible=0, use_xauth=True) display.start() new_xauth = os.getenv('XAUTHORITY') ok_(new_xauth is not None) ok_(os.path.isfile(new_xauth)) filename = os.path.basename(new_xauth) ok_(filename.startswith('PyVirtualDisplay.')) ok_(filename.endswith('Xauthority')) display.stop() eq_(old_xauth, os.getenv('XAUTHORITY')) ok_(not os.path.isfile(new_xauth))
def run(self, url, retries=1, tests=None): logger.info('Starting getting data ({0} runs) with selenium for url: {1}'.format(retries, url)) results = {} if tests is None: tests = self.get_test_list(url) display = None try: if settings.SELENIUM_USE_VIRTUAL_DISPLAY: display = Display() display.start() def run_test(): return self.run_test(test) for test in tests: test_results = zip(range(retries),collect_results(run_test,retries)) results[test['test_name']] = [ { 'run': i + 1, 'result': test_result } for i, test_result in test_results ] finally: if display is not None: display.stop() return { 'generator': 'selenium', 'context': { 'origin': 'selenium' }, 'data': results }
class ProtonmailClient: """ This class contains the core functions that are used by both protonmail-cli and interactive session. example usage for reading inbox mails >>> client = core.ProtonmailClient() >>> client.login(settings.username, settings.password) >>> inbox = client.get_mails("inbox") """ web_driver = None virtual_display = None def __init__(self): utilities.log("Initiating ProtonMail client") try: if not settings.show_browser: self.virtual_display = Display(visible=0, size=(1366, 768)) self.virtual_display.start() self.web_driver = webdriver.Firefox() atexit.register(self.destroy) except Exception as e: utilities.log( "Unable to initiate Protonmail Client. Reason: " + str(e)) def login(self, username, password): """Login to ProtonMail panel Raises Exception on failure :param username: your ProtonMail username - email :param password: your ProtonMail password """ try: utilities.log("Logging in...") self.web_driver.get(variables.url) utilities.wait_for_elem( self.web_driver, variables.element_login['username_id'], "id") utilities.log("Login page loaded...", "DEBUG") username_input = self.web_driver.find_element_by_id( variables.element_login['username_id']) password_input = self.web_driver.find_element_by_id( variables.element_login['password_id']) username_input.send_keys(username) password_input.send_keys(password) password_input.send_keys(Keys.RETURN) utilities.log("Login credentials sent [" + username + "]", "DEBUG") time.sleep(1) two_factor = False if "ng-hide" not in self.web_driver.find_element_by_id( variables.element_twofactor['detection_id']).get_attribute('class'): two_factor = True if two_factor: utilities.log("Two-factor authentication enabled", "DEBUG") two_factor_input = self.web_driver.find_element_by_id( variables.element_twofactor['code_id']) two_factor_input.send_keys( input("Enter two-factor authentication code: ")) two_factor_input.send_keys(Keys.RETURN) if utilities.wait_for_elem(self.web_driver, variables.element_login['after_login_detection_class'], "class"): utilities.log("Logged in successfully") else: raise Exception() except Exception as ignored_err: utilities.log("Login failed!") raise Exception("Unable to login") def parse_mails(self): """ Reads and returns a list of Mails inside the current web driver's page :return: a list of Mail objects """ if not utilities.wait_for_elem(self.web_driver, variables.element_list_inbox['email_list_wrapper_id'], "id"): # for some reason the wrapper wasn't loaded return None utilities.wait_for_elem( self.web_driver, variables.element_list_inbox["individual_email_soupclass"][1:], "class", max_retries=3) soup = BeautifulSoup(self.web_driver.page_source, "html.parser") mails_soup = soup.select( variables.element_list_inbox['individual_email_soupclass']) mails = [] subject_class = variables.element_list_inbox['individual_email_subject_soupclass'] time_class = variables.element_list_inbox['individual_email_time_soupclass'] sender_name_class = variables.element_list_inbox['individual_email_sender_name_soupclass'] for m in mails_soup: # @TODO mails without subject or title, etc.. are ignored try: new_mail = mail.Mail( subject=m.select(subject_class)[0].get("title"), time_received=m.select(time_class)[0].string, mail_alias=m.select(sender_name_class)[0].get("title"), mail=m.select(sender_name_class)[0].string, ) mails.append(new_mail) except Exception as e: utilities.log("Skip mail... " + str(e)) continue if settings.mails_read_num > 0: mails = mails[:settings.mails_read_num] if settings.date_order == "asc": return list(reversed(mails)) return mails def get_mails(self, page): """ Get a list of mails that are into the given page :param page: One of the pages listed in variables.py > page_urls :return: a list of Mail objects """ url = variables.page_urls.get(page) if not url: raise ValueError("Page doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) return self.parse_mails() def has_new_mail(self): """Generates a unique hash from the mail inbox If the hash is different from the previous call of this function then a new mail was received. :returns: True if a new mail was arrived else False @TODO in case we delete an email then the hash will be changed and we'll get a new mail notification. """ mails = self.get_mails("inbox") old_hash = utilities.get_hash() mails_str = "" for m in mails: mails_str += str(m) mails_str += str(m) new_hash = hashlib.sha256(mails_str.encode()).hexdigest() utilities.write_hash(new_hash) if old_hash and new_hash != old_hash: return True return False def change_name(self, new_name): """ Change name of your account. The name is the name that appears on recipients inbox. <Your Name> [email protected] :param new_name: str (the updated user's name) """ url = variables.page_urls.get('account') if not url: raise ValueError("Page doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) # type the new user name utilities.wait_for_elem( self.web_driver, variables.element_account['display_name'] ) el = self.web_driver.find_element_by_id( variables.element_account['display_name']) el.clear() # clear old name el.send_keys(new_name) # write new name # click save button utilities.wait_for_elem( self.web_driver, variables.element_account['save_btn'], "class" ) el = self.web_driver.find_element_by_class_name( variables.element_account['save_btn']) el.click() time.sleep(settings.load_wait) #click back button el = self.web_driver.find_element_by_class_name( variables.element_account['back_btn']) el.click() def send_mail(self, to, subject, message): """Sends email. :param to: [str] (list of mail addresses - recipients) :param message: str (subject of the mail) :param subject: str (message of the mail) """ # click new mail button el = self.web_driver.find_element_by_class_name( variables.element_send_mail['open_composer_class']) el.click() # wait for mail dialog to appear utilities.wait_for_elem( self.web_driver, variables.element_send_mail['composer_detection_class'], "class") # type receivers list el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['to_field_css']) for address in to: el.send_keys(address + ";") time.sleep(0.2) # type subject el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['subject_field_css']) el.send_keys(subject) # type message self.web_driver.switch_to.frame( self.web_driver.find_element_by_class_name(variables.element_send_mail['switch_to_message_field_class'])) el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['message_field_css']) el.send_keys(message) self.web_driver.switch_to.default_content() # click send el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['send_button_css']) el.click() time.sleep(settings.load_wait) def destroy(self): """ atexit handler; automatically executed upon normal interpreter termination. Should be called after any work done with client """ if self.web_driver is not None: self.web_driver.close() self.web_driver.quit() self.web_driver = None if self.virtual_display is not None: self.virtual_display.stop() self.virtual_display = None
class ProtonmailClient: """ This class contains the core functions that are used by both protonmail-cli and interactive session. example usage for reading inbox mails >>> client = core.ProtonmailClient() >>> client.login(settings.username, settings.password) >>> inbox = client.get_mails("inbox") """ web_driver = None virtual_display = None def __init__(self): utilities.log("Initiating ProtonMail client") try: if not settings.show_browser: self.virtual_display = Display(visible=0, size=(1366, 768)) self.virtual_display.start() self.web_driver = webdriver.Firefox() atexit.register(self.destroy) except Exception as e: utilities.log( "Unable to initiate Protonmail Client. Reason: " + str(e)) def login(self, username, password): """Login to ProtonMail panel Raises Exception on failure :param username: your ProtonMail username - email :param password: your ProtonMail password """ if self.web_driver is None: utilities.log("Client is not initialized") return try: utilities.log("Logging in...") self.web_driver.get(variables.url) utilities.wait_for_elem( self.web_driver, variables.element_login['username_id'], "id") utilities.log("Login page loaded...", "DEBUG") username_input = self.web_driver.find_element_by_id( variables.element_login['username_id']) password_input = self.web_driver.find_element_by_id( variables.element_login['password_id']) username_input.send_keys(username) password_input.send_keys(password) password_input.send_keys(Keys.RETURN) utilities.log("Login credentials sent [" + username + "]", "DEBUG") time.sleep(1) two_factor = False try: self.web_driver.find_element_by_id(variables.element_twofactor['detection_id']).get_attribute('class') two_factor = True except: pass if two_factor: utilities.log("Two-factor authentication enabled", "DEBUG") two_factor_input = self.web_driver.find_element_by_id( variables.element_twofactor['code_id']) two_factor_input.send_keys( input("Enter two-factor authentication code: ")) two_factor_input.send_keys(Keys.RETURN) if utilities.wait_for_elem(self.web_driver, variables.element_login['after_login_detection_class'], "class"): utilities.log("Logged in successfully") else: raise Exception() except Exception as ignored_err: utilities.log("Login failed!") raise Exception("Unable to login") def parse_mails(self): """ Reads and returns a list of Mails inside the current web driver's page :return: a list of Mail objects """ if not utilities.wait_for_elem(self.web_driver, variables.element_list_inbox['email_list_wrapper_id'], "id"): # for some reason the wrapper wasn't loaded return None utilities.wait_for_elem( self.web_driver, variables.element_list_inbox["individual_email_soupclass"][1:], "class", max_retries=3) soup = BeautifulSoup(self.web_driver.page_source, "html.parser") mails_soup = soup.select( variables.element_list_inbox['individual_email_soupclass']) mails = [] subject_class = variables.element_list_inbox['individual_email_subject_soupclass'] time_class = variables.element_list_inbox['individual_email_time_soupclass'] sender_name_class = variables.element_list_inbox['individual_email_sender_name_soupclass'] for m in mails_soup: # @TODO mails without subject or title, etc.. are ignored try: new_mail = mail.Mail( subject=m.select(subject_class)[0].string, time_received=m.select(time_class)[0].string, mail_alias=m.select(sender_name_class)[0].get("title"), mail=m.select(sender_name_class)[0].string, ) mails.append(new_mail) except Exception as e: utilities.log("Skip mail... " + str(e)) continue if settings.mails_read_num > 0: mails = mails[:settings.mails_read_num] if settings.date_order == "asc": return list(reversed(mails)) return mails def get_mails(self, page): """ Get a list of mails that are into the given page, folder, or label :param page: One of the pages listed in variables.py > page_urls, or a user defined folder or label :return: a list of Mail objects """ url = variables.page_urls.get(page) or self.get_folders_and_labels().get(page.lower()) if not url: raise ValueError("Page doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) return self.parse_mails() def get_mails_in_folder(self, folder): """ Get a list of mails that are in the given folder :param page: A user defined folder, populated at runtime by get_folders() :return: a list of Mail objects """ # this is valid because ProtonMail folders are case-insensitive folder = folder.lower() url = self.get_folders().get(folder) if not url: raise ValueError("Folder doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) return self.parse_mails() def get_mails_in_label(self, label): """ Get a list of mails that are in the given label :param label: A user defined label, populated at runtime by get_labels() :return: a list of Mail objects """ # this is valid because ProtonMail labels are case-insensitive label = label.lower() url = self.get_labels().get(label) if not url: raise ValueError("Label doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) return self.parse_mails() def get_folders_and_labels(self): """ Get a list of the user's mail folders and labels :return: a dict of mail folder and label urls, similar to page_urls """ all_items = dict() soup = BeautifulSoup(self.web_driver.page_source, "html.parser") folders_and_labels = soup.select( variables.element_folders_labels['list_element_title_selector'] ) for folder_or_label in folders_and_labels: # this is valid because ProtonMail folders and labels are case-insensitive name = folder_or_label.text.lower() path = folder_or_label.parent['href'] all_items[name] = variables.base_url + path return all_items def get_folders(self): """ Get a list of mail folders (not labels!) :return: a dict of mail folder urls, similar to page_urls """ all_folders = dict() soup = BeautifulSoup(self.web_driver.page_source, "html.parser") folders = soup.select( variables.element_folders_labels['folder_element_selector'] ) for folder in folders: # this is valid because ProtonMail folders are case-insensitive name = folder.find_next_sibling("span", class_="menuLabel-title").text.lower() path = folder.parent['href'] all_folders[name] = variables.base_url + path return all_folders def get_labels(self): """ Get a list of mail labels (not folders!) :return: a dict of mail label urls, similar to page_urls """ all_labels = dict() soup = BeautifulSoup(self.web_driver.page_source, "html.parser") labels = soup.select( variables.element_folders_labels['label_element_selector'] ) for label in labels: # this is valid because ProtonMail labels are case-insensitive name = label.find_next_sibling("span", class_="menuLabel-title").text.lower() path = label.parent['href'] all_labels[name] = variables.base_url + path return all_labels def has_new_mail(self): """Generates a unique hash from the mail inbox If the hash is different from the previous call of this function then a new mail was received. :returns: True if a new mail was arrived else False @TODO in case we delete an email then the hash will be changed and we'll get a new mail notification. """ mails = self.get_mails("inbox") old_hash = utilities.get_hash() mails_str = "" for m in mails: mails_str += str(m) mails_str += str(m) new_hash = hashlib.sha256(mails_str.encode()).hexdigest() utilities.write_hash(new_hash) if old_hash and new_hash != old_hash: return True return False def change_name(self, new_name): """ Change name of your account. The name is the name that appears on recipients inbox. <Your Name> [email protected] :param new_name: str (the updated user's name) """ url = variables.page_urls.get('account') if not url: raise ValueError("Page doesn't exist") if self.web_driver.current_url != url: utilities.log("Opening %s" % url) self.web_driver.get(url) # type the new user name utilities.wait_for_elem( self.web_driver, variables.element_account['display_name'] ) el = self.web_driver.find_element_by_id( variables.element_account['display_name']) el.clear() # clear old name el.send_keys(new_name) # write new name # click save button utilities.wait_for_elem( self.web_driver, variables.element_account['save_btn'], "class" ) el = self.web_driver.find_element_by_class_name( variables.element_account['save_btn']) el.click() time.sleep(settings.load_wait) # click back button el = self.web_driver.find_element_by_class_name( variables.element_account['back_btn']) el.click() def send_mail(self, to, subject, message, as_html=False, attachments=[]): """Sends email. :param to: [str] (list of mail addresses - recipients) :param message: str (subject of the mail) :param subject: str (message of the mail) :param as_html: bool (whether or not to render :message as html) :param attachments: [str] (list of files to upload as attachments) """ def upload_attachments(attachments): # wait for files to be uploaded initial_send_text = self.web_driver.find_element_by_css_selector( variables.element_send_mail['send_button_css'] ).text el = self.web_driver.find_element_by_css_selector( 'input[type=file][multiple=multiple]' ) el.send_keys("\n".join(attachments)) time.sleep(settings.load_wait) try: # this dialog appears on images for inline placing self.web_driver.find_element_by_css_selector( variables.element_send_mail['as_attachment_btn'] ).click() except: pass # wait for files to be uploaded while True: time.sleep(settings.load_wait) curr_send_text = self.web_driver.find_element_by_css_selector( variables.element_send_mail['send_button_css'] ).text if curr_send_text == initial_send_text: break # click new mail button el = self.web_driver.find_element_by_class_name( variables.element_send_mail['open_composer_class']) el.click() # wait for mail dialog to appear utilities.wait_for_elem( self.web_driver, variables.element_send_mail['composer_detection_class'], "class") # type receivers list el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['to_field_css']) for address in to: el.send_keys(address + ";") time.sleep(0.2) # type subject el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['subject_field_css']) el.send_keys(subject) self.web_driver.switch_to.frame( self.web_driver.find_element_by_class_name( variables.element_send_mail['switch_to_message_field_class']) ) el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['message_field_css']) if not as_html: # type message if we don't want to render as html el.send_keys(message) else: # render as html by executing a js oneliner to alter the DOM self.web_driver.execute_script(""" document.querySelector('.angular-squire-iframe body>div').innerHTML="%s" """ % (message)) el.send_keys() self.web_driver.switch_to.default_content() if attachments: upload_attachments(attachments) # click send el = self.web_driver.find_element_by_css_selector( variables.element_send_mail['send_button_css']) el.click() time.sleep(settings.load_wait) def destroy(self): """ atexit handler; automatically executed upon normal interpreter termination. Should be called after any work done with client """ if self.web_driver is not None: self.web_driver.quit() self.web_driver = None if self.virtual_display is not None: self.virtual_display.stop() self.virtual_display = None
class Nexis(ClosableObject): """ The actual Nexis database wrapper. It fetches data from the Uni Hamburg nexis database via selenium. """ COUNT_REGEX = re.compile(r"([0-9]+) Dokument.* und ([0-9]+) Duplikat.*") def __init__(self, user, password, hide_window=True, printer=PrimitiveLogPrinter(), ignore_big_queries=True): """ Creates a new database proxy. :param user: User name. :param password: Password. :param hide_window: Wether or not to show the browser window (showing might be useful for debugging.) :param printer: A PrimitiveLogPrinter which will be used for indicating actions and logging. :param ignore_big_queries: If set to True, queries resulting in more than 3000 results will be ignored. """ ClosableObject.__init__(self) # If an exception occurs later those members have to exist for _close() self.browser = None self.display = None self.tempdir = None # For (primitive) logging self.printer = printer self.ignore_big_queries = ignore_big_queries self.user = user self.password = password if hide_window: with self.printer.do_safe_action( 'Starting virtual display', "Browser window cannot be hidden. You might need to " "install Xvfb. Continuing with visible browser window."): self.display = Display(visible=0) self.display.start() self.tempdir = mkdtemp() self.browser = webdriver.Firefox(DirectDownloadProfile(self.tempdir)) # Retrieve token for this session with self.printer.do_safe_action('Authenticating at Nexis', reraise=True): self.authenticate() sleep(5) self.browser.find_element_by_xpath( '//a[@title="Profisuche"]').click() sleep(10 + random()*5) self.home_url = self.browser.current_url def authenticate(self): self.browser.get( "http://rzblx10.uni-regensburg.de/dbinfo/warpto.php?bib_id=" "sub_hh&color=2&titel_id=1670&url=http%3A%2F%2Femedien.sub." "uni-hamburg.de%2Fhan%2Flexis") sleep(5 + random()*5) self.browser.find_element_by_name('User').send_keys(self.user) self.browser.find_element_by_name('Password').send_keys(self.password) self.browser.find_element_by_name("submitimg").click() # accept the terms, if required try: sleep(10 + random()*5) self.browser.find_element_by_css_selector( "a[href^='/auth/submitterms.do']").click() except NoSuchElementException: pass # Press ok on relogin prompt if needed try: sleep(5 + random()*5) self.browser.find_element_by_xpath( '//td/input[@title="OK"]').click() self.printer.debug("Relogin needed for {}. Executed successfully." .format(self.user)) except NoSuchElementException: pass def query(self, search_term: str, from_date: date=None, to_date: date=None, languages: str='us', company_canonical_name=' '): """ Performs a query to the NexisLexis database. :param search_term: The search term. :param from_date: Lower date limit. :param to_date: Upper date limit. :param languages: One of "all", "german", "english" or "us". """ with self.printer.do_safe_action( "Querying for '{}'".format(search_term), reraise=True): if self.browser.current_url != self.home_url: self.browser.get(self.home_url) from_date = from_date or date(2005, 1, 1) to_date = to_date or date.today() # Fill and submit query self._fill_query_data(search_term, from_date, to_date, languages) sleep(5 + random()*5) self.browser.find_element_by_css_selector( "img[title='Suche']").click() sleep(5 + random()*5) if self._no_results: return [] if self.too_many_results: if self.ignore_big_queries: return {'error': 'Too many results (>3000)', 'error_code': 1} self.printer.debug("Number of results for query '{}' exceeds " "3000. The query will be split.".format( search_term)) # Split up logarithmically middle = from_date + (to_date-from_date)/2 one_after_middle = middle + timedelta(days=1) results = self.query(search_term, from_date, middle,company_canonical_name=company_canonical_name) results += self.query(search_term, one_after_middle, to_date,company_canonical_name=company_canonical_name) return results return self._get_results(company_canonical_name, search_term) def element_exists_by_xpath(self, path): try: element = self.browser.find_element_by_xpath(path) return element except NoSuchElementException: return None @property def _no_results(self): return (self.element_exists_by_xpath("//h1[@class='zeroMsgHeader']") is not None) def _get_results(self, company_canonical_name, search_term): def press_forward(): """ Presses the forward button so lexisnexis updates the document count. """ try: self.browser.find_element_by_xpath( '//div/ol/li[@class="last"]/a').click() except: pass def get_document_count(): """ Retrieves the document count. Though it might raise over time when LexisNexis decides to analyze more documents. """ sleep(10) count_text = self.browser.find_element_by_xpath( "//div/dl/dd[last()]").text try: matches = self.COUNT_REGEX.search(count_text).groups() except AttributeError: # The one result case return 1 documents = int(matches[0]) duplicates = int(matches[1]) return documents-duplicates results = [] # Give nexis some time for duplication analysis sleep(10) press_forward() document_count = get_document_count() downloaded_documents = 0 while downloaded_documents < document_count: batch_start = downloaded_documents + 1 batch_end = min(downloaded_documents + 200, document_count) results += self._download_results(batch_start, batch_end, company_canonical_name, search_term) sleep(2) downloaded_documents = batch_end if len(results) != downloaded_documents: print("Got", len(results), "results, expecting", downloaded_documents, "instead.") # Updates document count, lexis will do that on the server while # were already downloading stuff press_forward() document_count = get_document_count() return results def _download_results(self, batch_start, batch_end, company_canonical_name, search_term, retry=3): def download_in_progress(): # if found a file without .part extension then we're done for file_ in listdir(self.tempdir): if ".part" in file_: return True else: return False # No files at all? Download hasn't started yet. return True # Open Download Popover, it'll have three "tabs" with options self.browser.find_element_by_id("delivery_DnldRender").click() sleep(5 + random()*5) # First tab: range and full text # Range field won't be there for only one document, selection irrelevant if not (batch_start == 1 == batch_end): range_text = "{}-{}".format(batch_start, batch_end) self.browser.find_element_by_id("sel").click() self.browser.find_element_by_id( "rangetextbox").send_keys(range_text) # full text with indexing Select(self.browser.find_element_by_name( "delView")).select_by_index(3) # Switch to tab 2: deactivate cover page self.browser.find_element_by_xpath("//div/ul/li/a[" "@href='#tabs-2']").click() if self.browser.find_element_by_id("cvpg").is_selected(): self.browser.find_element_by_id("cvpg").click() # Switch to tab 3: download as txt, ignore the rest (not applicable # to txt) self.browser.find_element_by_xpath("//div/ul/li/a[" "@href='#tabs-3']").click() Select(self.browser.find_element_by_id("delFmt")).select_by_index(3) # Actually click Download self.browser.find_element_by_class_name("deliverBtn").click() while True: close_btn = self.element_exists_by_xpath("//*[@id='closeBtn']") if close_btn is not None: # Download is started, click the OK button to close popover close_btn.click() break partial_content = self.element_exists_by_xpath( "//h1[text()=\"Partial Content\"]") arbitrary_error = self.element_exists_by_xpath( "//span[text()=\"Fehler bei der Anfrage\"]") if (partial_content or arbitrary_error) is not None: sleep(5) for elem in self.browser.find_elements_by_xpath( "//a/span[text()='close']"): try: elem.click() break except: pass else: raise RuntimeError("Closing the window impossible") sleep(2 + random()*5) if retry == 0: raise ServerError("Unable to fetch data.") # Retry return self._download_results(batch_start, batch_end, company_canonical_name, search_term, retry=retry-1) # Do not busy loop the whole time sleep(1) sleep(1) while download_in_progress(): sleep(1) sleep(1) with open(path.join(self.tempdir, listdir(self.tempdir)[0]), 'r') as file: content = file.read() articles = list(Article.from_nexis_text(content, company_canonical_name, search_term)) for file in listdir(self.tempdir): unlink(path.join(self.tempdir, file)) return articles @property def too_many_results(self): try: container = self.browser.find_element_by_css_selector( '#popupContainer') container.find_element_by_css_selector("span.l0") return True except NoSuchElementException: return False def _fill_query_data(self, search_term, from_date, to_date, languages): """ Fills the search form with the given data. """ #wait = WebDriverWait(self.browser, 10) #term_input = wait.until(self.browser.find_element_by_name("searchTermsTextArea")) #confirm.click() sleep(10) term_input = self.browser.find_element_by_name("searchTermsTextArea") term_input.clear() term_input.send_keys('"' + search_term + '"') language_value_dict = { 'all': 'All English and German Language News', 'english': 'All English Language News', 'german': 'German Language News', 'us': 'US Publications'} source_term = language_value_dict.get(languages, 'US Publications') self.browser.find_element_by_xpath("//div[@rel='more_sources']").click() # Activate JS in the text field self.browser.find_element_by_id('selected_source').send_keys('') sleep(5 + random() * 5) self.browser.find_element_by_id('selected_source').send_keys( source_term) # Page needs some time to show autocompletion box sleep(10 + random() * 5) self.browser.find_element_by_id('selected_source').send_keys( Keys.ARROW_DOWN) self.browser.find_element_by_id('selected_source').send_keys( Keys.ENTER) date_selector = self.browser.find_element_by_name("dateSelector") ''' # select custom date Select(date_selector).select_by_index(11) date_selector.send_keys(Keys.ENTER) from_date_field = self.browser.find_element_by_name("fromDate") from_date_field.clear() from_date_field.send_keys(from_date.strftime('%d/%m/%Y')) to_date_field = self.browser.find_element_by_name("toDate") to_date_field.clear() to_date_field.send_keys(to_date.strftime('%d/%m/%Y')) ''' # Filter group duplicates if not self.browser.find_element_by_name("gDuplicates").is_selected(): self.browser.find_element_by_name("gDuplicates").click() # Set group duplicate filter to "similar" self.browser.find_element_by_id('duplicatesModal').click() try: self.browser.find_element_by_xpath( "//input[@value='search.common.threshold.broadrange']").click() except: pass self.browser.find_element_by_id('saveBooksBtn').click() if not self.browser.find_element_by_name( "excludeObituariesChecked").is_selected(): self.browser.find_element_by_name( "excludeObituariesChecked").click() def _close(self): with self.printer.do_safe_action('Cleaning up'): if self.browser is not None: self.browser.quit() if self.display is not None: self.display.stop() if self.tempdir: rmtree(self.tempdir, ignore_errors=True)
def setup_func(): 'set up test fixtures' global process, screen screen = Display(visible=0) screen.start() process = EasyProcess('gnumeric').start().sleep(3)
time.sleep(settings.check_mail_period) elif op == "help": print_usage() else: print("Operation not valid") print_usage() return print_usage() if __name__ == "__main__": display = None if not settings.show_browser: display = Display(visible=0, size=(1366, 768)) display.start() driver = webdriver.Firefox() try: run() except Exception as e: log(str(e), "ERROR") driver.close() if display is not None: display.stop()
class seleniumClient(): def __init__(self, agg, appId, scenarioId, manage, scenario, baseImgDir): self.scenario = scenario self.manage = manage self.appId = appId self.scenarioId = scenarioId self.aggClient = agg self.baseImgDir = baseImgDir self.logger = getLogger('testmanager') # Inicializacia virtualneho displeja def initDisplay(self): if (not 'DISPLAY' in environ) or environ['DISPLAY'] == '': self.display = Display(backend='xvnc', rfbport=5904, visible=0, size=(4096, 2160)) self.display.start() else: self.display = None # Zrusenie vritaulneho displeja def stopDisplay(self): if not self.display is None: self.display.stop() # Inicializacia prehliadaca def initDriver(self): chromeOptions = webdriver.ChromeOptions() chromeOptions.add_argument('--user-data-dir=' + self.generateNewProfile()) self.driver = webdriver.Chrome(options=chromeOptions) self.driver.set_window_position(0, 0) self.driver.get(self.scenario[0]['url']) # Nastavenie priecinka pre obrazky z prehliadaca def initScreenShotDir(self): imgDir = self.baseImgDir + '/' + \ self.appId + '/' + \ self.scenarioId + '/' + \ str(self.testId) try: makedirs(imgDir) except OSError as exc: if exc.errno == errno.EEXIST and path.isdir(imgDir): pass else: raise self.screenshotDir = imgDir # Zrusenie testovacieho prostredia def endTest(self): try: self.driver.close() except: pass self.stopDisplay() rmtree(self.generateNewProfile(), True) # Nastavenie prveho regresneho testu def setRegressTest(self): response = self.aggClient.sendCommand('setRegressTest', { 'appId': self.appId, 'scenarioId': self.scenarioId, 'testId': 1 }) if not response['status']: self.logger.critical('Cannot set reggressTestId') raise RuntimeError(testState.FAILED) # Aktualizovanie velkosti okna prehliadaca podla udalosti def updateWindowSize(self, screenX, screenY): self.driver.set_window_position(0, 0) self.driver.set_window_size(screenX, screenY) # Nastavenie Id testu def setTestId(self): currentTestId = 1 if self.manage is None or not 'lastTestId' in self.manage: self.setRegressTest() self.manage = {'regressTestId': 1, 'lastTestId': 1} else: currentTestId = self.manage['lastTestId'] + 1 response = self.aggClient.sendCommand( 'setTestId', { 'appId': self.appId, 'scenarioId': self.scenarioId, 'testId': currentTestId }) if not response['status']: self.logger.critical('Cannot set testId') raise RuntimeError(testState.FAILED) self.testId = currentTestId # Zaznamenanie regresneho testu pre aktualny test def setRegressTestForTest(self): self.aggClient.sendCommand( 'setRegressTestForTest', { 'appId': self.appId, 'scenarioId': self.scenarioId, 'testId': self.testId, 'regressTestId': self.manage['regressTestId'] }) # Nastavenie noveho cache priecinka pre prehliadac def generateNewProfile(self): return '{}/.browserData/{:.10}-{}'.format(getcwd(), self.scenarioId, self.testId) def run(self): self.initDisplay() self.scenario.sort(key=lambda x: x['timestamp']) # Nastavenie testovacieho prostredia try: self.setTestId() self.initScreenShotDir() self.setRegressTestForTest() self.initDriver() processedEvent = 0 # Simulovanie udalosti processedEvent = self.processScenario() except Exception as e: self.logger.critical(str(e)) self.endTest() # Kontrola vykonania vsetkych udalosti if processedEvent != len(self.scenario): self.logger.critical('Cannot perform all events in scenario') raise RuntimeError(testState.FAILED) return (self.testId, self.manage['regressTestId']) # Ziskanie elementu na zaklade CSS selektora def getElementSelector(self, selector): return self.driver.find_element_by_css_selector(selector) # Vyber akcie na zaklade typu udalosti def selectAction(self, event): return getattr(self, 'action_%s' % event['type'])(event) def action_click(self, event): action = ActionChains(self.driver) elem = self.getElementSelector(event['locator']) action.move_to_element(elem) action.click(elem) return action def action_focusout(self, event): action = ActionChains(self.driver) elem = self.getElementSelector(event['locator']) action.move_to_element(elem) action.send_keys(event['content']) return action def action_keypress(self, event): action = ActionChains(self.driver) return action def action_mouseover(self, event): action = ActionChains(self.driver) elem = self.getElementSelector(event['locator']) action.move_to_element(elem) return action # Ulozenie obrazku a vysledku o simulovaniu udalosti def saveScreenShot(self, event, performTime): image = self.screenshotDir + '/' + str(event['timestamp']) + '.png' self.driver.get_screenshot_as_file(image) self.aggClient.sendCommand( 'createTest', { 'appId': self.appId, 'scenarioId': self.scenarioId, 'image': image, 'performTime': performTime, 'testId': self.testId }) def getPageLoaded(self): return self.driver.execute_script( 'return document.readyState;') == 'complete' # Replikacia udalosti def processScenario(self): pageTime = 0 sleepTime = -1 performTime = 0 startTime = 0 processedEvent = 0 for event in self.scenario: # Simulovanie uzivatelskeho casu straveneho na stranke if (sleepTime > 0): sleep(sleepTime) sleepTime = 0 try: # Vykonanie udalosti action = self.selectAction(event) if not action is None: self.updateWindowSize(event['screenX'], event['screenY']) startTime = time() * 1000 action.perform() except Exception as e: self.logger.warning(str(e)) self.saveScreenShot(event, performTime) processedEvent = processedEvent + 1 continue # Vypocitanie casu pre simolovanie uzivatelskeho chovania endTime = time() * 1000 performTime = endTime - startTime pageTime = pageTime + performTime sleepTime = round((event['pageTime'] - pageTime) / 100000, 2) self.logger.critical(sleepTime) # Ulozenie vysledkov self.saveScreenShot(event, performTime) processedEvent = processedEvent + 1 self.logger.warning(processedEvent) return processedEvent