def test_nest(self):
vd = Display().start()
nd = Display(visible=1).start().stop()
self.assertEquals(nd.return_code, 0)
vd.stop()
def test_nest(self):
vd = Display().start()
ok_(vd.is_alive())
nd = Display(visible=1).start().stop()
# self.assertEquals(nd.return_code, 0)
vd.stop()
ok_(not vd.is_alive())
def test_disp(self):
vd = Display().start()
d = Display(visible=1).start().sleep(2).stop()
self.assertEquals(d.return_code, 0)
d = Display(visible=0).start().stop()
self.assertEquals(d.return_code, 0)
vd.stop()
def test_deadlock():
d = Display(visible=VISIBLE, size=(600, 400))
d.start()
p = EasyProcess([python, '-c', 'import Image;Image.new("RGB",(99, 99)).show()'])
p.start()
p.sleep(1)
# hangs with pipes
p.stop()
d.stop()
def test_disp(self):
vd = Display().start()
ok_(vd.is_alive())
d = Display(visible=1).start().sleep(2).stop()
# self.assertEquals(d.return_code, 0)
d = Display(visible=0).start().stop()
# self.assertEquals(d.return_code, 0)
vd.stop()
ok_(not vd.is_alive())
def run(self, result_uri, url, retries=1, tests=None, **params):
logger.info('Starting getting data ({0} runs) with selenium for url: {1}'.format(retries, url))
self.position = params.get('task_position', self.MIDDLE)
self.on_start(params['task_uri'])
results = {}
if tests is None:
tests = self.get_test_list(url)
display = None
try:
if settings.SELENIUM_USE_VIRTUAL_DISPLAY:
display = Display()
display.start()
def run_test():
return self.run_test(test)
for test in tests:
test_results = zip(range(retries), collect_results(run_test, retries))
results[test['test_name']] = [
{
'run': i + 1,
'result': test_result
}
for i, test_result in test_results
]
finally:
if display is not None:
display.stop()
logger.info('Sending results from Selenium for url: {0}'.format(url))
# posting raw results
ApiClient.post(params['raw_result_uri'], {
'result': result_uri,
'generator': 'selenium',
'context': {'url': url, 'origin': 'selenium', },
'data': results,
})
return result_uri
def test_xauth():
'''
Test that a Xauthority file is created.
'''
if not xauth.is_installed():
raise SkipTest('This test needs xauth installed')
old_xauth = os.getenv('XAUTHORITY')
display = Display(visible=0, use_xauth=True)
display.start()
new_xauth = os.getenv('XAUTHORITY')
ok_(new_xauth is not None)
ok_(os.path.isfile(new_xauth))
filename = os.path.basename(new_xauth)
ok_(filename.startswith('PyVirtualDisplay.'))
ok_(filename.endswith('Xauthority'))
display.stop()
eq_(old_xauth, os.getenv('XAUTHORITY'))
ok_(not os.path.isfile(new_xauth))
def run(self, url, retries=1, tests=None):
logger.info('Starting getting data ({0} runs) with selenium for url: {1}'.format(retries, url))
results = {}
if tests is None:
tests = self.get_test_list(url)
display = None
try:
if settings.SELENIUM_USE_VIRTUAL_DISPLAY:
display = Display()
display.start()
def run_test():
return self.run_test(test)
for test in tests:
test_results = zip(range(retries),collect_results(run_test,retries))
results[test['test_name']] = [
{
'run': i + 1,
'result': test_result
}
for i, test_result in test_results
]
finally:
if display is not None:
display.stop()
return {
'generator': 'selenium',
'context': {
'origin': 'selenium'
},
'data': results
}
class ProtonmailClient:
"""
This class contains the core functions
that are used by both protonmail-cli and interactive session.
example usage for reading inbox mails
>>> client = core.ProtonmailClient()
>>> client.login(settings.username, settings.password)
>>> inbox = client.get_mails("inbox")
"""
web_driver = None
virtual_display = None
def __init__(self):
utilities.log("Initiating ProtonMail client")
try:
if not settings.show_browser:
self.virtual_display = Display(visible=0, size=(1366, 768))
self.virtual_display.start()
self.web_driver = webdriver.Firefox()
atexit.register(self.destroy)
except Exception as e:
utilities.log("Unable to initiate Protonmail Client. Reason: " +
str(e))
def login(self, username, password):
"""Login to ProtonMail panel
Raises Exception on failure
:param username: your ProtonMail username - email
:param password: your ProtonMail password
"""
try:
utilities.log("Logging in...")
self.web_driver.get(variables.url)
utilities.wait_for_elem(self.web_driver,
variables.element_login['username_id'],
"id")
utilities.log("Login page loaded...", "DEBUG")
username_input = self.web_driver.find_element_by_id(
variables.element_login['username_id'])
password_input = self.web_driver.find_element_by_id(
variables.element_login['password_id'])
username_input.send_keys(username)
password_input.send_keys(password)
password_input.send_keys(Keys.RETURN)
utilities.log("Login credentials sent [" + username + "]", "DEBUG")
time.sleep(1)
two_factor = False
try:
self.web_driver.find_element_by_id(
variables.element_twofactor['detection_id']).get_attribute(
'class')
two_factor = True
except:
pass
if two_factor:
utilities.log("Two-factor authentication enabled", "DEBUG")
two_factor_input = self.web_driver.find_element_by_id(
variables.element_twofactor['code_id'])
two_factor_input.send_keys(
input("Enter two-factor authentication code: "))
two_factor_input.send_keys(Keys.RETURN)
if utilities.wait_for_elem(
self.web_driver,
variables.element_login['after_login_detection_class'],
"class"):
utilities.log("Logged in successfully")
else:
raise Exception()
except Exception as ignored_err:
utilities.log("Login failed!")
raise Exception("Unable to login")
def parse_mails(self):
"""
Reads and returns a list of Mails inside the current web driver's page
:return: a list of Mail objects
"""
if not utilities.wait_for_elem(
self.web_driver,
variables.element_list_inbox['email_list_wrapper_id'], "id"):
# for some reason the wrapper wasn't loaded
return None
utilities.wait_for_elem(
self.web_driver,
variables.element_list_inbox["individual_email_soupclass"][1:],
"class",
max_retries=3)
soup = BeautifulSoup(self.web_driver.page_source, "html.parser")
mails_soup = soup.select(
variables.element_list_inbox['individual_email_soupclass'])
mails = []
subject_class = variables.element_list_inbox[
'individual_email_subject_soupclass']
time_class = variables.element_list_inbox[
'individual_email_time_soupclass']
sender_name_class = variables.element_list_inbox[
'individual_email_sender_name_soupclass']
for m in mails_soup:
# @TODO mails without subject or title, etc.. are ignored
try:
new_mail = mail.Mail(
subject=m.select(subject_class)[0].get("title"),
time_received=m.select(time_class)[0].string,
mail_alias=m.select(sender_name_class)[0].get("title"),
mail=m.select(sender_name_class)[0].string,
)
mails.append(new_mail)
except Exception as e:
utilities.log("Skip mail... " + str(e))
continue
if settings.mails_read_num > 0:
mails = mails[:settings.mails_read_num]
if settings.date_order == "asc":
return list(reversed(mails))
return mails
def get_mails(self, page):
"""
Get a list of mails that are into the given page, folder, or label
:param page: One of the pages listed in variables.py > page_urls, or a user defined folder or label
:return: a list of Mail objects
"""
url = variables.page_urls.get(
page) or self.get_folders_and_labels().get(page.lower())
if not url:
raise ValueError("Page doesn't exist")
if self.web_driver.current_url != url:
utilities.log("Opening %s" % url)
self.web_driver.get(url)
return self.parse_mails()
def get_mails_in_folder(self, folder):
"""
Get a list of mails that are in the given folder
:param page: A user defined folder, populated at runtime by get_folders()
:return: a list of Mail objects
"""
# this is valid because ProtonMail folders are case-insensitive
folder = folder.lower()
url = self.get_folders().get(folder)
if not url:
raise ValueError("Folder doesn't exist")
if self.web_driver.current_url != url:
utilities.log("Opening %s" % url)
self.web_driver.get(url)
return self.parse_mails()
def get_mails_in_label(self, label):
"""
Get a list of mails that are in the given label
:param label: A user defined label, populated at runtime by get_labels()
:return: a list of Mail objects
"""
# this is valid because ProtonMail labels are case-insensitive
label = label.lower()
url = self.get_labels().get(label)
if not url:
raise ValueError("Label doesn't exist")
if self.web_driver.current_url != url:
utilities.log("Opening %s" % url)
self.web_driver.get(url)
return self.parse_mails()
def get_folders_and_labels(self):
"""
Get a list of the user's mail folders and labels
:return: a dict of mail folder and label urls, similar to page_urls
"""
all_items = dict()
soup = BeautifulSoup(self.web_driver.page_source, "html.parser")
folders_and_labels = soup.select(
variables.element_folders_labels['list_element_title_selector'])
for folder_or_label in folders_and_labels:
# this is valid because ProtonMail folders and labels are case-insensitive
name = folder_or_label.text.lower()
path = folder_or_label.parent['href']
all_items[name] = variables.base_url + path
return all_items
def get_folders(self):
"""
Get a list of mail folders (not labels!)
:return: a dict of mail folder urls, similar to page_urls
"""
all_folders = dict()
soup = BeautifulSoup(self.web_driver.page_source, "html.parser")
folders = soup.select(
variables.element_folders_labels['folder_element_selector'])
for folder in folders:
# this is valid because ProtonMail folders are case-insensitive
name = folder.find_next_sibling(
"span", class_="menuLabel-title").text.lower()
path = folder.parent['href']
all_folders[name] = variables.base_url + path
return all_folders
def get_labels(self):
"""
Get a list of mail labels (not folders!)
:return: a dict of mail label urls, similar to page_urls
"""
all_labels = dict()
soup = BeautifulSoup(self.web_driver.page_source, "html.parser")
labels = soup.select(
variables.element_folders_labels['label_element_selector'])
for label in labels:
# this is valid because ProtonMail labels are case-insensitive
name = label.find_next_sibling(
"span", class_="menuLabel-title").text.lower()
path = label.parent['href']
all_labels[name] = variables.base_url + path
return all_labels
def has_new_mail(self):
"""Generates a unique hash from the mail inbox
If the hash is different from the previous call of this function
then a new mail was received.
:returns: True if a new mail was arrived else False
@TODO in case we delete an email then the hash will be
changed and we'll get a new mail notification.
"""
mails = self.get_mails("inbox")
old_hash = utilities.get_hash()
mails_str = ""
for m in mails:
mails_str += str(m)
mails_str += str(m)
new_hash = hashlib.sha256(mails_str.encode()).hexdigest()
utilities.write_hash(new_hash)
if old_hash and new_hash != old_hash:
return True
return False
def change_name(self, new_name):
""" Change name of your account.
The name is the name that appears on recipients inbox.
<Your Name> [email protected]
:param new_name: str (the updated user's name)
"""
url = variables.page_urls.get('account')
if not url:
raise ValueError("Page doesn't exist")
if self.web_driver.current_url != url:
utilities.log("Opening %s" % url)
self.web_driver.get(url)
# type the new user name
utilities.wait_for_elem(self.web_driver,
variables.element_account['display_name'])
el = self.web_driver.find_element_by_id(
variables.element_account['display_name'])
el.clear() # clear old name
el.send_keys(new_name) # write new name
# click save button
utilities.wait_for_elem(self.web_driver,
variables.element_account['save_btn'], "class")
el = self.web_driver.find_element_by_class_name(
variables.element_account['save_btn'])
el.click()
time.sleep(settings.load_wait)
# click back button
el = self.web_driver.find_element_by_class_name(
variables.element_account['back_btn'])
el.click()
def send_mail(self, to, subject, message, as_html=False, attachments=[]):
"""Sends email.
:param to: [str] (list of mail addresses - recipients)
:param message: str (subject of the mail)
:param subject: str (message of the mail)
:param as_html: bool (whether or not to render :message as html)
:param attachments: [str] (list of files to upload as attachments)
"""
def upload_attachments(attachments):
# wait for files to be uploaded
initial_send_text = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['send_button_css']).text
el = self.web_driver.find_element_by_css_selector(
'input[type=file][multiple=multiple]')
el.send_keys("\n".join(attachments))
time.sleep(settings.load_wait)
try:
# this dialog appears on images for inline placing
self.web_driver.find_element_by_css_selector(
variables.element_send_mail['as_attachment_btn']).click()
except:
pass
# wait for files to be uploaded
while True:
time.sleep(settings.load_wait)
curr_send_text = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['send_button_css']).text
if curr_send_text == initial_send_text:
break
# click new mail button
el = self.web_driver.find_element_by_class_name(
variables.element_send_mail['open_composer_class'])
el.click()
# wait for mail dialog to appear
utilities.wait_for_elem(
self.web_driver,
variables.element_send_mail['composer_detection_class'], "class")
# type receivers list
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['to_field_css'])
for address in to:
el.send_keys(address + ";")
time.sleep(0.2)
# type subject
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['subject_field_css'])
el.send_keys(subject)
self.web_driver.switch_to.frame(
self.web_driver.find_element_by_class_name(
variables.element_send_mail['switch_to_message_field_class']))
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['message_field_css'])
if not as_html:
# type message if we don't want to render as html
el.send_keys(message)
else:
# render as html by executing a js oneliner to alter the DOM
self.web_driver.execute_script("""
document.querySelector('.angular-squire-iframe body>div').innerHTML="%s"
""" % (message.replace('"', '\\"')))
self.web_driver.switch_to.default_content()
if attachments:
upload_attachments(attachments)
# click send
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['send_button_css'])
el.click()
time.sleep(settings.load_wait)
def destroy(self):
"""
atexit handler; automatically executed upon normal interpreter termination.
Should be called after any work done with client
"""
if self.web_driver is not None:
self.web_driver.close()
self.web_driver.quit()
self.web_driver = None
if self.virtual_display is not None:
self.virtual_display.stop()
self.virtual_display = None
time.sleep(settings.check_mail_period)
elif op == "help":
print_usage()
else:
print("Operation not valid")
print_usage()
return
print_usage()
if __name__ == "__main__":
display = None
if not settings.show_browser:
display = Display(visible=0, size=(1366, 768))
display.start()
driver = webdriver.Firefox()
try:
run()
except Exception as e:
log(str(e), "ERROR")
driver.close()
if display is not None:
display.stop()
class seleniumClient():
def __init__(self, agg, appId, scenarioId, manage, scenario, baseImgDir):
self.scenario = scenario
self.manage = manage
self.appId = appId
self.scenarioId = scenarioId
self.aggClient = agg
self.baseImgDir = baseImgDir
self.logger = getLogger('testmanager')
# Inicializacia virtualneho displeja
def initDisplay(self):
if (not 'DISPLAY' in environ) or environ['DISPLAY'] == '':
self.display = Display(backend='xvnc',
rfbport=5904,
visible=0,
size=(4096, 2160))
self.display.start()
else:
self.display = None
# Zrusenie vritaulneho displeja
def stopDisplay(self):
if not self.display is None:
self.display.stop()
# Inicializacia prehliadaca
def initDriver(self):
chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_argument('--user-data-dir=' +
self.generateNewProfile())
self.driver = webdriver.Chrome(options=chromeOptions)
self.driver.set_window_position(0, 0)
self.driver.get(self.scenario[0]['url'])
# Nastavenie priecinka pre obrazky z prehliadaca
def initScreenShotDir(self):
imgDir = self.baseImgDir + '/' + \
self.appId + '/' + \
self.scenarioId + '/' + \
str(self.testId)
try:
makedirs(imgDir)
except OSError as exc:
if exc.errno == errno.EEXIST and path.isdir(imgDir):
pass
else:
raise
self.screenshotDir = imgDir
# Zrusenie testovacieho prostredia
def endTest(self):
try:
self.driver.close()
except:
pass
self.stopDisplay()
rmtree(self.generateNewProfile(), True)
# Nastavenie prveho regresneho testu
def setRegressTest(self):
response = self.aggClient.sendCommand('setRegressTest', {
'appId': self.appId,
'scenarioId': self.scenarioId,
'testId': 1
})
if not response['status']:
self.logger.critical('Cannot set reggressTestId')
raise RuntimeError(testState.FAILED)
# Aktualizovanie velkosti okna prehliadaca podla udalosti
def updateWindowSize(self, screenX, screenY):
self.driver.set_window_position(0, 0)
self.driver.set_window_size(screenX, screenY)
# Nastavenie Id testu
def setTestId(self):
currentTestId = 1
if self.manage is None or not 'lastTestId' in self.manage:
self.setRegressTest()
self.manage = {'regressTestId': 1, 'lastTestId': 1}
else:
currentTestId = self.manage['lastTestId'] + 1
response = self.aggClient.sendCommand(
'setTestId', {
'appId': self.appId,
'scenarioId': self.scenarioId,
'testId': currentTestId
})
if not response['status']:
self.logger.critical('Cannot set testId')
raise RuntimeError(testState.FAILED)
self.testId = currentTestId
# Zaznamenanie regresneho testu pre aktualny test
def setRegressTestForTest(self):
self.aggClient.sendCommand(
'setRegressTestForTest', {
'appId': self.appId,
'scenarioId': self.scenarioId,
'testId': self.testId,
'regressTestId': self.manage['regressTestId']
})
# Nastavenie noveho cache priecinka pre prehliadac
def generateNewProfile(self):
return '{}/.browserData/{:.10}-{}'.format(getcwd(), self.scenarioId,
self.testId)
def run(self):
self.initDisplay()
self.scenario.sort(key=lambda x: x['timestamp'])
# Nastavenie testovacieho prostredia
try:
self.setTestId()
self.initScreenShotDir()
self.setRegressTestForTest()
self.initDriver()
processedEvent = 0
# Simulovanie udalosti
processedEvent = self.processScenario()
except Exception as e:
self.logger.critical(str(e))
self.endTest()
# Kontrola vykonania vsetkych udalosti
if processedEvent != len(self.scenario):
self.logger.critical('Cannot perform all events in scenario')
raise RuntimeError(testState.FAILED)
return (self.testId, self.manage['regressTestId'])
# Ziskanie elementu na zaklade CSS selektora
def getElementSelector(self, selector):
return self.driver.find_element_by_css_selector(selector)
# Vyber akcie na zaklade typu udalosti
def selectAction(self, event):
return getattr(self, 'action_%s' % event['type'])(event)
def action_click(self, event):
action = ActionChains(self.driver)
elem = self.getElementSelector(event['locator'])
action.move_to_element(elem)
action.click(elem)
return action
def action_focusout(self, event):
action = ActionChains(self.driver)
elem = self.getElementSelector(event['locator'])
action.move_to_element(elem)
action.send_keys(event['content'])
return action
def action_keypress(self, event):
action = ActionChains(self.driver)
return action
def action_mouseover(self, event):
action = ActionChains(self.driver)
elem = self.getElementSelector(event['locator'])
action.move_to_element(elem)
return action
# Ulozenie obrazku a vysledku o simulovaniu udalosti
def saveScreenShot(self, event, performTime):
image = self.screenshotDir + '/' + str(event['timestamp']) + '.png'
self.driver.get_screenshot_as_file(image)
self.aggClient.sendCommand(
'createTest', {
'appId': self.appId,
'scenarioId': self.scenarioId,
'image': image,
'performTime': performTime,
'testId': self.testId
})
def getPageLoaded(self):
return self.driver.execute_script(
'return document.readyState;') == 'complete'
# Replikacia udalosti
def processScenario(self):
pageTime = 0
sleepTime = -1
performTime = 0
startTime = 0
processedEvent = 0
for event in self.scenario:
# Simulovanie uzivatelskeho casu straveneho na stranke
if (sleepTime > 0):
sleep(sleepTime)
sleepTime = 0
try:
# Vykonanie udalosti
action = self.selectAction(event)
if not action is None:
self.updateWindowSize(event['screenX'], event['screenY'])
startTime = time() * 1000
action.perform()
except Exception as e:
self.logger.warning(str(e))
self.saveScreenShot(event, performTime)
processedEvent = processedEvent + 1
continue
# Vypocitanie casu pre simolovanie uzivatelskeho chovania
endTime = time() * 1000
performTime = endTime - startTime
pageTime = pageTime + performTime
sleepTime = round((event['pageTime'] - pageTime) / 100000, 2)
self.logger.critical(sleepTime)
# Ulozenie vysledkov
self.saveScreenShot(event, performTime)
processedEvent = processedEvent + 1
self.logger.warning(processedEvent)
return processedEvent
class Nexis(ClosableObject):
"""
The actual Nexis database wrapper. It fetches data from the Uni Hamburg
nexis database via selenium.
"""
COUNT_REGEX = re.compile(r"([0-9]+) Dokument.* und ([0-9]+) Duplikat.*")
def __init__(self,
user,
password,
hide_window=True,
printer=PrimitiveLogPrinter(),
ignore_big_queries=True):
"""
Creates a new database proxy.
:param user: User name.
:param password: Password.
:param hide_window: Wether or not to show the browser window (showing
might be useful for debugging.)
:param printer: A PrimitiveLogPrinter which will be used for indicating
actions and logging.
:param ignore_big_queries: If set to True, queries resulting in more
than 3000 results will be ignored.
"""
ClosableObject.__init__(self)
# If an exception occurs later those members have to exist for _close()
self.browser = None
self.display = None
self.tempdir = None
# For (primitive) logging
self.printer = printer
self.ignore_big_queries = ignore_big_queries
self.user = user
self.password = password
if hide_window:
with self.printer.do_safe_action(
'Starting virtual display',
"Browser window cannot be hidden. You might need to "
"install Xvfb. Continuing with visible browser window."):
self.display = Display(visible=0)
self.display.start()
self.tempdir = mkdtemp()
self.browser = webdriver.Firefox(DirectDownloadProfile(self.tempdir))
# Retrieve token for this session
with self.printer.do_safe_action('Authenticating at Nexis',
reraise=True):
self.authenticate()
self.home_url = self.browser.current_url
def authenticate(self):
self.browser.get(
"http://rzblx10.uni-regensburg.de/dbinfo/warpto.php?bib_id="
"sub_hh&color=2&titel_id=1670&url=http%3A%2F%2Femedien.sub."
"uni-hamburg.de%2Fhan%2Flexis")
sleep(5 + random() * 5)
self.browser.find_element_by_name('User').send_keys(self.user)
self.browser.find_element_by_name('Password').send_keys(self.password)
self.browser.find_element_by_name("submitimg").click()
# accept the terms, if required
try:
sleep(5 + random() * 5)
self.browser.find_element_by_css_selector(
"a[href^='/auth/submitterms.do']").click()
except NoSuchElementException:
pass
# Press ok on relogin prompt if needed
try:
sleep(5 + random() * 5)
self.browser.find_element_by_xpath(
'//td/input[@title="OK"]').click()
self.printer.debug(
"Relogin needed for {}. Executed successfully.".format(
self.user))
except NoSuchElementException:
pass
sleep(5 + random() * 5)
self.browser.find_element_by_xpath('//a[@title="Profisuche"]').click()
sleep(5 + random() * 5)
def query(self,
search_term: str,
from_date: date = None,
to_date: date = None,
languages: str = 'us'):
"""
Performs a query to the NexisLexis database.
:param search_term: The search term.
:param from_date: Lower date limit.
:param to_date: Upper date limit.
:param languages: One of "all", "german", "english" or "us".
"""
with self.printer.do_safe_action(
"Querying for '{}'".format(search_term), reraise=True):
if self.browser.current_url != self.home_url:
self.browser.get(self.home_url)
from_date = from_date or date(2005, 1, 1)
to_date = to_date or date.today()
# Fill and submit query
self._fill_query_data(search_term, from_date, to_date, languages)
sleep(5 + random() * 5)
self.browser.find_element_by_css_selector(
"img[title='Suche']").click()
sleep(5 + random() * 5)
if self._no_results:
return []
if self.too_many_results:
if self.ignore_big_queries:
return {
'error': 'Too many results (>3000)',
'error_code': 1
}
self.printer.debug(
"Number of results for query '{}' exceeds "
"3000. The query will be split.".format(search_term))
# Split up logarithmically
middle = from_date + (to_date - from_date) / 2
one_after_middle = middle + timedelta(days=1)
results = self.query(search_term, from_date, middle)
results += self.query(search_term, one_after_middle, to_date)
return results
return self._get_results()
def element_exists_by_xpath(self, path):
try:
element = self.browser.find_element_by_xpath(path)
return element
except NoSuchElementException:
return None
@property
def _no_results(self):
return (self.element_exists_by_xpath("//h1[@class='zeroMsgHeader']")
is not None)
def _get_results(self):
def press_forward():
"""
Presses the forward button so lexisnexis updates the document count.
"""
try:
self.browser.find_element_by_xpath(
'//div/ol/li[@class="last"]/a').click()
except:
pass
def get_document_count():
"""
Retrieves the document count. Though it might raise over time when
LexisNexis decides to analyze more documents.
"""
count_text = self.browser.find_element_by_xpath(
"//div/dl/dd[last()]").text
try:
matches = self.COUNT_REGEX.search(count_text).groups()
except AttributeError:
# The one result case
return 1
documents = int(matches[0])
duplicates = int(matches[1])
return documents - duplicates
results = []
# Give nexis some time for duplication analysis
sleep(10)
press_forward()
document_count = get_document_count()
downloaded_documents = 0
while downloaded_documents < document_count:
batch_start = downloaded_documents + 1
batch_end = min(downloaded_documents + 200, document_count)
results += self._download_results(batch_start, batch_end)
sleep(2)
downloaded_documents = batch_end
if len(results) != downloaded_documents:
print("Got", len(results), "results, expecting",
downloaded_documents, "instead.")
# Updates document count, lexis will do that on the server while
# were already downloading stuff
press_forward()
document_count = get_document_count()
return results
def _download_results(self, batch_start, batch_end, retry=3):
def download_in_progress():
# if found a file without .part extension then we're done
for file_ in listdir(self.tempdir):
if ".part" in file_:
return True
else:
return False
# No files at all? Download hasn't started yet.
return True
# Open Download Popover, it'll have three "tabs" with options
self.browser.find_element_by_id("delivery_DnldRender").click()
sleep(5 + random() * 5)
# First tab: range and full text
# Range field won't be there for only one document, selection irrelevant
if not (batch_start == 1 == batch_end):
range_text = "{}-{}".format(batch_start, batch_end)
self.browser.find_element_by_id("sel").click()
self.browser.find_element_by_id("rangetextbox").send_keys(
range_text)
# full text with indexing
Select(self.browser.find_element_by_name(
"delView")).select_by_index(3)
# Switch to tab 2: deactivate cover page
self.browser.find_element_by_xpath("//div/ul/li/a["
"@href='#tabs-2']").click()
if self.browser.find_element_by_id("cvpg").is_selected():
self.browser.find_element_by_id("cvpg").click()
# Switch to tab 3: download as txt, ignore the rest (not applicable
# to txt)
self.browser.find_element_by_xpath("//div/ul/li/a["
"@href='#tabs-3']").click()
Select(self.browser.find_element_by_id("delFmt")).select_by_index(3)
# Actually click Download
self.browser.find_element_by_class_name("deliverBtn").click()
while True:
close_btn = self.element_exists_by_xpath("//*[@id='closeBtn']")
if close_btn is not None:
# Download is started, click the OK button to close popover
close_btn.click()
break
partial_content = self.element_exists_by_xpath(
"//h1[text()=\"Partial Content\"]")
arbitrary_error = self.element_exists_by_xpath(
"//span[text()=\"Fehler bei der Anfrage\"]")
if (partial_content or arbitrary_error) is not None:
sleep(5)
for elem in self.browser.find_elements_by_xpath(
"//a/span[text()='close']"):
try:
elem.click()
break
except:
pass
else:
raise RuntimeError("Closing the window impossible")
sleep(2 + random() * 5)
if retry == 0:
raise ServerError("Unable to fetch data.")
# Retry
return self._download_results(batch_start,
batch_end,
retry=retry - 1)
# Do not busy loop the whole time
sleep(1)
sleep(1)
while download_in_progress():
sleep(1)
sleep(1)
with open(path.join(self.tempdir,
listdir(self.tempdir)[0]), 'r') as file:
content = file.read()
articles = list(Article.from_nexis_text(content))
for file in listdir(self.tempdir):
unlink(path.join(self.tempdir, file))
return articles
@property
def too_many_results(self):
try:
container = self.browser.find_element_by_css_selector(
'#popupContainer')
container.find_element_by_css_selector("span.l0")
return True
except NoSuchElementException:
return False
def _fill_query_data(self, search_term, from_date, to_date, languages):
"""
Fills the search form with the given data.
"""
term_input = self.browser.find_element_by_name("searchTermsTextArea")
term_input.clear()
term_input.send_keys('"' + search_term + '"')
language_value_dict = {
'all': 'All English and German Language News',
'english': 'All English Language News',
'german': 'German Language News',
'us': 'US Publications'
}
source_term = language_value_dict.get(languages, 'US Publications')
self.browser.find_element_by_xpath(
"//div[@rel='more_sources']").click()
# Activate JS in the text field
self.browser.find_element_by_id('selected_source').send_keys('')
sleep(5 + random() * 5)
self.browser.find_element_by_id('selected_source').send_keys(
source_term)
# Page needs some time to show autocompletion box
sleep(10 + random() * 5)
self.browser.find_element_by_id('selected_source').send_keys(
Keys.ARROW_DOWN)
self.browser.find_element_by_id('selected_source').send_keys(
Keys.ENTER)
date_selector = self.browser.find_element_by_name("dateSelector")
# select custom date
Select(date_selector).select_by_index(11)
date_selector.send_keys(Keys.ENTER)
from_date_field = self.browser.find_element_by_name("fromDate")
from_date_field.clear()
from_date_field.send_keys(from_date.strftime('%d/%m/%Y'))
to_date_field = self.browser.find_element_by_name("toDate")
to_date_field.clear()
to_date_field.send_keys(to_date.strftime('%d/%m/%Y'))
# Filter group duplicates
if not self.browser.find_element_by_name("gDuplicates").is_selected():
self.browser.find_element_by_name("gDuplicates").click()
# Set group duplicate filter to "similar"
self.browser.find_element_by_id('duplicatesModal').click()
try:
self.browser.find_element_by_xpath(
"//input[@value='search.common.threshold.broadrange']").click(
)
except:
pass
self.browser.find_element_by_id('saveBooksBtn').click()
if not self.browser.find_element_by_name(
"excludeObituariesChecked").is_selected():
self.browser.find_element_by_name(
"excludeObituariesChecked").click()
def _close(self):
with self.printer.do_safe_action('Cleaning up'):
if self.browser is not None:
self.browser.quit()
if self.display is not None:
self.display.stop()
if self.tempdir:
rmtree(self.tempdir, ignore_errors=True)
def test_repr():
display = Display()
print((repr(display)))
'''
Example for Xvnc backend
'''
from easyprocess import EasyProcess
from pyvirtualdisplay.display import Display
if __name__ == "__main__":
with Display(backend='xvnc', rfbport=5904) as disp:
with EasyProcess('xmessage hello') as proc:
proc.wait()
def f1():
ls[0] = Display()
def setup_func():
'set up test fixtures'
global process, screen
screen = Display(visible=0)
screen.start()
process = EasyProcess('gnumeric').start().sleep(3)
def test_virt(self):
vd = Display().start().stop()
# self.assertEquals(vd.return_code, 0)
ok_(not vd.is_alive())
def print_version():
with Display(visible=False):
print(version())
def test_random(self):
r = Randomizer()
vd = Display(randomizer=r).start().stop()
self.assertEquals(vd.return_code, 0)
assert r.min <= vd.display <= r.min + r.delta
ok_(not vd.is_alive())
def test_speedtest():
with Display(visible=0, size=(800, 600)):
speedtest(virtual_display=True)
def test_screenshot3():
with Display(visible=VISIBLE):
with EasyProcess('python -m pyvirtualdisplay.examples.screenshot3'):
time.sleep(1)
def test_virt(self):
vd = Display().start().stop()
# self.assertEquals(vd.return_code, 0)
ok_(not vd.is_alive())
def test_speedtest():
with Display(visible=0, size=(800, 600)):
speedtest()
def f2():
ls[1] = Display()
def main(rfbport=5904):
with Display(backend='xvnc', rfbport=rfbport) as disp:
with EasyProcess('xmessage hello') as proc:
proc.wait()
class ProtonmailClient:
web_driver = None
virtual_display = None
def __init__(self):
utilities.log("Initiating ProtonMail client")
try:
if not settings.show_browser:
self.virtual_display = Display(visible=0, size=(1366, 768))
self.virtual_display.start()
self.web_driver = webdriver.Firefox()
atexit.register(self.destroy)
except Exception as e:
utilities.log(str(e), "ERROR")
def login(self, username, password):
"""Login to ProtonMail panel
Raises Exception on failure
:param username: your ProtonMail username - email
:param password: your ProtonMail password
"""
try:
utilities.log("Open login page")
self.web_driver.get(variables.url)
utilities.wait_for_elem(self.web_driver,
variables.element_login['username_id'],
"id")
utilities.log("Login page loaded")
username_input = self.web_driver.find_element_by_id(
variables.element_login['username_id'])
password_input = self.web_driver.find_element_by_id(
variables.element_login['password_id'])
username_input.send_keys(username)
password_input.send_keys(password)
password_input.send_keys(Keys.RETURN)
utilities.log("Login credentials sent [" + username + "]")
time.sleep(1)
two_factor = False
if "ng-hide" not in self.web_driver.find_element_by_id(
variables.element_twofactor['detection_id']).get_attribute(
'class'):
two_factor = True
if two_factor:
utilities.log("Two-factor authentication enabled")
two_factor_input = self.web_driver.find_element_by_id(
variables.element_twofactor['code_id'])
two_factor_input.send_keys(
input("Enter two-factor authentication code: "))
two_factor_input.send_keys(Keys.RETURN)
if utilities.wait_for_elem(
self.web_driver,
variables.element_login['after_login_detection_class'],
"class"):
utilities.log("Logged in successfully")
else:
raise Exception()
except Exception as e:
utilities.log("Unable to login", "ERROR")
raise Exception("Unable to login")
def parse_mails(self):
"""
Reads and returns a list of Mails inside the current web driver's page
:return: a list of Mail objects
"""
if not utilities.wait_for_elem(
self.web_driver,
variables.element_list_inbox['email_list_wrapper_id'], "id"):
# for some reason the wrapper wasn't loaded
return None
utilities.wait_for_elem(
self.web_driver,
variables.element_list_inbox["individual_email_soupclass"][1:],
"class",
max_retries=3)
soup = BeautifulSoup(self.web_driver.page_source, "html.parser")
mails_soup = soup.select(
variables.element_list_inbox['individual_email_soupclass'])
mails = []
subject_class = variables.element_list_inbox[
'individual_email_subject_soupclass']
time_class = variables.element_list_inbox[
'individual_email_time_soupclass']
sender_name_class = variables.element_list_inbox[
'individual_email_sender_name_soupclass']
for mail in mails_soup:
# @TODO mails without subject or title, etc.. are ignored
try:
new_mail = Mail(
subject=mail.select(subject_class)[0].get("title"),
time_received=mail.select(time_class)[0].string,
mail_alias=mail.select(sender_name_class)[0].get("title"),
mail=mail.select(sender_name_class)[0].string,
)
mails.append(new_mail)
except Exception as e:
utilities.log("Skip mail... " + str(e), "ERROR")
continue
if settings.mails_read_num > 0:
mails = mails[:settings.mails_read_num]
if settings.date_order == "asc":
return list(reversed(mails))
return mails
def get_mails(self, page):
"""
Get a list of mails that are into the given page
:param page: One of the pages listed in variables.py > page_urls
:return: a list of Mail objects
"""
url = variables.page_urls.get(page)
if not url:
raise ValueError("Page doesn't exist")
if self.web_driver.current_url != url:
utilities.log("Opening %s" % url)
self.web_driver.get(url)
return self.parse_mails()
def has_new_mail(self):
"""Generates a unique hash from the mail inbox
If the hash is different from the previous call of this function
then a new mail was received.
:returns: True if a new mail was arrived else False
@TODO in case we delete an email then the hash will be
changed and we'll get a new mail notification.
"""
mails = self.get_mails("inbox")
old_hash = utilities.get_hash()
mails_str = ""
for mail in mails:
mails_str += str(mail)
mails_str += str(mail)
new_hash = hashlib.sha256(mails_str.encode()).hexdigest()
utilities.write_hash(new_hash)
if old_hash and new_hash != old_hash:
return True
return False
def send_mail(self, to, subject, message):
"""Sends email.
:param to: [str] (list of mail addresses - recipients)
:param message: str (subject of the mail)
:param subject: str (message of the mail)
"""
# click new mail button
el = self.web_driver.find_element_by_class_name(
variables.element_send_mail['open_composer_class'])
el.click()
# wait for mail dialog to appear
utilities.wait_for_elem(
self.web_driver,
variables.element_send_mail['composer_detection_class'], "class")
# type receivers list
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['to_field_css'])
for address in to:
el.send_keys(address + ";")
time.sleep(0.2)
# type subject
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['subject_field_css'])
el.send_keys(subject)
# type message
self.web_driver.switch_to.frame(
self.web_driver.find_element_by_class_name(
variables.element_send_mail['switch_to_message_field_class']))
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['message_field_css'])
el.send_keys(message)
self.web_driver.switch_to.default_content()
# click send
el = self.web_driver.find_element_by_css_selector(
variables.element_send_mail['send_button_css'])
el.click()
time.sleep(settings.load_wait)
def destroy(self):
"""
atexit handler; automatically executed upon normal interpreter termination.
Should be called after any work done with client
"""
if self.web_driver is not None:
self.web_driver.close()
self.web_driver.quit()
self.web_driver = None
if self.virtual_display is not None:
self.virtual_display.stop()
self.virtual_display = None
def test_dpi():
with Display(backend="xvfb", size=(800, 600), dpi=99) as vd:
ok_(vd.is_alive())
eq_(vd.return_code, 0)
ok_(not vd.is_alive())
def test_with():
with Display(visible=0, size=(800, 600)) as vd:
ok_(vd.is_alive())
eq_(vd.return_code, 0)
ok_(not vd.is_alive())
def test_showgrabfullscreen():
with Display(visible=0, size=(800, 600)):
run_in_childprocess(target=showgrabfullscreen_subprocess)
