def bot_benchmark(self, browser_name, driver_mode, is_headless):
"""Allow to open and close any browser_name with
any driver_mode configuration
Arguments:
browser_name {str} -- browser name
driver_mode {str} -- driver mode (local, remote)
"""
settings = SETTINGS.copy()
settings.get('bot').update({
'browser': str(browser_name),
'mode': str(driver_mode),
'options': {
'headless': is_headless
}
})
if browser_name == 'edge':
browser_name = 'MicrosoftEdge'
pytest.skip(msg="Browser not configured")
if browser_name == 'iexplorer':
browser_name = 'internet explorer'
self.bot = BotBase(**settings)
ASSERT.is_instance(self.bot, BotBase)
ASSERT.equals(self.bot.settings.get('browser'),
settings.get('bot').get('browser'))
ASSERT.equals(self.bot.settings.get('mode'), driver_mode)
ASSERT.equals(self.bot.curr_caps['browserName'], browser_name)
try:
if self.bot:
self.bot.close()
except Exception:
print("Fail at try to close bot, maybe never opened")
def test_007_bot_remote_chrome(self):
"""Testcase: test_007_bot_remote_chrome"""
self.log.debug("TestBotBase: REMOTE started for CHROME")
self.bot_config.config['browser'] = 'chrome'
self.bot_config.config['mode'] = 'remote'
bot = BotBase(self.bot_config)
self.assertEqual(bot.curr_caps['browserName'], 'chrome')
self.timer(wait=WAIT_TO_CLOSE)
bot.close()
self.log.debug("TestBotBase: REMOTE terminated for CHROME")
def test_004_bot_local_iexplorer(self):
"""Testcase: test_004_bot_local_iexplorer"""
self.log.debug("TestBotBase: LOCAL started for IEXPLORER")
self.bot_config.config['browser'] = 'iexplorer'
self.bot_config.config['mode'] = 'local'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'internet explorer')
bot.close()
self.log.debug("TestBotBase: LOCAL terminated for IEXPLORER")
def test_005_bot_local_edge(self):
"""Testcase: test_005_bot_local_edge"""
self.log.debug("TestBotBase: LOCAL started for EDGE")
self.bot_config.config['browser'] = 'edge'
self.bot_config.config['mode'] = 'local'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'MicrosoftEdge')
bot.close()
self.log.debug("TestBotBase: LOCAL terminated for EDGE")
def test_002_bot_local_firefox(self):
"""Testcase: test_002_bot_local_firefox"""
self.log.debug("TestBotBase: LOCAL started for FIREFOX")
self.bot_config.config['browser'] = 'firefox'
self.bot_config.config['mode'] = 'local'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'firefox')
bot.close()
self.log.debug("TestBotBase: LOCAL terminated for FIREFOX")
def test_003_bot_local_phantomjs(self):
"""Testcase: test_003_bot_local_phantomjs"""
self.log.debug("TestBotBase: LOCAL started for PHANTOMJS")
self.bot_config.config['browser'] = 'phantomjs'
self.bot_config.config['mode'] = 'local'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'phantomjs')
bot.close()
self.log.debug("TestBotBase: LOCAL terminated for PHANTOMJS")
def test_001_bot_local_chrome(self):
"""Testcase: test_001_bot_local_chrome"""
self.log.debug('TestBotBase: LOCAL started for CHROME')
self.bot_config.config['browser'] = 'chrome'
self.bot_config.config['mode'] = 'local'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'chrome')
bot.close()
self.log.debug("TestBotBase: LOCAL terminated for CHROME")
def test_009_bot_remote_phantomjs(self):
"""Testcase: test_009_bot_remote_phantomjs"""
self.log.debug("TestBotBase: REMOTE started for PHANTOMJS")
self.bot_config.config['browser'] = "phantomjs"
self.bot_config.config['mode'] = "remote"
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'phantomjs')
bot.close()
self.log.debug("TestBotBase: REMOTE terminated for PHANTOMJS")
def browser():
CFG = settings(file_path="qacode/configs/", file_name="settings.json")
bot = None
try:
bot = BotBase(**CFG)
yield bot
except Exception as err:
raise err
finally:
if bot:
bot.close()
def test_012_bot_remote_opera(self):
"""Testcase: test_012_bot_remote_opera
ISSUE OPENED :
https://github.com/operasoftware/operachromiumdriver/issues/9
"""
self.log.debug("TestBotBase: REMOTE started for OPERA")
self.bot_config.config['browser'] = 'opera'
self.bot_config.config['mode'] = 'remote'
bot = BotBase(self.bot_config)
self.timer(wait=WAIT_TO_CLOSE)
self.assertEqual(bot.curr_caps['browserName'], 'opera')
bot.close()
self.log.debug("TestBotBase: REMOTE terminated for OPERA")
def test_bot_modes_headless(self, driver_mode, browser_name):
"""Testcase: test_bot_modes_headless"""
if SKIP[browser_name][driver_mode]:
pytest.skip(SKIP_MSG.format(browser_name))
settings = SETTINGS.copy()
settings.get('bot').update({
'browser': str(browser_name),
'mode': str(driver_mode),
'options': {"headless": True}
})
self.bot = BotBase(**settings)
self.timer(wait=WAIT_TO_CLOSE)
ASSERT.is_instance(self.bot, BotBase)
ASSERT.equals(
self.bot.settings.get('browser'),
settings.get('bot').get('browser'))
ASSERT.equals(self.bot.settings.get('mode'), driver_mode)
ASSERT.equals(self.bot.curr_caps['browserName'], browser_name)
def test_bot_modes_and_names(self, driver_mode, browser_name):
"""Testcase: test_001_bot_local_chrome"""
if SKIP[browser_name][driver_mode]:
pytest.skip(SKIP_MSG.format(browser_name))
settings = SETTINGS.copy()
settings.get('bot').update({
'browser': str(browser_name),
'mode': str(driver_mode)
})
if browser_name == 'edge':
browser_name = 'MicrosoftEdge'
pytest.skip(msg="Browser not configured")
if browser_name == 'iexplorer':
browser_name = 'internet explorer'
self.bot = BotBase(**settings)
self.timer(wait=WAIT_TO_CLOSE)
ASSERT.is_instance(self.bot, BotBase)
ASSERT.equals(
self.bot.settings.get('browser'),
settings.get('bot').get('browser'))
ASSERT.equals(self.bot.settings.get('mode'), driver_mode)
ASSERT.equals(self.bot.curr_caps['browserName'], browser_name)
def test_botbase_drivernamefilter_ok(self, browser, is_win, is_64bits):
"""Testcase: test_botbase_drivernamefilter_ok"""
if 'bot' not in dir(self):
settings = SETTINGS.copy()
self.add_property('bot', BotBase(**settings))
# end setup
self.bot.IS_WIN = is_win
self.bot.IS_64BITS = is_64bits
name_formatted = self.bot.driver_name_filter(browser)
if is_win and not is_64bits:
ASSERT.equals(
name_formatted, "{}driver_32.exe".format(browser))
if is_win and is_64bits:
ASSERT.equals(
name_formatted, "{}driver_64.exe".format(browser))
if not is_win and not is_64bits:
ASSERT.equals(
name_formatted, "{}driver_32".format(browser))
if not is_win and is_64bits:
ASSERT.equals(
name_formatted, "{}driver_64".format(browser))
self.try_bot_close()
class TestBotBase(TestInfoBase):
"""Testcases for class BotBase"""
def setup_method(self, test_method):
"""Setup test_method to be executed"""
super(TestBotBase,
self).setup_method(test_method,
config=settings(file_path="qacode/configs/"),
skip_force=SKIP)
def bot_benchmark(self, browser_name, driver_mode, is_headless):
"""Allow to open and close any browser_name with
any driver_mode configuration
Arguments:
browser_name {str} -- browser name
driver_mode {str} -- driver mode (local, remote)
"""
settings = SETTINGS.copy()
settings.get('bot').update({
'browser': str(browser_name),
'mode': str(driver_mode),
'options': {
'headless': is_headless
}
})
if browser_name == 'edge':
browser_name = 'MicrosoftEdge'
pytest.skip(msg="Browser not configured")
if browser_name == 'iexplorer':
browser_name = 'internet explorer'
self.bot = BotBase(**settings)
ASSERT.is_instance(self.bot, BotBase)
ASSERT.equals(self.bot.settings.get('browser'),
settings.get('bot').get('browser'))
ASSERT.equals(self.bot.settings.get('mode'), driver_mode)
ASSERT.equals(self.bot.curr_caps['browserName'], browser_name)
try:
if self.bot:
self.bot.close()
except Exception:
print("Fail at try to close bot, maybe never opened")
@pytest.mark.benchmark(group='BROWSERS')
@pytest.mark.parametrize("browser_name",
["chrome", "firefox", "iexplorer", "edge"])
@pytest.mark.parametrize("driver_mode", ["local", "remote"])
@pytest.mark.skipIf(SKIP, SKIP_MSG)
def test_benchmark_browsers(self, benchmark, browser_name, driver_mode):
"""Testcase: test_benchmark_local_chrome"""
if SKIP:
pytest.skip(SKIP_MSG)
cfg_local = {
'browser_name': browser_name,
'driver_mode': driver_mode,
'is_headless': False
}
benchmark.pedantic(self.bot_benchmark,
kwargs=cfg_local,
iterations=ITERATIONS,
rounds=ROUNDS)
@pytest.mark.benchmark(group='BROWSERS_HEADLESS')
@pytest.mark.parametrize("browser_name", ["chrome", "firefox"])
@pytest.mark.parametrize("driver_mode", ["local", "remote"])
@pytest.mark.skipIf(SKIP, SKIP_MSG)
def test_benchmark_browsers_headless(self, benchmark, browser_name,
driver_mode):
"""Testcase: test_benchmark_local_chrome"""
if SKIP:
pytest.skip(SKIP_MSG)
cfg_local = {
'browser_name': browser_name,
'driver_mode': driver_mode,
'is_headless': True
}
benchmark.pedantic(self.bot_benchmark,
kwargs=cfg_local,
iterations=ITERATIONS,
rounds=ROUNDS)
def test_botbase_invalidbrowser(self):
"""Testcase: test_botbase_invalidbrowser"""
settings = SETTINGS.copy()
settings.get('bot').update({"browser": "must_raises"})
with pytest.raises(CoreException):
self.bot = BotBase(**settings)
def main():
bot = None
try:
curr_date = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
results = []
# START
bot = BotBase(**SETTINGS)
for url in URLS:
page_config = PAGE_CONFIG.copy()
page_config.update({"url": url})
page = PageBase(bot, **page_config)
# flats found : .i
# tems-container .item
flats = page.flat_containers.elements
bot.log.info("Found flats : {}".format(len(flats)))
for flat_container in flats:
# title
_title = flat_container.find_element(BY,
".item-info-container>a")
bot.navigation.execute_js("arguments[0].scrollIntoView();",
_title)
title_href = _title.get_attribute("href")
title = _title.text
# price
price = flat_container.find_element(
BY, ".item-info-container .item-price.h2-simulated").text
# details
details = ''
detail_elements = flat_container.find_elements(
BY, ".item-info-container .item-detail")
for detail in detail_elements:
details += detail.text
# contact
try:
contact = flat_container.find_element(
BY, ".item-toolbar-contact a span").text
except:
contact = "FAIL at obtain CONTACT for this flat"
# Parse
result = {
"price": price,
"details": details,
"contact_href": contact.replace(" ", ""),
"contact": contact,
"title_href": title_href,
"title": title
}
results.append(result)
# END
if len(flats) <= 0:
page.error_ddos.reload()
if page.error_ddos.element is None:
bot.log.warning(
"Not going to send an email for 0 flats founds")
raise InterruptedError(
1, "IDEALISTA think you are performing DDos attack")
email_rows_str = ''
bot.log.info("EMAIL: building...")
for result in results:
email_row = EMAIL_TEMPLATE_ROW.format(
result['price'], result['contact_href'], result['contact'],
result['details'], result['title_href'], result['title'])
email_rows_str += (email_row)
bot.log.info("EMAIL: adding HTML content...")
for email_dst in EMAILS_DST:
bot.log.debug("Building email to='{}'".format(email_dst))
email_html = EMAIL_TEMPLATE.format(email_rows_str)
email = EmailMessage.Message()
email['Subject'] = "IDEALISTA | Flat search - {}".format(
curr_date)
email['From'] = EMAIL_SRC
email['To'] = email_dst
email.add_header('Content-Type', 'text/html')
email.set_payload(email_html)
bot.log.info("EMAIL: added HTML content...")
bot.log.info("SMTP: connecting...")
server = smtplib.SMTP(SMTP_HOST, SMTP_PORT)
server.ehlo()
server.starttls()
server.login(USER, PASS)
bot.log.info("SMTP: connected!")
bot.log.info("SMTP: send an email")
server.sendmail(email['From'], email['To'],
email.as_string().encode())
bot.log.info("SMTP: email sent!")
server.close()
# END, next iteration
bot.log.info(
"Hardcoded waiting to avoid idealista think it's DDos Error")
bot.log.info("Waiting 15 seconds before continue")
time.sleep(15)
except Exception as err:
bot.log.error("ERROR: {}".format(err))
finally:
bot.log.info("BOT: closing...")
bot.close()
bot.log.info("BOT: closed!")
def test_botbase_invalidsettingskey(self):
"""Testcase: test_botbase_invalidsettingskey"""
settings = SETTINGS.copy()
settings.get('bot').update({"must_raises": "test"})
with pytest.raises(CoreException):
BotBase(**settings)
def bot_open(cls, test_config, logger_manager):
"""Open bot and return it"""
bot_config = BotConfig(test_config, logger_manager)
return BotBase(bot_config)