def test_016_method_getelements_raises_noneselectors(self):
"""Testcase: test_016_method_getelements_raises_noneselectors"""
page = PageBase(self.bot, self.url)
self.assertRaises(PageException,
page.get_elements,
None,
as_controls=True)
def test_instance_url(self):
"""Testcase: test_001_instance_url"""
cfg = self.page.copy()
cfg.update({"go_url": True})
page = PageBase(self.bot, **cfg)
ASSERT.is_instance(page, PageBase)
ASSERT.equals(self.bot.curr_driver.current_url, self.page.get('url'))
def test_023_method_isurl_raises_false_notignoreraises(self):
"""Testcase: test_023_method_isurl_raises_false_notignoreraises"""
page = PageBase(self.bot, self.url)
self.assertRaises(PageException,
page.is_url,
url=self.url_other,
ignore_raises=False)
def test_instance_maximized(self):
"""Testcase: test_004_instance_maximized"""
cfg = self.page.copy()
cfg.update({"go_url": True, "maximize": True, "controls": []})
page = PageBase(self.bot, **cfg)
ASSERT.is_instance(page, PageBase)
ASSERT.equals(self.bot.curr_driver.current_url, cfg.get('url'))
def test_002_instance_notgourl(self):
"""Testcase: test_002_instance_notgourl"""
self.bot.navigation.get_url(self.url_other)
time.sleep(3)
page = PageBase(self.bot, self.url, go_url=False)
self.assertIsInstance(page, PageBase)
self.assert_not_equals_url(self.bot.curr_driver.current_url, self.url)
def test_instance_notgourl(self):
"""Testcase: test_002_instance_notgourl"""
cfg = self.page.copy()
cfg.update({"go_url": False, "controls": []})
page = PageBase(self.bot, **cfg)
ASSERT.is_instance(page, PageBase)
ASSERT.not_equals(self.bot.curr_driver.current_url, cfg.get('url'))
def test_instance_element(self):
"""Testcase: test_003_instance_element"""
cfg = self.page.copy()
page = PageBase(self.bot, **cfg)
ASSERT.is_instance(page, PageBase)
ASSERT.equals(self.bot.curr_driver.current_url, cfg.get('url'))
for control in self.page.get('controls'):
name = control.get('name')
ASSERT.in_list(name, dir(page))
element = page.__dict__[name]
ASSERT.is_instance(element, ControlBase)
def test_method_getelement(self):
"""Testcase: test_004_instance_maximized"""
cfg = self.page.copy()
page = PageBase(self.bot, **cfg)
ASSERT.is_instance(page, PageBase)
ASSERT.equals(self.bot.curr_driver.current_url, cfg.get('url'))
for config_control in self.page.get('controls'):
name = config_control.get('name')
instance_name = config_control.get('instance')
ctl = page.get_element(config_control)
ASSERT.in_list(name, dir(page))
if instance_name == 'ControlBase':
ASSERT.is_instance(ctl, ControlBase)
def test_003_instance_element(self):
"""Testcase: test_003_instance_element"""
page = PageBase(self.bot, self.url, selectors=[self.selectors[0]])
self.assertIsInstance(page, PageBase)
self.assertIsInstance(page.elements[0], ControlBase)
self.assertIsInstance(page.elements[0].element, WebElement)
def test_001_instance_url(self):
"""Testcase: test_001_instance_url"""
page = PageBase(self.bot, self.url)
self.assertIsInstance(page, PageBase)
self.assert_equals_url(self.bot.curr_driver.current_url, self.url)
def test_022_method_isurl_true_notignoreraises(self):
"""Testcase: test_022_method_isurl_true_notignoreraises"""
page = PageBase(self.bot, self.url)
self.assertTrue(page.is_url(url=self.url, ignore_raises=False))
def main():
bot = None
try:
curr_date = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
results = []
# START
bot = BotBase(**SETTINGS)
for url in URLS:
page_config = PAGE_CONFIG.copy()
page_config.update({"url": url})
page = PageBase(bot, **page_config)
# flats found : .i
# tems-container .item
flats = page.flat_containers.elements
bot.log.info("Found flats : {}".format(len(flats)))
for flat_container in flats:
# title
_title = flat_container.find_element(BY,
".item-info-container>a")
bot.navigation.execute_js("arguments[0].scrollIntoView();",
_title)
title_href = _title.get_attribute("href")
title = _title.text
# price
price = flat_container.find_element(
BY, ".item-info-container .item-price.h2-simulated").text
# details
details = ''
detail_elements = flat_container.find_elements(
BY, ".item-info-container .item-detail")
for detail in detail_elements:
details += detail.text
# contact
try:
contact = flat_container.find_element(
BY, ".item-toolbar-contact a span").text
except:
contact = "FAIL at obtain CONTACT for this flat"
# Parse
result = {
"price": price,
"details": details,
"contact_href": contact.replace(" ", ""),
"contact": contact,
"title_href": title_href,
"title": title
}
results.append(result)
# END
if len(flats) <= 0:
page.error_ddos.reload()
if page.error_ddos.element is None:
bot.log.warning(
"Not going to send an email for 0 flats founds")
raise InterruptedError(
1, "IDEALISTA think you are performing DDos attack")
email_rows_str = ''
bot.log.info("EMAIL: building...")
for result in results:
email_row = EMAIL_TEMPLATE_ROW.format(
result['price'], result['contact_href'], result['contact'],
result['details'], result['title_href'], result['title'])
email_rows_str += (email_row)
bot.log.info("EMAIL: adding HTML content...")
for email_dst in EMAILS_DST:
bot.log.debug("Building email to='{}'".format(email_dst))
email_html = EMAIL_TEMPLATE.format(email_rows_str)
email = EmailMessage.Message()
email['Subject'] = "IDEALISTA | Flat search - {}".format(
curr_date)
email['From'] = EMAIL_SRC
email['To'] = email_dst
email.add_header('Content-Type', 'text/html')
email.set_payload(email_html)
bot.log.info("EMAIL: added HTML content...")
bot.log.info("SMTP: connecting...")
server = smtplib.SMTP(SMTP_HOST, SMTP_PORT)
server.ehlo()
server.starttls()
server.login(USER, PASS)
bot.log.info("SMTP: connected!")
bot.log.info("SMTP: send an email")
server.sendmail(email['From'], email['To'],
email.as_string().encode())
bot.log.info("SMTP: email sent!")
server.close()
# END, next iteration
bot.log.info(
"Hardcoded waiting to avoid idealista think it's DDos Error")
bot.log.info("Waiting 15 seconds before continue")
time.sleep(15)
except Exception as err:
bot.log.error("ERROR: {}".format(err))
finally:
bot.log.info("BOT: closing...")
bot.close()
bot.log.info("BOT: closed!")
def test_019_method_isurl_true(self):
"""Testcase: test_019_method_isurl_true"""
page = PageBase(self.bot, self.url)
self.assertTrue(page.is_url(self.url))
def test_020_method_isurl_false(self):
"""Testcase: test_020_method_isurl_false"""
page = PageBase(self.bot, self.url)
self.assertFalse(page.is_url(self.url_other))
def test_011_method_gopageurl(self):
"""Testcase: test_011_method_gopageurl"""
page = PageBase(self.bot, '', go_url=False)
page.go_page_url(self.url)
self.assert_equals_url(self.bot.curr_driver.current_url, self.url)
def test_018_method_isurl_default(self):
"""Testcase: test_018_method_isurl_default"""
page = PageBase(self.bot, self.url)
self.assertTrue(page.is_url())
def test_004_instance_elements(self):
"""Testcase: test_004_instance_elements"""
page = PageBase(self.bot, self.url, selectors=self.selectors)
self.assertIsInstance(page, PageBase)
self.assertIsInstance(page.elements[1], ControlBase)
self.assertIsInstance(page.elements[1].element, WebElement)
def test_015_method_getelements_ascontrols(self):
"""Testcase: test_015_method_getelements_ascontrols"""
page = PageBase(self.bot, self.url)
elements = page.get_elements(self.selectors, as_controls=True)
self.assertIsInstance(elements[1], ControlBase)
def test_014_method_getelements_aswebelements(self):
"""Testcase: test_014_method_getelements_aswebelements"""
page = PageBase(self.bot, self.url)
elements = page.get_elements(self.selectors)
self.assertIsInstance(elements[1], WebElement)
def test_013_method_getelement_ascontrol(self):
"""Testcase: test_013_method_getelement_ascontrol"""
page = PageBase(self.bot, self.url)
element = page.get_element(self.selectors[0], as_control=True)
self.assertIsInstance(element, ControlBase)
def test_012_method_getelement_aswebelement(self):
"""Testcase: test_012_method_getelement_aswebelement"""
page = PageBase(self.bot, self.url)
element = page.get_element(self.selectors[0])
self.assertIsInstance(element, WebElement)
def test_005_instance_maximized(self):
"""Testcase: test_005_instance_maximized"""
page = PageBase(self.bot, self.url, maximize=True)
self.assertIsInstance(page, PageBase)
def test_017_method_getelements_raises_emptyselector(self):
"""Testcase: test_017_method_getelements_raises_emptyselector"""
page = PageBase(self.bot, self.url)
self.assertRaises(PageException, page.get_elements, [''])