def _fetch_page_content(self) -> BeautifulSoup:
""" Function which fetchs HTML page content. """
log.debug(f"Fetching '{self.url}' content.")
response = requests.get(self.url)
log.debug(f"Response code: {str(response.status_code)}")
if response.status_code == 200:
content = BeautifulSoup(response.text, "html.parser")
return content
def _validate_yaml_content(
self, content: Dict[str, Dict[str, Union[str, List[str]]]]
) -> Optional[SystemExit]:
""" Function which validates config file content with schema,
and exits program when exception occours. """
log.debug("Validating config file...")
try:
jsonschema.validate(content, self.schema)
except jsonschema.ValidationError as err:
log.error(err)
sys.exit(0)
log.debug("Validation succeeded!")
def _save_advert(self, link: Link) -> None:
""" Function that's creates Advert object, and save to the database. """
advert = Advert(name=link.name, url=link.url)
self.session.add(advert)
try:
self.session.commit()
except sqlalchemy.exc.IntegrityError:
self.session.rollback()
log.warning(
f"There is already same advert in 'adverts' table.\nInvolved url: '{link.url}'"
)
else:
log.debug(f"Saved advert in database.")
def send_message(self, text: str) -> bool:
""" Function which sends Slack message as HTTP POST request. """
status = None
log.info("Sending message...")
response = requests.post(self.webhook_url,
data=self._to_json(text),
headers=self.headers)
log.debug(f"Response code: {str(response.status_code)}")
if response.status_code == 200:
log.info("Message succesfully sent!")
status = True
else:
log.info("Message cannot be sent.")
status = False
return status
def proccess_page_content(self, advert_database: AdvertDatabase,
slack: Slack) -> None:
""" Function which fetches HTML page, search in it
for the adverts links, store them in the database
and send Slack notification. """
content = self._fetch_page_content()
if content:
log.debug(f"Parsing page content...")
if self.portal == 'olx' and self._check_matching_olx_adverts(
content):
links = self._get_olx_adverts_links(content)
self._proccess_adverts_links(links, advert_database, slack)
log.debug(f"Parsing finished.")
else:
log.debug(f"No content to parse.")
def _is_file_exist(self, path: str) -> bool:
""" Function which checks is file exists. """
log.debug(f"Checking is '{path}' file exists.")
return os.path.isfile(path)