def main():
"""Run all checks and handle fatal errors."""
try:
sites_data = sites_from_file(config.MONITORED_SITES)
RunChecks(sites_data).check_all()
except Exception as fatal_err:
# Fatal error during CSV read or RunChecks
fatal_err_msg = logs.get_err_str(fatal_err, "Fatal error.")
logs.log_error(fatal_err_msg)
email_subject = "Robots.txt Check Fatal Error"
email_content = "There was a fatal error during the latest robots.txt checks which " \
"caused the program to terminate unexpectedly."
email_body = emails.get_admin_email_body(email_content)
emails.admin_email.append(
(config.ADMIN_EMAIL, email_subject, email_body))
finally:
if config.EMAILS_ENABLED:
emails.send_emails(emails.admin_email)
else:
print(
"Note: emails are disabled. Details of the program run have been printed "
"and/or logged. Set 'EMAILS_ENABLED' to equal 'True' to send/receive emails."
)
logs.update_main_log("\n{}END OF RUN{}\n".format("-" * 20, "-" * 20),
timestamp=False)
def create_reports(self):
"""Update site log, update main log, print result, and prepare email."""
log_content = "Error: {}. {}".format(self.url, self.err_message)
# Only create/update site log if site directory exists
if (os.path.isdir(self.dir)) and (self.dir[-5:] != "data/"):
self.update_site_log(log_content)
logs.update_main_log(log_content)
print(log_content)
email_subject = "{} Robots.txt Check Error".format(self.name)
email_content = "There was an error while checking the {} robots.txt file. " \
"The check was not completed. The details are shown below.\n\n{}" \
"".format(self.url, self.err_message)
email_content = emails.replace_angle_brackets(email_content)
email_body = emails.get_site_email_body(email_content)
emails.site_emails.append((self.email, email_subject, email_body))
def create_reports(self):
"""Update site log, update main log, print result, create snapshot, and prepare email."""
log_content = "First run: {}. First successful check of robots.txt file.".format(
self.url)
self.update_site_log(log_content)
logs.update_main_log(log_content)
print(log_content)
self.create_snapshot()
email_subject = "First {} Robots.txt Check Complete".format(self.name)
email_content = "The first successful check of the {} robots.txt file is complete. " \
"Going forwards, you'll receive an email if the robots.txt file changes " \
"or if there's an error during the check. Otherwise, you can assume " \
"that the file has not changed.\n\nThe extracted content is shown below:" \
"\n\n-----START OF FILE-----\n\n{}\n\n-----END OF FILE-----\n\n" \
"".format(self.url, self.new_content)
email_body = emails.get_site_email_body(email_content)
emails.site_emails.append((self.email, email_subject, email_body))
def create_reports(self):
"""Update site logs, print result, create snapshot, create diff, and prepare email."""
log_content = "Change: {}. Change detected in the robots.txt file.".format(
self.url)
self.update_site_log(log_content)
logs.update_main_log(log_content)
print(log_content)
self.create_snapshot()
diff_file = self.create_diff_file()
email_subject = "{} Robots.txt Change".format(self.name)
link = "<a href=\"{}\">{}</a>".format(self.url + "robots.txt",
self.url + "robots.txt")
email_content = "A change has been detected in the {} robots.txt file. " \
"Copies of the old file and new file are attached. " \
"\n\nView the live robots.txt file: {}" \
"".format(self.url, link)
email_body = emails.get_site_email_body(email_content)
emails.site_emails.append((self.email, email_subject, email_body,
self.old_file, self.new_file, diff_file))
def check_all(self):
"""Run robots.txt checks and reports for all sites."""
start_content = "Starting checks on {} sites.".format(len(self.sites))
logs.update_main_log(start_content)
print(start_content)
self.reset_counts()
for site_attributes in self.sites:
self.check_site(site_attributes)
summary = "Checks and reports complete. No change: {}. Change: {}. First run: {}. " \
"Error: {}.".format(self.no_change, self.change, self.first_run, self.error)
print("\n" + summary)
logs.update_main_log(summary, blank_before=True)
emails.send_emails(emails.site_emails)
email_subject = "Robots.txt Checks Complete"
email_body = emails.get_admin_email_body(summary)
emails.admin_email.append(
(config.ADMIN_EMAIL, email_subject, email_body, config.MAIN_LOG))
def save_unsent_email(address, subject, body):
"""Save the key details of an email which couldn't be sent to a timestamped .txt file.
Args:
address (str): the destination email address.
subject (str): the subject line of the email.
body (str): the main body content of the email.
"""
unsent_dir = config.PATH + 'data/_unsent_emails'
if not os.path.isdir(unsent_dir):
os.mkdir(unsent_dir)
file_name = logs.get_timestamp(str_format="%d-%m-%y T %H-%M-%S") + ".txt"
with open(unsent_dir + "/" + file_name, 'x') as f:
f.write(address + "\n\n" + subject + "\n\n" + body)
success_msg = "Unsent email content successfully saved in /data/_unsent_emails/."
print(success_msg)
logs.update_main_log(success_msg)
# Ensure timestamp is unique
time.sleep(1)