def do_report(db, mail_server, cyhy_report_dir, tmail_report_dir,
https_report_dir, cybex_report_dir, summary_to):
"""Given the parameters, send out Cyber Hygiene, Trustworthy
Email, HTTPS reports, and a summary email out as appropriate.
Parameters
----------
db : MongoDatabase
The Mongo database from which Cyber Hygiene agency data can
be retrieved.
mail_server : smtplib.SMTP
The mail server via which outgoing mail should be sent.
cyhy_report_dir : str
The directory where the Cyber Hygiene reports can be found.
If None then no Cyber Hygiene reports will be sent.
tmail_report_dir : str
The directory where the Trustworthy Email reports can be
found. If None then no Trustworthy Email reports will be
sent.
https_report_dir : str
The directory where the HTTPS reports can be found. If None
then no HTTPS reports will be sent.
cybex_report_dir : str
The directory where the Cybex report can be found. If None
then no Cybex report will be sent.
summary_to : str
A comma-separated list of email addresses to which the
summary statistics should be sent at the end of the run. If
None then no summary will be sent.
"""
try:
requests = get_cyhy_requests(db)
except TypeError:
return 4
try:
total_agencies = requests.count()
except pymongo.errors.OperationFailure:
logging.critical(
'Mongo database error while counting the number of request documents returned',
exc_info=True)
agencies_emailed_cyhy_reports = 0
agencies_emailed_tmail_reports = 0
agencies_emailed_https_reports = 0
cybex_report_emailed = False
for request in requests:
id = request['_id']
acronym = request['agency']['acronym']
to_emails = get_emails_from_request(request)
# to_emails should contain at least one email
if not to_emails:
continue
###
# Find and mail the CYHY report, if necessary
###
if cyhy_report_dir:
cyhy_report_glob = '{}/cyhy-{}-*.pdf'.format(cyhy_report_dir, id)
cyhy_report_filenames = glob.glob(cyhy_report_glob)
# Exactly one CYHY report should match
if len(cyhy_report_filenames) > 1:
logging.warn(
'More than one Cyber Hygiene report found for agency with ID {}'
.format(id))
elif not cyhy_report_filenames:
# This is an error since we are starting from the list
# of CYHY agencys and they should all have reports
logging.error(
'No Cyber Hygiene report found for agency with ID {}'.
format(id))
if cyhy_report_filenames:
# We take the last filename since, if there happens to
# be more than one, we hope it is the latest.
cyhy_attachment_filename = cyhy_report_filenames[-1]
# Extract the report date from the report filename
match = re.search(r'-(?P<date>\d{4}-[01]\d-[0-3]\d)T',
cyhy_attachment_filename)
report_date = datetime.datetime.strptime(
match.group('date'), '%Y-%m-%d').strftime('%B %d, %Y')
# Construct the CYHY message to send
message = CyhyMessage(to_emails, cyhy_attachment_filename,
acronym, report_date)
try:
agencies_emailed_cyhy_reports = send_message(
mail_server, message, agencies_emailed_cyhy_reports)
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error(
'Unable to send Cyber Hygiene report for agency with ID {}'
.format(id),
exc_info=True,
stack_info=True)
###
# Find and mail the trustymail report, if necessary
#
# This is very similar to the CYHY block but slightly
# different. I need to figure out how to isolate the common
# functionality into a class or functions.
###
if tmail_report_dir:
tmail_report_glob = '{}/cyhy-{}-*.pdf'.format(tmail_report_dir, id)
tmail_report_filenames = glob.glob(tmail_report_glob)
# At most one Tmail report should match
if len(tmail_report_filenames) > 1:
logging.warn(
'More than one Trustworthy Email report found for agency with ID {}'
.format(id))
elif not tmail_report_filenames:
# This is only at info since we are starting from the
# list of CYHY agencys. Many of them will not have
# Tmail reports.
logging.info(
'No Trustworthy Email report found for agency with ID {}'.
format(id))
if tmail_report_filenames:
# We take the last filename since, if there happens to
# be more than one, we hope it is the latest.
tmail_attachment_filename = tmail_report_filenames[-1]
# Extract the report date from the report filename
match = re.search(
r'-(?P<date>\d{4}-[01]\d-[0-3]\d)-tmail-report',
tmail_attachment_filename)
report_date = datetime.datetime.strptime(
match.group('date'), '%Y-%m-%d').strftime('%B %d, %Y')
# Construct the Tmail message to send
message = TmailMessage(to_emails, tmail_attachment_filename,
acronym, report_date)
try:
agencies_emailed_tmail_reports = send_message(
mail_server, message, agencies_emailed_tmail_reports)
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error(
'Unable to send Trustworthy Email report for agency with ID {}'
.format(id),
exc_info=True,
stack_info=True)
###
# Find and mail the https report, if necessary
#
# This is very similar to the CYHY block but slightly
# different. I need to figure out how to isolate the common
# functionality into a class or functions.
###
if https_report_dir:
https_report_glob = '{}/cyhy-{}-*.pdf'.format(https_report_dir, id)
https_report_filenames = glob.glob(https_report_glob)
# At most one HTTPS report should match
if len(https_report_filenames) > 1:
logging.warn(
'More than one HTTPS report found for agency with ID {}'.
format(id))
elif not https_report_filenames:
# This is only at info since we are starting from the
# list of CYHY agencys. Many of them will not have
# HTTPS reports.
logging.info(
'No HTTPS report found for agency with ID {}'.format(id))
if https_report_filenames:
# We take the last filename since, if there happens to
# be more than one, we hope it is the latest.
https_attachment_filename = https_report_filenames[-1]
# Extract the report date from the report filename
match = re.search(
r'-(?P<date>\d{4}-[01]\d-[0-3]\d)-https-report',
https_attachment_filename)
report_date = datetime.datetime.strptime(
match.group('date'), '%Y-%m-%d').strftime('%B %d, %Y')
# Construct the HTTPS message to send
message = HttpsMessage(to_emails, https_attachment_filename,
acronym, report_date)
try:
agencies_emailed_https_reports = send_message(
mail_server, message, agencies_emailed_https_reports)
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error(
'Unable to send HTTPS report for agency with ID {}'.
format(id),
exc_info=True,
stack_info=True)
###
# Find and mail the Cybex report, if necessary
#
# This is very similar to the CYHY block but slightly different.
# I need to figure out how to isolate the common functionality
# into a class or functions.
###
if cybex_report_dir:
cybex_report_glob = '{}/Federal_Cyber_Exposure_Scorecard-*.pdf'.format(
cybex_report_dir)
cybex_report_filenames = glob.glob(cybex_report_glob)
cybex_critical_open_csv_glob = '{}/cybex_open_tickets_critical_*.csv'.format(
cybex_report_dir)
cybex_critical_open_csv_filenames = glob.glob(
cybex_critical_open_csv_glob)
cybex_critical_closed_csv_glob = '{}/cybex_closed_tickets_critical_*.csv'.format(
cybex_report_dir)
cybex_critical_closed_csv_filenames = glob.glob(
cybex_critical_closed_csv_glob)
cybex_high_open_csv_glob = '{}/cybex_open_tickets_high_*.csv'.format(
cybex_report_dir)
cybex_high_open_csv_filenames = glob.glob(cybex_high_open_csv_glob)
cybex_high_closed_csv_glob = '{}/cybex_closed_tickets_high_*.csv'.format(
cybex_report_dir)
cybex_high_closed_csv_filenames = glob.glob(cybex_high_closed_csv_glob)
# At most one Cybex report and CSV should match
if len(cybex_report_filenames) > 1:
logging.warn('More than one Cybex report found')
elif not cybex_report_filenames:
logging.error('No Cybex report found')
if len(cybex_critical_open_csv_filenames) > 1:
logging.warn('More than one Cybex critical open CSV found')
elif not cybex_critical_open_csv_filenames:
logging.error('No Cybex critical open CSV found')
if len(cybex_critical_closed_csv_filenames) > 1:
logging.warn('More than one Cybex critical closed CSV found')
elif not cybex_critical_closed_csv_filenames:
logging.error('No Cybex critical closed CSV found')
if len(cybex_high_open_csv_filenames) > 1:
logging.warn('More than one Cybex high open CSV found')
elif not cybex_high_open_csv_filenames:
logging.error('No Cybex high open CSV found')
if len(cybex_high_closed_csv_filenames) > 1:
logging.warn('More than one Cybex high closed CSV found')
elif not cybex_high_closed_csv_filenames:
logging.error('No Cybex high closed CSV found')
if cybex_report_filenames and cybex_critical_open_csv_filenames and cybex_critical_closed_csv_filenames and cybex_high_open_csv_filenames and cybex_high_closed_csv_filenames:
# We take the last filename since, if there happens to be
# more than one, we hope it is the latest.
cybex_report_filename = cybex_report_filenames[-1]
cybex_critical_open_csv_filename = cybex_critical_open_csv_filenames[
-1]
cybex_critical_closed_csv_filename = cybex_critical_closed_csv_filenames[
-1]
cybex_high_open_csv_filename = cybex_high_open_csv_filenames[-1]
cybex_high_closed_csv_filename = cybex_high_closed_csv_filenames[
-1]
# Extract the report date from the report filename
match = re.search(
r'Federal_Cyber_Exposure_Scorecard-(?P<date>\d{4}-[01]\d-[0-3]\d)',
cybex_report_filename)
report_date = datetime.datetime.strptime(
match.group('date'), '%Y-%m-%d').strftime('%B %d, %Y')
# Construct the Cybex message to send
message = CybexMessage(cybex_report_filename,
cybex_critical_open_csv_filename,
cybex_critical_closed_csv_filename,
cybex_high_open_csv_filename,
cybex_high_closed_csv_filename, report_date)
try:
cybex_report_emailed = bool(
send_message(mail_server, message, 0))
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error('Unable to send Cybex report',
exc_info=True,
stack_info=True)
###
# Find and mail the CYHY sample report, if it is present
###
sample_cyhy_report_emailed = False
if cyhy_report_dir:
cyhy_sample_report_glob = '{}/cyhy-SAMPLE-*.pdf'.format(
cyhy_report_dir)
cyhy_sample_report_filenames = glob.glob(cyhy_sample_report_glob)
# Exactly one CYHY sample report should match
if len(cyhy_sample_report_filenames) > 1:
logging.warn('More than one Cyber Hygiene sample report found')
elif not cyhy_sample_report_filenames:
logging.warn('No Cyber Hygiene sample report found')
if cyhy_sample_report_filenames:
# We take the last filename since, if there happens to be
# more than one, we hope it is the latest.
cyhy_attachment_filename = cyhy_sample_report_filenames[-1]
# Extract the report date from the report filename
match = re.search(r'-(?P<date>\d{4}-[01]\d-[0-3]\d)T',
cyhy_attachment_filename)
report_date = datetime.datetime.strptime(
match.group('date'), '%Y-%m-%d').strftime('%B %d, %Y')
# Construct the report message to send
subject = 'Sample Cyber Hygiene Report - {}'.format(report_date)
message = ReportMessage(['*****@*****.**'],
subject,
None,
None,
cyhy_attachment_filename,
cc_addrs=None)
try:
sample_cyhy_report_emailed = bool(
send_message(mail_server, message, 0))
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error('Unable to send sample Cyber Hygiene report',
exc_info=True,
stack_info=True)
# Print out and log some statistics
cyhy_stats_string = 'Out of {} Cyber Hygiene agencies, {} ({:.2f}%) were emailed Cyber Hygiene reports.'.format(
total_agencies, agencies_emailed_cyhy_reports,
100.0 * agencies_emailed_cyhy_reports / total_agencies)
tmail_stats_string = 'Out of {} Cyber Hygiene agencies, {} ({:.2f}%) were emailed Trustworthy Email reports.'.format(
total_agencies, agencies_emailed_tmail_reports,
100.0 * agencies_emailed_tmail_reports / total_agencies)
https_stats_string = 'Out of {} Cyber Hygiene agencies, {} ({:.2f}%) were emailed HTTPS reports.'.format(
total_agencies, agencies_emailed_https_reports,
100.0 * agencies_emailed_https_reports / total_agencies)
if cybex_report_emailed:
cybex_stats_string = 'Cybex report was emailed.'
else:
cybex_stats_string = 'Cybex report was not emailed.'
if sample_cyhy_report_emailed:
sample_cyhy_stats_string = 'Sample Cyber Hygiene report was emailed.'
else:
sample_cyhy_stats_string = 'Sample Cyber Hygiene report was not emailed.'
logging.info(cyhy_stats_string)
logging.info(tmail_stats_string)
logging.info(https_stats_string)
logging.info(cybex_stats_string)
logging.info(sample_cyhy_stats_string)
print(cyhy_stats_string)
print(tmail_stats_string)
print(https_stats_string)
print(cybex_stats_string)
print(sample_cyhy_stats_string)
###
# Email the summary statistics, if necessary
###
if summary_to:
message = StatsMessage(summary_to.split(','), [
cyhy_stats_string, tmail_stats_string, https_stats_string,
cybex_stats_string, sample_cyhy_stats_string
])
try:
send_message(mail_server, message)
except (smtplib.SMTPRecipientsRefused, smtplib.SMTPHeloError,
smtplib.SMTPSenderRefused, smtplib.SMTPDataError,
smtplib.SMTPNotSupportedError):
logging.error('Unable to send cyhy-mailer report summary',
exc_info=True,
stack_info=True)
def test_four_params_multiple_recipients(self):
to = ['*****@*****.**', '*****@*****.**']
pdf = './tests/data/pdf-sample.pdf'
agency_acronym = 'CLARKE'
report_date = 'December 15, 2001'
message = CyhyMessage(to, pdf, agency_acronym, report_date)
self.assertEqual(message['From'], '*****@*****.**')
self.assertEqual(
message['Subject'],
'CLARKE - Cyber Hygiene Report - December 15, 2001 Results')
self.assertEqual(message['CC'], '*****@*****.**')
self.assertEqual(message['To'],
'[email protected],[email protected]')
# Grab the bytes that comprise the attachment
bytes = open(pdf, 'rb').read()
# Make sure the correct body and PDF attachments were added
for part in message.walk():
# multipart/* are just containers
if part.get_content_type() == 'application/pdf':
self.assertEqual(part.get_payload(decode=True), bytes)
self.assertEqual(part.get_filename(), 'pdf-sample.pdf')
elif part.get_content_type() == 'text/plain':
body = '''Greetings CLARKE,
The Cyber Hygiene scan results are attached for your review. Same password as before. (If this is your first report and you have yet to receive a password, please let us know!)
If you have any questions, please contact our office.
Cheers,
The NCATS team
National Cybersecurity Assessments and Technical Services (NCATS)
National Cybersecurity and Communications Integration Center
U.S. Department of Homeland Security
[email protected]
WARNING: This document is FOR OFFICIAL USE ONLY (FOUO). It contains information that may be exempt from public release under the Freedom of Information Act (5 U.S.G. 552). It is to be controlled, stored, handled, transmitted, distributed, and disposed of in accordance with DHS policy relating to FOUO information and is not to be released to the public or other personnel who do not have a valid 'need-to-know' without prior approval of an authorized DHS official.
'''
self.assertEqual(part.get_payload(), body)
elif part.get_content_type() == 'text/html':
html_body = '''<html>
<head></head>
<body>
<p>Greetings CLARKE,</p>
<p>The Cyber Hygiene scan results are attached for your review. Same password as before. (If this is your first report and you have yet to receive a password, please let us know!)</p>
<p>If you have any questions, please contact our office.</p>
<p>Cheers,<br>
The NCATS team</p>
<p>National Cybersecurity Assessments and Technical Services (NCATS)<br>
National Cybersecurity and Communications Integration Center<br>
U.S. Department of Homeland Security<br>
<a href="mailto:[email protected]">[email protected]</a></p>
<p>WARNING: This document is FOR OFFICIAL USE ONLY (FOUO). It contains information that may be exempt from public release under the Freedom of Information Act (5 U.S.G. 552). It is to be controlled, stored, handled, transmitted, distributed, and disposed of in accordance with DHS policy relating to FOUO information and is not to be released to the public or other personnel who do not have a valid 'need-to-know' without prior approval of an authorized DHS official.</p>
</body>
</html>
'''
self.assertEqual(part.get_payload(), html_body)