def test_generate_incomplete_data(self) -> None:
"""Test that upload_string_to_storage is not called and a KeyError is raised if the recipient data is missing
a key needed for the HTML template."""
with self.assertRaises(KeyError):
self.recipient_data.pop('officer_given_name')
report_context = PoMonthlyReportContext(self.state_code,
self.recipient_data)
generate(report_context)
self.mock_upload_string_to_storage.assert_not_called()
def test_generate(self) -> None:
"""Test that the prepared html is added to Google Cloud Storage with the correct bucket name, filepath,
and prepared html template for the report context."""
prepared_html = self.report_context.render_html()
generate(self.report_context)
bucket_name = "recidiviz-test-report-html"
bucket_filepath = f"{self.state_code.value}/{self.mock_batch_id}/html/{self.recipient.email_address}.html"
path = GcsfsFilePath.from_absolute_path(f"gs://{bucket_name}/{bucket_filepath}")
self.assertEqual(self.gcs_file_system.download_as_string(path), prepared_html)
def test_generate(self) -> None:
"""Test that upload_string_to_storage is called with the correct bucket name, filepath, and prepared
html template for the report context."""
template_path = self.report_context.get_html_template_filepath()
with open(template_path) as html_file:
prepared_data = self.report_context.get_prepared_data()
html_template = Template(html_file.read())
prepared_html = html_template.substitute(prepared_data)
generate(self.report_context)
bucket_name = 'recidiviz-test-report-html'
bucket_filepath = f'{self.mock_batch_id}/{self.recipient_data["email_address"]}.html'
self.mock_upload_string_to_storage.assert_called_with(
bucket_name, bucket_filepath, prepared_html, 'text/html')
def test_generate(self) -> None:
"""Test that the prepared html is added to Google Cloud Storage with the correct bucket name, filepath,
and prepared html template for the report context."""
template_path = self.report_context.get_html_template_filepath()
with open(template_path) as html_file:
prepared_data = self.report_context.get_prepared_data()
html_template = Template(html_file.read())
prepared_html = html_template.substitute(prepared_data)
generate(self.report_context)
bucket_name = "recidiviz-test-report-html"
bucket_filepath = (
f"{self.mock_batch_id}/html/{self.recipient.email_address}.html"
)
path = GcsfsFilePath.from_absolute_path(f"gs://{bucket_name}/{bucket_filepath}")
self.assertEqual(self.gcs_file_system.download_as_string(path), prepared_html)
def test_generate_incomplete_data(self) -> None:
"""Test that no files are added to Google Cloud Storage and a KeyError is raised
if the recipient data is missing a key needed for the HTML template."""
with self.assertRaises(KeyError):
recipient = Recipient.from_report_json(
{
"email_address": "*****@*****.**",
"state_code": "US_ID",
"district": "DISTRICT OFFICE 3",
}
)
report_context = self.report_context_type(self.state_code, recipient)
generate(report_context)
self.assertEqual(self.gcs_file_system.all_paths, [])
def start(state_code: str,
report_type: str,
test_address: Optional[str] = None) -> str:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
test_address: Optional email address for which to generate all emails
Returns: The batch id for the newly started batch
"""
batch_id = utils.generate_batch_id()
logging.info("New batch started for %s and %s. Batch id = %s", state_code,
report_type, batch_id)
recipient_data = retrieve_data(state_code, report_type, batch_id)
for recipient in recipient_data:
if test_address:
recipient_email_address = recipient[utils.KEY_EMAIL_ADDRESS]
formatted_test_address = utils.format_test_address(
test_address, recipient_email_address)
# Override the recipient's address with the test address
recipient[utils.KEY_EMAIL_ADDRESS] = formatted_test_address
logging.info("Generating email for [%s] with test address: %s",
recipient_email_address, formatted_test_address)
recipient[utils.KEY_BATCH_ID] = batch_id
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
return batch_id
def start(
state_code: str,
report_type: str,
batch_id: Optional[str] = None,
test_address: Optional[str] = None,
region_code: Optional[str] = None,
email_allowlist: Optional[List[str]] = None,
message_body: Optional[str] = None,
) -> Tuple[int, int]:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
batch_id: The batch id to save the newly started batch to
test_address: Optional email address for which to generate all emails
region_code: Optional region code which specifies the sub-region of the state in which to
generate reports. If empty, this generates reports for all regions.
email_allowlist: Optional list of email_addresses to generate for; all other recipients are skipped
recipient_emails: Optional list of email_addresses to generate for; all other recipients are skipped
message_body: Optional override for the message body in the email.
Returns: Tuple containing:
- Number of failed email generations
- Number of successful email generations
"""
if batch_id is None:
batch_id = utils.generate_batch_id()
logging.info(
"New batch started for %s (region: %s) and %s. Batch id = %s",
state_code,
region_code,
report_type,
batch_id,
)
recipients: List[Recipient] = retrieve_data(state_code, report_type, batch_id)
recipients = filter_recipients(recipients, region_code, email_allowlist)
if test_address:
logging.info("Overriding batch emails with test address: %s", test_address)
recipients = [
recipient.create_derived_recipient(
{
"email_address": utils.format_test_address(
test_address, recipient.email_address
),
}
)
for recipient in recipients
]
if message_body is not None:
logging.info(
"Overriding default message body in batch emails (batch id = %s)", batch_id
)
recipients = [
recipient.create_derived_recipient({"message_body": message_body})
for recipient in recipients
]
failure_count = 0
success_count = 0
for recipient in recipients:
try:
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
except Exception as e:
failure_count += 1
logging.error("Failed to generate report email for %s %s", recipient, e)
else:
success_count += 1
return failure_count, success_count
def start(
state_code: StateCode,
report_type: ReportType,
batch_id: Optional[str] = None,
test_address: Optional[str] = None,
region_code: Optional[str] = None,
email_allowlist: Optional[List[str]] = None,
message_body_override: Optional[str] = None,
) -> MultiRequestResult[str, str]:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
batch_id: The batch id to save the newly started batch to
test_address: Optional email address for which to generate all emails
region_code: Optional region code which specifies the sub-region of the state in which to
generate reports. If empty, this generates reports for all regions.
email_allowlist: Optional list of email_addresses to generate for; all other recipients are skipped
recipient_emails: Optional list of email_addresses to generate for; all other recipients are skipped
message_body_override: Optional override for the message body in the email.
Returns: A MultiRequestResult containing the email addresses for which reports were successfully generated for
and failed to generate for
"""
if batch_id is None:
batch_id = utils.generate_batch_id()
logging.info(
"New batch started for %s (region: %s) and %s. Batch id = %s",
state_code,
region_code,
report_type,
batch_id,
)
recipients: List[Recipient] = retrieve_data(state_code, report_type, batch_id)
recipients = filter_recipients(recipients, region_code, email_allowlist)
if test_address:
logging.info("Overriding batch emails with test address: %s", test_address)
recipients = [
recipient.create_derived_recipient(
{
"email_address": utils.format_test_address(
test_address, recipient.email_address
),
}
)
for recipient in recipients
]
if message_body_override is not None:
logging.info(
"Overriding default message body in batch emails (batch id = %s)", batch_id
)
recipients = [
recipient.create_derived_recipient(
{"message_body_override": message_body_override}
)
for recipient in recipients
]
failed_email_addresses: List[str] = []
succeeded_email_addresses: List[str] = []
# Currently this is only used to pass review month & year, but when we need to add
# more, the way that we do this should likely be changed/refactored.
metadata: Dict[str, str] = {}
for recipient in recipients:
try:
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
except Exception as e:
failed_email_addresses.append(recipient.email_address)
logging.error("Failed to generate report email for %s %s", recipient, e)
else:
succeeded_email_addresses.append(recipient.email_address)
if report_type == ReportType.POMonthlyReport and len(metadata) == 0:
metadata["review_year"] = recipient.data["review_year"]
metadata["review_month"] = recipient.data["review_month"]
_write_batch_metadata(
batch_id=batch_id,
state_code=state_code,
report_type=report_type,
**metadata,
)
return MultiRequestResult(
successes=succeeded_email_addresses, failures=failed_email_addresses
)