def start(state_code: str,
report_type: str,
test_address: Optional[str] = None) -> str:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
test_address: Optional email address for which to generate all emails
Returns: The batch id for the newly started batch
"""
batch_id = utils.generate_batch_id()
logging.info("New batch started for %s and %s. Batch id = %s", state_code,
report_type, batch_id)
recipient_data = retrieve_data(state_code, report_type, batch_id)
for recipient in recipient_data:
if test_address:
recipient_email_address = recipient[utils.KEY_EMAIL_ADDRESS]
formatted_test_address = utils.format_test_address(
test_address, recipient_email_address)
# Override the recipient's address with the test address
recipient[utils.KEY_EMAIL_ADDRESS] = formatted_test_address
logging.info("Generating email for [%s] with test address: %s",
recipient_email_address, formatted_test_address)
recipient[utils.KEY_BATCH_ID] = batch_id
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
return batch_id
def _generate_emails(state_code_str: str) -> Tuple[str, HTTPStatus]:
try:
data = request.json
state_code = StateCode(state_code_str)
if state_code not in EMAIL_STATE_CODES:
raise ValueError("State code is invalid for PO monthly reports")
# TODO(#7790): Support more email types.
report_type = ReportType(data.get("reportType"))
if report_type != ReportType.POMonthlyReport:
raise ValueError(f"{report_type.value} is not a valid ReportType")
test_address = data.get("testAddress")
region_code = data.get("regionCode")
message_body_override = data.get("messageBodyOverride")
email_allowlist = data.get("emailAllowlist")
validate_email_address(test_address)
if email_allowlist is not None:
for recipient_email in email_allowlist:
validate_email_address(recipient_email)
except (ValueError, JSONDecodeError) as error:
logging.error(error)
return str(error), HTTPStatus.BAD_REQUEST
if test_address == "":
test_address = None
if region_code not in REGION_CODES:
region_code = None
try:
batch_id = generate_batch_id()
if in_development():
with local_project_id_override(GCP_PROJECT_STAGING):
result: MultiRequestResult[str, str] = data_retrieval.start(
state_code=state_code,
report_type=report_type,
batch_id=batch_id,
test_address=test_address,
region_code=region_code,
email_allowlist=email_allowlist,
message_body_override=message_body_override,
)
else:
result = data_retrieval.start(
state_code=state_code,
report_type=report_type,
batch_id=batch_id,
test_address=test_address,
region_code=region_code,
email_allowlist=email_allowlist,
message_body_override=message_body_override,
)
except InvalidRegionCodeException:
return "Invalid region code provided", HTTPStatus.BAD_REQUEST
new_batch_text = f"New batch started for {state_code} and {report_type}. Batch id = {batch_id}."
test_address_text = (
f"Emails generated for test address: {test_address}" if test_address else ""
)
counts_text = f"Successfully generated {len(result.successes)} email(s)"
success_text = f"{new_batch_text} {test_address_text} {counts_text}."
if result.failures and not result.successes:
return (
f"{success_text}"
f" Failed to generate all emails. Retry the request again."
), HTTPStatus.INTERNAL_SERVER_ERROR
if result.failures:
return (
f"{success_text}"
f" Failed to generate {len(result.failures)} email(s): {', '.join(result.failures)}"
), HTTPStatus.MULTI_STATUS
return (
jsonify(
{
"batchId": batch_id,
"statusText": f"{success_text}",
}
),
HTTPStatus.OK,
)
def start(
state_code: StateCode,
report_type: ReportType,
batch_id: Optional[str] = None,
test_address: Optional[str] = None,
region_code: Optional[str] = None,
email_allowlist: Optional[List[str]] = None,
message_body_override: Optional[str] = None,
) -> MultiRequestResult[str, str]:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
batch_id: The batch id to save the newly started batch to
test_address: Optional email address for which to generate all emails
region_code: Optional region code which specifies the sub-region of the state in which to
generate reports. If empty, this generates reports for all regions.
email_allowlist: Optional list of email_addresses to generate for; all other recipients are skipped
recipient_emails: Optional list of email_addresses to generate for; all other recipients are skipped
message_body_override: Optional override for the message body in the email.
Returns: A MultiRequestResult containing the email addresses for which reports were successfully generated for
and failed to generate for
"""
if batch_id is None:
batch_id = utils.generate_batch_id()
logging.info(
"New batch started for %s (region: %s) and %s. Batch id = %s",
state_code,
region_code,
report_type,
batch_id,
)
recipients: List[Recipient] = retrieve_data(state_code, report_type, batch_id)
recipients = filter_recipients(recipients, region_code, email_allowlist)
if test_address:
logging.info("Overriding batch emails with test address: %s", test_address)
recipients = [
recipient.create_derived_recipient(
{
"email_address": utils.format_test_address(
test_address, recipient.email_address
),
}
)
for recipient in recipients
]
if message_body_override is not None:
logging.info(
"Overriding default message body in batch emails (batch id = %s)", batch_id
)
recipients = [
recipient.create_derived_recipient(
{"message_body_override": message_body_override}
)
for recipient in recipients
]
failed_email_addresses: List[str] = []
succeeded_email_addresses: List[str] = []
# Currently this is only used to pass review month & year, but when we need to add
# more, the way that we do this should likely be changed/refactored.
metadata: Dict[str, str] = {}
for recipient in recipients:
try:
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
except Exception as e:
failed_email_addresses.append(recipient.email_address)
logging.error("Failed to generate report email for %s %s", recipient, e)
else:
succeeded_email_addresses.append(recipient.email_address)
if report_type == ReportType.POMonthlyReport and len(metadata) == 0:
metadata["review_year"] = recipient.data["review_year"]
metadata["review_month"] = recipient.data["review_month"]
_write_batch_metadata(
batch_id=batch_id,
state_code=state_code,
report_type=report_type,
**metadata,
)
return MultiRequestResult(
successes=succeeded_email_addresses, failures=failed_email_addresses
)
def start(
state_code: str,
report_type: str,
batch_id: Optional[str] = None,
test_address: Optional[str] = None,
region_code: Optional[str] = None,
email_allowlist: Optional[List[str]] = None,
message_body: Optional[str] = None,
) -> Tuple[int, int]:
"""Begins data retrieval for a new batch of email reports.
Start with collection of data from the calculation pipelines.
If a test_address is provided, it overrides the recipient's email_address with a formatted test_address
and uses the formatted test_address to save the generated emails.
Args:
state_code: The state for which to generate reports
report_type: The type of report to send
batch_id: The batch id to save the newly started batch to
test_address: Optional email address for which to generate all emails
region_code: Optional region code which specifies the sub-region of the state in which to
generate reports. If empty, this generates reports for all regions.
email_allowlist: Optional list of email_addresses to generate for; all other recipients are skipped
recipient_emails: Optional list of email_addresses to generate for; all other recipients are skipped
message_body: Optional override for the message body in the email.
Returns: Tuple containing:
- Number of failed email generations
- Number of successful email generations
"""
if batch_id is None:
batch_id = utils.generate_batch_id()
logging.info(
"New batch started for %s (region: %s) and %s. Batch id = %s",
state_code,
region_code,
report_type,
batch_id,
)
recipients: List[Recipient] = retrieve_data(state_code, report_type, batch_id)
recipients = filter_recipients(recipients, region_code, email_allowlist)
if test_address:
logging.info("Overriding batch emails with test address: %s", test_address)
recipients = [
recipient.create_derived_recipient(
{
"email_address": utils.format_test_address(
test_address, recipient.email_address
),
}
)
for recipient in recipients
]
if message_body is not None:
logging.info(
"Overriding default message body in batch emails (batch id = %s)", batch_id
)
recipients = [
recipient.create_derived_recipient({"message_body": message_body})
for recipient in recipients
]
failure_count = 0
success_count = 0
for recipient in recipients:
try:
report_context = get_report_context(state_code, report_type, recipient)
email_generation.generate(report_context)
except Exception as e:
failure_count += 1
logging.error("Failed to generate report email for %s %s", recipient, e)
else:
success_count += 1
return failure_count, success_count
def start_new_batch() -> Tuple[str, HTTPStatus]:
"""Start a new batch of email generation for the indicated state.
Validates the test address provided in the params.
Query parameters:
state_code: (required) A valid state code for which reporting is enabled (ex: "US_ID")
report_type: (required) A valid report type identifier (ex: "po_monthly_report)
test_address: (optional) Should only be used for testing. When provided, the test_address is used to generate
the email filenames, ensuring that all emails in the batch can only be delivered to the test_address and not
to the usual recipients of the report. The email filenames will include the original recipient's email
username, for example: [email protected].
region_code: (optional) Indicates the sub-region of the state to generate emails for. If
omitted, we generate emails for all sub-regions of the state.
email_allowlist: (optional) A json list of emails we should generate emails for. Emails that do not exist in the
report will be silently skipped.
message_body: (optional) If included, overrides the default message body.
Returns:
Text indicating the results of the run and an HTTP status
Raises:
Nothing. Catch everything so that we can always return a response to the request
"""
try:
state_code = get_only_str_param_value("state_code", request.args)
report_type = get_only_str_param_value("report_type", request.args)
test_address = get_only_str_param_value("test_address", request.args)
region_code = get_only_str_param_value("region_code", request.args)
raw_email_allowlist = get_only_str_param_value("email_allowlist",
request.args)
message_body = get_only_str_param_value("message_body",
request.args,
preserve_case=True)
validate_email_address(test_address)
email_allowlist: Optional[List[str]] = (
json.loads(raw_email_allowlist) if raw_email_allowlist else None)
if email_allowlist is not None:
for recipient_email in email_allowlist:
validate_email_address(recipient_email)
except (ValueError, JSONDecodeError) as error:
logging.error(error)
return str(error), HTTPStatus.BAD_REQUEST
if not state_code or not report_type:
msg = "Request does not include 'state_code' and/or 'report_type' parameters"
logging.error(msg)
return msg, HTTPStatus.BAD_REQUEST
# Normalize query param inputs
state_code = state_code.upper()
if test_address == "":
test_address = None
region_code = None if not region_code else region_code.upper()
try:
batch_id = email_reporting_utils.generate_batch_id()
failure_count, success_count = data_retrieval.start(
state_code=state_code,
report_type=report_type,
batch_id=batch_id,
test_address=test_address,
region_code=region_code,
email_allowlist=email_allowlist,
message_body=message_body,
)
except InvalidRegionCodeException:
return "Invalid region code provided", HTTPStatus.BAD_REQUEST
else:
test_address_text = (
f"Emails generated for test address: {test_address}"
if test_address else "")
counts_text = f"Successfully generated {success_count} email(s)"
if failure_count:
counts_text += f" Failed to generate {failure_count} email(s)"
return (
f"New batch started for {state_code} and {report_type}. Batch "
f"id = {batch_id}. {test_address_text} {counts_text}"
), HTTPStatus.OK