def test_filter_recipients(self) -> None:
dev_from_idaho = Recipient.from_report_json(
{
"email_address": "*****@*****.**",
"state_code": "US_ID",
"district": REGION_CODES["US_ID_D3"],
}
)
dev_from_iowa = Recipient.from_report_json(
{"email_address": "*****@*****.**", "state_code": "US_IA", "district": None}
)
recipients = [dev_from_idaho, dev_from_iowa]
self.assertEqual(filter_recipients(recipients), recipients)
self.assertEqual(
filter_recipients(recipients, region_code="US_ID_D3"), [dev_from_idaho]
)
self.assertEqual(
filter_recipients(
recipients, region_code="US_ID_D3", email_allowlist=["*****@*****.**"]
),
[],
)
self.assertEqual(
filter_recipients(recipients, email_allowlist=["*****@*****.**"]),
[dev_from_iowa],
)
self.assertEqual(
filter_recipients(recipients, email_allowlist=["*****@*****.**"]), []
)
with self.assertRaises(InvalidRegionCodeException):
filter_recipients(recipients, region_code="gibberish")
def _retrieve_data_for_top_opportunities(state_code: StateCode) -> List[Recipient]:
"""Fetches list of recipients from the Case Triage backend where we store information
about which opportunities are active via the OpportunityPresenter."""
recipients = []
for officer_email in _top_opps_email_recipient_addresses():
mismatches = _get_mismatch_data_for_officer(officer_email)
if mismatches is not None:
with SessionFactory.using_database(
SQLAlchemyDatabaseKey.for_schema(SchemaType.CASE_TRIAGE),
autocommit=False,
) as session:
officer = CaseTriageQuerier.officer_for_email(session, officer_email)
recipients.append(
Recipient.from_report_json(
{
utils.KEY_EMAIL_ADDRESS: officer_email,
utils.KEY_STATE_CODE: state_code.value,
utils.KEY_DISTRICT: None,
OFFICER_GIVEN_NAME: officer.given_names,
"mismatches": mismatches,
}
)
)
return recipients
def setUp(self) -> None:
with open(os.path.join(os.path.dirname(__file__),
FIXTURE_FILE)) as fixture_file:
self.recipient = Recipient.from_report_json(
json.loads(fixture_file.read()))
self.recipient.data["batch_id"] = "20201105123033"
project_id = "RECIDIVIZ_TEST"
cdn_static_ip = "123.456.7.8"
test_secrets = {"po_report_cdn_static_IP": cdn_static_ip}
self.get_secret_patcher = patch("recidiviz.utils.secrets.get_secret")
self.project_id_patcher = patch("recidiviz.utils.metadata.project_id")
self.get_secret_patcher.start().side_effect = test_secrets.get
self.project_id_patcher.start().return_value = project_id
def test_generate_incomplete_data(self) -> None:
"""Test that no files are added to Google Cloud Storage and a KeyError is raised
if the recipient data is missing a key needed for the HTML template."""
with self.assertRaises(KeyError):
recipient = Recipient.from_report_json(
{
"email_address": "*****@*****.**",
"state_code": "US_ID",
"district": "DISTRICT OFFICE 3",
}
)
report_context = self.report_context_type(self.state_code, recipient)
generate(report_context)
self.assertEqual(self.gcs_file_system.all_paths, [])
def setUp(self) -> None:
self.project_id_patcher = patch("recidiviz.utils.metadata.project_id")
self.get_secret_patcher = patch("recidiviz.utils.secrets.get_secret")
self.gcs_file_system_patcher = patch(
"recidiviz.reporting.email_generation.GcsfsFactory.build"
)
test_secrets = {"po_report_cdn_static_IP": "123.456.7.8"}
self.get_secret_patcher.start().side_effect = test_secrets.get
self.project_id_patcher.start().return_value = "recidiviz-test"
self.gcs_file_system = FakeGCSFileSystem()
self.mock_gcs_file_system = self.gcs_file_system_patcher.start()
self.mock_gcs_file_system.return_value = self.gcs_file_system
with open(self.fixture_file_path()) as fixture_file:
self.recipient = Recipient.from_report_json(json.loads(fixture_file.read()))
self.state_code = StateCode.US_ID
self.mock_batch_id = "1"
self.recipient.data["batch_id"] = self.mock_batch_id
self.report_context = self.report_context_type(self.state_code, self.recipient)
def setUp(self) -> None:
self.project_id_patcher = patch("recidiviz.utils.metadata.project_id")
self.get_secret_patcher = patch("recidiviz.utils.secrets.get_secret")
self.gcs_file_system_patcher = patch(
"recidiviz.reporting.email_generation.GcsfsFactory.build"
)
test_secrets = {"po_report_cdn_static_IP": "123.456.7.8"}
self.get_secret_patcher.start().side_effect = test_secrets.get
self.project_id_patcher.start().return_value = "recidiviz-test"
self.gcs_file_system = FakeGCSFileSystem()
self.mock_gcs_file_system = self.gcs_file_system_patcher.start()
self.mock_gcs_file_system.return_value = self.gcs_file_system
with open(
os.path.join(
f"{os.path.dirname(__file__)}/context/po_monthly_report", FIXTURE_FILE
)
) as fixture_file:
self.recipient = Recipient.from_report_json(json.loads(fixture_file.read()))
self.state_code = "US_ID"
self.mock_batch_id = "1"
self.recipient.data["batch_id"] = self.mock_batch_id
self.report_context = PoMonthlyReportContext(self.state_code, self.recipient)
Recipient.from_report_json({
utils.KEY_EMAIL_ADDRESS:
"*****@*****.**",
utils.KEY_STATE_CODE:
"US_ID",
utils.KEY_DISTRICT:
"US_ID_D3",
OFFICER_GIVEN_NAME:
"Alex",
"assessment_name":
"LSI-R",
"mismatches": [
{
"name": "Tonye Thompson",
"person_external_id": "189472",
"last_score": 14,
"last_assessment_date": "10/12/20",
"current_supervision_level": "Medium",
"recommended_level": "Low",
},
{
"name": "Linet Hansen",
"person_external_id": "47228",
"last_assessment_date": "1/12/21",
"last_score": 8,
"current_supervision_level": "Medium",
"recommended_level": "Low",
},
{
"name": "Rebekah Cortes",
"person_external_id": "132878",
"last_assessment_date": "3/14/20",
"last_score": 10,
"current_supervision_level": "High",
"recommended_level": "Medium",
},
{
"name": "Taryn Berry",
"person_external_id": "147872",
"last_assessment_date": "3/13/20",
"last_score": 4,
"current_supervision_level": "High",
"recommended_level": "Low",
},
],
}),
Recipient.from_report_json({
utils.KEY_EMAIL_ADDRESS:
"*****@*****.**",
utils.KEY_STATE_CODE:
"US_ID",
utils.KEY_DISTRICT:
"US_ID_D3",
"pos_discharges":
0,
"earned_discharges":
0,
"supervision_downgrades":
0,
"technical_revocations":
0,
"crime_revocations":
0,
"absconsions":
0,
"pos_discharges_district_average":
0,
"pos_discharges_state_average":
0,
"earned_discharges_district_average":
0,
"earned_discharges_state_average":
0,
"supervision_downgrades_district_average":
0,
"supervision_downgrades_state_average":
0,
"technical_revocations_district_average":
0,
"technical_revocations_state_average":
0,
"crime_revocations_district_average":
0,
"crime_revocations_state_average":
0,
"absconsions_district_average":
0,
"absconsions_state_average":
0,
"pos_discharges_last_month":
0,
"earned_discharges_last_month":
0,
"supervision_downgrades_last_month":
0,
"technical_revocations_last_month":
0,
"crime_revocations_last_month":
0,
"absconsions_last_month":
0,
"pos_discharges_clients":
0,
"earned_discharges_clients":
0,
"supervision_downgrades_clients":
0,
"absconsions_clients":
0,
"assessments_out_of_date_clients":
0,
"facetoface_out_of_date_clients":
0,
"revocations_clients":
0,
"assessments":
0,
"assessments_percent":
0,
"facetoface":
0,
"facetoface_percent":
0,
"officer_external_id":
0,
"officer_given_name":
"Clementine",
"review_month":
4,
"mismatches": [
{
"name": "Tonye Thompson",
"person_external_id": "189472",
"last_score": 14,
"last_assessment_date": "10/12/20",
"current_supervision_level": "Medium",
"recommended_level": "Low",
},
{
"name": "Linet Hansen",
"person_external_id": "47228",
"last_assessment_date": "1/12/21",
"last_score": 8,
"current_supervision_level": "Medium",
"recommended_level": "Low",
},
{
"name": "Rebekah Cortes",
"person_external_id": "132878",
"last_assessment_date": "3/14/20",
"last_score": 10,
"current_supervision_level": "High",
"recommended_level": "Medium",
},
{
"name": "Taryn Berry",
"person_external_id": "147872",
"last_assessment_date": "3/13/20",
"last_score": 4,
"current_supervision_level": "High",
"recommended_level": "Low",
},
],
}),
def setUp(self) -> None:
with open(
os.path.join(os.path.dirname(__file__), FIXTURE_FILE)
) as fixture_file:
self.recipient = Recipient.from_report_json(json.loads(fixture_file.read()))
self.recipient.data["batch_id"] = "20201105123033"
def retrieve_data(state_code: str, report_type: str, batch_id: str) -> List[Recipient]:
"""Retrieves the data for email generation of the given report type for the given state.
Get the data from Cloud Storage and return it in a list of dictionaries. Saves the data file into an archive
bucket on completion, so that we have the ability to troubleshoot or re-generate a previous batch of emails
later on.
Args:
state_code: State identifier used to retrieve appropriate data
report_type: The type of report, used to determine the data file name
batch_id: The identifier for this batch
Returns:
A list of recipient data dictionaries
Raises:
Non-recoverable errors that should stop execution. Attempts to catch and handle errors that are recoverable.
Provides logging for debug purposes whenever possible.
"""
data_bucket = utils.get_data_storage_bucket_name()
data_filename = ""
gcs_file_system = GcsfsFactory.build()
try:
data_filename = utils.get_data_filename(state_code, report_type)
path = GcsfsFilePath.from_absolute_path(f"gs://{data_bucket}/{data_filename}")
file_contents = gcs_file_system.download_as_string(path)
except BaseException:
logging.info("Unable to load data file %s/%s", data_bucket, data_filename)
raise
archive_bucket = utils.get_data_archive_bucket_name()
archive_filename = ""
try:
archive_filename = utils.get_data_archive_filename(batch_id)
archive_path = GcsfsFilePath.from_absolute_path(
f"gs://{archive_bucket}/{archive_filename}"
)
gcs_file_system.upload_from_string(
path=archive_path, contents=file_contents, content_type="text/json"
)
except Exception:
logging.error(
"Unable to archive the data file to %s/%s", archive_bucket, archive_filename
)
raise
json_list = file_contents.splitlines()
recipient_data: List[dict] = []
for json_str in json_list:
try:
item = json.loads(json_str)
except Exception as err:
logging.error(
"Unable to parse JSON found in the file %s. Offending json string is: '%s'. <%s> %s",
data_filename,
json_str,
type(err).__name__,
err,
)
else:
recipient_data.append(item)
logging.info(
"Retrieved %s recipients from data file %s", len(recipient_data), data_filename
)
return [
Recipient.from_report_json(
{
**recipient,
utils.KEY_BATCH_ID: batch_id,
}
)
for recipient in recipient_data
]
err,
)
else:
if email := item.get("email_address"):
mismatches = _get_mismatch_data_for_officer(email)
if mismatches is not None:
item["mismatches"] = mismatches
recipient_data.append(item)
logging.info(
"Retrieved %s recipients from data file %s", len(recipient_data), data_filename
)
return [
Recipient.from_report_json(
{
**recipient,
utils.KEY_BATCH_ID: batch_id,
}
)
for recipient in recipient_data
]
def _write_batch_metadata(
*,
batch_id: str,
state_code: StateCode,
report_type: ReportType,
**metadata_fields: str,
) -> None:
gcsfs = GcsfsFactory.build()
gcsfs.upload_from_string(