def setUp(self) -> None:
self.database_key = SQLAlchemyDatabaseKey.canonical_for_schema(
SchemaType.STATE)
local_postgres_helpers.use_on_disk_postgresql_database(
SQLAlchemyDatabaseKey.canonical_for_schema(SchemaType.STATE))
self.state_code = "US_XX"
def test_enums_match_schema(self) -> None:
with runner(self.default_config(), self.engine) as r:
r.migrate_up_to("head")
# Fetch enum values
migration_enums = self.fetch_all_enums()
# Doing teardown/setup to generate a new postgres instance
local_postgres_helpers.restore_local_env_vars(self.overridden_env_vars)
local_postgres_helpers.stop_and_clear_on_disk_postgresql_database(
self.db_dir)
self.db_dir = local_postgres_helpers.start_on_disk_postgresql_database(
)
self.overridden_env_vars = (
local_postgres_helpers.update_local_sqlalchemy_postgres_env_vars())
local_postgres_helpers.use_on_disk_postgresql_database(
self.database_key)
# Check enum values
schema_enums = self.fetch_all_enums()
# Assert that they all match
self.assertEqual(len(migration_enums), len(schema_enums))
for enum_name, migration_values in migration_enums.items():
schema_values = schema_enums[enum_name]
self.assertCountEqual(migration_values, schema_values)
# Cleanup needed for this method.
local_postgres_helpers.teardown_on_disk_postgresql_database(
self.database_key)
def setUp(self) -> None:
self.maxDiff = 250000
self.metadata_patcher = patch("recidiviz.utils.metadata.project_id")
self.mock_project_id_fn = self.metadata_patcher.start()
self.mock_project_id_fn.return_value = "recidiviz-staging"
local_postgres_helpers.use_on_disk_postgresql_database(
self.schema_base())
local_postgres_helpers.use_on_disk_postgresql_database(OperationsBase)
self.controller = build_gcsfs_controller_for_tests(
self.controller_cls(),
self.fixture_path_prefix(),
run_async=False,
max_delay_sec_between_files=0,
regions_module=regions,
)
# Set entity matching error threshold to a diminishingly small number
# for tests. We cannot set it to 0 because we throw when errors *equal*
# the error threshold.
self.entity_matching_error_threshold_patcher = patch.dict(
"recidiviz.persistence.persistence.SYSTEM_TYPE_TO_ERROR_THRESHOLD",
{
SystemLevel.STATE: {
OVERALL_THRESHOLD: 0,
ENUM_THRESHOLD: 0,
ENTITY_MATCHING_THRESHOLD: 0,
DATABASE_INVARIANT_THRESHOLD: 0,
}
},
)
self.entity_matching_error_threshold_patcher.start()
def setUp(self) -> None:
super().setUp()
self.database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(
self.database_key)
def test_enums_match_schema(self):
with runner(self.default_config()) as r:
r.migrate_up_to('head')
# Fetch enum values
migration_enums = self.fetch_all_enums()
# Doing teardown/setup to generate a new postgres instance
local_postgres_helpers.restore_local_env_vars(self.overridden_env_vars)
local_postgres_helpers.stop_and_clear_on_disk_postgresql_database(
self.db_dir)
self.db_dir = local_postgres_helpers.start_on_disk_postgresql_database(
)
self.overridden_env_vars = local_postgres_helpers.update_local_sqlalchemy_postgres_env_vars(
)
local_postgres_helpers.use_on_disk_postgresql_database(
SQLAlchemyEngineManager.declarative_method_for_schema(
self.schema_type))
# Check enum values
schema_enums = self.fetch_all_enums()
# Assert that they all match
self.assertEqual(len(migration_enums), len(schema_enums))
for enum_name, migration_values in migration_enums.items():
schema_values = schema_enums[enum_name]
self.assertEqual(len(migration_values),
len(schema_values),
msg=f'{enum_name} lengths differ')
self.assertEqual(len(migration_values),
len(migration_values.intersection(schema_values)),
msg=f'{enum_name} values differ')
def setUp(self) -> None:
self.user_1_email = "*****@*****.**"
self.mock_instance_id = "mock_instance_id"
self.cloud_sql_client_patcher = patch(
"recidiviz.cloud_sql.gcs_import_to_cloud_sql.CloudSQLClientImpl")
self.mock_cloud_sql_client = MagicMock()
self.cloud_sql_client_patcher.start(
).return_value = self.mock_cloud_sql_client
self.mock_sqlalchemy_engine_manager = SQLAlchemyEngineManager
setattr(
self.mock_sqlalchemy_engine_manager,
"get_stripped_cloudsql_instance_id",
Mock(return_value=self.mock_instance_id),
)
self.database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(
self.database_key)
self.table_name = DashboardUserRestrictions.__tablename__
self.columns = [
col.name for col in DashboardUserRestrictions.__table__.columns
]
self.gcs_uri = GcsfsFilePath.from_absolute_path(
"US_MO/dashboard_user_restrictions.csv")
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(StateBase)
self.get_region_patcher = patch(
"recidiviz.persistence.entity_matching.state."
"base_state_matching_delegate.get_region",
new=self.get_fake_region)
self.get_region_patcher.start()
self.addCleanup(self.get_region_patcher.stop)
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(CaseTriageBase)
self.mock_officer = generate_fake_officer("id_1")
self.mock_client = generate_fake_client(
"person_id_1",
last_assessment_date=date(2021, 2, 1),
)
def setUp(self) -> None:
self.operations_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.OPERATIONS)
local_postgres_helpers.use_on_disk_postgresql_database(
self.operations_key)
self.us_xx_manager = DirectIngestInstanceStatusManager.add_instance(
"US_XX", DirectIngestInstance.PRIMARY, True)
self.us_ww_manager = DirectIngestInstanceStatusManager.add_instance(
"US_WW", DirectIngestInstance.PRIMARY, False)
def run_justice_counts_ingest_locally(manifest_file: str,
clean_up_db: bool) -> None:
tmp_db_dir = local_postgres_helpers.start_on_disk_postgresql_database()
local_postgres_helpers.use_on_disk_postgresql_database(
SQLAlchemyDatabaseKey.for_schema(SchemaType.JUSTICE_COUNTS))
fs = FakeGCSFileSystem()
try:
manual_upload.ingest(fs, test_utils.prepare_files(fs, manifest_file))
finally:
cleanup_run(tmp_db_dir, clean_up_db)
def setUp(self) -> None:
self.maxDiff = 250000
local_postgres_helpers.use_on_disk_postgresql_database(
self.schema_base())
local_postgres_helpers.use_on_disk_postgresql_database(OperationsBase)
self.controller = build_gcsfs_controller_for_tests(
self.controller_cls(),
self.fixture_path_prefix(),
run_async=False,
max_delay_sec_between_files=0)
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(StateBase)
# State persistence ends up having to instantiate the us_nd_controller to
# get enum overrides, and the controller goes on to create bigquery,
# storage, and tasks clients.
self.bq_client_patcher = patch("google.cloud.bigquery.Client")
self.storage_client_patcher = patch("google.cloud.storage.Client")
self.task_client_patcher = patch(
"google.cloud.tasks_v2.CloudTasksClient")
self.bq_client_patcher.start()
self.storage_client_patcher.start()
self.task_client_patcher.start()
def setUp(self) -> None:
self.database_key = SQLAlchemyDatabaseKey.canonical_for_schema(SchemaType.STATE)
local_postgres_helpers.use_on_disk_postgresql_database(self.database_key)
# State persistence ends up having to instantiate the us_nd_controller to
# get enum overrides, and the controller goes on to create bigquery,
# storage, and tasks clients.
self.bq_client_patcher = patch("google.cloud.bigquery.Client")
self.storage_client_patcher = patch("google.cloud.storage.Client")
self.task_client_patcher = patch("google.cloud.tasks_v2.CloudTasksClient")
self.bq_client_patcher.start()
self.storage_client_patcher.start()
self.task_client_patcher.start()
def main(
repo_directory: str,
system: schema.System,
base_drive_folder_id: str,
credentials_directory: str,
app_url: Optional[str],
filter_type: Optional[FilterType],
regions: Optional[List[str]],
) -> None:
"""
Downloads, tests, and ingests specified regions
"""
regions_to_ingest = _get_list_of_regions(filter_type, regions)
logging.info("Starting ingest of regions...")
logging.info(regions_to_ingest)
tmp_db_dir = local_postgres_helpers.start_on_disk_postgresql_database()
local_postgres_helpers.use_on_disk_postgresql_database(
SQLAlchemyDatabaseKey.for_schema(SchemaType.JUSTICE_COUNTS)
)
fs = FakeGCSFileSystem()
region_ingest_summary = []
try:
for region in regions_to_ingest:
region_ingest_summary.append(
_full_ingest_region(
fs,
region,
repo_directory,
system,
base_drive_folder_id,
credentials_directory,
app_url,
)
)
finally:
cleanup_run(tmp_db_dir, True)
for ingest_result in region_ingest_summary:
if ingest_result.success:
logging.info("%s: success", ingest_result.region_code)
else:
logging.error(
"%s: failed - %s", ingest_result.region_code, ingest_result.error
)
def setUp(self) -> None:
self.database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(
self.database_key)
self.mock_officer = generate_fake_officer("id_1")
self.mock_client = generate_fake_client(
"person_id_1",
last_assessment_date=date(2021, 2, 1),
)
self.mock_context = UserContext(
email=self.mock_officer.email_address,
authorization_store=AuthorizationStore(),
current_user=self.mock_officer,
)
def setUp(self) -> None:
self.maxDiff = 250000
self.metadata_patcher = patch("recidiviz.utils.metadata.project_id")
self.mock_project_id_fn = self.metadata_patcher.start()
self.mock_project_id_fn.return_value = "recidiviz-staging"
self.main_database_key = self._main_database_key()
self.operations_database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.OPERATIONS)
local_postgres_helpers.use_on_disk_postgresql_database(
self.main_database_key)
local_postgres_helpers.use_on_disk_postgresql_database(
self.operations_database_key)
self.controller = build_gcsfs_controller_for_tests(
self.controller_cls(),
ingest_instance=self._main_ingest_instance(),
run_async=False,
regions_module=regions,
)
# Set entity matching error threshold to a diminishingly small number
# for tests. We cannot set it to 0 because we throw when errors *equal*
# the error threshold.
self.entity_matching_error_threshold_patcher = patch.dict(
"recidiviz.persistence.persistence.SYSTEM_TYPE_TO_ERROR_THRESHOLD",
{
SystemLevel.STATE: {
OVERALL_THRESHOLD: 0,
ENUM_THRESHOLD: 0,
ENTITY_MATCHING_THRESHOLD: 0,
DATABASE_INVARIANT_THRESHOLD: 0,
}
},
)
self.entity_matching_error_threshold_patcher.start()
for instance in DirectIngestInstance:
DirectIngestInstanceStatusManager.add_instance(
self.region_code(), instance,
(instance != self._main_ingest_instance()))
def setUp(self) -> None:
self.app = Flask(__name__)
self.app.register_blueprint(auth_endpoint_blueprint)
self.app.config["TESTING"] = True
self.client = self.app.test_client()
self.headers: Dict[str, Dict[Any, Any]] = {
"x-goog-iap-jwt-assertion": {}
}
self.region_code = "US_MO"
self.bucket = "test-project-dashboard-user-restrictions"
self.filename = "dashboard_user_restrictions.json"
self.gcs_csv_uri = GcsfsFilePath.from_absolute_path(
f"{self.bucket}/{self.region_code}/dashboard_user_restrictions.csv"
)
self.columns = [
col.name for col in DashboardUserRestrictions.__table__.columns
]
# Setup database
self.database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(
self.database_key)
# Mock Auth0Client
self.auth0_client_patcher = patch(
"recidiviz.auth.auth_endpoint.Auth0Client")
self.mock_auth0_client = MagicMock()
self.auth0_client_patcher.start().return_value = self.mock_auth0_client
with self.app.test_request_context():
self.import_user_restrictions_csv_to_sql_url = flask.url_for(
"auth_endpoint_blueprint.import_user_restrictions_csv_to_sql")
self.dashboard_user_restrictions_by_email_url = flask.url_for(
"auth_endpoint_blueprint.dashboard_user_restrictions_by_email")
self.update_auth0_user_metadata_url = flask.url_for(
"auth_endpoint_blueprint.update_auth0_user_metadata")
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(StateBase)
self.system_level = SystemLevel.STATE
self.state_code = 'US_XX'
def setUp(self) -> None:
self.get_local_patcher = mock.patch(
"recidiviz.case_triage.authorization.get_local_file",
new=_test_get_local_file,
)
self.get_local_patcher.start()
self.auth_store = AuthorizationStore()
self.auth_store.refresh()
self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(self.database_key)
self.case_triage_user = generate_fake_user_restrictions(
"US_XX",
"*****@*****.**",
can_access_leadership_dashboard=False,
can_access_case_triage=True,
)
self.dashboard_user = generate_fake_user_restrictions(
"US_XX",
"*****@*****.**",
can_access_leadership_dashboard=True,
can_access_case_triage=False,
)
self.both_user = generate_fake_user_restrictions(
"US_XX",
"*****@*****.**",
can_access_leadership_dashboard=True,
can_access_case_triage=True,
)
self.overridden_user = generate_fake_user_restrictions(
"US_XX",
"*****@*****.**",
can_access_leadership_dashboard=True,
can_access_case_triage=False,
)
self.both_user_different_state = generate_fake_user_restrictions(
"US_YY",
"*****@*****.**",
can_access_leadership_dashboard=True,
can_access_case_triage=True,
)
self.officer = generate_fake_officer(
"test", "*****@*****.**", state_code="US_XX"
)
with SessionFactory.using_database(self.database_key) as session:
session.expire_on_commit = False
session.add_all(
[
self.case_triage_user,
self.dashboard_user,
self.both_user,
self.overridden_user,
self.both_user_different_state,
self.officer,
]
)
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(
JusticeCountsBase)
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(StateBase)
def setUp(self) -> None:
self.project_id_patcher = patch("recidiviz.utils.metadata.project_id")
self.email_generation_patcher = patch(
"recidiviz.reporting.email_generation.generate"
)
self.gcs_file_system_patcher = patch(
"recidiviz.cloud_storage.gcsfs_factory.GcsfsFactory.build"
)
self.project_id_patcher.start().return_value = "recidiviz-test"
self.mock_email_generation = self.email_generation_patcher.start()
self.gcs_file_system = FakeGCSFileSystem()
self.mock_gcs_file_system = self.gcs_file_system_patcher.start()
self.mock_gcs_file_system.return_value = self.gcs_file_system
self.get_secret_patcher = patch("recidiviz.utils.secrets.get_secret")
self.get_secret_patcher.start()
self.state_code = StateCode.US_ID
self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.CASE_TRIAGE)
local_postgres_helpers.use_on_disk_postgresql_database(self.database_key)
self.officer = generate_fake_officer("officer_id_1", "*****@*****.**")
self.client_downgradable_high = generate_fake_client(
client_id="client_1",
supervising_officer_id=self.officer.external_id,
supervision_level=StateSupervisionLevel.HIGH,
last_assessment_date=date(2021, 1, 2),
assessment_score=1,
)
self.client_downgradable_medium_1 = generate_fake_client(
client_id="client_2",
supervising_officer_id=self.officer.external_id,
supervision_level=StateSupervisionLevel.MEDIUM,
last_assessment_date=date(2021, 1, 2),
assessment_score=1,
)
self.client_downgradable_medium_2 = generate_fake_client(
client_id="client_3",
supervising_officer_id=self.officer.external_id,
supervision_level=StateSupervisionLevel.MEDIUM,
last_assessment_date=date(2021, 1, 2),
assessment_score=1,
)
self.client_no_downgrade = generate_fake_client(
client_id="client_4",
supervising_officer_id=self.officer.external_id,
supervision_level=StateSupervisionLevel.HIGH,
last_assessment_date=date(2021, 1, 2),
assessment_score=100,
)
self.opportunities = [
generate_fake_etl_opportunity(
officer_id=self.officer.external_id,
person_external_id=client.person_external_id,
opportunity_type=OpportunityType.OVERDUE_DOWNGRADE,
opportunity_metadata={
"assessmentScore": client.assessment_score,
"latestAssessmentDate": str(client.most_recent_assessment_date),
"recommendedSupervisionLevel": "MINIMUM",
},
)
for client in [
self.client_downgradable_high,
self.client_downgradable_medium_1,
self.client_downgradable_medium_2,
]
]
with SessionFactory.using_database(self.database_key) as session:
session.expire_on_commit = False
session.add_all(
[
self.officer,
self.client_downgradable_high,
self.client_downgradable_medium_1,
self.client_downgradable_medium_2,
self.client_no_downgrade,
*self.opportunities,
]
)
self.top_opps_email_recipient_patcher = patch(
"recidiviz.reporting.data_retrieval._top_opps_email_recipient_addresses"
)
self.top_opps_email_recipient_patcher.start().return_value = [
self.officer.email_address
]
root = logging.getLogger()
root.setLevel(level)
if __name__ == '__main__':
arg_parser = _create_parser()
arguments = arg_parser.parse_args()
_configure_logging(arguments.log)
# TODO(#4386): Currently this starts a local postgres and talks to it as we aren't able to talk to a postgres
# instance in GCP from local. Long-term there are a few options:
# - Upload the files to a gcs bucket to be processed by the app
# - This is straightforward, but to see what is happening you have to go to GCP
# - Create an endpoint that takes the parsed `Report` object
# - This allows any conversion errors to show up locally, and persistence errors to be communicated back
# - However, then `Report` becomes an api and can't evolve as easily (which maybe is true no matter what once
# manifest files have been created?)
recidiviz.called_from_test = True
tmp_db_dir = local_postgres_helpers.start_on_disk_postgresql_database()
local_postgres_helpers.use_on_disk_postgresql_database(JusticeCountsBase)
manual_upload.ingest(arguments.manifest_file)
# Don't cleanup the database so that user can query the data afterward.
logging.info(
"To query the data, connect to the local database with `psql --dbname=recidiviz_test_db`"
)
logging.info("For future cleanup, the postgres data directory is at %s.",
tmp_db_dir)
def setUp(self) -> None:
local_postgres_helpers.use_on_disk_postgresql_database(
SQLAlchemyDatabaseKey.canonical_for_schema(SchemaType.STATE))
