def test_add_behavior_2(self):
fakes.use_in_memory_sqlite_database(TestBase)
parent = entities.Parent.new_with_defaults(
full_name='Krusty the Clown',
)
converter = TestSchemaEntityConverter()
schema_parent = converter.convert(parent)
session = SessionFactory.for_schema_base(TestBase)
session.add(schema_parent)
session.commit()
parents = session.query(schema.Parent).all()
self.assertEqual(len(parents), 1)
root = entities.Root.new_with_defaults(
type=RootType.SIMPSONS,
)
db_root = converter.convert(root)
db_root.parents.append(parents[0])
session.add(db_root)
session.commit()
roots = session.query(schema.Root).all()
self.assertEqual(len(roots), 1)
parents = session.query(schema.Parent).all()
self.assertEqual(len(parents), 1)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(TestBase)
session = SessionFactory.for_schema_base(TestBase)
self.assertEqual(len(session.query(schema.Root).all()), 0)
self.assertEqual(len(session.query(schema.Parent).all()), 0)
self.assertEqual(len(session.query(schema.Child).all()), 0)
def setUp(self) -> None:
self.metadata_patcher = mock.patch(
"recidiviz.utils.metadata.project_id")
self.mock_project_id_fn = self.metadata_patcher.start()
self.mock_project_id_fn.return_value = "recidiviz-456"
fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self):
fakes.use_in_memory_sqlite_database(JailsBase)
sample_views = [
{
'view_id': 'my_fake_view',
'view_query': 'SELECT NULL LIMIT 0'
},
{
'view_id': 'my_other_fake_view',
'view_query': 'SELECT NULL LIMIT 0'
},
]
self.mock_views = [
bqview.BigQueryView(**view) for view in sample_views
]
self.mock_project_id = 'fake-recidiviz-project'
self.mock_view_dataset_name = 'my_views_dataset'
self.mock_dataset = bigquery.dataset.DatasetReference(
self.mock_project_id, self.mock_view_dataset_name)
self.bq_utils_patcher = mock.patch(
'recidiviz.calculator.query.county.view_manager.bq_utils')
self.mock_bq_utils = self.bq_utils_patcher.start()
self.client_patcher = mock.patch(
'recidiviz.calculator.query.county.view_manager.bq_utils.client')
self.mock_client = self.client_patcher.start().return_value
def setUp(self) -> None:
self.metadata_patcher = mock.patch("recidiviz.utils.metadata.project_id")
self.mock_project_id_fn = self.metadata_patcher.start()
self.mock_project_id_fn.return_value = "recidiviz-456"
self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.JAILS)
fakes.use_in_memory_sqlite_database(self.database_key)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(StateBase)
self.get_region_patcher = patch(
"recidiviz.persistence.entity_matching.state."
"base_state_matching_delegate.get_region", new=self.get_fake_region)
self.get_region_patcher.start()
self.addCleanup(self.get_region_patcher.stop)
def run_scraper(args: argparse.Namespace) -> None:
use_in_memory_sqlite_database(JailsBase)
region_codes = validate_regions(args.region.split(","))
if not region_codes:
sys.exit(1)
failed_regions = []
valid_region_codes = cast(Set[str], region_codes)
for region_code in valid_region_codes:
logging.info("***")
logging.info("***")
logging.info("Starting scraper for region: [%s]", region_code)
logging.info("***")
logging.info("***")
try:
run_scraper_for_region(regions.get_region(region_code), args)
except Exception:
print(traceback.format_exc())
failed_regions.append(region_code)
if failed_regions:
logging.info("***")
logging.info(
"The following regions raised errors during scraping: " "[%s]",
failed_regions,
)
def setUp(self) -> None:
self.database_key = create_autospec(SQLAlchemyDatabaseKey)
self.database_key.declarative_meta = TestBase
self.database_key.isolation_level = "SERIALIZABLE"
self.database_key.poolclass = None
fakes.use_in_memory_sqlite_database(self.database_key)
with SessionFactory.using_database(self.database_key,
autocommit=False) as session:
self.assertEqual(len(session.query(schema.Root).all()), 0)
self.assertEqual(len(session.query(schema.Parent).all()), 0)
self.assertEqual(len(session.query(schema.Child).all()), 0)
def setUp(self) -> None:
self.project_id = "recidiviz-456"
self.region = "us_xx"
self.project_id_patcher = patch("recidiviz.utils.metadata.project_id")
self.project_id_patcher.start().return_value = self.project_id
self.operations_database_key = SQLAlchemyDatabaseKey.for_schema(
SchemaType.OPERATIONS)
fakes.use_in_memory_sqlite_database(self.operations_database_key)
self.us_xx_manager = DirectIngestInstanceStatusManager.add_instance(
self.region, DirectIngestInstance.PRIMARY, is_paused=False)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(StateBase)
# State persistence ends up having to instantiate the us_nd_controller to
# get enum overrides, and the controller goes on to create bigquery,
# storage, and tasks clients.
self.bq_client_patcher = patch('google.cloud.bigquery.Client')
self.storage_client_patcher = patch('google.cloud.storage.Client')
self.task_client_patcher = patch('google.cloud.tasks_v2.CloudTasksClient')
self.bq_client_patcher.start()
self.storage_client_patcher.start()
self.task_client_patcher.start()
def setUp(self) -> None:
self.metadata_patcher = patch("recidiviz.utils.metadata.project_id")
self.mock_project_id_fn = self.metadata_patcher.start()
self.mock_project_id = "recidiviz-456"
self.mock_project_id_fn.return_value = self.mock_project_id
fakes.use_in_memory_sqlite_database(OperationsBase)
self.client_patcher = patch(
"recidiviz.big_query.big_query_client.BigQueryClient")
self.mock_client = self.client_patcher.start().return_value
project_id_mock = mock.PropertyMock(return_value=self.mock_project_id)
type(self.mock_client).project_id = project_id_mock
def test_add_behavior(self):
fakes.use_in_memory_sqlite_database(TestBase)
session = SessionFactory.for_schema_base(TestBase)
self.assertEqual(len(session.query(schema.Root).all()), 0)
self.assertEqual(len(session.query(schema.Parent).all()), 0)
self.assertEqual(len(session.query(schema.Child).all()), 0)
parent = entities.Parent.new_with_defaults(
full_name='Krusty the Clown',
)
converter = TestSchemaEntityConverter()
schema_parent = converter.convert(parent)
session = SessionFactory.for_schema_base(TestBase)
session.add(schema_parent)
session.commit()
parents = session.query(schema.Parent).all()
self.assertEqual(len(parents), 1)
child = entities.Child.new_with_defaults(
full_name='Child name'
)
schema_child = converter.convert(child)
parents[0].children.append(schema_child)
session.commit()
children = session.query(schema.Child).all()
self.assertEqual(len(children), 1)
self.assertEqual(len(children[0].parents), 1)
parent2 = entities.Parent.new_with_defaults(
full_name='Krusty the Clown 2',
)
children[0].parents = [
converter.convert(parent2)
]
session.commit()
parents = session.query(schema.Parent).all()
self.assertEqual(len(parents), 2)
children = session.query(schema.Child).all()
self.assertEqual(len(children), 1)
self.assertEqual(len(children[0].parents), 1)
def setUp(self) -> None:
self.maxDiff = 250000
fakes.use_in_memory_sqlite_database(self.schema_base())
self.controller = build_gcsfs_controller_for_tests(
self.controller_cls(),
self.fixture_path_prefix(),
run_async=False,
max_delay_sec_between_files=0)
# Set entity matching error threshold to a diminishingly small number
# for tests. We cannot set it to 0 because we throw when errors *equal*
# the error threshold.
self.entity_matching_error_threshold_patcher = patch(
'recidiviz.persistence.persistence.ERROR_THRESHOLD', pow(1, -10))
self.entity_matching_error_threshold_patcher.start()
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(OperationsBase)
self.metadata_manager = PostgresDirectIngestFileMetadataManager(
region_code='us_xx')
self.metadata_manager_other_region = PostgresDirectIngestFileMetadataManager(
region_code='us_yy')
def fake_eq(e1, e2) -> bool:
def _should_ignore_field_cb(_: Type, field_name: str) -> bool:
return field_name == 'file_id'
return entity_graph_eq(e1, e2, _should_ignore_field_cb)
self.entity_eq_patcher = patch(
'recidiviz.persistence.entity.operations.entities.OperationsEntity.__eq__',
fake_eq)
self.entity_eq_patcher.start()
def setUp(self) -> None:
self.maxDiff = 250000
# TODO(3289): Fix hanging state table queries so we can use an on-disk postgres DB for the State/Jails schemas
# as well. Currently, using postgres for StateBase causes a hang when we go t drop the tables in
# stop_and_clear_on_disk_postgresql_database()
fakes.use_in_memory_sqlite_database(self.schema_base())
fakes.use_on_disk_postgresql_database(OperationsBase)
self.controller = build_gcsfs_controller_for_tests(
self.controller_cls(),
self.fixture_path_prefix(),
run_async=False,
max_delay_sec_between_files=0)
# Set entity matching error threshold to a diminishingly small number
# for tests. We cannot set it to 0 because we throw when errors *equal*
# the error threshold.
self.entity_matching_error_threshold_patcher = patch(
'recidiviz.persistence.persistence.ERROR_THRESHOLD', pow(1, -10))
self.entity_matching_error_threshold_patcher.start()
def run_scraper(args):
use_in_memory_sqlite_database(JailsBase)
region_codes = validate_regions(args.region.split(','))
if not region_codes:
sys.exit(1)
failed_regions = []
for region_code in region_codes:
logging.info('***')
logging.info('***')
logging.info("Starting scraper for region: [%s]", region_code)
logging.info('***')
logging.info('***')
try:
run_scraper_for_region(regions.get_region(region_code), args)
except Exception:
print(traceback.format_exc())
failed_regions.append(region_code)
if failed_regions:
logging.info('***')
logging.info("The following regions raised errors during scraping: "
"[%s]",
failed_regions)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(JailsBase)
def setup_method(self, _test_method):
fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None:
self.client = app.test_client()
fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(OperationsBase)
self.client_patcher = patch('recidiviz.big_query.big_query_client.BigQueryClient')
self.mock_client = self.client_patcher.start().return_value
project_id_mock = mock.PropertyMock(return_value='recidiviz-456')
type(self.mock_client).project_id = project_id_mock
def setup_method(self, _test_method):
self.client = app.test_client()
fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None:
self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.JAILS)
fakes.use_in_memory_sqlite_database(self.database_key)
def setUp(self) -> None:
fakes.use_in_memory_sqlite_database(OperationsBase)
