def test_add_behavior_2(self): fakes.use_in_memory_sqlite_database(TestBase) parent = entities.Parent.new_with_defaults( full_name='Krusty the Clown', ) converter = TestSchemaEntityConverter() schema_parent = converter.convert(parent) session = SessionFactory.for_schema_base(TestBase) session.add(schema_parent) session.commit() parents = session.query(schema.Parent).all() self.assertEqual(len(parents), 1) root = entities.Root.new_with_defaults( type=RootType.SIMPSONS, ) db_root = converter.convert(root) db_root.parents.append(parents[0]) session.add(db_root) session.commit() roots = session.query(schema.Root).all() self.assertEqual(len(roots), 1) parents = session.query(schema.Parent).all() self.assertEqual(len(parents), 1)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(TestBase) session = SessionFactory.for_schema_base(TestBase) self.assertEqual(len(session.query(schema.Root).all()), 0) self.assertEqual(len(session.query(schema.Parent).all()), 0) self.assertEqual(len(session.query(schema.Child).all()), 0)
def setUp(self) -> None: self.metadata_patcher = mock.patch( "recidiviz.utils.metadata.project_id") self.mock_project_id_fn = self.metadata_patcher.start() self.mock_project_id_fn.return_value = "recidiviz-456" fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self): fakes.use_in_memory_sqlite_database(JailsBase) sample_views = [ { 'view_id': 'my_fake_view', 'view_query': 'SELECT NULL LIMIT 0' }, { 'view_id': 'my_other_fake_view', 'view_query': 'SELECT NULL LIMIT 0' }, ] self.mock_views = [ bqview.BigQueryView(**view) for view in sample_views ] self.mock_project_id = 'fake-recidiviz-project' self.mock_view_dataset_name = 'my_views_dataset' self.mock_dataset = bigquery.dataset.DatasetReference( self.mock_project_id, self.mock_view_dataset_name) self.bq_utils_patcher = mock.patch( 'recidiviz.calculator.query.county.view_manager.bq_utils') self.mock_bq_utils = self.bq_utils_patcher.start() self.client_patcher = mock.patch( 'recidiviz.calculator.query.county.view_manager.bq_utils.client') self.mock_client = self.client_patcher.start().return_value
def setUp(self) -> None: self.metadata_patcher = mock.patch("recidiviz.utils.metadata.project_id") self.mock_project_id_fn = self.metadata_patcher.start() self.mock_project_id_fn.return_value = "recidiviz-456" self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.JAILS) fakes.use_in_memory_sqlite_database(self.database_key)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(StateBase) self.get_region_patcher = patch( "recidiviz.persistence.entity_matching.state." "base_state_matching_delegate.get_region", new=self.get_fake_region) self.get_region_patcher.start() self.addCleanup(self.get_region_patcher.stop)
def run_scraper(args: argparse.Namespace) -> None: use_in_memory_sqlite_database(JailsBase) region_codes = validate_regions(args.region.split(",")) if not region_codes: sys.exit(1) failed_regions = [] valid_region_codes = cast(Set[str], region_codes) for region_code in valid_region_codes: logging.info("***") logging.info("***") logging.info("Starting scraper for region: [%s]", region_code) logging.info("***") logging.info("***") try: run_scraper_for_region(regions.get_region(region_code), args) except Exception: print(traceback.format_exc()) failed_regions.append(region_code) if failed_regions: logging.info("***") logging.info( "The following regions raised errors during scraping: " "[%s]", failed_regions, )
def setUp(self) -> None: self.database_key = create_autospec(SQLAlchemyDatabaseKey) self.database_key.declarative_meta = TestBase self.database_key.isolation_level = "SERIALIZABLE" self.database_key.poolclass = None fakes.use_in_memory_sqlite_database(self.database_key) with SessionFactory.using_database(self.database_key, autocommit=False) as session: self.assertEqual(len(session.query(schema.Root).all()), 0) self.assertEqual(len(session.query(schema.Parent).all()), 0) self.assertEqual(len(session.query(schema.Child).all()), 0)
def setUp(self) -> None: self.project_id = "recidiviz-456" self.region = "us_xx" self.project_id_patcher = patch("recidiviz.utils.metadata.project_id") self.project_id_patcher.start().return_value = self.project_id self.operations_database_key = SQLAlchemyDatabaseKey.for_schema( SchemaType.OPERATIONS) fakes.use_in_memory_sqlite_database(self.operations_database_key) self.us_xx_manager = DirectIngestInstanceStatusManager.add_instance( self.region, DirectIngestInstance.PRIMARY, is_paused=False)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(StateBase) # State persistence ends up having to instantiate the us_nd_controller to # get enum overrides, and the controller goes on to create bigquery, # storage, and tasks clients. self.bq_client_patcher = patch('google.cloud.bigquery.Client') self.storage_client_patcher = patch('google.cloud.storage.Client') self.task_client_patcher = patch('google.cloud.tasks_v2.CloudTasksClient') self.bq_client_patcher.start() self.storage_client_patcher.start() self.task_client_patcher.start()
def setUp(self) -> None: self.metadata_patcher = patch("recidiviz.utils.metadata.project_id") self.mock_project_id_fn = self.metadata_patcher.start() self.mock_project_id = "recidiviz-456" self.mock_project_id_fn.return_value = self.mock_project_id fakes.use_in_memory_sqlite_database(OperationsBase) self.client_patcher = patch( "recidiviz.big_query.big_query_client.BigQueryClient") self.mock_client = self.client_patcher.start().return_value project_id_mock = mock.PropertyMock(return_value=self.mock_project_id) type(self.mock_client).project_id = project_id_mock
def test_add_behavior(self): fakes.use_in_memory_sqlite_database(TestBase) session = SessionFactory.for_schema_base(TestBase) self.assertEqual(len(session.query(schema.Root).all()), 0) self.assertEqual(len(session.query(schema.Parent).all()), 0) self.assertEqual(len(session.query(schema.Child).all()), 0) parent = entities.Parent.new_with_defaults( full_name='Krusty the Clown', ) converter = TestSchemaEntityConverter() schema_parent = converter.convert(parent) session = SessionFactory.for_schema_base(TestBase) session.add(schema_parent) session.commit() parents = session.query(schema.Parent).all() self.assertEqual(len(parents), 1) child = entities.Child.new_with_defaults( full_name='Child name' ) schema_child = converter.convert(child) parents[0].children.append(schema_child) session.commit() children = session.query(schema.Child).all() self.assertEqual(len(children), 1) self.assertEqual(len(children[0].parents), 1) parent2 = entities.Parent.new_with_defaults( full_name='Krusty the Clown 2', ) children[0].parents = [ converter.convert(parent2) ] session.commit() parents = session.query(schema.Parent).all() self.assertEqual(len(parents), 2) children = session.query(schema.Child).all() self.assertEqual(len(children), 1) self.assertEqual(len(children[0].parents), 1)
def setUp(self) -> None: self.maxDiff = 250000 fakes.use_in_memory_sqlite_database(self.schema_base()) self.controller = build_gcsfs_controller_for_tests( self.controller_cls(), self.fixture_path_prefix(), run_async=False, max_delay_sec_between_files=0) # Set entity matching error threshold to a diminishingly small number # for tests. We cannot set it to 0 because we throw when errors *equal* # the error threshold. self.entity_matching_error_threshold_patcher = patch( 'recidiviz.persistence.persistence.ERROR_THRESHOLD', pow(1, -10)) self.entity_matching_error_threshold_patcher.start()
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(OperationsBase) self.metadata_manager = PostgresDirectIngestFileMetadataManager( region_code='us_xx') self.metadata_manager_other_region = PostgresDirectIngestFileMetadataManager( region_code='us_yy') def fake_eq(e1, e2) -> bool: def _should_ignore_field_cb(_: Type, field_name: str) -> bool: return field_name == 'file_id' return entity_graph_eq(e1, e2, _should_ignore_field_cb) self.entity_eq_patcher = patch( 'recidiviz.persistence.entity.operations.entities.OperationsEntity.__eq__', fake_eq) self.entity_eq_patcher.start()
def setUp(self) -> None: self.maxDiff = 250000 # TODO(3289): Fix hanging state table queries so we can use an on-disk postgres DB for the State/Jails schemas # as well. Currently, using postgres for StateBase causes a hang when we go t drop the tables in # stop_and_clear_on_disk_postgresql_database() fakes.use_in_memory_sqlite_database(self.schema_base()) fakes.use_on_disk_postgresql_database(OperationsBase) self.controller = build_gcsfs_controller_for_tests( self.controller_cls(), self.fixture_path_prefix(), run_async=False, max_delay_sec_between_files=0) # Set entity matching error threshold to a diminishingly small number # for tests. We cannot set it to 0 because we throw when errors *equal* # the error threshold. self.entity_matching_error_threshold_patcher = patch( 'recidiviz.persistence.persistence.ERROR_THRESHOLD', pow(1, -10)) self.entity_matching_error_threshold_patcher.start()
def run_scraper(args): use_in_memory_sqlite_database(JailsBase) region_codes = validate_regions(args.region.split(',')) if not region_codes: sys.exit(1) failed_regions = [] for region_code in region_codes: logging.info('***') logging.info('***') logging.info("Starting scraper for region: [%s]", region_code) logging.info('***') logging.info('***') try: run_scraper_for_region(regions.get_region(region_code), args) except Exception: print(traceback.format_exc()) failed_regions.append(region_code) if failed_regions: logging.info('***') logging.info("The following regions raised errors during scraping: " "[%s]", failed_regions)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(JailsBase)
def setup_method(self, _test_method): fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None: self.client = app.test_client() fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(OperationsBase) self.client_patcher = patch('recidiviz.big_query.big_query_client.BigQueryClient') self.mock_client = self.client_patcher.start().return_value project_id_mock = mock.PropertyMock(return_value='recidiviz-456') type(self.mock_client).project_id = project_id_mock
def setup_method(self, _test_method): self.client = app.test_client() fakes.use_in_memory_sqlite_database(JailsBase)
def setUp(self) -> None: self.database_key = SQLAlchemyDatabaseKey.for_schema(SchemaType.JAILS) fakes.use_in_memory_sqlite_database(self.database_key)
def setUp(self) -> None: fakes.use_in_memory_sqlite_database(OperationsBase)