def run_parse_file_test(self, expected: IngestInfo,
fixture_file_name: str) -> IngestInfo:
"""Runs a test that reads and parses a given fixture file. Returns the
parsed IngestInfo object for tests to run further validations."""
args = ingest_args_for_fixture_file(self.controller,
f'{fixture_file_name}.csv')
if not isinstance(self.controller.fs, FakeDirectIngestGCSFileSystem):
raise ValueError(f"Controller fs must have type "
f"FakeDirectIngestGCSFileSystem. Found instead "
f"type [{type(self.controller.fs)}]")
self.controller.fs.test_add_path(args.file_path)
# pylint:disable=protected-access
fixture_contents_handle = self.controller._get_contents_handle(args)
final_info = self.controller._parse(args, fixture_contents_handle)
print_visible_header_label('FINAL')
print(final_info)
print_visible_header_label('EXPECTED')
print(expected)
self.assertEqual(expected, final_info)
return final_info
def run_parse_file_test(self, expected: IngestInfo,
fixture_file_name: str) -> IngestInfo:
"""Runs a test that reads and parses a given fixture file. Returns the
parsed IngestInfo object for tests to run further validations."""
args = ingest_args_for_fixture_file(self.controller,
f"{fixture_file_name}.csv")
if not isinstance(self.controller.fs.gcs_file_system,
FakeGCSFileSystem):
raise ValueError(
f"Controller fs must have type "
f"FakeGCSFileSystem. Found instead "
f"type [{type(self.controller.fs.gcs_file_system)}]")
if self.controller.region.is_ingest_launched_in_env():
now = datetime.datetime.now()
yesterday = now - datetime.timedelta(days=1)
ingest_file_export_job_args = GcsfsIngestViewExportArgs(
ingest_view_name=fixture_file_name,
upper_bound_datetime_to_export=now,
upper_bound_datetime_prev=yesterday,
output_bucket_name=self.controller.ingest_bucket_path.
bucket_name,
)
self.controller.file_metadata_manager.register_ingest_file_export_job(
ingest_file_export_job_args)
self.controller.ingest_view_export_manager.export_view_for_args(
ingest_file_export_job_args)
else:
fixture_util.add_direct_ingest_path(
self.controller.fs.gcs_file_system,
args.file_path,
region_code=self.controller.region_code(),
)
# pylint:disable=protected-access
fixture_contents_handle = self.controller._get_contents_handle(args)
if fixture_contents_handle is None:
self.fail("fixture_contents_handle should not be None")
final_info = self.controller._parse(args, fixture_contents_handle)
print_visible_header_label("FINAL")
print(final_info)
print_visible_header_label("EXPECTED")
print(expected)
self.assertEqual(expected, final_info)
return final_info
def assert_people_match(self,
expected_people: List[StatePerson],
matched_people: List[schema.StatePerson],
debug: bool = False):
converted_matched = converter.convert_schema_objects_to_entity(
matched_people)
db_expected_with_backedges = converter.convert_entity_people_to_schema_people(
expected_people)
expected_with_backedges = converter.convert_schema_objects_to_entity(
db_expected_with_backedges)
if debug:
print_visible_header_label('EXPECTED')
print_entity_trees(expected_with_backedges)
print_visible_header_label('FINAL')
print_entity_trees(converted_matched)
self.assertEqual(expected_with_backedges, converted_matched)
def run_parse_file_test(self, expected: IngestInfo,
fixture_file_name: str) -> IngestInfo:
"""Runs a test that reads and parses a given fixture file. Returns the
parsed IngestInfo object for tests to run further validations."""
args = ingest_args_for_fixture_file(self.controller,
f'{fixture_file_name}.csv')
if not isinstance(self.controller.fs, FakeDirectIngestGCSFileSystem):
raise ValueError(f"Controller fs must have type "
f"FakeDirectIngestGCSFileSystem. Found instead "
f"type [{type(self.controller.fs)}]")
if self.controller.region.are_ingest_view_exports_enabled_in_env():
ingest_file_export_job_args = GcsfsIngestViewExportArgs(
ingest_view_name=fixture_file_name,
upper_bound_datetime_to_export=datetime.datetime.utcnow(),
upper_bound_datetime_prev=None)
self.controller.file_metadata_manager.register_ingest_file_export_job(
ingest_file_export_job_args)
self.controller.ingest_view_export_manager.export_view_for_args(
ingest_file_export_job_args)
else:
self.controller.fs.test_add_path(args.file_path)
# pylint:disable=protected-access
fixture_contents_handle = self.controller._get_contents_handle(args)
final_info = self.controller._parse(args, fixture_contents_handle)
print_visible_header_label('FINAL')
print(final_info)
print_visible_header_label('EXPECTED')
print(expected)
self.assertEqual(expected, final_info)
return final_info
def assert_expected_db_people(
self,
expected_db_people: List[StatePerson],
debug: bool = False,
single_person_to_debug: Optional[str] = None,
# TODO(#2492): Once we properly clean up dangling placeholders,
# delete this.
ignore_dangling_placeholders: bool = False,
print_tree_structure_only: bool = False,
) -> None:
"""Asserts that the set of expected people matches all the people that currently exist in the database.
Args:
debug: (bool) If true, prints out both the found and expected entity trees.
single_person_to_debug: (str) A string external_id of a person. If debug=True and this is not None, this
will only check for equality between the people with that external_id. This should be used for debugging
only and this function will throw if this value is set in CI.
ignore_dangling_placeholders: (bool) If True, eliminates dangling placeholder objects (i.e. placeholders
with no non-placeholder children) from both the result and expected trees before doing a comparison.
print_tree_structure_only: (bool) If True and debug=True, then the printed result only shows the tree
structure - external ids and parent-child relationships.
"""
if debug:
print("\n\n************** ASSERTING *************")
session = SessionFactory.for_schema_base(StateBase)
found_people_from_db = dao.read_people(session)
found_people = cast(
List[StatePerson],
self.convert_and_clear_db_ids(found_people_from_db))
if ignore_dangling_placeholders:
pruned_found_people = []
for person in found_people:
pruned_person = cast(
StatePerson, prune_dangling_placeholders_from_tree(person))
if pruned_person is not None:
pruned_found_people.append(pruned_person)
found_people = pruned_found_people
pruned_expected_people: List[StatePerson] = []
for person in expected_db_people:
pruned_expected_person = cast(
StatePerson, prune_dangling_placeholders_from_tree(person))
if pruned_expected_person is not None:
pruned_expected_people.append(pruned_expected_person)
expected_db_people = pruned_expected_people
if debug:
if is_running_in_ci():
self.fail(
"The |debug| flag should only be used for local debugging."
)
if single_person_to_debug is not None:
found_people = [
p for p in found_people
if person_has_id(p, single_person_to_debug)
]
expected_db_people = [
p for p in expected_db_people
if person_has_id(p, single_person_to_debug)
]
print_visible_header_label("FINAL")
print_entity_trees(
found_people,
print_tree_structure_only=print_tree_structure_only)
print_visible_header_label("EXPECTED")
print_entity_trees(
expected_db_people,
print_tree_structure_only=print_tree_structure_only)
self.assertCountEqual(found_people, expected_db_people)
assert_no_unexpected_entities_in_db(found_people_from_db, session)