def test_persist_to_db_different_regions(self, mock_write, _mock_region,
                                             mock_session_return):
        scrape_key1 = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)
        scrape_key2 = ScrapeKey(REGIONS[1], constants.ScrapeType.BACKGROUND)

        ii = ingest_info.IngestInfo()
        ii.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME).create_booking(booking_id=TEST_ID)

        ii2 = ingest_info.IngestInfo()
        ii2.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME2).create_booking(booking_id=TEST_ID)

        t = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        t2 = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        mock_session_1 = mock_session_return.return_value = create_mock_session(
        )

        batch_persistence.write(ii, scrape_key1, t)
        expected_proto = serialization.convert_ingest_info_to_proto(ii)
        batch_persistence.persist_to_database(scrape_key1.region_code,
                                              mock_session_1.start)

        result_proto = mock_write.call_args[0][0]
        self.assertEqual(result_proto, expected_proto)

        # We expect the region that we persisted to have no more ingest infos.
        ingest_infos_1 = datastore_ingest_info.batch_get_ingest_infos_for_region(
            REGIONS[0], mock_session_1.start)
        self.assertEqual(len(ingest_infos_1), 0)

        mock_session_2 = mock_session_return.return_value = create_mock_session(
        )

        batch_persistence.write(ii2, scrape_key2, t2)
        ingest_infos_2 = datastore_ingest_info.batch_get_ingest_infos_for_region(
            REGIONS[1], mock_session_2.start)
        self.assertEqual(len(ingest_infos_2), 1)

        expected_proto = serialization.convert_ingest_info_to_proto(ii2)
        batch_persistence.persist_to_database(scrape_key2.region_code,
                                              mock_session_2.start)

        result_proto = mock_write.call_args[0][0]
        self.assertEqual(result_proto, expected_proto)

        self.assertEqual(mock_write.call_count, 2)
    def test_write_to_datastore(self, mock_session_return):
        mock_session = mock_session_return.return_value = create_mock_session()

        scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)

        ii = ingest_info.IngestInfo()
        ii.create_person(full_name=TEST_NAME).create_booking(
            booking_id=TEST_ID)

        t = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )
        task_hash = hash(json.dumps(t.to_serializable(), sort_keys=True))

        expected_batch = BatchIngestInfoData(ingest_info=ii,
                                             task_hash=task_hash)

        batch_persistence.write(ii, scrape_key, t)

        batch_ingest_info_list = batch_persistence._get_batch_ingest_info_list(
            scrape_key.region_code, mock_session.start)

        self.assertEqual(len(batch_ingest_info_list), 1)
        self.assertEqual(expected_batch, batch_ingest_info_list[0])
    def test_persist_to_db(self, mock_write, _mock_region,
                           mock_session_return):
        mock_session = mock_session_return.return_value = create_mock_session()
        scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)

        ii = ingest_info.IngestInfo()
        ii.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME).create_booking(booking_id=TEST_ID)

        t = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        batch_persistence.write(ii, scrape_key, t)

        expected_proto = serialization.convert_ingest_info_to_proto(ii)

        batch_persistence.persist_to_database(scrape_key.region_code,
                                              mock_session.start)

        result_proto = mock_write.call_args[0][0]
        self.assertEqual(result_proto, expected_proto)

        # After we persist, there should no longer be ingest infos on Datastore
        ingest_infos = datastore_ingest_info.batch_get_ingest_infos_for_region(
            REGIONS[0], mock_session.start)
        self.assertEqual(len(ingest_infos), 0)
Exemple #4
0
    def validate_and_return_populate_data(self,
                                          content,
                                          expected_ingest_info=None,
                                          expected_single_counts=None,
                                          expected_persist=True,
                                          task=None,
                                          info=None):
        """This function runs populate_data and runs some extra validation
        on the output.

        Args:
            content: the content of the page to pass into get_more_tasks
            expected_ingest_info: the ingest info expected to be returned from
                `populate_data`. If `expected_ingest_info` is `None`, then
                expects the return value of `populate_data` to be `None`.
            expected_single_counts: the list of SingleCounts expected to be
            returned from `populate_data`.
            expected_persist: the expected value of persist to be returned from
                `populate_data`.
            task: the task that is being processed, optional.
            info: an ingest_info to use if provided.

        Returns:
            The result from populate_data in case the user needs to do any
            extra validations on the output.
        """
        info = info or ingest_info.IngestInfo()
        task = task or Task(task_type=constants.TaskType.SCRAPE_DATA,
                            endpoint='')

        scrape_data = self.scraper.populate_data(content, task, info)

        print('FINAL')
        print(scrape_data.ingest_info)
        print('EXPECTED')
        print(expected_ingest_info)

        if expected_ingest_info is None and expected_single_counts is None:
            if scrape_data:
                self.assertFalse(scrape_data.persist)
            else:
                self.assertIsNone(scrape_data)

        self.assertCountEqual(scrape_data.single_counts, expected_single_counts
                              or [])

        metadata = IngestMetadata(self.scraper.region.region_code,
                                  self.scraper.region.jurisdiction_id,
                                  _FAKE_SCRAPER_START_TIME,
                                  self.scraper.get_enum_overrides())

        self.validate_ingest(scrape_data.ingest_info, expected_ingest_info,
                             metadata)

        assert scrape_data.persist == expected_persist

        return scrape_data
Exemple #5
0
    def test_multipleOpenBookings_raisesPersistenceError(self):
        ingest_info = ii.IngestInfo()
        person = ingest_info.create_person(full_name=FULL_NAME_1)
        person.create_booking(admission_date=DATE_RAW)
        person.create_booking(admission_date=DATE_RAW)

        self.assertFalse(
            persistence.write(convert_ingest_info_to_proto(ingest_info),
                              DEFAULT_METADATA))
    def test_yaml_is_correct(self) -> None:
        if self.yaml:
            with open(self.yaml, "r") as ymlfile:
                manifest = yaml.full_load(ymlfile)

            person = ingest_info.IngestInfo().create_person()
            booking = person.create_booking()
            charge = booking.create_charge()
            arrest = booking.create_arrest()
            sentence = charge.create_sentence()
            bond = charge.create_bond()
            object_verification_map = {
                "person": person,
                "booking": booking,
                "charge": charge,
                "arrest": arrest,
                "bond": bond,
                "sentence": sentence,
            }

            # Validate that key_mappings exists
            if "key_mappings" not in manifest:
                raise AttributeError("key_mappings must exist in the manifest")

            # Make sure there are no unknown keys
            for key in manifest:
                if key not in [
                        "key_mappings",
                        "multi_key_mapping",
                        "keys_to_ignore",
                        "css_key_mappings",
                ]:
                    raise AttributeError("Unknown yaml key %s" % key)

            # Make sure every mapped value in the yaml file exists as a variable
            # in the relevant class.
            for value in manifest["key_mappings"].values():
                class_to_set, attr = value.split(".")
                if attr not in vars(object_verification_map[class_to_set]):
                    raise AttributeError(
                        "Attribute %s is unknown on %s, found in key_mappings"
                        % (attr, class_to_set))

            if "multi_key_mappings" in manifest:
                for value in manifest["multi_key_mappings"].values():
                    class_to_set, attr = value.split(".")
                    if attr not in vars(object_verification_map[class_to_set]):
                        raise AttributeError(
                            "Attribute %s is unknown on %s, found in "
                            "multi_key_mappings" % (attr, class_to_set))
    def test_persist_duplicates_to_db(self, mock_write, _mock_region,
                                      mock_session_return):
        """Tests that duplicate ingest_info.Person objects are merged before
        write."""
        mock_session = mock_session_return.return_value = create_mock_session()
        scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)

        # Arrange
        ii = ingest_info.IngestInfo()
        ii.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME).create_booking(booking_id=TEST_ID)

        ii_2 = ingest_info.IngestInfo()
        ii.create_person(person_id=TEST_ID2, full_name=TEST_NAME2)

        ii_1_dup = copy.deepcopy(ii)

        t1, t2, t3 = (Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT + str(i),
            response_type=constants.ResponseType.TEXT,
        ) for i in range(3))

        batch_persistence.write(ii, scrape_key, t1)
        batch_persistence.write(ii_2, scrape_key, t2)
        batch_persistence.write(ii_1_dup, scrape_key, t3)

        batch_persistence.persist_to_database(scrape_key.region_code,
                                              mock_session.start)

        expected_ii = ingest_info.IngestInfo(people=ii.people + ii_2.people)
        expected_proto = serialization.convert_ingest_info_to_proto(
            expected_ii)
        result_proto = mock_write.call_args[0][0]
        self.assertEqual(result_proto, expected_proto)
Exemple #8
0
    def test_yaml_is_correct(self):
        if self.yaml:
            with open(self.yaml, 'r') as ymlfile:
                manifest = yaml.full_load(ymlfile)

            person = ingest_info.IngestInfo().create_person()
            booking = person.create_booking()
            charge = booking.create_charge()
            arrest = booking.create_arrest()
            sentence = charge.create_sentence()
            bond = charge.create_bond()
            object_verification_map = {
                'person': person,
                'booking': booking,
                'charge': charge,
                'arrest': arrest,
                'bond': bond,
                'sentence': sentence
            }

            # Validate that key_mappings exists
            if 'key_mappings' not in manifest:
                raise AttributeError("key_mappings must exist in the manifest")

            # Make sure there are no unknown keys
            for key in manifest:
                if (key not in [
                        'key_mappings', 'multi_key_mapping', 'keys_to_ignore',
                        'css_key_mappings'
                ]):
                    raise AttributeError("Unknown yaml key %s" % key)

            # Make sure every mapped value in the yaml file exists as a variable
            # in the relevant class.
            for value in manifest['key_mappings'].values():
                class_to_set, attr = value.split('.')
                if attr not in vars(object_verification_map[class_to_set]):
                    raise AttributeError(
                        "Attribute %s is unknown on %s, found in key_mappings"
                        % (attr, class_to_set))

            if 'multi_key_mappings' in manifest:
                for value in manifest['multi_key_mappings'].values():
                    class_to_set, attr = value.split('.')
                    if attr not in vars(object_verification_map[class_to_set]):
                        raise AttributeError(
                            "Attribute %s is unknown on %s, found in "
                            "multi_key_mappings" % (attr, class_to_set))
    def test_convert_ingest_info_one_charge_to_one_bond(
        self, mock_create: Mock
    ) -> None:
        mock_create.side_effect = self._create_generated_id
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = "id1"

        booking = person.create_booking()
        booking.booking_id = "id1"
        charge = booking.create_charge()
        charge.charge_id = "id1"
        bond1 = charge.create_bond()
        bond1.amount = "$1"
        charge = booking.create_charge()
        charge.charge_id = "id2"
        bond2 = charge.create_bond()
        bond2.amount = "$1"

        expected_proto = ingest_info_pb2.IngestInfo()
        proto_person = expected_proto.people.add()
        proto_person.person_id = "id1"
        proto_person.booking_ids.append("id1")
        proto_booking = expected_proto.bookings.add()
        proto_booking.booking_id = "id1"
        proto_booking.charge_ids.extend(["id1", "id2"])
        proto_charge = expected_proto.charges.add()
        proto_charge.charge_id = "id1"
        proto_bond1 = expected_proto.bonds.add()
        proto_bond1.amount = "$1"
        proto_bond1.bond_id = "1_GENERATE"
        proto_charge.bond_id = proto_bond1.bond_id
        proto_charge = expected_proto.charges.add()
        proto_charge.charge_id = "id2"
        proto_bond2 = expected_proto.bonds.add()
        proto_bond2.amount = "$1"
        proto_bond2.bond_id = "2_GENERATE"
        proto_charge.bond_id = proto_bond2.bond_id

        proto = serialization.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        info_back = serialization.convert_proto_to_ingest_info(proto)
        assert info_back == info
    def test_convert_ingest_info_one_charge_to_one_bond(self, mock_create):
        mock_create.side_effect = self._create_generated_id
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = 'id1'

        booking = person.create_booking()
        booking.booking_id = 'id1'
        charge = booking.create_charge()
        charge.charge_id = 'id1'
        bond1 = charge.create_bond()
        bond1.amount = '$1'
        charge = booking.create_charge()
        charge.charge_id = 'id2'
        bond2 = charge.create_bond()
        bond2.amount = '$1'

        expected_proto = ingest_info_pb2.IngestInfo()
        person = expected_proto.people.add()
        person.person_id = 'id1'
        person.booking_ids.append('id1')
        booking = expected_proto.bookings.add()
        booking.booking_id = 'id1'
        booking.charge_ids.extend(['id1', 'id2'])
        charge = expected_proto.charges.add()
        charge.charge_id = 'id1'
        proto_bond1 = expected_proto.bonds.add()
        proto_bond1.amount = '$1'
        proto_bond1.bond_id = '1_GENERATE'
        charge.bond_id = proto_bond1.bond_id
        charge = expected_proto.charges.add()
        charge.charge_id = 'id2'
        proto_bond2 = expected_proto.bonds.add()
        proto_bond2.amount = '$1'
        proto_bond2.bond_id = '2_GENERATE'
        charge.bond_id = proto_bond2.bond_id

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == info
    def test_convert_ingest_info_id_is_generated(self, mock_create):
        mock_create.side_effect = self._create_generated_id
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.surname = "testname"
        person.create_booking()

        expected_proto = ingest_info_pb2.IngestInfo()
        proto_person = expected_proto.people.add()
        proto_person.surname = "testname"
        proto_person.person_id = "1_GENERATE"
        proto_booking = expected_proto.bookings.add()
        proto_booking.booking_id = "2_GENERATE"
        proto_person.booking_ids.append(proto_booking.booking_id)

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert proto == expected_proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == info
    def test_serializable(self):
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = 'id1'

        booking = person.create_booking()
        booking.booking_id = 'id1'
        charge = booking.create_charge()
        charge.charge_id = 'id1'
        bond1 = charge.create_bond()
        bond1.amount = '$1'
        charge = booking.create_charge()
        charge.charge_id = 'id2'
        bond2 = charge.create_bond()
        bond2.amount = '$1'

        converted_info = ingest_utils.ingest_info_from_serializable(
            ingest_utils.ingest_info_to_serializable(info))

        assert converted_info == info
    def test_persist_to_db_same_task_one_fail_one_pass(self, mock_write,
                                                       _mock_region,
                                                       mock_session_return):
        mock_session = mock_session_return.return_value = create_mock_session()
        scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)
        mock_write.return_value = True

        ii = ingest_info.IngestInfo()
        ii.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME).create_booking(booking_id=TEST_ID)

        t = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        # Because the tasks are the same, we expect that to be counted as a
        # pass.
        t2 = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        batch_persistence.write(ii, scrape_key, t)
        batch_persistence.write_error(TEST_ERROR, TEST_TRACE, t2, scrape_key)

        expected_proto = serialization.convert_ingest_info_to_proto(ii)

        self.assertTrue(
            batch_persistence.persist_to_database(scrape_key.region_code,
                                                  mock_session.start))

        result_proto = mock_write.call_args[0][0]
        self.assertEqual(result_proto, expected_proto)

        ingest_infos = datastore_ingest_info.batch_get_ingest_infos_for_region(
            REGIONS[0], mock_session.start)
        self.assertEqual(len(ingest_infos), 0)
    def test_serializable(self) -> None:
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = "id1"

        booking = person.create_booking()
        booking.booking_id = "id1"
        charge = booking.create_charge()
        charge.charge_id = "id1"
        bond1 = charge.create_bond()
        bond1.amount = "$1"
        charge = booking.create_charge()
        charge.charge_id = "id2"
        bond2 = charge.create_bond()
        bond2.amount = "$1"

        converted_info = serialization.ingest_info_from_serializable(
            serialization.ingest_info_to_serializable(info)
        )

        assert converted_info == info
    def test_convert_ingest_info_many_charge_to_one_bond(self, mock_create):
        mock_create.side_effect = self._create_generated_id
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = "id1"

        booking = person.create_booking()
        booking.booking_id = "id1"
        charge = booking.create_charge()
        charge.charge_id = "id1"
        bond1 = charge.create_bond()
        bond1.amount = "$1"
        charge = booking.create_charge()
        charge.charge_id = "id2"
        charge.bond = bond1

        expected_proto = ingest_info_pb2.IngestInfo()
        person = expected_proto.people.add()
        person.person_id = "id1"
        person.booking_ids.append("id1")
        booking = expected_proto.bookings.add()
        booking.booking_id = "id1"
        booking.charge_ids.extend(["id1", "id2"])
        charge = expected_proto.charges.add()
        charge.charge_id = "id1"
        proto_bond = expected_proto.bonds.add()
        proto_bond.amount = "$1"
        proto_bond.bond_id = "1_GENERATE"
        charge.bond_id = proto_bond.bond_id
        charge = expected_proto.charges.add()
        charge.charge_id = "id2"
        charge.bond_id = proto_bond.bond_id

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert len(proto.bonds) == 1
        assert expected_proto == proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == info
    def test_persist_to_db_failed_no_write(self, mock_write, _mock_region,
                                           mock_session_return):
        mock_session = mock_session_return.return_value = create_mock_session()
        scrape_key = ScrapeKey(REGIONS[0], constants.ScrapeType.BACKGROUND)

        ii = ingest_info.IngestInfo()
        ii.create_person(
            person_id=TEST_ID,
            full_name=TEST_NAME).create_booking(booking_id=TEST_ID)

        t = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
        )

        # Because the tasks are different, we should fail.
        t2 = Task(
            task_type=constants.TaskType.SCRAPE_DATA,
            endpoint=TEST_ENDPOINT,
            response_type=constants.ResponseType.TEXT,
            params=TEST_PARAMS,
        )

        batch_persistence.write(ii, scrape_key, t)
        batch_persistence.write_error(TEST_ERROR, TEST_TRACE, t2, scrape_key)

        self.assertFalse(
            batch_persistence.persist_to_database(scrape_key.region_code,
                                                  mock_session.start))

        self.assertEqual(mock_write.call_count, 0)

        # We should still have both items still on Datastore because they
        # weren't persisted.
        batch_ingest_info_data_list = batch_persistence._get_batch_ingest_info_list(
            scrape_key.region_code, mock_session.start)
        self.assertEqual(len(batch_ingest_info_data_list), 2)
    def test_convert_ingest_info_id_is_not_generated(self):
        info = ingest_info.IngestInfo()
        person = info.create_person()
        person.person_id = "id1"
        person.surname = "testname"
        booking = person.create_booking()
        booking.booking_id = "id2"
        booking.admission_date = "testdate"

        expected_proto = ingest_info_pb2.IngestInfo()
        person = expected_proto.people.add()
        person.person_id = "id1"
        person.surname = "testname"
        person.booking_ids.append("id2")
        booking = expected_proto.bookings.add()
        booking.booking_id = "id2"
        booking.admission_date = "testdate"

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == info
    def test_convert_ingest_info_state_entities(self):
        # Arrange Python ingest info
        info = ingest_info.IngestInfo()
        person = info.create_state_person()
        person.state_person_id = 'person1'
        person.surname = 'testname'

        race = person.create_state_person_race()
        race.state_person_race_id = 'race1'
        race.race = 'white'
        ethnicity = person.create_state_person_ethnicity()
        ethnicity.state_person_ethnicity_id = 'ethnicity1'
        ethnicity.ethnicity = 'non-hispanic'
        external_id = person.create_state_person_external_id()
        external_id.state_person_external_id_id = 'external_id1'
        external_id.id_type = 'contrived'
        alias = person.create_state_alias()
        alias.state_alias_id = 'alias1'
        alias.surname = 'testerson'
        assessment = person.create_state_assessment()
        assessment.state_assessment_id = 'assessment1'
        assessment.assessment_score = '42'
        supervising_officer = person.create_state_agent()
        supervising_officer.state_agent_id = 'supervising_officer1'
        supervising_officer.full_name = 'Officer Supervising'

        assessment_agent = assessment.create_state_agent()
        assessment_agent.state_agent_id = 'agent1'
        assessment_agent.full_name = 'Officer Jones'

        program_assignment = person.create_state_program_assignment()
        program_assignment.state_program_assignment_id = 'assignment1'
        program_assignment.program_id = 'program_id1'

        program_assignment_agent = program_assignment.create_state_agent()
        program_assignment_agent.state_agent_id = 'program_agent1'
        program_assignment_agent.full_name = 'Officer Program'

        group = person.create_state_sentence_group()
        group.state_sentence_group_id = 'group1'

        fine = group.create_state_fine()
        fine.state_fine_id = 'fine1'

        incarceration_sentence = group.create_state_incarceration_sentence()
        incarceration_sentence.state_incarceration_sentence_id = 'is1'
        early_discharge1 = incarceration_sentence.create_state_early_discharge(
        )
        early_discharge1.state_early_discharge_id = 'early_discharge1'
        charge1 = incarceration_sentence.create_state_charge()
        charge1.state_charge_id = 'charge1'
        charge1.classification_type = 'F'
        incarceration_period = incarceration_sentence. \
            create_state_incarceration_period()
        incarceration_period.state_incarceration_period_id = 'ip1'
        incarceration_period.status = 'IN_CUSTODY'
        incarceration_period.specialized_purpose_for_incarceration = \
            'SHOCK INCARCERATION'
        incarceration_period.state_program_assignments = [program_assignment]
        incident = incarceration_period.create_state_incarceration_incident()
        incident.state_incarceration_incident_id = 'incident1'
        incident.incident_type = 'FISTICUFFS'
        incident_outcome = \
            incident.create_state_incarceration_incident_outcome()
        incident_outcome.state_incarceration_incident_outcome_id = 'incident1-1'
        incident_outcome.outcome_type = 'FINE'

        incident_agent = incident.create_state_agent()
        incident_agent.state_agent_id = 'agent2'
        incident_agent.full_name = 'Officer Thompson'

        decision = incarceration_period.create_state_parole_decision()
        decision.state_parole_decision_id = 'decision1'

        decision_agent = decision.create_state_agent()
        decision_agent.state_agent_id = 'agent3'
        decision_agent.full_name = 'Officer Barkley'

        supervision_sentence = group.create_state_supervision_sentence()
        supervision_sentence.state_supervision_sentence_id = 'ss1'
        early_discharge2 = supervision_sentence.create_state_early_discharge()
        early_discharge2.state_early_discharge_id = 'early_discharge2'
        charge2 = supervision_sentence.create_state_charge()
        charge2.state_charge_id = 'charge2'
        charge2.classification_type = 'M'
        supervision_period = supervision_sentence. \
            create_state_supervision_period()
        supervision_period.state_supervision_period_id = 'sp1'
        supervision_period.status = 'TERMINATED'
        supervision_period_agent = supervision_period.create_state_agent()
        supervision_period_agent.state_agent_id = 'agentPO'
        supervision_period_agent.full_name = 'Officer Paroley'
        supervision_period.state_program_assignments = [program_assignment]

        supervision_case_type_entry = supervision_period.create_state_supervision_case_type_entry(
        )
        supervision_case_type_entry.case_type = 'case_type'
        supervision_case_type_entry.state_supervision_case_type_entry_id = 'case_type_entry_id'

        supervision_contact = supervision_period.create_state_supervision_contact(
        )
        supervision_contact.state_supervision_contact_id = 'supervision_contact_id'
        supervision_contact.contact_type = 'contact_type'
        supervision_contacted_agent = supervision_contact.create_state_agent()
        supervision_contacted_agent.state_agent_id = 'agentPO'
        supervision_contacted_agent.full_name = 'Officer Paroley'

        violation = supervision_period.create_state_supervision_violation()
        violation.state_supervision_violation_id = 'violation1'
        violation.violated_conditions = 'cond'
        violation.is_violent = 'false'

        violation_type = violation.\
            create_state_supervision_violation_type_entry()
        violation_type.state_supervision_violation_type_entry_id =\
            'violation_type_id'
        violation_type.violation_type = 'FELONY'

        violated_condition = \
            violation.create_state_supervision_violated_condition_entry()
        violated_condition.state_supervision_violated_condition_entry_id =\
            'condition_id'
        violated_condition.condition = 'CURFEW'

        response = violation.create_state_supervision_violation_response()
        response.state_supervision_violation_response_id = 'response1'
        response_decision_agent = response.create_state_agent()
        response_decision_agent.state_agent_id = 'agentTERM'
        response_decision_agent.full_name = 'Officer Termy'

        response_decision = response.\
            create_state_supervision_violation_response_decision_entry()
        response_decision.\
            state_supervision_violation_response_decision_entry_id =\
            'response_decision_id'
        response_decision.decision = 'REVOCATION'
        response_decision.revocation_type = 'REINCARCERATION'

        bond = charge1.create_state_bond()
        bond.state_bond_id = 'bond1'

        court_case = charge2.create_state_court_case()
        court_case.state_court_case_id = 'case1'

        court_case_agent = court_case.create_state_agent()
        court_case_agent.state_agent_id = 'agentJ'
        court_case_agent.full_name = 'Judge Agent'

        # Arrange Proto ingest info
        expected_proto = ingest_info_pb2.IngestInfo()
        person_pb = expected_proto.state_people.add()
        person_pb.state_person_id = 'person1'
        person_pb.surname = 'testname'

        person_pb.state_person_race_ids.append('race1')
        race_pb = expected_proto.state_person_races.add()
        race_pb.state_person_race_id = 'race1'
        race_pb.race = 'white'
        person_pb.state_person_ethnicity_ids.append('ethnicity1')
        ethnicity_pb = expected_proto.state_person_ethnicities.add()
        ethnicity_pb.state_person_ethnicity_id = 'ethnicity1'
        ethnicity_pb.ethnicity = 'non-hispanic'
        person_pb.state_person_external_ids_ids.append(
            'contrived:external_id1')
        external_id_pb = expected_proto.state_person_external_ids.add()
        external_id_pb.state_person_external_id_id = 'contrived:external_id1'
        external_id_pb.id_type = 'contrived'
        person_pb.state_alias_ids.append('alias1')
        alias_pb = expected_proto.state_aliases.add()
        alias_pb.state_alias_id = 'alias1'
        alias_pb.surname = 'testerson'
        person_pb.state_assessment_ids.append('assessment1')
        assessment_pb = expected_proto.state_assessments.add()
        assessment_pb.state_assessment_id = 'assessment1'
        assessment_pb.assessment_score = '42'
        person_pb.supervising_officer_id = 'supervising_officer1'
        supervising_officer_pb = expected_proto.state_agents.add()
        supervising_officer_pb.state_agent_id = 'supervising_officer1'
        supervising_officer_pb.full_name = 'Officer Supervising'

        assessment_pb.conducting_agent_id = 'agent1'
        assessment_agent_pb = expected_proto.state_agents.add()
        assessment_agent_pb.state_agent_id = 'agent1'
        assessment_agent_pb.full_name = 'Officer Jones'

        person_pb.state_program_assignment_ids.append('assignment1')
        program_assignment_pb = expected_proto.state_program_assignments.add()
        program_assignment_pb.state_program_assignment_id = 'assignment1'
        program_assignment_pb.program_id = 'program_id1'
        program_assignment_pb.referring_agent_id = 'program_agent1'
        program_assignment_agent_pb = expected_proto.state_agents.add()
        program_assignment_agent_pb.state_agent_id = 'program_agent1'
        program_assignment_agent_pb.full_name = 'Officer Program'

        person_pb.state_sentence_group_ids.append('group1')
        group_pb = expected_proto.state_sentence_groups.add()
        group_pb.state_sentence_group_id = 'group1'

        group_pb.state_fine_ids.append('fine1')
        fine_pb = expected_proto.state_fines.add()
        fine_pb.state_fine_id = 'fine1'

        group_pb.state_supervision_sentence_ids.append('ss1')
        supervision_sentence_pb = \
            expected_proto.state_supervision_sentences.add()
        supervision_sentence_pb.state_supervision_sentence_id = 'ss1'
        supervision_sentence_pb.state_early_discharge_ids.append(
            'early_discharge2')
        early_discharge2_pb = expected_proto.state_early_discharges.add()
        early_discharge2_pb.state_early_discharge_id = 'early_discharge2'
        supervision_sentence_pb.state_charge_ids.append('charge2')
        charge2_pb = expected_proto.state_charges.add()
        charge2_pb.state_charge_id = 'charge2'
        charge2_pb.classification_type = 'M'
        supervision_sentence_pb.state_supervision_period_ids.append('sp1')
        supervision_period_pb = expected_proto.state_supervision_periods.add()
        supervision_period_pb.state_supervision_period_id = 'sp1'
        supervision_period_pb.status = 'TERMINATED'
        supervision_period_pb.state_program_assignment_ids.append(
            'assignment1')

        # An ordering requirement in the proto equality check at the end of this
        # test requires that this agent be added after agent1 and before agentPO
        court_case_agent_pb = expected_proto.state_agents.add()
        court_case_agent_pb.state_agent_id = 'agentJ'
        court_case_agent_pb.full_name = 'Judge Agent'

        supervision_period_pb.supervising_officer_id = 'agentPO'
        supervision_period_agent_pb = expected_proto.state_agents.add()
        supervision_period_agent_pb.state_agent_id = 'agentPO'
        supervision_period_agent_pb.full_name = 'Officer Paroley'

        supervision_case_type_entry_pb = expected_proto.state_supervision_case_type_entries.add(
        )
        supervision_case_type_entry_pb.state_supervision_case_type_entry_id = 'case_type_entry_id'
        supervision_case_type_entry_pb.case_type = 'case_type'
        supervision_period_pb.state_supervision_case_type_entry_ids.append(
            'case_type_entry_id')

        supervision_contact_pb = expected_proto.state_supervision_contacts.add(
        )
        supervision_contact_pb.state_supervision_contact_id = 'supervision_contact_id'
        supervision_contact_pb.contact_type = 'contact_type'
        supervision_contact_pb.contacted_agent_id = 'agentPO'
        supervision_period_pb.state_supervision_contact_ids.append(
            'supervision_contact_id')

        supervision_period_pb.state_supervision_violation_entry_ids.append(
            'violation1')
        violation_pb = expected_proto.state_supervision_violations.add()
        violation_pb.state_supervision_violation_id = 'violation1'
        violation_pb.is_violent = 'false'
        violation_pb.violated_conditions = 'cond'
        violation_pb.state_supervision_violation_type_entry_ids.append(
            'violation_type_id')
        violation_type_pb = \
            expected_proto.state_supervision_violation_type_entries.add()
        violation_type_pb.state_supervision_violation_type_entry_id = \
            'violation_type_id'
        violation_type_pb.violation_type = 'FELONY'

        violation_pb.state_supervision_violated_condition_entry_ids.append(
            'condition_id')
        violation_type_pb = \
            expected_proto.state_supervision_violated_condition_entries.add()
        violation_type_pb.state_supervision_violated_condition_entry_id = \
            'condition_id'
        violation_type_pb.condition = 'CURFEW'

        violation_pb.state_supervision_violation_response_ids.append(
            'response1')
        response_pb = expected_proto.state_supervision_violation_responses.add(
        )
        response_pb.state_supervision_violation_response_id = 'response1'
        response_pb.decision_agent_ids.append('agentTERM')
        response_decision_agent_pb = expected_proto.state_agents.add()
        response_decision_agent_pb.state_agent_id = 'agentTERM'
        response_decision_agent_pb.full_name = 'Officer Termy'
        response_decision_pb = \
            expected_proto.\
            state_supervision_violation_response_decision_entries.add()
        response_decision_pb.\
            state_supervision_violation_response_decision_entry_id = \
            'response_decision_id'
        response_decision_pb.decision = 'REVOCATION'
        response_decision_pb.revocation_type = 'REINCARCERATION'
        response_pb.\
            state_supervision_violation_response_decision_entry_ids.append(
                'response_decision_id'
            )

        group_pb.state_incarceration_sentence_ids.append('is1')
        incarceration_sentence_pb = \
            expected_proto.state_incarceration_sentences.add()
        incarceration_sentence_pb.state_incarceration_sentence_id = 'is1'
        incarceration_sentence_pb.state_early_discharge_ids.append(
            'early_discharge1')
        early_discharge1_pb = expected_proto.state_early_discharges.add()
        early_discharge1_pb.state_early_discharge_id = 'early_discharge1'
        incarceration_sentence_pb.state_charge_ids.append('charge1')
        charge1_pb = expected_proto.state_charges.add()
        charge1_pb.state_charge_id = 'charge1'
        charge1_pb.classification_type = 'F'
        incarceration_sentence_pb.state_incarceration_period_ids.append('ip1')
        incarceration_period_pb = \
            expected_proto.state_incarceration_periods.add()
        incarceration_period_pb.state_incarceration_period_id = 'ip1'
        incarceration_period_pb.status = 'IN_CUSTODY'
        incarceration_period_pb.specialized_purpose_for_incarceration = \
            'SHOCK INCARCERATION'
        incarceration_period_pb.state_incarceration_incident_ids \
            .append('incident1')
        incident_pb = expected_proto.state_incarceration_incidents.add()
        incident_pb.state_incarceration_incident_id = 'incident1'
        incident_pb.incident_type = 'FISTICUFFS'
        incarceration_period_pb.state_program_assignment_ids.append(
            'assignment1')

        incident_pb.responding_officer_id = 'agent2'
        incident_agent_pb = expected_proto.state_agents.add()
        incident_agent_pb.state_agent_id = 'agent2'
        incident_agent_pb.full_name = 'Officer Thompson'

        incident_pb.state_incarceration_incident_outcome_ids.append(
            'incident1-1')
        incident_outcome_pb = \
            expected_proto.state_incarceration_incident_outcomes.add()
        incident_outcome_pb.state_incarceration_incident_outcome_id = \
            'incident1-1'
        incident_outcome_pb.outcome_type = 'FINE'

        incarceration_period_pb.state_parole_decision_ids.append('decision1')
        decision_pb = expected_proto.state_parole_decisions.add()
        decision_pb.state_parole_decision_id = 'decision1'

        decision_pb.decision_agent_ids.append('agent3')
        decision_agent_pb = expected_proto.state_agents.add()
        decision_agent_pb.state_agent_id = 'agent3'
        decision_agent_pb.full_name = 'Officer Barkley'

        charge1_pb.state_bond_id = 'bond1'
        bond_pb = expected_proto.state_bonds.add()
        bond_pb.state_bond_id = 'bond1'

        charge2_pb.state_court_case_id = 'case1'
        court_case_pb = expected_proto.state_court_cases.add()
        court_case_pb.state_court_case_id = 'case1'

        court_case_pb.judge_id = 'agentJ'

        expected_info = copy.deepcopy(info)
        # Act & Assert

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == expected_info

        # Assert that none of the proto's collections are empty, i.e. we've
        # tested all of the object graph
        proto_classes = [field.name for field in proto.DESCRIPTOR.fields]
        for cls in proto_classes:
            if cls.startswith('state_'):
                assert proto.__getattribute__(cls)
    def validate_and_return_populate_data(
        self,
        content: Optional[html.HtmlElement],
        expected_ingest_info: Optional[IngestInfo] = None,
        expected_single_counts: Optional[List[SingleCount]] = None,
        expected_persist: bool = True,
        task: Optional[Task] = None,
        info: Optional[IngestInfo] = None,
    ) -> ScrapedData:
        """This function runs populate_data and runs some extra validation
        on the output.

        Args:
            content: the content of the page to pass into get_more_tasks
            expected_ingest_info: the ingest info expected to be returned from
                `populate_data`. If `expected_ingest_info` is `None`, then
                expects the return value of `populate_data` to be `None`.
            expected_single_counts: the list of SingleCounts expected to be
            returned from `populate_data`.
            expected_persist: the expected value of persist to be returned from
                `populate_data`.
            task: the task that is being processed, optional.
            info: an ingest_info to use if provided.

        Returns:
            The result from populate_data in case the user needs to do any
            extra validations on the output.
        """
        info_to_ingest: IngestInfo = info or ingest_info.IngestInfo()
        task_to_process: Task = task or Task(
            task_type=constants.TaskType.SCRAPE_DATA, endpoint="")

        if self.scraper:
            scrape_data = self.scraper.populate_data(content, task_to_process,
                                                     info_to_ingest)

            print("FINAL")
            print(scrape_data.ingest_info)
            print("EXPECTED")
            print(expected_ingest_info)

            if expected_ingest_info is None and expected_single_counts is None:
                if scrape_data:
                    assert scrape_data.persist is False
                else:
                    assert scrape_data is None

            if expected_single_counts and scrape_data.single_counts:
                assert len(
                    scrape_data.single_counts) == len(expected_single_counts)
                diff = set(expected_single_counts) ^ set(
                    scrape_data.single_counts)
                assert not diff

            metadata: IngestMetadata = IngestMetadata(
                region=self.scraper.region.region_code,
                jurisdiction_id=self.scraper.region.jurisdiction_id,
                ingest_time=_FAKE_SCRAPER_START_TIME,
                enum_overrides=self.scraper.get_enum_overrides(),
                system_level=SystemLevel.COUNTY,
                database_key=SQLAlchemyDatabaseKey.for_schema(
                    SchemaType.JAILS),
            )

            if scrape_data.ingest_info and expected_ingest_info:
                self.validate_ingest(scrape_data.ingest_info,
                                     expected_ingest_info, metadata)

            assert scrape_data.persist == expected_persist

        if scrape_data:
            return scrape_data
        raise ValueError("Scrape data was not provided ingest info")
Exemple #20
0
def convert_proto_to_ingest_info(
        proto: ingest_info_pb2.IngestInfo) -> ingest_info.IngestInfo:
    """Populates an `IngestInfo` python object from the given proto """

    person_map: Dict[str, ingest_info.Person] = \
        dict(_proto_to_py(person, ingest_info.Person, 'person_id')
             for person in proto.people)
    booking_map: Dict[str, ingest_info.Booking] = \
        dict(_proto_to_py(booking, ingest_info.Booking, 'booking_id')
             for booking in proto.bookings)
    charge_map: Dict[str, ingest_info.Charge] = \
        dict(_proto_to_py(charge, ingest_info.Charge, 'charge_id')
             for charge in proto.charges)
    hold_map: Dict[str, ingest_info.Hold] = \
        dict(_proto_to_py(hold, ingest_info.Hold, 'hold_id')
             for hold in proto.holds)
    arrest_map: Dict[str, ingest_info.Arrest] = \
        dict(_proto_to_py(arrest, ingest_info.Arrest, 'arrest_id')
             for arrest in proto.arrests)
    bond_map: Dict[str, ingest_info.Bond] = \
        dict(_proto_to_py(bond, ingest_info.Bond, 'bond_id')
             for bond in proto.bonds)
    sentence_map: Dict[str, ingest_info.Sentence] = \
        dict(_proto_to_py(sentence, ingest_info.Sentence, 'sentence_id')
             for sentence in proto.sentences)

    state_person_map: Dict[str, ingest_info.StatePerson] = \
        dict(_proto_to_py(state_person, ingest_info.StatePerson,
                          'state_person_id')
             for state_person in proto.state_people)
    state_person_race_map: Dict[str, ingest_info.StatePersonRace] = \
        dict(_proto_to_py(race, ingest_info.StatePersonRace,
                          'state_person_race_id')
             for race in proto.state_person_races)
    state_person_ethnicity_map: Dict[str, ingest_info.StatePersonEthnicity] = \
        dict(_proto_to_py(ethnicity, ingest_info.StatePersonEthnicity,
                          'state_person_ethnicity_id')
             for ethnicity in proto.state_person_ethnicities)
    state_person_external_id_map: Dict[str,
                                       ingest_info.StatePersonExternalId] = \
        dict(_proto_to_py(external_id, ingest_info.StatePersonExternalId,
                          'state_person_external_id_id')
             for external_id in proto.state_person_external_ids)
    state_alias_map: Dict[str, ingest_info.StateAlias] = \
        dict(_proto_to_py(alias, ingest_info.StateAlias, 'state_alias_id')
             for alias in proto.state_aliases)
    state_assessment_map: Dict[str, ingest_info.StateAssessment] = \
        dict(_proto_to_py(assessment,
                          ingest_info.StateAssessment,
                          'state_assessment_id')
             for assessment in proto.state_assessments)
    state_sentence_group_map: Dict[str, ingest_info.StateSentenceGroup] = \
        dict(_proto_to_py(sentence_group, ingest_info.StateSentenceGroup,
                          'state_sentence_group_id')
             for sentence_group in proto.state_sentence_groups)
    state_supervision_sentence_map: \
        Dict[str, ingest_info.StateSupervisionSentence] = \
        dict(_proto_to_py(supervision_sentence,
                          ingest_info.StateSupervisionSentence,
                          'state_supervision_sentence_id')
             for supervision_sentence in proto.state_supervision_sentences)
    state_incarceration_sentence_map: \
        Dict[str, ingest_info.StateIncarcerationSentence] = \
        dict(_proto_to_py(incarceration_sentence,
                          ingest_info.StateIncarcerationSentence,
                          'state_incarceration_sentence_id')
             for incarceration_sentence in proto.state_incarceration_sentences)
    state_fine_map: Dict[str, ingest_info.StateFine] = \
        dict(_proto_to_py(fine, ingest_info.StateFine, 'state_fine_id')
             for fine in proto.state_fines)
    state_charge_map: Dict[str, ingest_info.StateCharge] = \
        dict(_proto_to_py(state_charge, ingest_info.StateCharge,
                          'state_charge_id')
             for state_charge in proto.state_charges)
    state_court_case_map: Dict[str, ingest_info.StateCourtCase] = \
        dict(_proto_to_py(court_case,
                          ingest_info.StateCourtCase,
                          'state_court_case_id')
             for court_case in proto.state_court_cases)
    state_bond_map: Dict[str, ingest_info.StateBond] = \
        dict(_proto_to_py(state_bond,
                          ingest_info.StateBond,
                          'state_bond_id')
             for state_bond in proto.state_bonds)
    state_incarceration_period_map: \
        Dict[str, ingest_info.StateIncarcerationPeriod] \
        = dict(_proto_to_py(incarceration_period,
                            ingest_info.StateIncarcerationPeriod,
                            'state_incarceration_period_id')
               for incarceration_period in proto.state_incarceration_periods)
    state_supervision_period_map: \
        Dict[str, ingest_info.StateSupervisionPeriod] = \
        dict(_proto_to_py(supervision_period,
                          ingest_info.StateSupervisionPeriod,
                          'state_supervision_period_id')
             for supervision_period in proto.state_supervision_periods)
    state_incarceration_incident_map: \
        Dict[str, ingest_info.StateIncarcerationIncident] = \
        dict(_proto_to_py(incarceration_incident,
                          ingest_info.StateIncarcerationIncident,
                          'state_incarceration_incident_id')
             for incarceration_incident in proto.state_incarceration_incidents)
    state_incarceration_incident_outcome_map: \
        Dict[str, ingest_info.StateIncarcerationIncidentOutcome] = \
        dict(_proto_to_py(incarceration_incident_outcome,
                          ingest_info.StateIncarcerationIncidentOutcome,
                          'state_incarceration_incident_outcome_id')
             for incarceration_incident_outcome in
             proto.state_incarceration_incident_outcomes)
    state_parole_decision_map: Dict[str, ingest_info.StateParoleDecision] = \
        dict(_proto_to_py(parole_decision,
                          ingest_info.StateParoleDecision,
                          'state_parole_decision_id')
             for parole_decision in proto.state_parole_decisions)
    state_supervision_violation_map: \
        Dict[str, ingest_info.StateSupervisionViolation] = \
        dict(_proto_to_py(supervision_violation,
                          ingest_info.StateSupervisionViolation,
                          'state_supervision_violation_id')
             for supervision_violation in proto.state_supervision_violations)
    state_supervision_violation_response_map: \
        Dict[str, ingest_info.StateSupervisionViolationResponse] = \
        dict(_proto_to_py(violation_response,
                          ingest_info.StateSupervisionViolationResponse,
                          'state_supervision_violation_response_id')
             for violation_response
             in proto.state_supervision_violation_responses)
    state_agent_map: Dict[str, ingest_info.StateAgent] = \
        dict(_proto_to_py(agent, ingest_info.StateAgent, 'state_agent_id')
             for agent in proto.state_agents)

    # Wire bonds and sentences to respective charges
    for proto_charge in proto.charges:
        charge = charge_map[proto_charge.charge_id]
        if proto_charge.bond_id:
            charge.bond = bond_map[proto_charge.bond_id]
        if proto_charge.sentence_id:
            charge.sentence = sentence_map[proto_charge.sentence_id]

    # Wire arrests, charges, and holds to respective bookings
    for proto_booking in proto.bookings:
        booking = booking_map[proto_booking.booking_id]
        if proto_booking.arrest_id:
            booking.arrest = arrest_map[proto_booking.arrest_id]
        booking.charges = [
            charge_map[proto_id] for proto_id in proto_booking.charge_ids
        ]
        booking.holds = [
            hold_map[proto_id] for proto_id in proto_booking.hold_ids
        ]

    # Wire bookings to respective people
    for proto_person in proto.people:
        person = person_map[proto_person.person_id]
        person.bookings = [
            booking_map[proto_id] for proto_id in proto_person.booking_ids
        ]

    def _wire_sentence_proto(proto_sentence, proto_sentence_id,
                             proto_sentence_map):
        """Wires up child entities to their respective sentence types."""
        sentence = proto_sentence_map[proto_sentence_id]

        sentence.state_charges = \
            [state_charge_map[proto_id] for proto_id
             in proto_sentence.state_charge_ids]

        sentence.state_supervision_periods = \
            [state_supervision_period_map[proto_id] for proto_id
             in proto_sentence.state_supervision_period_ids]

        sentence.state_incarceration_periods = \
            [state_incarceration_period_map[proto_id] for proto_id
             in proto_sentence.state_incarceration_period_ids]

    # Wire agents to respective parent entities
    for proto_parole_decision in proto.state_parole_decisions:
        parole_decision = state_parole_decision_map[
            proto_parole_decision.state_parole_decision_id]
        parole_decision.decision_agents = \
            [state_agent_map[proto_id] for proto_id
             in proto_parole_decision.decision_agent_ids]

    for proto_court_case in proto.state_court_cases:
        state_court_case = state_court_case_map[
            proto_court_case.state_court_case_id]
        if proto_court_case.judge_id:
            state_court_case.judge = state_agent_map[proto_court_case.judge_id]

    for proto_incident in proto.state_incarceration_incidents:
        incarceration_incident = state_incarceration_incident_map[
            proto_incident.state_incarceration_incident_id]
        if proto_incident.responding_officer_id:
            incarceration_incident.responding_officer = \
                state_agent_map[proto_incident.responding_officer_id]

        incarceration_incident_outcomes = []
        for proto_incident_outcome in \
                proto.state_incarceration_incident_outcomes:
            incarceration_incident_outcomes.append(
                state_incarceration_incident_outcome_map[
                    proto_incident_outcome.
                    state_incarceration_incident_outcome_id])

        incarceration_incident.state_incarceration_incident_outcomes = \
            incarceration_incident_outcomes

    for proto_assessment in proto.state_assessments:
        assessment = state_assessment_map[proto_assessment.state_assessment_id]
        if proto_assessment.conducting_agent_id:
            assessment.conducting_agent = \
                state_agent_map[proto_assessment.conducting_agent_id]

    # Wire child entities to respective violations
    for proto_supervision_violation in proto.state_supervision_violations:
        supervision_violation = state_supervision_violation_map[
            proto_supervision_violation.state_supervision_violation_id]
        supervision_violation.state_supervision_violation_responses = \
            [state_supervision_violation_response_map[proto_id] for proto_id
             in proto_supervision_violation.
             state_supervision_violation_response_ids]

    # Wire child entities to respective supervision periods
    for proto_supervision_period in proto.state_supervision_periods:
        supervision_period = state_supervision_period_map[
            proto_supervision_period.state_supervision_period_id]
        supervision_period.state_supervision_violations = \
            [state_supervision_violation_map[proto_id] for proto_id
             in proto_supervision_period.state_supervision_violation_ids]
        supervision_period.state_assessments = \
            [state_assessment_map[proto_id] for proto_id
             in proto_supervision_period.state_assessment_ids]

    # Wire child entities to respective incarceration periods
    for proto_incarceration_period in proto.state_incarceration_periods:
        incarceration_period = state_incarceration_period_map[
            proto_incarceration_period.state_incarceration_period_id]
        incarceration_period.state_incarceration_incidents = \
            [state_incarceration_incident_map[proto_id] for proto_id
             in proto_incarceration_period.state_incarceration_incident_ids]
        incarceration_period.state_parole_decisions = \
            [state_parole_decision_map[proto_id] for proto_id
             in proto_incarceration_period.state_parole_decision_ids]
        incarceration_period.state_assessments = \
            [state_assessment_map[proto_id] for proto_id
             in proto_incarceration_period.state_assessment_ids]

    # Wire child entities to respective incarceration incidents
    for proto_incarceration_incident in proto.state_incarceration_incidents:
        incarceration_incident = state_incarceration_incident_map[
            proto_incarceration_incident.state_incarceration_incident_id]
        incarceration_incident.state_incarceration_incident_outcomes = \
            [state_incarceration_incident_outcome_map[proto_id] for proto_id
             in proto_incarceration_incident.
             state_incarceration_incident_outcome_ids]

    # Wire court cases and state bonds to respective state charges
    for proto_state_charge in proto.state_charges:
        state_charge = state_charge_map[proto_state_charge.state_charge_id]
        if proto_state_charge.state_court_case_id:
            state_charge.state_court_case = state_court_case_map[
                proto_state_charge.state_court_case_id]

        if proto_state_charge.state_bond_id:
            state_charge.state_bond = state_bond_map[
                proto_state_charge.state_bond_id]

    # Wire all state charges and period types to respective sentence types
    for proto_fine in proto.state_fines:
        fine = state_fine_map[proto_fine.state_fine_id]
        fine.state_charges = [
            state_charge_map[proto_id]
            for proto_id in proto_fine.state_charge_ids
        ]

    for proto_incarceration_sentence in proto.state_incarceration_sentences:
        _wire_sentence_proto(
            proto_incarceration_sentence,
            proto_incarceration_sentence.state_incarceration_sentence_id,
            state_incarceration_sentence_map)

    for proto_supervision_sentence in proto.state_supervision_sentences:
        _wire_sentence_proto(
            proto_supervision_sentence,
            proto_supervision_sentence.state_supervision_sentence_id,
            state_supervision_sentence_map)

    # Wire all sentence types to respective sentence groups
    for proto_sentence_group in proto.state_sentence_groups:
        sentence_group = state_sentence_group_map[
            proto_sentence_group.state_sentence_group_id]

        sentence_group.state_fines = [
            state_fine_map[proto_id]
            for proto_id in proto_sentence_group.state_fine_ids
        ]

        sentence_group.state_incarceration_sentences = \
            [state_incarceration_sentence_map[proto_id] for proto_id
             in proto_sentence_group.state_incarceration_sentence_ids]

        sentence_group.state_supervision_sentences = \
            [state_supervision_sentence_map[proto_id] for proto_id
             in proto_sentence_group.state_supervision_sentence_ids]

    # Wire child entities to respective state people
    for proto_state_person in proto.state_people:
        state_person = state_person_map[proto_state_person.state_person_id]

        state_person.state_person_races = \
            [state_person_race_map[proto_id] for proto_id
             in proto_state_person.state_person_race_ids]

        state_person.state_person_ethnicities = \
            [state_person_ethnicity_map[proto_id] for proto_id
             in proto_state_person.state_person_ethnicity_ids]

        state_person.state_person_external_ids = \
            [state_person_external_id_map[proto_id] for proto_id
             in proto_state_person.state_person_external_ids_ids]

        state_person.state_aliases = \
            [state_alias_map[proto_id] for proto_id
             in proto_state_person.state_alias_ids]

        state_person.state_assessments = \
            [state_assessment_map[proto_id] for proto_id
             in proto_state_person.state_assessment_ids]

        state_person.state_sentence_groups = \
            [state_sentence_group_map[proto_id] for proto_id
             in proto_state_person.state_sentence_group_ids]

    def _process_external_ids(ii: ingest_info.IngestInfo) -> None:
        # Undo preprocessing on external_ids performed when converting from
        # py -> proto
        for p in ii.state_people:
            for ex_id in p.state_person_external_ids:
                existing_id = cast(str, ex_id.state_person_external_id_id)
                ex_id.state_person_external_id_id = get_external_id(
                    synthetic_id=existing_id)

    # Wire people to ingest info
    ingest_info_py = ingest_info.IngestInfo()
    ingest_info_py.people.extend(person_map.values())
    ingest_info_py.state_people.extend(state_person_map.values())
    _process_external_ids(ingest_info_py)
    return ingest_info_py
    def test_convert_ingest_info_state_entities(self):
        # Arrange Python ingest info
        info = ingest_info.IngestInfo()
        person = info.create_state_person()
        person.state_person_id = "person1"
        person.surname = "testname"

        race = person.create_state_person_race()
        race.state_person_race_id = "race1"
        race.race = "white"
        ethnicity = person.create_state_person_ethnicity()
        ethnicity.state_person_ethnicity_id = "ethnicity1"
        ethnicity.ethnicity = "non-hispanic"
        external_id = person.create_state_person_external_id()
        external_id.state_person_external_id_id = "external_id1"
        external_id.id_type = "contrived"
        alias = person.create_state_alias()
        alias.state_alias_id = "alias1"
        alias.surname = "testerson"
        assessment = person.create_state_assessment()
        assessment.state_assessment_id = "assessment1"
        assessment.assessment_score = "42"
        supervising_officer = person.create_state_agent()
        supervising_officer.state_agent_id = "supervising_officer1"
        supervising_officer.full_name = "Officer Supervising"

        assessment_agent = assessment.create_state_agent()
        assessment_agent.state_agent_id = "agent1"
        assessment_agent.full_name = "Officer Jones"

        program_assignment = person.create_state_program_assignment()
        program_assignment.state_program_assignment_id = "assignment1"
        program_assignment.program_id = "program_id1"

        program_assignment_agent = program_assignment.create_state_agent()
        program_assignment_agent.state_agent_id = "program_agent1"
        program_assignment_agent.full_name = "Officer Program"

        group = person.create_state_sentence_group()
        group.state_sentence_group_id = "group1"

        fine = group.create_state_fine()
        fine.state_fine_id = "fine1"

        incarceration_sentence = group.create_state_incarceration_sentence()
        incarceration_sentence.state_incarceration_sentence_id = "is1"
        early_discharge1 = incarceration_sentence.create_state_early_discharge()
        early_discharge1.state_early_discharge_id = "early_discharge1"
        charge1 = incarceration_sentence.create_state_charge()
        charge1.state_charge_id = "charge1"
        charge1.classification_type = "F"
        incarceration_period = (
            incarceration_sentence.create_state_incarceration_period()
        )
        incarceration_period.state_incarceration_period_id = "ip1"
        incarceration_period.status = "IN_CUSTODY"
        incarceration_period.specialized_purpose_for_incarceration = (
            "SHOCK INCARCERATION"
        )
        incarceration_period.state_program_assignments = [program_assignment]
        incident = incarceration_period.create_state_incarceration_incident()
        incident.state_incarceration_incident_id = "incident1"
        incident.incident_type = "FISTICUFFS"
        incident_outcome = incident.create_state_incarceration_incident_outcome()
        incident_outcome.state_incarceration_incident_outcome_id = "incident1-1"
        incident_outcome.outcome_type = "FINE"

        incident_agent = incident.create_state_agent()
        incident_agent.state_agent_id = "agent2"
        incident_agent.full_name = "Officer Thompson"

        decision = incarceration_period.create_state_parole_decision()
        decision.state_parole_decision_id = "decision1"

        decision_agent = decision.create_state_agent()
        decision_agent.state_agent_id = "agent3"
        decision_agent.full_name = "Officer Barkley"

        supervision_sentence = group.create_state_supervision_sentence()
        supervision_sentence.state_supervision_sentence_id = "ss1"
        early_discharge2 = supervision_sentence.create_state_early_discharge()
        early_discharge2.state_early_discharge_id = "early_discharge2"
        charge2 = supervision_sentence.create_state_charge()
        charge2.state_charge_id = "charge2"
        charge2.classification_type = "M"
        supervision_period = supervision_sentence.create_state_supervision_period()
        supervision_period.state_supervision_period_id = "sp1"
        supervision_period.status = "TERMINATED"
        supervision_period_agent = supervision_period.create_state_agent()
        supervision_period_agent.state_agent_id = "agentPO"
        supervision_period_agent.full_name = "Officer Paroley"
        supervision_period.state_program_assignments = [program_assignment]

        supervision_case_type_entry = (
            supervision_period.create_state_supervision_case_type_entry()
        )
        supervision_case_type_entry.case_type = "case_type"
        supervision_case_type_entry.state_supervision_case_type_entry_id = (
            "case_type_entry_id"
        )

        supervision_contact = supervision_period.create_state_supervision_contact()
        supervision_contact.state_supervision_contact_id = "supervision_contact_id"
        supervision_contact.contact_type = "contact_type"
        supervision_contacted_agent = supervision_contact.create_state_agent()
        supervision_contacted_agent.state_agent_id = "agentPO"
        supervision_contacted_agent.full_name = "Officer Paroley"

        violation = supervision_period.create_state_supervision_violation()
        violation.state_supervision_violation_id = "violation1"
        violation.violated_conditions = "cond"
        violation.is_violent = "false"

        violation_type = violation.create_state_supervision_violation_type_entry()
        violation_type.state_supervision_violation_type_entry_id = "violation_type_id"
        violation_type.violation_type = "FELONY"

        violated_condition = (
            violation.create_state_supervision_violated_condition_entry()
        )
        violated_condition.state_supervision_violated_condition_entry_id = (
            "condition_id"
        )
        violated_condition.condition = "CURFEW"

        response = violation.create_state_supervision_violation_response()
        response.state_supervision_violation_response_id = "response1"
        response_decision_agent = response.create_state_agent()
        response_decision_agent.state_agent_id = "agentTERM"
        response_decision_agent.full_name = "Officer Termy"

        response_decision = (
            response.create_state_supervision_violation_response_decision_entry()
        )
        response_decision.state_supervision_violation_response_decision_entry_id = (
            "response_decision_id"
        )
        response_decision.decision = "REVOCATION"
        response_decision.revocation_type = "REINCARCERATION"

        bond = charge1.create_state_bond()
        bond.state_bond_id = "bond1"

        court_case = charge2.create_state_court_case()
        court_case.state_court_case_id = "case1"

        court_case_agent = court_case.create_state_agent()
        court_case_agent.state_agent_id = "agentJ"
        court_case_agent.full_name = "Judge Agent"

        # Arrange Proto ingest info
        expected_proto = ingest_info_pb2.IngestInfo()
        person_pb = expected_proto.state_people.add()
        person_pb.state_person_id = "person1"
        person_pb.surname = "testname"

        person_pb.state_person_race_ids.append("race1")
        race_pb = expected_proto.state_person_races.add()
        race_pb.state_person_race_id = "race1"
        race_pb.race = "white"
        person_pb.state_person_ethnicity_ids.append("ethnicity1")
        ethnicity_pb = expected_proto.state_person_ethnicities.add()
        ethnicity_pb.state_person_ethnicity_id = "ethnicity1"
        ethnicity_pb.ethnicity = "non-hispanic"
        person_pb.state_person_external_ids_ids.append("contrived:external_id1")
        external_id_pb = expected_proto.state_person_external_ids.add()
        external_id_pb.state_person_external_id_id = "contrived:external_id1"
        external_id_pb.id_type = "contrived"
        person_pb.state_alias_ids.append("alias1")
        alias_pb = expected_proto.state_aliases.add()
        alias_pb.state_alias_id = "alias1"
        alias_pb.surname = "testerson"
        person_pb.state_assessment_ids.append("assessment1")
        assessment_pb = expected_proto.state_assessments.add()
        assessment_pb.state_assessment_id = "assessment1"
        assessment_pb.assessment_score = "42"
        person_pb.supervising_officer_id = "supervising_officer1"
        supervising_officer_pb = expected_proto.state_agents.add()
        supervising_officer_pb.state_agent_id = "supervising_officer1"
        supervising_officer_pb.full_name = "Officer Supervising"

        assessment_pb.conducting_agent_id = "agent1"
        assessment_agent_pb = expected_proto.state_agents.add()
        assessment_agent_pb.state_agent_id = "agent1"
        assessment_agent_pb.full_name = "Officer Jones"

        person_pb.state_program_assignment_ids.append("assignment1")
        program_assignment_pb = expected_proto.state_program_assignments.add()
        program_assignment_pb.state_program_assignment_id = "assignment1"
        program_assignment_pb.program_id = "program_id1"
        program_assignment_pb.referring_agent_id = "program_agent1"
        program_assignment_agent_pb = expected_proto.state_agents.add()
        program_assignment_agent_pb.state_agent_id = "program_agent1"
        program_assignment_agent_pb.full_name = "Officer Program"

        person_pb.state_sentence_group_ids.append("group1")
        group_pb = expected_proto.state_sentence_groups.add()
        group_pb.state_sentence_group_id = "group1"

        group_pb.state_fine_ids.append("fine1")
        fine_pb = expected_proto.state_fines.add()
        fine_pb.state_fine_id = "fine1"

        group_pb.state_supervision_sentence_ids.append("ss1")
        supervision_sentence_pb = expected_proto.state_supervision_sentences.add()
        supervision_sentence_pb.state_supervision_sentence_id = "ss1"
        supervision_sentence_pb.state_early_discharge_ids.append("early_discharge2")
        early_discharge2_pb = expected_proto.state_early_discharges.add()
        early_discharge2_pb.state_early_discharge_id = "early_discharge2"
        supervision_sentence_pb.state_charge_ids.append("charge2")
        charge2_pb = expected_proto.state_charges.add()
        charge2_pb.state_charge_id = "charge2"
        charge2_pb.classification_type = "M"
        supervision_sentence_pb.state_supervision_period_ids.append("sp1")
        supervision_period_pb = expected_proto.state_supervision_periods.add()
        supervision_period_pb.state_supervision_period_id = "sp1"
        supervision_period_pb.status = "TERMINATED"
        supervision_period_pb.state_program_assignment_ids.append("assignment1")

        # An ordering requirement in the proto equality check at the end of this
        # test requires that this agent be added after agent1 and before agentPO
        court_case_agent_pb = expected_proto.state_agents.add()
        court_case_agent_pb.state_agent_id = "agentJ"
        court_case_agent_pb.full_name = "Judge Agent"

        supervision_period_pb.supervising_officer_id = "agentPO"
        supervision_period_agent_pb = expected_proto.state_agents.add()
        supervision_period_agent_pb.state_agent_id = "agentPO"
        supervision_period_agent_pb.full_name = "Officer Paroley"

        supervision_case_type_entry_pb = (
            expected_proto.state_supervision_case_type_entries.add()
        )
        supervision_case_type_entry_pb.state_supervision_case_type_entry_id = (
            "case_type_entry_id"
        )
        supervision_case_type_entry_pb.case_type = "case_type"
        supervision_period_pb.state_supervision_case_type_entry_ids.append(
            "case_type_entry_id"
        )

        supervision_contact_pb = expected_proto.state_supervision_contacts.add()
        supervision_contact_pb.state_supervision_contact_id = "supervision_contact_id"
        supervision_contact_pb.contact_type = "contact_type"
        supervision_contact_pb.contacted_agent_id = "agentPO"
        supervision_period_pb.state_supervision_contact_ids.append(
            "supervision_contact_id"
        )

        supervision_period_pb.state_supervision_violation_entry_ids.append("violation1")
        violation_pb = expected_proto.state_supervision_violations.add()
        violation_pb.state_supervision_violation_id = "violation1"
        violation_pb.is_violent = "false"
        violation_pb.violated_conditions = "cond"
        violation_pb.state_supervision_violation_type_entry_ids.append(
            "violation_type_id"
        )
        violation_type_pb = (
            expected_proto.state_supervision_violation_type_entries.add()
        )
        violation_type_pb.state_supervision_violation_type_entry_id = (
            "violation_type_id"
        )
        violation_type_pb.violation_type = "FELONY"

        violation_pb.state_supervision_violated_condition_entry_ids.append(
            "condition_id"
        )
        violation_type_pb = (
            expected_proto.state_supervision_violated_condition_entries.add()
        )
        violation_type_pb.state_supervision_violated_condition_entry_id = "condition_id"
        violation_type_pb.condition = "CURFEW"

        violation_pb.state_supervision_violation_response_ids.append("response1")
        response_pb = expected_proto.state_supervision_violation_responses.add()
        response_pb.state_supervision_violation_response_id = "response1"
        response_pb.decision_agent_ids.append("agentTERM")
        response_decision_agent_pb = expected_proto.state_agents.add()
        response_decision_agent_pb.state_agent_id = "agentTERM"
        response_decision_agent_pb.full_name = "Officer Termy"
        response_decision_pb = (
            expected_proto.state_supervision_violation_response_decision_entries.add()
        )
        response_decision_pb.state_supervision_violation_response_decision_entry_id = (
            "response_decision_id"
        )
        response_decision_pb.decision = "REVOCATION"
        response_decision_pb.revocation_type = "REINCARCERATION"
        response_pb.state_supervision_violation_response_decision_entry_ids.append(
            "response_decision_id"
        )

        group_pb.state_incarceration_sentence_ids.append("is1")
        incarceration_sentence_pb = expected_proto.state_incarceration_sentences.add()
        incarceration_sentence_pb.state_incarceration_sentence_id = "is1"
        incarceration_sentence_pb.state_early_discharge_ids.append("early_discharge1")
        early_discharge1_pb = expected_proto.state_early_discharges.add()
        early_discharge1_pb.state_early_discharge_id = "early_discharge1"
        incarceration_sentence_pb.state_charge_ids.append("charge1")
        charge1_pb = expected_proto.state_charges.add()
        charge1_pb.state_charge_id = "charge1"
        charge1_pb.classification_type = "F"
        incarceration_sentence_pb.state_incarceration_period_ids.append("ip1")
        incarceration_period_pb = expected_proto.state_incarceration_periods.add()
        incarceration_period_pb.state_incarceration_period_id = "ip1"
        incarceration_period_pb.status = "IN_CUSTODY"
        incarceration_period_pb.specialized_purpose_for_incarceration = (
            "SHOCK INCARCERATION"
        )
        incarceration_period_pb.state_incarceration_incident_ids.append("incident1")
        incident_pb = expected_proto.state_incarceration_incidents.add()
        incident_pb.state_incarceration_incident_id = "incident1"
        incident_pb.incident_type = "FISTICUFFS"
        incarceration_period_pb.state_program_assignment_ids.append("assignment1")

        incident_pb.responding_officer_id = "agent2"
        incident_agent_pb = expected_proto.state_agents.add()
        incident_agent_pb.state_agent_id = "agent2"
        incident_agent_pb.full_name = "Officer Thompson"

        incident_pb.state_incarceration_incident_outcome_ids.append("incident1-1")
        incident_outcome_pb = expected_proto.state_incarceration_incident_outcomes.add()
        incident_outcome_pb.state_incarceration_incident_outcome_id = "incident1-1"
        incident_outcome_pb.outcome_type = "FINE"

        incarceration_period_pb.state_parole_decision_ids.append("decision1")
        decision_pb = expected_proto.state_parole_decisions.add()
        decision_pb.state_parole_decision_id = "decision1"

        decision_pb.decision_agent_ids.append("agent3")
        decision_agent_pb = expected_proto.state_agents.add()
        decision_agent_pb.state_agent_id = "agent3"
        decision_agent_pb.full_name = "Officer Barkley"

        charge1_pb.state_bond_id = "bond1"
        bond_pb = expected_proto.state_bonds.add()
        bond_pb.state_bond_id = "bond1"

        charge2_pb.state_court_case_id = "case1"
        court_case_pb = expected_proto.state_court_cases.add()
        court_case_pb.state_court_case_id = "case1"

        court_case_pb.judge_id = "agentJ"

        expected_info = copy.deepcopy(info)
        # Act & Assert

        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        assert info_back == expected_info

        # Assert that none of the proto's collections are empty, i.e. we've
        # tested all of the object graph
        proto_classes = [field.name for field in proto.DESCRIPTOR.fields]
        for cls in proto_classes:
            if cls.startswith("state_"):
                assert proto.__getattribute__(cls)
Exemple #22
0
    def test_convert_ingest_info_duplicate_incarceration_incidents(self):
        # Arrange Python ingest info
        info = ingest_info.IngestInfo()
        person = info.create_state_person()
        person.state_person_id = 'person1'
        person.surname = 'testname'

        group = person.create_state_sentence_group()
        group.state_sentence_group_id = 'group1'

        incarceration_sentence = group.create_state_incarceration_sentence()
        incarceration_sentence.state_incarceration_sentence_id = 'is1'
        incarceration_period = incarceration_sentence. \
            create_state_incarceration_period()
        incarceration_period.state_incarceration_period_id = 'ip1'
        incarceration_period.status = 'IN_CUSTODY'

        incident = incarceration_period.create_state_incarceration_incident()
        incident.state_incarceration_incident_id = 'incident1'
        incident.incident_type = 'FISTICUFFS'
        incident_outcome = \
            incident.create_state_incarceration_incident_outcome()
        incident_outcome.state_incarceration_incident_outcome_id = 'incident1-1'
        incident_outcome.outcome_type = 'FINE'

        incident_dup = \
            incarceration_period.create_state_incarceration_incident()
        incident_dup.state_incarceration_incident_id = 'incident1'
        incident_dup.incident_type = 'FISTICUFFS'
        incident_outcome_2 = \
            incident_dup.create_state_incarceration_incident_outcome()
        incident_outcome_2.state_incarceration_incident_outcome_id = \
            'incident1-2'
        incident_outcome_2.outcome_type = 'FINE'

        # Arrange Proto ingest info
        expected_proto = ingest_info_pb2.IngestInfo()
        pb_person = expected_proto.state_people.add()
        pb_person.state_person_id = 'person1'
        pb_person.surname = 'testname'
        pb_person.state_sentence_group_ids.append('group1')
        pb_group = expected_proto.state_sentence_groups.add()
        pb_group.state_sentence_group_id = 'group1'
        pb_group.state_incarceration_sentence_ids.append('is1')

        pb_incarceration_sentence = \
            expected_proto.state_incarceration_sentences.add()
        pb_incarceration_sentence.state_incarceration_sentence_id = 'is1'
        pb_incarceration_sentence.state_incarceration_period_ids.append('ip1')
        pb_incarceration_period = \
            expected_proto.state_incarceration_periods.add()
        pb_incarceration_period.state_incarceration_period_id = 'ip1'
        pb_incarceration_period.status = 'IN_CUSTODY'
        pb_incarceration_period.state_incarceration_incident_ids \
            .append('incident1')

        pb_incident = expected_proto.state_incarceration_incidents.add()
        pb_incident.state_incarceration_incident_id = 'incident1'
        pb_incident.incident_type = 'FISTICUFFS'

        pb_incident.state_incarceration_incident_outcome_ids.append(
            'incident1-1')
        pb_incident_outcome = \
            expected_proto.state_incarceration_incident_outcomes.add()
        pb_incident_outcome.state_incarceration_incident_outcome_id = \
            'incident1-1'
        pb_incident_outcome.outcome_type = 'FINE'

        pb_incident.state_incarceration_incident_outcome_ids.append(
            'incident1-2')
        pb_incident_outcome = \
            expected_proto.state_incarceration_incident_outcomes.add()
        pb_incident_outcome.state_incarceration_incident_outcome_id = \
            'incident1-2'
        pb_incident_outcome.outcome_type = 'FINE'

        # Act & Assert
        proto = ingest_utils.convert_ingest_info_to_proto(info)
        assert expected_proto == proto

        # Duplicate IncarcerationIncident is gone.
        info_back = ingest_utils.convert_proto_to_ingest_info(proto)
        incarceration_period.state_incarceration_incidents = [incident]
        incident.state_incarceration_incident_outcomes = [
            incident_outcome, incident_outcome_2
        ]

        assert info_back == info