def _match_from_charges(*, db_booking: entities.Booking, ingested_booking: entities.Booking, name: str, orphaned_entities: List[Entity]) -> None: """Helper function that, within a booking, matches objects that are children of the booking's charges. |name| should be 'bond' or 'sentence'. Any entities that are orphaned as a part of this process are added to the given |orphaned_entities|. """ db_obj_map, db_relationship_map = _build_maps_from_charges( db_booking.charges, name) ing_obj_map, ing_relationship_map = _build_maps_from_charges( ingested_booking.charges, name) def _is_match_with_relationships(*, db_entity: Entity, ingested_entity: Entity) -> bool: ing_entity_id = generate_id_from_obj(ingested_entity) db_entity_id = db_entity.get_id() matcher = getattr(county_matching_utils, "is_{}_match".format(name)) obj_match = matcher(db_entity=db_entity, ingested_entity=ingested_entity) # The relationships "match" if new relationships have been added # since the last scrape, but not if relationships have been removed. parents_of_db_entity = db_relationship_map[db_entity_id] parents_of_ing_entity = ing_relationship_map[ing_entity_id] relationship_match = parents_of_db_entity.issubset( parents_of_ing_entity) return obj_match and relationship_match matched_ing_objs_by_db_id: Dict[int, Entity] = {} for ing_obj in ing_obj_map.values(): db_obj = get_next_available_match( ing_obj, list(db_obj_map.values()), matched_ing_objs_by_db_id, _is_match_with_relationships, ) if db_obj: db_id = db_obj.get_id() logging.debug("successfully matched to %s with id %s", name, db_id) setattr(ing_obj, name + "_id", db_id) matched_ing_objs_by_db_id[cast(int, db_id)] = ing_obj for db_obj in db_obj_map.values(): db_obj_id = db_obj.get_id() if db_obj_id not in matched_ing_objs_by_db_id: logging.debug("Did not match %s to any ingested %s, dropping", db_obj_id, name) drop_fn = globals()["_drop_" + name] drop_fn(db_obj) orphaned_entities.append(db_obj)
def match_charges(*, db_booking: entities.Booking, ingested_booking: entities.Booking): """ Attempts to match all charges on the |ingested_booking| with charges on the |db_booking|. For any ingested charge, if a matching charge exists on |db_booking|, the primary key is updated on the ingested charge. All db charges that are not matched to an ingested charge are marked dropped and added to the |ingested_booking|. Note about charge matching: Our matching scheme here is designed to reduce turnover with our charge, sentence, an bond entities (i.e. preferring to update entities rather than replacing old ones). If possible, we match ingested_charges to db_charges while considering the equality of their children (bonds/sentences). If we cannot match an ingested_charge while considering its children, we attempt to match it only based on the charge fields. If the ingested_charge still has no match, it will be marked dropped. Because we can match two charges whose children are not equal, we sort our ingested_charges, attempting to match those with more children before those with fewer or none. Examples: 1. Two identical charges are ingested, one with a bond (A) and one without (B). The same charges were also scraped yesterday, and are in our DB as charge C (with a bond) and D (without a bond) Because A could match either C or D, we sort our ingested charges by the number of children, so that we always attempt to match A before B. This ensures A matches to C, and then B matches to D. If we attempted to match B before A, B could match to C, causing a new bond to be added to B. Then A would match to D, and its bond would be dropped. This turnover is not desired. 2. We scrape one charge from the website (A), and this charge is exactly the same as a charge in our DB (B), except that the ingested charge A now has a bond. In this case we'll first try to match A considering the child bond, and when no matches are found, we'll match just considering the charge fields. At this point A matches to B, and B gets a new bond created in the DB. """ matched_charges_by_db_id: Dict[int, entities.Charge] = {} ing_charges_sorted_by_child_count = sorted(ingested_booking.charges, key=_charge_relationship_count, reverse=True) for ingested_charge in ing_charges_sorted_by_child_count: db_charge: entities.Charge = get_next_available_match( ingested_charge, db_booking.charges, matched_charges_by_db_id, is_charge_match_with_children, ) if not db_charge: db_charge = get_next_available_match( ingested_charge, db_booking.charges, matched_charges_by_db_id, is_charge_match, ) if db_charge: logging.debug("Successfully matched to charge with ID %s", db_charge.charge_id) matched_charges_by_db_id[cast( int, db_charge.charge_id)] = ingested_charge ingested_charge.charge_id = db_charge.charge_id dropped_charges = [] for db_charge in db_booking.charges: if db_charge.charge_id not in matched_charges_by_db_id: _drop_charge(db_charge) dropped_charges.append(db_charge) ingested_booking.charges.extend(dropped_charges)