def decide(collection_id, xref_id): """ --- post: summary: Give feedback about the veracity of an xref match. description: > This lets a user decide if they think a given xref match is a true or false match, and what group of users (context) should be privy to this insight. parameters: - in: path name: collection_id required: true schema: type: integer - in: path name: xref_id required: true schema: type: string requestBody: content: application/json: schema: $ref: '#/components/schemas/XrefDecide' responses: '202': content: application/json: schema: properties: status: description: accepted type: string type: object description: Accepted tags: - Xref - Profiles - EntitySet """ data = parse_request("XrefDecide") xref = obj_or_404(get_xref(xref_id, collection_id=collection_id)) require(request.authz.can(collection_id, request.authz.WRITE)) entity = get_index_entity(xref.get("entity_id")) match = get_index_entity(xref.get("match_id")) if entity is None and match is None: # This will raise a InvalidData error if the two types are not compatible model.common_schema(entity.get("schema"), match.get("schema")) decide_xref(xref, judgement=data.get("decision"), authz=request.authz) return jsonify({"status": "ok"}, status=204)
def parse_party(emitter, doc, distinct_party): profile = distinct_party.find(qpath('Profile')) sub_type_ = profile.get('PartySubTypeID') sub_type = deref(doc, 'PartySubType', sub_type_) type_ = deref(doc, 'PartySubType', sub_type_, 'PartyTypeID') type_ = deref(doc, 'PartyType', type_) schema = TYPES.get(type_, TYPES.get(sub_type)) if schema is None: emitter.log.error("Unknown type: %s", type_) return party = emitter.make(schema) party.make_id('Profile', profile.get('ID')) party.add('notes', distinct_party.findtext(qpath('Comment'))) for identity in profile.findall(qpath('Identity')): for alias in identity.findall(qpath('Alias')): parse_alias(party, doc, alias) identity_id = identity.get('ID') query = '//%s[@IdentityID="%s"]' % (qtag('IDRegDocument'), identity_id) for idreg in doc.findall(query): authority = idreg.findtext(qpath('IssuingAuthority')) number = idreg.findtext(qpath('IDRegistrationNo')) type_ = deref(doc, 'IDRegDocType', idreg.get('IDRegDocTypeID')) if authority == 'INN': party.add('innCode', number) continue if authority == 'OGRN': party.schema = model.get('Company') party.add('ogrnCode', number) continue schema, attr = REGISTRATIONS.get(type_) party.schema = model.common_schema(party.schema, schema) if len(attr): party.add(attr, number) for feature in profile.findall(qpath('Feature')): feature_type = deref(doc, 'FeatureType', feature.get('FeatureTypeID')) attr, schema = FEATURES.get(feature_type) party.schema = model.common_schema(party.schema, schema) if len(attr): value = parse_feature(doc, feature) if isinstance(value, tuple): value, country_code = value if party.schema.get(attr).type == registry.country: value = country_code else: party.add('country', country_code) party.add(attr, value, quiet=True) emitter.emit(party) emitter.log.info("[%s] %s", party.schema.name, party.caption)
def _merge_schemata(proxy, schemata): for other in schemata: try: other = model.get(other) proxy.schema = model.common_schema(proxy.schema, other) except InvalidData: proxy.schema = model.get(Entity.LEGAL_ENTITY)
def add_schema(self, schema: Union[str, Schema]) -> None: """Try to apply the given schema to the current entity, making it more specific (e.g. turning a `LegalEntity` into a `Company`). This raises an exception if the current and new type are incompatible.""" try: self.schema = model.common_schema(self.schema, schema) except InvalidData as exc: raise InvalidData(f"{self.id}: {exc}") from exc
def add_schema(entity, addition): try: entity.schema = model.common_schema(entity.schema, addition) except InvalidData: for schema in model.schemata.values(): if schema.is_a(entity.schema) and schema.is_a(addition): entity.schema = schema return raise
def decide_pairwise(collection, entity, match_collection, match, judgement, authz): """Store user feedback from an pairwise judgement as an profile-type EntitySet The problem here is that we're trying to translate a single pair-wise user judgement into a merge or split judgement regarding a cluster of entities. This works for most cases, with the exception that a profile, once established, cannot be split in a way that preserves what entities were linked to what other entities originally.""" if not isinstance(judgement, Judgement): judgement = Judgement(judgement) # This will raise a InvalidData error if the two types are not compatible model.common_schema(entity.get("schema"), match.get("schema")) profile = EntitySet.by_entity_id( entity.get("id"), collection_ids=[collection.id], types=[EntitySet.PROFILE], judgements=[Judgement.POSITIVE], ).first() if profile is None: data = {"type": EntitySet.PROFILE, "label": "profile"} profile = EntitySet.create(data, collection, authz) item = save_entityset_item( profile, collection, entity.get("id"), judgement=Judgement.POSITIVE, added_by_id=authz.id, ) item = save_entityset_item( profile, match_collection, match.get("id"), judgement=judgement, compared_to_entity_id=entity.get("id"), added_by_id=authz.id, ) db.session.commit() if item is not None: return item.entityset
def add_schema(entity, addition): try: entity.schema = model.common_schema(entity.schema, addition) except InvalidData: # FIXME: this might make vessels out of companies!!! for schema in model.schemata.values(): if schema.is_a(entity.schema) and schema.is_a(addition): entity.schema = schema return raise
def test_model_common_schema(self): assert model.common_schema('Thing', 'Thing') == 'Thing' assert model.common_schema('Thing', 'Person') == 'Person' assert model.common_schema('Person', 'Thing') == 'Person' assert model.common_schema('Person', 'Company') == 'LegalEntity' assert model.common_schema('LegalEntity', 'Company') == 'Company' with assert_raises(InvalidData): model.common_schema('Person', 'Directorship')
def add_cast(self, schema, prop, value): """Set a property on an entity. If the entity is of a schema that doesn't have the given property, also modify the schema (e.g. if something has a birthDate, assume it's a Person, not a LegalEntity). """ if self.schema.get(prop) is not None: return self.add(prop, value) schema = model.get(schema) prop_ = schema.get(prop) if prop_.type.clean(value) is None: return self.schema = model.common_schema(self.schema, schema) return self.add(prop, value)
def make_pair(pair, judgement, source): from followthemoney import model (left, right) = pair if judgement is False and left["id"] == right["id"]: return None try: schema = model.common_schema(left["schema"], right["schema"]) except InvalidData: return None return { "left": left, "right": right, "judgement": judgement, "schema": schema.name, "source": source, }
def test_model_common_schema(self): assert model.common_schema("Thing", "Thing") == "Thing" assert model.common_schema("Thing", "Person") == "Person" assert model.common_schema("Person", "Thing") == "Person" assert model.common_schema("LegalEntity", "Company") == "Company" assert model.common_schema("Interval", "Ownership") == "Ownership" # assert model.common_schema("LegalEntity", "Asset") == "Company" with assert_raises(InvalidData): model.common_schema("Person", "Directorship") with assert_raises(InvalidData): model.common_schema("Person", "Company") with assert_raises(InvalidData): model.common_schema("Membership", "Thing")
def test_model_common_schema(self): assert model.common_schema("Thing", "Thing") == model["Thing"] assert model.common_schema("Thing", "Person") == model["Person"] assert model.common_schema("Person", "Thing") == model["Person"] assert model.common_schema("LegalEntity", "Company") == model["Company"] assert model.common_schema("Interval", "Ownership") == model["Ownership"] # This behaviour turned out the be a really bad idea: # assert model.common_schema("LegalEntity", "Asset") == "Company" with raises(InvalidData): model.common_schema("Person", "Directorship") with raises(InvalidData): model.common_schema("Person", "Company") with raises(InvalidData): model.common_schema("Membership", "Thing")