def decide(collection_id, xref_id):
"""
---
post:
summary: Give feedback about the veracity of an xref match.
description: >
This lets a user decide if they think a given xref match is a true or
false match, and what group of users (context) should be privy to this
insight.
parameters:
- in: path
name: collection_id
required: true
schema:
type: integer
- in: path
name: xref_id
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/XrefDecide'
responses:
'202':
content:
application/json:
schema:
properties:
status:
description: accepted
type: string
type: object
description: Accepted
tags:
- Xref
- Profiles
- EntitySet
"""
data = parse_request("XrefDecide")
xref = obj_or_404(get_xref(xref_id, collection_id=collection_id))
require(request.authz.can(collection_id, request.authz.WRITE))
entity = get_index_entity(xref.get("entity_id"))
match = get_index_entity(xref.get("match_id"))
if entity is None and match is None:
# This will raise a InvalidData error if the two types are not compatible
model.common_schema(entity.get("schema"), match.get("schema"))
decide_xref(xref, judgement=data.get("decision"), authz=request.authz)
return jsonify({"status": "ok"}, status=204)
def parse_party(emitter, doc, distinct_party):
profile = distinct_party.find(qpath('Profile'))
sub_type_ = profile.get('PartySubTypeID')
sub_type = deref(doc, 'PartySubType', sub_type_)
type_ = deref(doc, 'PartySubType', sub_type_, 'PartyTypeID')
type_ = deref(doc, 'PartyType', type_)
schema = TYPES.get(type_, TYPES.get(sub_type))
if schema is None:
emitter.log.error("Unknown type: %s", type_)
return
party = emitter.make(schema)
party.make_id('Profile', profile.get('ID'))
party.add('notes', distinct_party.findtext(qpath('Comment')))
for identity in profile.findall(qpath('Identity')):
for alias in identity.findall(qpath('Alias')):
parse_alias(party, doc, alias)
identity_id = identity.get('ID')
query = '//%s[@IdentityID="%s"]' % (qtag('IDRegDocument'), identity_id)
for idreg in doc.findall(query):
authority = idreg.findtext(qpath('IssuingAuthority'))
number = idreg.findtext(qpath('IDRegistrationNo'))
type_ = deref(doc, 'IDRegDocType', idreg.get('IDRegDocTypeID'))
if authority == 'INN':
party.add('innCode', number)
continue
if authority == 'OGRN':
party.schema = model.get('Company')
party.add('ogrnCode', number)
continue
schema, attr = REGISTRATIONS.get(type_)
party.schema = model.common_schema(party.schema, schema)
if len(attr):
party.add(attr, number)
for feature in profile.findall(qpath('Feature')):
feature_type = deref(doc, 'FeatureType', feature.get('FeatureTypeID'))
attr, schema = FEATURES.get(feature_type)
party.schema = model.common_schema(party.schema, schema)
if len(attr):
value = parse_feature(doc, feature)
if isinstance(value, tuple):
value, country_code = value
if party.schema.get(attr).type == registry.country:
value = country_code
else:
party.add('country', country_code)
party.add(attr, value, quiet=True)
emitter.emit(party)
emitter.log.info("[%s] %s", party.schema.name, party.caption)
def _merge_schemata(proxy, schemata):
for other in schemata:
try:
other = model.get(other)
proxy.schema = model.common_schema(proxy.schema, other)
except InvalidData:
proxy.schema = model.get(Entity.LEGAL_ENTITY)
def add_schema(self, schema: Union[str, Schema]) -> None:
"""Try to apply the given schema to the current entity, making it more
specific (e.g. turning a `LegalEntity` into a `Company`). This raises an
exception if the current and new type are incompatible."""
try:
self.schema = model.common_schema(self.schema, schema)
except InvalidData as exc:
raise InvalidData(f"{self.id}: {exc}") from exc
def add_schema(entity, addition):
try:
entity.schema = model.common_schema(entity.schema, addition)
except InvalidData:
for schema in model.schemata.values():
if schema.is_a(entity.schema) and schema.is_a(addition):
entity.schema = schema
return
raise
def decide_pairwise(collection, entity, match_collection, match, judgement,
authz):
"""Store user feedback from an pairwise judgement as an profile-type EntitySet
The problem here is that we're trying to translate a single pair-wise user
judgement into a merge or split judgement regarding a cluster of entities.
This works for most cases, with the exception that a profile, once
established, cannot be split in a way that preserves what entities
were linked to what other entities originally."""
if not isinstance(judgement, Judgement):
judgement = Judgement(judgement)
# This will raise a InvalidData error if the two types are not compatible
model.common_schema(entity.get("schema"), match.get("schema"))
profile = EntitySet.by_entity_id(
entity.get("id"),
collection_ids=[collection.id],
types=[EntitySet.PROFILE],
judgements=[Judgement.POSITIVE],
).first()
if profile is None:
data = {"type": EntitySet.PROFILE, "label": "profile"}
profile = EntitySet.create(data, collection, authz)
item = save_entityset_item(
profile,
collection,
entity.get("id"),
judgement=Judgement.POSITIVE,
added_by_id=authz.id,
)
item = save_entityset_item(
profile,
match_collection,
match.get("id"),
judgement=judgement,
compared_to_entity_id=entity.get("id"),
added_by_id=authz.id,
)
db.session.commit()
if item is not None:
return item.entityset
def add_schema(entity, addition):
try:
entity.schema = model.common_schema(entity.schema, addition)
except InvalidData:
# FIXME: this might make vessels out of companies!!!
for schema in model.schemata.values():
if schema.is_a(entity.schema) and schema.is_a(addition):
entity.schema = schema
return
raise
def test_model_common_schema(self):
assert model.common_schema('Thing', 'Thing') == 'Thing'
assert model.common_schema('Thing', 'Person') == 'Person'
assert model.common_schema('Person', 'Thing') == 'Person'
assert model.common_schema('Person', 'Company') == 'LegalEntity'
assert model.common_schema('LegalEntity', 'Company') == 'Company'
with assert_raises(InvalidData):
model.common_schema('Person', 'Directorship')
def add_cast(self, schema, prop, value):
"""Set a property on an entity. If the entity is of a schema that doesn't
have the given property, also modify the schema (e.g. if something has a
birthDate, assume it's a Person, not a LegalEntity).
"""
if self.schema.get(prop) is not None:
return self.add(prop, value)
schema = model.get(schema)
prop_ = schema.get(prop)
if prop_.type.clean(value) is None:
return
self.schema = model.common_schema(self.schema, schema)
return self.add(prop, value)
def make_pair(pair, judgement, source):
from followthemoney import model
(left, right) = pair
if judgement is False and left["id"] == right["id"]:
return None
try:
schema = model.common_schema(left["schema"], right["schema"])
except InvalidData:
return None
return {
"left": left,
"right": right,
"judgement": judgement,
"schema": schema.name,
"source": source,
}
def test_model_common_schema(self):
assert model.common_schema("Thing", "Thing") == "Thing"
assert model.common_schema("Thing", "Person") == "Person"
assert model.common_schema("Person", "Thing") == "Person"
assert model.common_schema("LegalEntity", "Company") == "Company"
assert model.common_schema("Interval", "Ownership") == "Ownership"
# assert model.common_schema("LegalEntity", "Asset") == "Company"
with assert_raises(InvalidData):
model.common_schema("Person", "Directorship")
with assert_raises(InvalidData):
model.common_schema("Person", "Company")
with assert_raises(InvalidData):
model.common_schema("Membership", "Thing")
def test_model_common_schema(self):
assert model.common_schema("Thing", "Thing") == model["Thing"]
assert model.common_schema("Thing", "Person") == model["Person"]
assert model.common_schema("Person", "Thing") == model["Person"]
assert model.common_schema("LegalEntity",
"Company") == model["Company"]
assert model.common_schema("Interval",
"Ownership") == model["Ownership"]
# This behaviour turned out the be a really bad idea:
# assert model.common_schema("LegalEntity", "Asset") == "Company"
with raises(InvalidData):
model.common_schema("Person", "Directorship")
with raises(InvalidData):
model.common_schema("Person", "Company")
with raises(InvalidData):
model.common_schema("Membership", "Thing")
