def setUp(self):
super(EntitiesTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Original Collection'
self.col.foreign_id = 'test_coll_entities'
db.session.add(self.col)
self.col_other = Collection()
self.col_other.label = 'Other Collection'
self.col_other.foreign_id = 'test_coll_entities_other'
db.session.add(self.col_other)
db.session.flush()
self.ent = Entity.save(
{
'name':
'Winnie the Pooh',
'jurisdiction_code':
'pa',
'summary':
'a fictional teddy bear created by author A. A. Milne',
'identifiers': [{
'scheme': 'wikipedia',
'identifier': 'en:Winnie-the-Pooh'
}],
'other_names': [{
'name': u'Puh der Bär'
}, {
'name': 'Pooh Bear'
}]
}, [self.col])
db.session.add(self.ent)
db.session.flush()
self.other = Entity.save(
{
'name':
'Pu der Bär',
'jurisdiction_code':
'de',
'description':
'he is a bear',
'identifiers': [{
'scheme': 'wikipedia',
'identifier': 'en:Winnie-the-Pooh'
}, {
'scheme': 'animals',
'identifier': 'bears.winnie.pooh'
}],
'other_names': [{
'name': u'Puh der Bär'
}]
}, [self.col_other])
db.session.add(self.other)
self.alert = Alert()
self.alert.entity = self.other
db.session.add(self.alert)
db.session.commit()
def setUp(self):
super(EntitiesTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Original Collection'
self.col.foreign_id = 'test_coll_entities'
db.session.add(self.col)
self.col_other = Collection()
self.col_other.label = 'Other Collection'
self.col_other.foreign_id = 'test_coll_entities_other'
db.session.add(self.col_other)
db.session.flush()
self.ent = Entity.save({
'name': 'Winnie the Pooh',
'collections': [self.col],
'jurisdiction_code': 'pa',
'summary': 'a fictional teddy bear created by author A. A. Milne',
'identifiers': [{
'scheme': 'wikipedia',
'identifier': 'en:Winnie-the-Pooh'
}],
'other_names': [{
'name': u'Puh der Bär'
}, {
'name': 'Pooh Bear'
}]
})
db.session.add(self.ent)
db.session.flush()
self.other = Entity.save({
'name': 'Pu der Bär',
'collections': [self.col_other],
'jurisdiction_code': 'de',
'description': 'he is a bear',
'identifiers': [{
'scheme': 'wikipedia',
'identifier': 'en:Winnie-the-Pooh'
}, {
'scheme': 'animals',
'identifier': 'bears.winnie.pooh'
}],
'other_names': [{
'name': u'Puh der Bär'
}]
})
db.session.add(self.other)
self.alert = Alert()
self.alert.entity = self.other
db.session.add(self.alert)
db.session.commit()
def emit_entity(self, collection, data):
entity = Entity.save(data, [collection], merge=True)
db.session.commit()
log.info("Entity [%s]: %s", entity.id, entity.name)
update_entity(entity)
self.increment_count()
return entity
def emit_entity(self, collection, data):
entity = Entity.save(data, [collection], merge=True)
db.session.commit()
log.info("Entity [%s]: %s", entity.id, entity.name)
update_entity(entity)
self.increment_count()
return entity
def load_entity(self, name, schema):
identifier = name.lower().strip()
q = db.session.query(EntityIdentifier)
q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst())
q = q.filter(EntityIdentifier.scheme == self.origin)
q = q.filter(EntityIdentifier.identifier == identifier)
ident = q.first()
if ident is not None:
if ident.deleted_at is None:
# TODO: add to collections? Security risk here.
return ident.entity_id
if ident.entity.deleted_at is None:
return None
data = {
'name': name,
'$schema': schema,
'state': Entity.STATE_PENDING,
'identifiers': [{
'scheme': self.origin,
'identifier': identifier
}]
}
entity = Entity.save(data, self.collections)
return entity.id
def load_entity(self, name, schema):
identifier = name.lower().strip()
q = db.session.query(EntityIdentifier)
q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst())
q = q.filter(EntityIdentifier.scheme == self.origin)
q = q.filter(EntityIdentifier.identifier == identifier)
ident = q.first()
if ident is not None:
if ident.deleted_at is None:
# TODO: add to collections? Security risk here.
return ident.entity_id
if ident.entity.deleted_at is None:
return None
data = {
'name': name,
'$schema': schema,
'state': Entity.STATE_PENDING,
'identifiers': [{
'scheme': self.origin,
'identifier': identifier
}]
}
entity = Entity.save(data, self.collections)
return entity.id
def load_entity(self, sa_id, name, full):
identifier = sa_id
log.debug("%s Loading %s %s", self, self.scheme, sa_id)
q = db.session.query(EntityIdentifier)
q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst())
q = q.filter(EntityIdentifier.scheme == self.scheme)
q = q.filter(EntityIdentifier.identifier == identifier)
ident = q.first()
if ident is not None:
if ident.deleted_at is None:
# TODO: add to collections? Security risk here.
return ident.entity_id
if ident.entity.deleted_at is None:
return None
data = {
'name': full,
'$schema': DEFAULT_SCHEMA,
'identifiers': [{
'scheme': self.scheme,
'identifier': identifier
}],
'other_names': [
{
'name': name
},
{
'name': sa_id
},
],
'company_number': sa_id
}
log.debug("%s Saving %s", self, full)
entity = Entity.save(data, self.collections)
return entity.id
def update(id):
entity = obj_or_404(Entity.by_id(id))
entity = Entity.save(get_data(entity=entity),
collection_id=entity.collection_id,
merge=arg_bool('merge'))
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
def emit_entity(self, collection, data):
data['collections'] = [collection]
entity = Entity.save(data, merge=True)
db.session.flush()
update_entity_full.delay(entity.id)
log.info("Entity [%s]: %s", entity.id, entity.name)
self.entity_cache[collection.id].append(entity)
return entity
def emit_entity(self, collection, data):
data['collections'] = [collection]
entity = Entity.save(data, merge=True)
db.session.flush()
update_entity_full.delay(entity.id)
log.info("Entity [%s]: %s", entity.id, entity.name)
self.entity_cache[collection.id].append(entity)
return entity
def update(id):
entity = obj_or_404(Entity.by_id(id))
entity = Entity.save(get_data(entity=entity),
collection_id=entity.collection_id,
merge=arg_bool('merge'))
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
def create():
data = request_data()
data.pop('id', None)
data['collections'] = get_collections(data)
for collection in data['collections']:
authz.require(authz.collection_write(collection.id))
entity = Entity.save(data)
db.session.commit()
update_entity(entity)
return view(entity.id)
def create():
data = request_data()
data.pop('id', None)
data['collections'] = get_collections(data)
for collection in data['collections']:
authz.require(authz.collection_write(collection.id))
entity = Entity.save(data)
db.session.commit()
update_entity(entity)
return view(entity.id)
def crawl_collection(self, collection):
if not len(collection.get('subjects', [])):
return
url = urljoin(self.URL, '/api/collections/%s' % collection.get('id'))
collection = Collection.by_foreign_id(url, {
'label': collection.get('title')
})
res = requests.get('%s/permissions' % url, headers=self.HEADERS)
for perm in res.json().get('results', []):
Permission.grant_foreign(collection, perm.get('role'),
perm.get('read'), perm.get('write'))
log.info(" > Spindle collection: %s", collection.label)
res = requests.get('%s/entities' % url, headers=self.HEADERS)
terms = set()
existing_entities = []
for entity in res.json().get('results', []):
if entity.get('name') is None:
continue
entity['$schema'] = SCHEMATA.get(entity.get('$schema'), OTHER)
if 'jurisdiction_code' in entity:
entity['jurisdiction_code'] = \
entity['jurisdiction_code'].lower()
entity.pop('members', None)
entity.pop('memberships', None)
entity.pop('assets', None)
entity.pop('owners', None)
entity.pop('family_first', None)
entity.pop('family_second', None)
entity.pop('social_first', None)
entity.pop('social_second', None)
for date_field in ['birth_date']:
if date_field in entity and 'T' in entity[date_field]:
entity[date_field], _ = entity[date_field].split('T', 1)
for on in entity.get('other_names', []):
name = on.pop('alias', None)
if name is not None:
on['name'] = name
entity['identifiers'] = [{
'scheme': 'spindle',
'identifier': entity.pop('id', None)
}]
ent = Entity.save(entity, collection_id=collection.id, merge=True)
db.session.flush()
terms.update(ent.terms)
existing_entities.append(ent.id)
log.info(" # %s", ent.name)
for entity in collection.entities:
if entity.id not in existing_entities:
entity.delete()
self.emit_collection(collection, terms)
def load_entity(self, fk, name, schema):
entity = Entity.by_foreign_id(fk, self.collection.id, deleted=True)
if entity is not None:
return entity
return Entity.save({
'name': name,
'schema': schema,
'foreign_ids': [fk],
'state': Entity.STATE_PENDING,
'data': {}
}, self.collection)
def update(id):
entity = obj_or_404(Entity.by_id(id))
check_authz(entity, authz.WRITE)
data = request_data()
data['id'] = entity.id
possible_collections = authz.collections(authz.WRITE)
possible_collections.extend([c.id for c in entity.collections])
data['collections'] = [c for c in get_collections(data)
if c.id in possible_collections]
entity = Entity.save(data, merge=arg_bool('merge'))
db.session.commit()
update_entity(entity)
return view(entity.id)
def setUp(self):
super(EntitiesTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Original Collection'
self.col.foreign_id = 'test_coll_entities'
db.session.add(self.col)
self.col_other = Collection()
self.col_other.label = 'Other Collection'
self.col_other.foreign_id = 'test_coll_entities_other'
db.session.add(self.col_other)
db.session.flush()
self.ent = Entity.save(
{
'schema': 'LegalEntity',
'name': 'Winnie the Pooh',
'data': {
'country': 'pa',
'summary': 'a fictional teddy bear created by A. A. Milne',
'alias': [u'Puh der Bär', 'Pooh Bear']
}
}, self.col)
self.other = Entity.save(
{
'schema': 'LegalEntity',
'name': 'Pu der Bär',
'data': {
'country': 'de',
'description': 'he is a bear',
'alias': [u'Puh der Bär']
}
}, self.col)
self.alert = Alert()
self.alert.entity = self.other
db.session.add(self.alert)
db.session.commit()
index_entity(self.ent)
index_entity(self.other)
def update(id):
entity = obj_or_404(Entity.by_id(id))
check_authz(entity, authz.WRITE)
data = request_data()
data['id'] = entity.id
possible_collections = authz.collections(authz.WRITE)
possible_collections.extend([c.id for c in entity.collections])
data['collections'] = [
c for c in get_collections(data) if c.id in possible_collections
]
entity = Entity.save(data, merge=arg_bool('merge'))
db.session.commit()
update_entity(entity)
return view(entity.id)
def create():
data = request_data()
data.pop('id', None)
collections = get_collections(data)
for collection in collections:
authz.require(authz.collection_write(collection.id))
entity = Entity.save(data, collections)
for collection in entity.collections:
collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def create():
data = request_data()
data.pop('id', None)
collections = get_collections(data)
for collection in collections:
authz.require(authz.collection_write(collection.id))
entity = Entity.save(data, collections)
for collection in entity.collections:
collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def update(id):
_, entity = get_entity(id, request.authz.WRITE)
try:
entity = Entity.save(request_data(),
entity.collection,
merge=arg_bool('merge'))
except (ValueError, TypeError) as ve:
raise BadRequest(ve.message)
entity.collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def setUp(self):
super(CollectionsApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Test Collection'
self.col.foreign_id = 'test_coll_entities_api'
self.col.category = 'leak'
self.col.countries = []
db.session.add(self.col)
db.session.flush()
self.ent = Entity.save({
'schema': 'Person',
'name': 'Winnie the Pooh',
}, self.col)
db.session.add(self.ent)
db.session.commit()
def setUp(self):
super(EntitiesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Test Collection'
self.col.foreign_id = 'test_coll_entities_api'
db.session.add(self.col)
db.session.flush()
self.ent = Entity.save({
'schema': 'LegalEntity',
'name': 'Winnie the Pooh',
'data': {
'country': 'pa',
}
}, self.col)
db.session.commit()
index_entity(self.ent)
def create():
data = request_data()
data.pop("id", None)
collections = get_collections(data)
for collection in collections:
authz.require(authz.collection_write(collection.id))
try:
entity = Entity.save(data, collections)
except ValueError as ve:
raise BadRequest(ve.message)
for collection in entity.collections:
collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def update(id):
entity = obj_or_404(Entity.by_id(id))
check_authz(entity, authz.WRITE)
data = request_data()
data["id"] = entity.id
possible_collections = authz.collections(authz.WRITE)
possible_collections.extend([c.id for c in entity.collections])
collections = [c for c in get_collections(data) if c.id in possible_collections]
try:
entity = Entity.save(data, collections, merge=arg_bool("merge"))
except ValueError as ve:
raise BadRequest(ve.message)
for collection in entity.collections:
collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def crawl_source(self, source):
if source.get('source_id') in IGNORE_SOURCES:
return
json_file = source.get('data', {}).get('json')
url = urljoin(JSON_PATH, json_file)
source_name = source.get('source') or source.get('source_id')
label = '%s - %s' % (source.get('publisher'), source_name)
collection = Collection.by_foreign_id(url, {'label': label})
Permission.grant_foreign(collection, Role.SYSTEM_GUEST, True, False)
log.info(" > OpenNames collection: %s", collection.label)
terms = set()
existing_entities = []
db.session.flush()
entities = requests.get(url).json().get('entities', [])
for entity in entities:
data = {
'identifiers': [{
'scheme':
'opennames:%s' % source.get('source_id'),
'identifier':
entity.get('uid')
}],
'other_names': [],
'name':
entity.get('name'),
'$schema':
SCHEMA.get(entity.get('type'), '/entity/entity.json#')
}
for on in entity.get('other_names', []):
on['name'] = on.pop('other_name', None)
data['other_names'].append(on)
ent = Entity.save(data, collection_id=collection.id, merge=True)
db.session.flush()
terms.update(ent.terms)
existing_entities.append(ent.id)
log.info(" # %s", ent.name)
for entity in collection.entities:
if entity.id not in existing_entities:
entity.delete()
self.emit_collection(collection, terms)
def crawl_source(self, source):
if source.get('source_id') in IGNORE_SOURCES:
return
json_file = source.get('data', {}).get('json')
url = urljoin(JSON_PATH, json_file)
source_name = source.get('source') or source.get('source_id')
label = '%s - %s' % (source.get('publisher'), source_name)
collection = Collection.by_foreign_id(url, {
'label': label
})
Permission.grant_foreign(collection, Role.SYSTEM_GUEST, True, False)
log.info(" > OpenNames collection: %s", collection.label)
terms = set()
existing_entities = []
db.session.flush()
entities = requests.get(url).json().get('entities', [])
for entity in entities:
data = {
'identifiers': [{
'scheme': 'opennames:%s' % source.get('source_id'),
'identifier': entity.get('uid')
}],
'other_names': [],
'name': entity.get('name'),
'$schema': SCHEMA.get(entity.get('type'),
'/entity/entity.json#')
}
for on in entity.get('other_names', []):
on['name'] = on.pop('other_name', None)
data['other_names'].append(on)
ent = Entity.save(data, collection_id=collection.id, merge=True)
db.session.flush()
terms.update(ent.terms)
existing_entities.append(ent.id)
log.info(" # %s", ent.name)
for entity in collection.entities:
if entity.id not in existing_entities:
entity.delete()
self.emit_collection(collection, terms)
def create():
data = request_data()
collection_id = data.get('collection_id')
try:
collection_id = int(collection_id)
except (ValueError, TypeError) as ve:
raise BadRequest("Invalid collection_id")
collection = obj_or_404(Collection.by_id(collection_id))
request.authz.require(request.authz.collection_write(collection.id))
try:
entity = Entity.save(data, collection)
except (ValueError, TypeError) as ve:
raise BadRequest(ve.message)
entity.collection.touch()
db.session.commit()
log_event(request, entity_id=entity.id)
update_entity(entity)
return view(entity.id)
def load_entity(self, name, schema):
q = db.session.query(EntityIdentifier)
q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst())
q = q.filter(EntityIdentifier.scheme == self.origin)
q = q.filter(EntityIdentifier.identifier == name)
ident = q.first()
if ident is not None:
if ident.deleted_at is None:
return ident.entity_id
if ident.entity.deleted_at is None:
return None
data = {
'name': name,
'$schema': schema,
'state': Entity.STATE_PENDING,
'identifiers': [{
'scheme': self.origin,
'identifier': name
}],
'collections': [self.load_collection()]
}
entity = Entity.save(data)
return entity.id
def update_entity(self, entity, collection):
category = REQUEST_TYPES.get(entity.get('ticket_type'))
if category is None:
return
data = {
'identifiers': [
{
'schema': 'idashboard',
'identifier': entity.get('id')
}
],
'other_names': [],
'name': entity.get('name'),
'$schema': category
}
ent = Entity.by_identifier('idashboard', entity.get('id'),
collection.id)
if ent is not None:
data['id'] = ent.id
ent = Entity.save(data, merge=True)
db.session.flush()
return ent
def update_entity(self, entity, collection):
category = REQUEST_TYPES.get(entity.get('ticket_type'))
if category is None:
return
data = {
'identifiers': [{
'schema': 'idashboard',
'identifier': entity.get('id')
}],
'other_names': [],
'name':
entity.get('name'),
'$schema':
category
}
ent = Entity.by_identifier('idashboard', entity.get('id'),
collection.id)
if ent is not None:
data['id'] = ent.id
ent = Entity.save(data, merge=True)
db.session.flush()
return ent
def load_entity(self, name, schema):
q = db.session.query(EntityIdentifier)
q = q.order_by(EntityIdentifier.deleted_at.desc().nullsfirst())
q = q.filter(EntityIdentifier.scheme == self.origin)
q = q.filter(EntityIdentifier.identifier == name)
ident = q.first()
if ident is not None:
if ident.deleted_at is None:
return ident.entity_id
if ident.entity.deleted_at is None:
return None
data = {
'name': name,
'$schema': schema,
'state': Entity.STATE_PENDING,
'identifiers': [{
'scheme': self.origin,
'identifier': name
}],
'collections': [self.load_collection()]
}
entity = Entity.save(data)
return entity.id
def create():
data = get_data()
entity = Entity.save(data, collection_id=data.get('collection_id'))
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
def create():
data = get_data()
entity = Entity.save(data, collection_id=data.get('collection_id'))
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
