def map_to_aggregator(collection, mapping, aggregator):
table = get_entity(mapping.table_id)
if table is None:
table = aggregator.get(mapping.table_id)
if table is None:
raise RuntimeError("Table cannot be found: %s" % mapping.table_id)
config = {"csv_url": _get_table_csv_link(table), "entities": mapping.query}
mapper = model.make_mapping(config, key_prefix=collection.foreign_id)
origin = mapping_origin(mapping.id)
aggregator.delete(origin=origin)
writer = aggregator.bulk()
idx = 0
for idx, record in enumerate(mapper.source.records, 1):
if idx > 0 and idx % 1000 == 0:
log.info("[%s] Mapped %s rows ...", mapping.id, idx)
for entity in mapper.map(record).values():
entity.context = mapping.get_proxy_context()
if entity.schema.is_a("Thing"):
entity.add("proof", mapping.table_id)
entity = collection.ns.apply(entity)
entity = remove_checksums(entity)
writer.put(entity, fragment=idx, origin=origin)
if mapping.entityset is not None:
EntitySetItem.save(
mapping.entityset,
entity.id,
collection_id=collection.id,
added_by_id=mapping.role_id,
)
writer.flush()
log.info("[%s] Mapping done (%s rows)", mapping.id, idx)
def prune_entity(collection, entity_id=None, job_id=None):
"""Prune handles the full deletion of an entity outside of the HTTP request
cycle. This involves cleaning up adjacent entities like xref results, notifications
and so on."""
# This is recursive and will also delete any entities which
# reference the given entity. Usually this is going to be child
# documents, or directoships referencing a person. It's a pretty
# dangerous operation, though.
log.info("[%s] Prune entity: %s", collection, entity_id)
for adjacent in index.iter_adjacent(collection.id, entity_id):
log.warning("Recursive delete: %s", adjacent.get("id"))
delete_entity(collection, adjacent, job_id=job_id)
flush_notifications(entity_id, clazz=Entity)
obj = Entity.by_id(entity_id, collection=collection)
if obj is not None:
obj.delete()
doc = Document.by_id(entity_id, collection=collection)
if doc is not None:
doc.delete()
EntitySetItem.delete_by_entity(entity_id)
Mapping.delete_by_table(entity_id)
xref_index.delete_xref(collection, entity_id=entity_id)
aggregator = get_aggregator(collection)
aggregator.delete(entity_id=entity_id)
refresh_entity(collection, entity_id)
collection.touch()
db.session.commit()
def create_entityset(collection, data, authz):
"""Create an entity set. This will create or update any entities
that already exist in the entityset and sign their IDs into the collection.
"""
old_to_new_id_map = {}
entity_ids = []
for entity in data.pop("entities", []):
old_id = entity.get("id")
new_id = upsert_entity(entity, collection, sync=True)
old_to_new_id_map[old_id] = new_id
entity_ids.append(new_id)
layout = data.get("layout", {})
data["layout"] = replace_layout_ids(layout, old_to_new_id_map)
entityset = EntitySet.create(data, collection, authz)
for entity_id in entity_ids:
EntitySetItem.save(entityset, entity_id)
publish(
Events.CREATE_ENTITYSET,
params={
"collection": collection,
"entityset": entityset
},
channels=[collection, authz.role],
actor_id=authz.id,
)
return entityset
def cleanup_deleted():
from aleph.model import Collection, Role
from aleph.model import EntitySet, EntitySetItem
EntitySetItem.cleanup_deleted()
EntitySet.cleanup_deleted()
Collection.cleanup_deleted()
Role.cleanup_deleted()
db.session.commit()
def delete_entity(collection, entity, deleted_at=None, sync=False):
# This is recursive and will also delete any entities which
# reference the given entity. Usually this is going to be child
# documents, or directoships referencing a person. It's a pretty
# dangerous operation, though.
entity_id = collection.ns.sign(entity.get("id"))
for adjacent in index.iter_adjacent(entity):
log.warning("Recursive delete: %r", adjacent)
delete_entity(collection, adjacent, deleted_at=deleted_at, sync=sync)
flush_notifications(entity_id, clazz=Entity)
obj = Entity.by_id(entity_id, collection=collection)
if obj is not None:
obj.delete()
doc = Document.by_id(entity_id, collection=collection)
if doc is not None:
doc.delete()
index.delete_entity(entity_id, sync=sync)
EntitySetItem.delete_by_entity(entity_id)
Mapping.delete_by_table(entity_id)
xref_index.delete_xref(collection, entity_id=entity_id, sync=sync)
delete_aggregator_entity(collection, entity_id)
refresh_entity(collection, entity_id)
def item_update(entityset_id):
"""Add an item to the entity set with id `entityset_id`, or change
the items judgement.
To delete an item from the entity set, apply the judgement: `no_judgement`.
---
post:
summary: Add item to an entityset
parameters:
- description: The entityset id.
in: path
name: entityset_id
required: true
schema:
type: string
example: 3a0d91ece2dce88ad3259594c7b642485235a048
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/EntitySetItemUpdate'
responses:
'200':
content:
application/json:
schema:
$ref: '#/components/schemas/EntitySetItem'
description: OK
'204':
description: Item removed
tags:
- EntitySetItem
"""
entityset = get_entityset(entityset_id, request.authz.WRITE)
data = parse_request("EntitySetItemUpdate")
entity = data.pop("entity", {})
entity_id = data.pop("entity_id", entity.get("id"))
entity = get_index_entity(entity_id, request.authz.READ)
data["collection_id"] = entity["collection_id"]
data["added_by_id"] = request.authz.id
item = EntitySetItem.save(entityset, entity_id, **data)
db.session.commit()
if item is None:
return ("", 204)
return EntitySetItemSerializer.jsonify(item)
def save_entityset_item(entityset, collection, entity_id, **data):
"""Change the association between an entity and an entityset. In the case of
a profile, this may require re-indexing of the entity to update the associated
profile_id.
"""
item = EntitySetItem.save(entityset,
entity_id,
collection_id=collection.id,
**data)
if entityset.type == EntitySet.PROFILE and entityset.collection_id == collection.id:
from aleph.logic.profiles import profile_fragments
aggregator = get_aggregator(collection)
profile_fragments(collection, aggregator, entity_id=entity_id)
index_aggregator(collection, aggregator, entity_ids=[entity_id])
refresh_entity(collection, entity_id)
refresh_entityset(entityset.id)
return item
def profile_add_entities(entityset, entity_id, collection_id,
compared_to_entity_id, judgement, authz):
pq = db.session.query(EntitySetItem)
pq = pq.filter(EntitySetItem.entityset_id == entityset.id)
pq = pq.filter(EntitySetItem.entity_id == entity_id)
pq = pq.filter(EntitySetItem.deleted_at == None) # noqa
pq.update({EntitySetItem.deleted_at: datetime.utcnow()},
synchronize_session=False)
esi = EntitySetItem(
entityset=entityset,
entity_id=entity_id,
compared_to_entity_id=compared_to_entity_id,
collection_id=collection_id,
added_by_id=authz.id,
judgement=judgement,
)
db.session.add(esi)
return esi
def entities_update(entityset_id):
"""
---
post:
summary: Update an entity and add it to the entity set.
description: >
Update the entity with id `entity_id`. If it does not exist it will be
created. If the user cannot edit the given entity, it is merely added
to the entity set. New entities are always created in the collection of
the entity set.
Aside from these idiosyncracies, this is the same as `/api/2/entities/<id>`,
but handles entity set membership transparently.
parameters:
- description: The entityset id.
in: path
name: entityset_id
required: true
schema:
type: string
example: 3a0d91ece2dce88ad3259594c7b642485235a048
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/EntityUpdate'
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/Entity'
tags:
- Entity
"""
entityset = get_entityset(entityset_id, request.authz.WRITE)
data = parse_request("EntityUpdate")
entity_id = data.get("id", make_textid())
try:
entity = get_index_entity(entity_id, request.authz.READ)
collection = get_db_collection(entity.get("collection_id"),
request.authz.READ)
except NotFound:
entity = None
collection = entityset.collection
tag_request(collection_id=entityset.collection_id)
if entity is None or check_write_entity(entity, request.authz):
if get_flag("validate", default=False):
validate_entity(data)
sync = get_flag("sync", default=True)
entity_id = upsert_entity(data,
collection,
authz=request.authz,
sync=sync)
EntitySetItem.save(
entityset,
entity_id,
collection_id=collection.id,
added_by_id=request.authz.id,
)
db.session.commit()
return entity_view(entity_id)
def bulk(collection_id):
"""
---
post:
summary: Load entities into a collection
description: >
Bulk load entities into the collection with id `collection_id`
parameters:
- description: The collection ID.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
- description: >-
This will disable checksum security measures in order to allow bulk
loading of document data.
in: query
name: unsafe
schema:
type: boolean
requestBody:
description: Entities to be loaded.
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/EntityUpdate'
responses:
'204':
description: No Content
tags:
- Collection
"""
collection = get_db_collection(collection_id, request.authz.WRITE)
require(request.authz.can_bulk_import())
job_id = get_session_id()
entityset = request.args.get("entityset_id")
if entityset is not None:
entityset = get_entityset(entityset, request.authz.WRITE)
# This will disable checksum security measures in order to allow bulk
# loading of document data:
safe = get_flag("safe", default=True)
# Flag is only available for admins:
if not request.authz.is_admin:
safe = True
# Let UI tools change the entities created by this:
mutable = get_flag("mutable", default=False)
role_id = request.authz.id
entities = ensure_list(request.get_json(force=True))
entity_ids = list()
for entity_id in bulk_write(collection,
entities,
safe=safe,
mutable=mutable,
role_id=role_id):
entity_ids.append(entity_id)
if entityset is not None:
EntitySetItem.save(
entityset,
entity_id,
collection_id=collection.id,
added_by_id=request.authz.id,
)
collection.touch()
db.session.commit()
data = {"entity_ids": entity_ids}
queue_task(collection, OP_INDEX, job_id=job_id, payload=data)
return ("", 204)
def get_entitysetitem(entitysetitem_id):
return EntitySetItem.by_id(entitysetitem_id)
def setUp(self):
super(ProfilesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id="rolex")
self.col1 = self.create_collection()
self.grant(self.col1, self.rolex, True, True)
authz = Authz.from_role(self.rolex)
self.profile = EntitySet.create(
{
"label": "x",
"type": EntitySet.PROFILE
}, self.col1, authz)
ent1 = {
"schema": "LegalEntity",
"properties": {
"name": "Donald Trump",
"address": "721 Fifth Avenue, New York, NY",
"phone": "+12024561414",
},
}
self.ent1 = self.create_entity(ent1, self.col1)
index_entity(self.ent1)
EntitySetItem.save(self.profile,
self.ent1.id,
collection_id=self.col1.id)
self.col2 = self.create_collection()
self.grant_publish(self.col2)
ent2 = {
"schema": "Person",
"properties": {
"name": "Donald J. Trump",
"position": "45th President of the US",
"phone": "+12024561414",
},
}
self.ent2 = self.create_entity(ent2, self.col2)
index_entity(self.ent2)
EntitySetItem.save(self.profile,
self.ent2.id,
collection_id=self.col2.id)
ent_false = {
"schema": "LegalEntity",
"properties": {
"name": "Donald Trump, Jr",
"email": "*****@*****.**"
},
}
self.ent_false = self.create_entity(ent_false, self.col2)
index_entity(self.ent_false)
EntitySetItem.save(
self.profile,
self.ent_false.id,
collection_id=self.col2.id,
judgement=Judgement.NEGATIVE,
)
self.col3 = self.create_collection()
ent3 = {
"schema": "LegalEntity",
"properties": {
"name": "Donald John Trump",
"birthDate": "1964"
},
}
self.ent3 = self.create_entity(ent3, self.col3)
index_entity(self.ent3)
EntitySetItem.save(self.profile,
self.ent3.id,
collection_id=self.col3.id)
db.session.commit()
