def setUp(self):
super(EntitiesTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Original Collection'
self.col.foreign_id = 'test_coll_entities'
db.session.add(self.col)
self.col_other = Collection()
self.col_other.label = 'Other Collection'
self.col_other.foreign_id = 'test_coll_entities_other'
db.session.add(self.col_other)
db.session.flush()
self.ent = Entity.create(
{
'schema': 'LegalEntity',
'properties': {
'name': 'Winnie the Pooh',
'country': 'pa',
'summary': 'a fictional teddy bear created by A. A. Milne',
'alias': ['Puh der Bär', 'Pooh Bear']
}
}, self.col)
self.other = Entity.create(
{
'schema': 'LegalEntity',
'properties': {
'name': 'Pu der Bär',
'country': 'de',
'description': 'he is a bear',
'alias': ['Puh der Bär']
}
}, self.col)
db.session.commit()
def setUp(self):
super(EntitiesTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Original Collection'
self.col.foreign_id = 'test_coll_entities'
db.session.add(self.col)
self.col_other = Collection()
self.col_other.label = 'Other Collection'
self.col_other.foreign_id = 'test_coll_entities_other'
db.session.add(self.col_other)
db.session.flush()
self.ent = Entity.create({
'schema': 'LegalEntity',
'properties': {
'name': 'Winnie the Pooh',
'country': 'pa',
'summary': 'a fictional teddy bear created by A. A. Milne',
'alias': ['Puh der Bär', 'Pooh Bear']
}
}, self.col)
self.other = Entity.create({
'schema': 'LegalEntity',
'properties': {
'name': 'Pu der Bär',
'country': 'de',
'description': 'he is a bear',
'alias': ['Puh der Bär']
}
}, self.col)
db.session.commit()
def setUp(self):
super(EntitiesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = self.create_collection()
book = {
'schema': 'PlainText',
'properties': {
'name': 'The Book',
'fileName': 'book.txt',
}
}
self.book = Entity.create(book, self.col)
self.book_id = self.col.ns.sign(self.book.id)
self.data = {
'schema': 'LegalEntity',
'properties': {
'name': 'Winnie the Pooh',
'country': 'pa',
'proof': self.book_id
}
}
self.ent = Entity.create(self.data, self.col)
self.id = self.col.ns.sign(self.ent.id)
db.session.commit()
self.col_id = str(self.col.id)
index_entity(self.book)
index_entity(self.ent)
def load_fixtures(self):
self.private_coll = Collection.create({
'foreign_id': 'test_private',
'label': "Private Collection",
'category': 'grey'
})
self._banana = Entity.create(
{
'schema': 'Person',
'properties': {
'name': ['Banana'],
}
}, self.private_coll)
user = Role.by_foreign_id(Role.SYSTEM_USER)
Permission.grant(self.private_coll, user, True, False)
self.public_coll = Collection.create({
'foreign_id': 'test_public',
'label': "Public Collection",
'category': 'news'
})
self._kwazulu = Entity.create(
{
'schema': 'Company',
'properties': {
'name': ['KwaZulu'],
'alias': ['kwazulu']
}
}, self.public_coll)
visitor = Role.by_foreign_id(Role.SYSTEM_GUEST)
Permission.grant(self.public_coll, visitor, True, False)
db.session.commit()
samples = read_entities(self.get_fixture_path('samples.ijson'))
index_entities(self.private_coll, samples)
process_collection(self.public_coll, ingest=False, reset=True)
process_collection(self.private_coll, ingest=False, reset=True)
def load_fixture(name):
dir_name = os.path.join(fixtures_path, name)
if not os.path.isdir(dir_name):
raise ValueError("No such directory: %r" % dir_name)
with open(os.path.join(dir_name, 'mapping.yaml'), 'rb') as fh:
data = yaml.load(fh)
lst = List.by_label(data.get('list'))
selectors = set()
if lst is not None:
selectors = lst.terms
lst.delete()
db.session.commit()
lst = List.create(
{
'label': data.get('list'),
'public': data.get('public'),
'users': []
}, None)
log.info("Loading %r", lst)
mapping = data.get('mapping')
default_category = data.get('default_category')
assert default_category in CATEGORIES, default_category
entities = defaultdict(set)
with open(os.path.join(dir_name, 'data.csv'), 'rb') as fh:
for row in unicodecsv.DictReader(fh):
label = row.get(mapping.get('label', 'label'))
if label is None:
continue
category = row.get(mapping.get('category', 'category'))
category = category or default_category
selectors = [row.get(mapping.get('selector', 'selector'))]
selectors = [s for s in selectors if s]
entities[(label, category)].update(selectors)
for (label, category), selectors in entities.items():
data = {
'label': label,
'category': category,
'selectors': selectors,
'list': lst
}
try:
Entity.create(data, None)
except Invalid, inv:
log.warn("Failed: %s", inv)
def load_fixtures(self):
self.admin = self.create_user(foreign_id='admin', is_admin=True)
self.private_coll = self.create_collection(foreign_id='test_private',
label="Private Collection",
category='grey',
casefile=False,
creator=self.admin)
self._banana = Entity.create(
{
'schema': 'Person',
'properties': {
'name': ['Banana'],
}
}, self.private_coll)
user = Role.by_foreign_id(Role.SYSTEM_USER)
Permission.grant(self.private_coll, user, True, False)
self.public_coll = self.create_collection(foreign_id='test_public',
label="Public Collection",
category='news',
casefile=False,
creator=self.admin)
self._kwazulu = Entity.create(
{
'schema': 'Company',
'properties': {
'name': ['KwaZulu'],
'alias': ['kwazulu']
}
}, self.public_coll)
visitor = Role.by_foreign_id(Role.SYSTEM_GUEST)
Permission.grant(self.public_coll, visitor, True, False)
db.session.commit()
drop_aggregator(self.public_coll)
stage = get_stage(self.public_coll, OP_PROCESS)
process_collection(stage, self.public_coll, ingest=False, sync=True)
aggregator = get_aggregator(self.private_coll)
aggregator.delete()
stage = get_stage(self.private_coll, OP_PROCESS)
for sample in read_entities(self.get_fixture_path('samples.ijson')):
aggregator.put(sample, fragment='sample')
index_aggregate(stage,
self.private_coll,
entity_id=sample.id,
sync=True)
aggregator.close()
process_collection(stage, self.private_coll, ingest=False, sync=True)
def upsert_entity(data,
collection,
authz=None,
sync=False,
sign=False,
job_id=None):
"""Create or update an entity in the database. This has a side effect of migrating
entities created via the _bulk API or a mapper to a database entity in the event
that it gets edited by the user.
"""
from aleph.logic.profiles import profile_fragments
entity = None
entity_id = collection.ns.sign(data.get("id"))
if entity_id is not None:
entity = Entity.by_id(entity_id, collection=collection)
if entity is None:
role_id = authz.id if authz is not None else None
entity = Entity.create(data, collection, sign=sign, role_id=role_id)
else:
entity.update(data, collection, sign=sign)
collection.touch()
proxy = entity.to_proxy()
aggregator = get_aggregator(collection)
aggregator.delete(entity_id=proxy.id)
aggregator.put(proxy, origin=MODEL_ORIGIN)
profile_fragments(collection, aggregator, entity_id=proxy.id)
index.index_proxy(collection, proxy, sync=sync)
refresh_entity(collection, proxy.id)
queue_task(collection, OP_UPDATE_ENTITY, job_id=job_id, entity_id=proxy.id)
return entity.id
def upsert_entity(data, collection, authz=None, sync=False):
"""Create or update an entity in the database. This has a side hustle
of migrating entities created via the _bulk API or a mapper to a
database entity in the event that it gets edited by the user.
"""
entity = None
entity_id = collection.ns.sign(data.get("id"))
if entity_id is not None:
entity = Entity.by_id(entity_id, collection=collection)
if entity is None:
role_id = authz.id if authz is not None else None
entity = Entity.create(data, collection, role_id=role_id)
else:
entity.update(data, collection)
# Inline name properties from adjacent entities. See the
# docstring on `inline_names` for a more detailed discussion.
proxy = entity.to_proxy()
entity_ids = proxy.get_type_values(registry.entity)
for rel in index.entities_by_ids(entity_ids):
inline_names(proxy, model.get_proxy(rel))
entity.data = proxy.properties
db.session.add(entity)
delete_aggregator_entity(collection, entity.id)
index.index_proxy(collection, proxy, sync=sync)
refresh_entity(collection, entity.id)
return entity.id
def create_entity(data, collection, role=None, sync=False):
entity = Entity.create(data, collection)
collection.touch()
db.session.commit()
index.index_entity(entity, sync=sync)
refresh_entity(entity)
return collection.ns.sign(entity.id)
def load_fixture(name):
dir_name = os.path.join(fixtures_path, name)
if not os.path.isdir(dir_name):
raise ValueError("No such directory: %r" % dir_name)
with open(os.path.join(dir_name, 'mapping.yaml'), 'rb') as fh:
data = yaml.load(fh)
lst = List.by_label(data.get('list'))
selectors = set()
if lst is not None:
selectors = lst.terms
lst.delete()
db.session.commit()
lst = List.create({
'label': data.get('list'),
'public': data.get('public'),
'users': []
}, None)
log.info("Loading %r", lst)
mapping = data.get('mapping')
default_category = data.get('default_category')
assert default_category in CATEGORIES, default_category
entities = defaultdict(set)
with open(os.path.join(dir_name, 'data.csv'), 'rb') as fh:
for row in unicodecsv.DictReader(fh):
label = row.get(mapping.get('label', 'label'))
if label is None:
continue
category = row.get(mapping.get('category', 'category'))
category = category or default_category
selectors = [row.get(mapping.get('selector', 'selector'))]
selectors = [s for s in selectors if s]
entities[(label, category)].update(selectors)
for (label, category), selectors in entities.items():
data = {'label': label, 'category': category,
'selectors': selectors, 'list': lst}
try:
Entity.create(data, None)
except Invalid, inv:
log.warn("Failed: %s", inv)
def create():
data = parse_request(EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
data = update_entity(entity)
update_collection(collection)
return jsonify(data, schema=CombinedSchema)
def create():
data = EntityForm().deserialize(request_data())
authz.require(data["list"])
authz.require(authz.list_write(data["list"].id))
entity = Entity.create(data, current_user)
db.session.commit()
refresh_selectors.delay(list(entity.terms))
return view(entity.id)
def create():
data = parse_request(schema=EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
update_entity(entity)
update_collection(collection)
return view(entity.id)
def create():
data = EntityForm().deserialize(request_data())
authz.require(data['list'])
authz.require(authz.list_write(data['list'].id))
entity = Entity.create(data, current_user)
db.session.commit()
refresh_selectors.delay(list(entity.terms))
return view(entity.id)
def create():
data = parse_request(EntityCreateSchema)
collection = get_db_collection(data['collection_id'], request.authz.WRITE)
entity = Entity.create(data, collection)
db.session.commit()
data = update_entity(entity)
update_collection(collection)
refresh_index(entities_index())
return serialize_data(data, CombinedSchema)
def create():
data = request_data()
collection = data.get('collection')
authz.require(collection)
authz.require(authz.collection_write(collection.id))
entity = Entity.create(data)
collection.touch()
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
def create():
data = EntityForm().deserialize(request_data())
watchlist = data.get("watchlist")
authz.require(watchlist)
authz.require(authz.watchlist_write(watchlist.id))
entity = Entity.create(data)
watchlist.touch()
db.session.commit()
analyze_entity.delay(entity.id)
return view(entity.id)
def setUp(self):
super(EntitiesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = self.create_collection()
self.ent = Entity.create({
'schema': 'LegalEntity',
'name': 'Winnie the Pooh',
'properties': {
'country': 'pa',
}
}, self.col)
db.session.commit()
index_entity(self.ent)
def setUp(self):
super(EntitiesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = self.create_collection()
self.data = {
'schema': 'LegalEntity',
'properties': {
'name': 'Winnie the Pooh',
'country': 'pa',
}
}
self.ent = Entity.create(self.data, self.col)
db.session.commit()
index_entity(self.ent)
def setUp(self):
super(CollectionsApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = self.create_collection(label='Test Collection',
foreign_id='test_coll_entities_api',
category='leak',
countries=[])
self.ent = Entity.create(
{
'schema': 'Person',
'name': 'Winnie the Pooh',
}, self.col)
db.session.add(self.ent)
db.session.commit()
def setUp(self):
super(CollectionsApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = self.create_collection(
label='Test Collection',
foreign_id='test_coll_entities_api',
category='leak',
countries=[]
)
self.ent = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Winnie the Pooh',
}
}, self.col)
db.session.add(self.ent)
db.session.commit()
def setUp(self):
super(EntitiesApiTestCase, self).setUp()
self.rolex = self.create_user(foreign_id='user_3')
self.col = Collection()
self.col.label = 'Test Collection'
self.col.foreign_id = 'test_coll_entities_api'
db.session.add(self.col)
db.session.flush()
self.ent = Entity.create(
{
'schema': 'LegalEntity',
'name': 'Winnie the Pooh',
'data': {
'country': 'pa',
}
}, self.col)
db.session.commit()
index_entity(self.ent)
def test_rdf(self):
coll = self.create_collection(
label='Test Collection',
foreign_id='test_coll_stream_api'
)
ent = Entity.create({
'schema': 'Person',
'name': 'Winnie the Pooh',
}, coll)
db.session.add(ent)
db.session.commit()
url = '/api/2/collections/%s/_rdf' % coll.id
res = self.client.get(url)
assert res.status_code == 403, res
_, headers = self.login(is_admin=True)
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
def upsert_entity(data, collection, sync=False):
entity = None
entity_id = collection.ns.sign(data.get('id'))
if entity_id is not None:
entity = Entity.by_id(entity_id,
collection=collection,
deleted=True)
# TODO: migrate softly from index.
if entity is None:
entity = Entity.create(data, collection)
else:
entity.update(data, collection)
collection.touch()
db.session.commit()
delete_aggregator_entity(collection, entity.id)
index.index_entity(entity, sync=sync)
refresh_entity(entity.id, sync=sync)
refresh_collection(collection.id, sync=sync)
return entity.id
def upsert_entity(data, collection, validate=True, sync=False):
"""Create or update an entity in the database. This has a side hustle
of migrating entities created via the _bulk API or a mapper to a
database entity in the event that it gets edited by the user.
"""
entity = None
entity_id = collection.ns.sign(data.get('id'))
if entity_id is not None:
entity = Entity.by_id(entity_id, collection=collection, deleted=True)
# TODO: migrate softly from index.
if entity is None:
entity = Entity.create(data, collection, validate=validate)
else:
entity.update(data, collection, validate=validate)
collection.touch()
db.session.commit()
delete_aggregator_entity(collection, entity.id)
index.index_entity(entity, sync=sync)
refresh_entity(entity.id, sync=sync)
refresh_collection(collection.id, sync=sync)
return entity.id
def setUp(self):
super(XrefApiTestCase, self).setUp()
self.creator = self.create_user(foreign_id='creator')
self.outsider = self.create_user(foreign_id='outsider')
# First public collection and entities
self.residents = self.create_collection(
label='Residents of Habitat Ring',
foreign_id='test_residents',
creator=self.creator)
self.grant_publish(self.residents)
self.ent = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Elim Garak',
}
}, self.residents)
db.session.add(self.ent)
self.ent2 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Leeta',
}
}, self.residents)
db.session.add(self.ent2)
# Second public collection and entities
self.dabo = self.create_collection(label='Dabo Girls',
foreign_id='test_dabo',
creator=self.creator)
self.grant_publish(self.dabo)
self.ent3 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'MPella',
}
}, self.dabo)
db.session.add(self.ent3)
self.ent4 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Leeta',
}
}, self.dabo)
db.session.add(self.ent4)
self.ent5 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Mardah',
}
}, self.dabo)
db.session.add(self.ent5)
# Private collection and entities
self.obsidian = self.create_collection(label='Obsidian Order',
foreign_id='test_obsidian',
creator=self.creator)
self.ent6 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Elim Garak',
}
}, self.obsidian)
db.session.add(self.ent6)
self.ent7 = Entity.create(
{
'schema': 'Person',
'properties': {
'name': 'Enabran Tain',
}
}, self.obsidian)
db.session.add(self.ent7)
db.session.commit()
index_entity(self.ent)
index_entity(self.ent2)
index_entity(self.ent3)
index_entity(self.ent4)
index_entity(self.ent5)
index_entity(self.ent6)
index_entity(self.ent7)
self.stage = get_stage(self.residents, OP_XREF)
def create_entity(self, data, collection):
return Entity.create(data, collection)
def create_entity(data, collection, role=None, sync=False):
entity = Entity.create(data, collection)
db.session.commit()
data = index.index_entity(entity, sync=sync)
refresh_entity(entity)
return data
def setUp(self):
super(XrefApiTestCase, self).setUp()
self.creator = self.create_user(foreign_id='creator')
self.outsider = self.create_user(foreign_id='outsider')
self.guest = self.create_user(foreign_id=Role.SYSTEM_GUEST)
# First public collection and entities
self.residents = self.create_collection(
label='Residents of Habitat Ring',
foreign_id='test_residents',
creator=self.creator)
self.grant(self.residents, self.guest, True, False)
self.ent = Entity.create({
'schema': 'Person',
'name': 'Elim Garak',
}, self.residents)
db.session.add(self.ent)
self.ent2 = Entity.create({
'schema': 'Person',
'name': 'Leeta',
}, self.residents)
db.session.add(self.ent2)
# Second public collection and entities
self.dabo = self.create_collection(label='Dabo Girls',
foreign_id='test_dabo',
creator=self.creator)
self.grant(self.dabo, self.guest, True, False)
self.ent3 = Entity.create({
'schema': 'Person',
'name': 'MPella',
}, self.dabo)
db.session.add(self.ent3)
self.ent4 = Entity.create({
'schema': 'Person',
'name': 'Leeta',
}, self.dabo)
db.session.add(self.ent4)
self.ent5 = Entity.create({
'schema': 'Person',
'name': 'Mardah',
}, self.dabo)
db.session.add(self.ent5)
# Private collection and entities
self.obsidian = self.create_collection(label='Obsidian Order',
foreign_id='test_obsidian',
creator=self.creator)
self.ent6 = Entity.create({
'schema': 'Person',
'name': 'Elim Garack',
}, self.obsidian)
db.session.add(self.ent6)
self.ent7 = Entity.create(
{
'schema': 'Person',
'name': 'Enabran Tain',
}, self.obsidian)
db.session.add(self.ent7)
db.session.commit()
index_entity(self.ent)
index_entity(self.ent2)
index_entity(self.ent3)
index_entity(self.ent4)
index_entity(self.ent5)
index_entity(self.ent6)
index_entity(self.ent7)
self.flush_index()
def create_entity(self, data, collection):
entity = Entity.create(data, collection)
entity.collection = collection
return entity
def create_entity(data, collection, role=None, sync=False):
entity = Entity.create(data, collection)
db.session.commit()
data = index.index_entity(entity, sync=sync)
refresh_entity(entity)
return data
def setUp(self):
super(XrefApiTestCase, self).setUp()
self.creator = self.create_user(foreign_id='creator')
self.outsider = self.create_user(foreign_id='outsider')
# First public collection and entities
self.residents = self.create_collection(
label='Residents of Habitat Ring',
foreign_id='test_residents',
creator=self.creator
)
self.grant_publish(self.residents)
self.ent = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Elim Garak',
}
}, self.residents)
db.session.add(self.ent)
self.ent2 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Leeta',
}
}, self.residents)
db.session.add(self.ent2)
# Second public collection and entities
self.dabo = self.create_collection(
label='Dabo Girls',
foreign_id='test_dabo',
creator=self.creator
)
self.grant_publish(self.dabo)
self.ent3 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'MPella',
}
}, self.dabo)
db.session.add(self.ent3)
self.ent4 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Leeta',
}
}, self.dabo)
db.session.add(self.ent4)
self.ent5 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Mardah',
}
}, self.dabo)
db.session.add(self.ent5)
# Private collection and entities
self.obsidian = self.create_collection(
label='Obsidian Order',
foreign_id='test_obsidian',
creator=self.creator
)
self.ent6 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Elim Garak',
}
}, self.obsidian)
db.session.add(self.ent6)
self.ent7 = Entity.create({
'schema': 'Person',
'properties': {
'name': 'Enabran Tain',
}
}, self.obsidian)
db.session.add(self.ent7)
db.session.commit()
index_entity(self.ent)
index_entity(self.ent2)
index_entity(self.ent3)
index_entity(self.ent4)
index_entity(self.ent5)
index_entity(self.ent6)
index_entity(self.ent7)
