def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.create( { 'schema': 'LegalEntity', 'properties': { 'name': 'Winnie the Pooh', 'country': 'pa', 'summary': 'a fictional teddy bear created by A. A. Milne', 'alias': ['Puh der Bär', 'Pooh Bear'] } }, self.col) self.other = Entity.create( { 'schema': 'LegalEntity', 'properties': { 'name': 'Pu der Bär', 'country': 'de', 'description': 'he is a bear', 'alias': ['Puh der Bär'] } }, self.col) db.session.commit()
def setUp(self): super(EntitiesTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Original Collection' self.col.foreign_id = 'test_coll_entities' db.session.add(self.col) self.col_other = Collection() self.col_other.label = 'Other Collection' self.col_other.foreign_id = 'test_coll_entities_other' db.session.add(self.col_other) db.session.flush() self.ent = Entity.create({ 'schema': 'LegalEntity', 'properties': { 'name': 'Winnie the Pooh', 'country': 'pa', 'summary': 'a fictional teddy bear created by A. A. Milne', 'alias': ['Puh der Bär', 'Pooh Bear'] } }, self.col) self.other = Entity.create({ 'schema': 'LegalEntity', 'properties': { 'name': 'Pu der Bär', 'country': 'de', 'description': 'he is a bear', 'alias': ['Puh der Bär'] } }, self.col) db.session.commit()
def setUp(self): super(EntitiesApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = self.create_collection() book = { 'schema': 'PlainText', 'properties': { 'name': 'The Book', 'fileName': 'book.txt', } } self.book = Entity.create(book, self.col) self.book_id = self.col.ns.sign(self.book.id) self.data = { 'schema': 'LegalEntity', 'properties': { 'name': 'Winnie the Pooh', 'country': 'pa', 'proof': self.book_id } } self.ent = Entity.create(self.data, self.col) self.id = self.col.ns.sign(self.ent.id) db.session.commit() self.col_id = str(self.col.id) index_entity(self.book) index_entity(self.ent)
def load_fixtures(self): self.private_coll = Collection.create({ 'foreign_id': 'test_private', 'label': "Private Collection", 'category': 'grey' }) self._banana = Entity.create( { 'schema': 'Person', 'properties': { 'name': ['Banana'], } }, self.private_coll) user = Role.by_foreign_id(Role.SYSTEM_USER) Permission.grant(self.private_coll, user, True, False) self.public_coll = Collection.create({ 'foreign_id': 'test_public', 'label': "Public Collection", 'category': 'news' }) self._kwazulu = Entity.create( { 'schema': 'Company', 'properties': { 'name': ['KwaZulu'], 'alias': ['kwazulu'] } }, self.public_coll) visitor = Role.by_foreign_id(Role.SYSTEM_GUEST) Permission.grant(self.public_coll, visitor, True, False) db.session.commit() samples = read_entities(self.get_fixture_path('samples.ijson')) index_entities(self.private_coll, samples) process_collection(self.public_coll, ingest=False, reset=True) process_collection(self.private_coll, ingest=False, reset=True)
def load_fixture(name): dir_name = os.path.join(fixtures_path, name) if not os.path.isdir(dir_name): raise ValueError("No such directory: %r" % dir_name) with open(os.path.join(dir_name, 'mapping.yaml'), 'rb') as fh: data = yaml.load(fh) lst = List.by_label(data.get('list')) selectors = set() if lst is not None: selectors = lst.terms lst.delete() db.session.commit() lst = List.create( { 'label': data.get('list'), 'public': data.get('public'), 'users': [] }, None) log.info("Loading %r", lst) mapping = data.get('mapping') default_category = data.get('default_category') assert default_category in CATEGORIES, default_category entities = defaultdict(set) with open(os.path.join(dir_name, 'data.csv'), 'rb') as fh: for row in unicodecsv.DictReader(fh): label = row.get(mapping.get('label', 'label')) if label is None: continue category = row.get(mapping.get('category', 'category')) category = category or default_category selectors = [row.get(mapping.get('selector', 'selector'))] selectors = [s for s in selectors if s] entities[(label, category)].update(selectors) for (label, category), selectors in entities.items(): data = { 'label': label, 'category': category, 'selectors': selectors, 'list': lst } try: Entity.create(data, None) except Invalid, inv: log.warn("Failed: %s", inv)
def load_fixtures(self): self.admin = self.create_user(foreign_id='admin', is_admin=True) self.private_coll = self.create_collection(foreign_id='test_private', label="Private Collection", category='grey', casefile=False, creator=self.admin) self._banana = Entity.create( { 'schema': 'Person', 'properties': { 'name': ['Banana'], } }, self.private_coll) user = Role.by_foreign_id(Role.SYSTEM_USER) Permission.grant(self.private_coll, user, True, False) self.public_coll = self.create_collection(foreign_id='test_public', label="Public Collection", category='news', casefile=False, creator=self.admin) self._kwazulu = Entity.create( { 'schema': 'Company', 'properties': { 'name': ['KwaZulu'], 'alias': ['kwazulu'] } }, self.public_coll) visitor = Role.by_foreign_id(Role.SYSTEM_GUEST) Permission.grant(self.public_coll, visitor, True, False) db.session.commit() drop_aggregator(self.public_coll) stage = get_stage(self.public_coll, OP_PROCESS) process_collection(stage, self.public_coll, ingest=False, sync=True) aggregator = get_aggregator(self.private_coll) aggregator.delete() stage = get_stage(self.private_coll, OP_PROCESS) for sample in read_entities(self.get_fixture_path('samples.ijson')): aggregator.put(sample, fragment='sample') index_aggregate(stage, self.private_coll, entity_id=sample.id, sync=True) aggregator.close() process_collection(stage, self.private_coll, ingest=False, sync=True)
def upsert_entity(data, collection, authz=None, sync=False, sign=False, job_id=None): """Create or update an entity in the database. This has a side effect of migrating entities created via the _bulk API or a mapper to a database entity in the event that it gets edited by the user. """ from aleph.logic.profiles import profile_fragments entity = None entity_id = collection.ns.sign(data.get("id")) if entity_id is not None: entity = Entity.by_id(entity_id, collection=collection) if entity is None: role_id = authz.id if authz is not None else None entity = Entity.create(data, collection, sign=sign, role_id=role_id) else: entity.update(data, collection, sign=sign) collection.touch() proxy = entity.to_proxy() aggregator = get_aggregator(collection) aggregator.delete(entity_id=proxy.id) aggregator.put(proxy, origin=MODEL_ORIGIN) profile_fragments(collection, aggregator, entity_id=proxy.id) index.index_proxy(collection, proxy, sync=sync) refresh_entity(collection, proxy.id) queue_task(collection, OP_UPDATE_ENTITY, job_id=job_id, entity_id=proxy.id) return entity.id
def upsert_entity(data, collection, authz=None, sync=False): """Create or update an entity in the database. This has a side hustle of migrating entities created via the _bulk API or a mapper to a database entity in the event that it gets edited by the user. """ entity = None entity_id = collection.ns.sign(data.get("id")) if entity_id is not None: entity = Entity.by_id(entity_id, collection=collection) if entity is None: role_id = authz.id if authz is not None else None entity = Entity.create(data, collection, role_id=role_id) else: entity.update(data, collection) # Inline name properties from adjacent entities. See the # docstring on `inline_names` for a more detailed discussion. proxy = entity.to_proxy() entity_ids = proxy.get_type_values(registry.entity) for rel in index.entities_by_ids(entity_ids): inline_names(proxy, model.get_proxy(rel)) entity.data = proxy.properties db.session.add(entity) delete_aggregator_entity(collection, entity.id) index.index_proxy(collection, proxy, sync=sync) refresh_entity(collection, entity.id) return entity.id
def create_entity(data, collection, role=None, sync=False): entity = Entity.create(data, collection) collection.touch() db.session.commit() index.index_entity(entity, sync=sync) refresh_entity(entity) return collection.ns.sign(entity.id)
def load_fixture(name): dir_name = os.path.join(fixtures_path, name) if not os.path.isdir(dir_name): raise ValueError("No such directory: %r" % dir_name) with open(os.path.join(dir_name, 'mapping.yaml'), 'rb') as fh: data = yaml.load(fh) lst = List.by_label(data.get('list')) selectors = set() if lst is not None: selectors = lst.terms lst.delete() db.session.commit() lst = List.create({ 'label': data.get('list'), 'public': data.get('public'), 'users': [] }, None) log.info("Loading %r", lst) mapping = data.get('mapping') default_category = data.get('default_category') assert default_category in CATEGORIES, default_category entities = defaultdict(set) with open(os.path.join(dir_name, 'data.csv'), 'rb') as fh: for row in unicodecsv.DictReader(fh): label = row.get(mapping.get('label', 'label')) if label is None: continue category = row.get(mapping.get('category', 'category')) category = category or default_category selectors = [row.get(mapping.get('selector', 'selector'))] selectors = [s for s in selectors if s] entities[(label, category)].update(selectors) for (label, category), selectors in entities.items(): data = {'label': label, 'category': category, 'selectors': selectors, 'list': lst} try: Entity.create(data, None) except Invalid, inv: log.warn("Failed: %s", inv)
def create(): data = parse_request(EntityCreateSchema) collection = get_db_collection(data['collection_id'], request.authz.WRITE) entity = Entity.create(data, collection) db.session.commit() data = update_entity(entity) update_collection(collection) return jsonify(data, schema=CombinedSchema)
def create(): data = EntityForm().deserialize(request_data()) authz.require(data["list"]) authz.require(authz.list_write(data["list"].id)) entity = Entity.create(data, current_user) db.session.commit() refresh_selectors.delay(list(entity.terms)) return view(entity.id)
def create(): data = parse_request(schema=EntityCreateSchema) collection = get_db_collection(data['collection_id'], request.authz.WRITE) entity = Entity.create(data, collection) db.session.commit() update_entity(entity) update_collection(collection) return view(entity.id)
def create(): data = EntityForm().deserialize(request_data()) authz.require(data['list']) authz.require(authz.list_write(data['list'].id)) entity = Entity.create(data, current_user) db.session.commit() refresh_selectors.delay(list(entity.terms)) return view(entity.id)
def create(): data = parse_request(EntityCreateSchema) collection = get_db_collection(data['collection_id'], request.authz.WRITE) entity = Entity.create(data, collection) db.session.commit() data = update_entity(entity) update_collection(collection) refresh_index(entities_index()) return serialize_data(data, CombinedSchema)
def create(): data = request_data() collection = data.get('collection') authz.require(collection) authz.require(authz.collection_write(collection.id)) entity = Entity.create(data) collection.touch() db.session.commit() analyze_entity.delay(entity.id) return view(entity.id)
def create(): data = EntityForm().deserialize(request_data()) watchlist = data.get("watchlist") authz.require(watchlist) authz.require(authz.watchlist_write(watchlist.id)) entity = Entity.create(data) watchlist.touch() db.session.commit() analyze_entity.delay(entity.id) return view(entity.id)
def setUp(self): super(EntitiesApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = self.create_collection() self.ent = Entity.create({ 'schema': 'LegalEntity', 'name': 'Winnie the Pooh', 'properties': { 'country': 'pa', } }, self.col) db.session.commit() index_entity(self.ent)
def setUp(self): super(EntitiesApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = self.create_collection() self.data = { 'schema': 'LegalEntity', 'properties': { 'name': 'Winnie the Pooh', 'country': 'pa', } } self.ent = Entity.create(self.data, self.col) db.session.commit() index_entity(self.ent)
def setUp(self): super(CollectionsApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = self.create_collection(label='Test Collection', foreign_id='test_coll_entities_api', category='leak', countries=[]) self.ent = Entity.create( { 'schema': 'Person', 'name': 'Winnie the Pooh', }, self.col) db.session.add(self.ent) db.session.commit()
def setUp(self): super(CollectionsApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = self.create_collection( label='Test Collection', foreign_id='test_coll_entities_api', category='leak', countries=[] ) self.ent = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Winnie the Pooh', } }, self.col) db.session.add(self.ent) db.session.commit()
def setUp(self): super(EntitiesApiTestCase, self).setUp() self.rolex = self.create_user(foreign_id='user_3') self.col = Collection() self.col.label = 'Test Collection' self.col.foreign_id = 'test_coll_entities_api' db.session.add(self.col) db.session.flush() self.ent = Entity.create( { 'schema': 'LegalEntity', 'name': 'Winnie the Pooh', 'data': { 'country': 'pa', } }, self.col) db.session.commit() index_entity(self.ent)
def test_rdf(self): coll = self.create_collection( label='Test Collection', foreign_id='test_coll_stream_api' ) ent = Entity.create({ 'schema': 'Person', 'name': 'Winnie the Pooh', }, coll) db.session.add(ent) db.session.commit() url = '/api/2/collections/%s/_rdf' % coll.id res = self.client.get(url) assert res.status_code == 403, res _, headers = self.login(is_admin=True) res = self.client.get(url, headers=headers) assert res.status_code == 200, res
def upsert_entity(data, collection, sync=False): entity = None entity_id = collection.ns.sign(data.get('id')) if entity_id is not None: entity = Entity.by_id(entity_id, collection=collection, deleted=True) # TODO: migrate softly from index. if entity is None: entity = Entity.create(data, collection) else: entity.update(data, collection) collection.touch() db.session.commit() delete_aggregator_entity(collection, entity.id) index.index_entity(entity, sync=sync) refresh_entity(entity.id, sync=sync) refresh_collection(collection.id, sync=sync) return entity.id
def upsert_entity(data, collection, validate=True, sync=False): """Create or update an entity in the database. This has a side hustle of migrating entities created via the _bulk API or a mapper to a database entity in the event that it gets edited by the user. """ entity = None entity_id = collection.ns.sign(data.get('id')) if entity_id is not None: entity = Entity.by_id(entity_id, collection=collection, deleted=True) # TODO: migrate softly from index. if entity is None: entity = Entity.create(data, collection, validate=validate) else: entity.update(data, collection, validate=validate) collection.touch() db.session.commit() delete_aggregator_entity(collection, entity.id) index.index_entity(entity, sync=sync) refresh_entity(entity.id, sync=sync) refresh_collection(collection.id, sync=sync) return entity.id
def setUp(self): super(XrefApiTestCase, self).setUp() self.creator = self.create_user(foreign_id='creator') self.outsider = self.create_user(foreign_id='outsider') # First public collection and entities self.residents = self.create_collection( label='Residents of Habitat Ring', foreign_id='test_residents', creator=self.creator) self.grant_publish(self.residents) self.ent = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Elim Garak', } }, self.residents) db.session.add(self.ent) self.ent2 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Leeta', } }, self.residents) db.session.add(self.ent2) # Second public collection and entities self.dabo = self.create_collection(label='Dabo Girls', foreign_id='test_dabo', creator=self.creator) self.grant_publish(self.dabo) self.ent3 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'MPella', } }, self.dabo) db.session.add(self.ent3) self.ent4 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Leeta', } }, self.dabo) db.session.add(self.ent4) self.ent5 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Mardah', } }, self.dabo) db.session.add(self.ent5) # Private collection and entities self.obsidian = self.create_collection(label='Obsidian Order', foreign_id='test_obsidian', creator=self.creator) self.ent6 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Elim Garak', } }, self.obsidian) db.session.add(self.ent6) self.ent7 = Entity.create( { 'schema': 'Person', 'properties': { 'name': 'Enabran Tain', } }, self.obsidian) db.session.add(self.ent7) db.session.commit() index_entity(self.ent) index_entity(self.ent2) index_entity(self.ent3) index_entity(self.ent4) index_entity(self.ent5) index_entity(self.ent6) index_entity(self.ent7) self.stage = get_stage(self.residents, OP_XREF)
def create_entity(self, data, collection): return Entity.create(data, collection)
def create_entity(data, collection, role=None, sync=False): entity = Entity.create(data, collection) db.session.commit() data = index.index_entity(entity, sync=sync) refresh_entity(entity) return data
def setUp(self): super(XrefApiTestCase, self).setUp() self.creator = self.create_user(foreign_id='creator') self.outsider = self.create_user(foreign_id='outsider') self.guest = self.create_user(foreign_id=Role.SYSTEM_GUEST) # First public collection and entities self.residents = self.create_collection( label='Residents of Habitat Ring', foreign_id='test_residents', creator=self.creator) self.grant(self.residents, self.guest, True, False) self.ent = Entity.create({ 'schema': 'Person', 'name': 'Elim Garak', }, self.residents) db.session.add(self.ent) self.ent2 = Entity.create({ 'schema': 'Person', 'name': 'Leeta', }, self.residents) db.session.add(self.ent2) # Second public collection and entities self.dabo = self.create_collection(label='Dabo Girls', foreign_id='test_dabo', creator=self.creator) self.grant(self.dabo, self.guest, True, False) self.ent3 = Entity.create({ 'schema': 'Person', 'name': 'MPella', }, self.dabo) db.session.add(self.ent3) self.ent4 = Entity.create({ 'schema': 'Person', 'name': 'Leeta', }, self.dabo) db.session.add(self.ent4) self.ent5 = Entity.create({ 'schema': 'Person', 'name': 'Mardah', }, self.dabo) db.session.add(self.ent5) # Private collection and entities self.obsidian = self.create_collection(label='Obsidian Order', foreign_id='test_obsidian', creator=self.creator) self.ent6 = Entity.create({ 'schema': 'Person', 'name': 'Elim Garack', }, self.obsidian) db.session.add(self.ent6) self.ent7 = Entity.create( { 'schema': 'Person', 'name': 'Enabran Tain', }, self.obsidian) db.session.add(self.ent7) db.session.commit() index_entity(self.ent) index_entity(self.ent2) index_entity(self.ent3) index_entity(self.ent4) index_entity(self.ent5) index_entity(self.ent6) index_entity(self.ent7) self.flush_index()
def create_entity(self, data, collection): entity = Entity.create(data, collection) entity.collection = collection return entity
def setUp(self): super(XrefApiTestCase, self).setUp() self.creator = self.create_user(foreign_id='creator') self.outsider = self.create_user(foreign_id='outsider') # First public collection and entities self.residents = self.create_collection( label='Residents of Habitat Ring', foreign_id='test_residents', creator=self.creator ) self.grant_publish(self.residents) self.ent = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Elim Garak', } }, self.residents) db.session.add(self.ent) self.ent2 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Leeta', } }, self.residents) db.session.add(self.ent2) # Second public collection and entities self.dabo = self.create_collection( label='Dabo Girls', foreign_id='test_dabo', creator=self.creator ) self.grant_publish(self.dabo) self.ent3 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'MPella', } }, self.dabo) db.session.add(self.ent3) self.ent4 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Leeta', } }, self.dabo) db.session.add(self.ent4) self.ent5 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Mardah', } }, self.dabo) db.session.add(self.ent5) # Private collection and entities self.obsidian = self.create_collection( label='Obsidian Order', foreign_id='test_obsidian', creator=self.creator ) self.ent6 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Elim Garak', } }, self.obsidian) db.session.add(self.ent6) self.ent7 = Entity.create({ 'schema': 'Person', 'properties': { 'name': 'Enabran Tain', } }, self.obsidian) db.session.add(self.ent7) db.session.commit() index_entity(self.ent) index_entity(self.ent2) index_entity(self.ent3) index_entity(self.ent4) index_entity(self.ent5) index_entity(self.ent6) index_entity(self.ent7)