def test_directory_with_file(self): _, headers = self.login(is_admin=True) meta = { 'file_name': 'directory', 'foreign_id': 'directory', 'schema': 'Folder', 'collection_id': self.col.id, } data = {'meta': json.dumps(meta)} res = self.client.post(self.url, data=data, headers=headers) assert res.status_code == 201, res assert 'id' in res.json, res.json directory = res.json['id'] meta = { 'file_name': 'subdirectory', 'foreign_id': 'subdirectory', 'parent': { 'id': directory }, 'collection_id': self.col.id, } data = {'meta': json.dumps(meta)} res = self.client.post(self.url, data=data, headers=headers) assert res.status_code == 201, res stage = get_stage(self.col, OP_PROCESS) process_collection(stage, self.col, ingest=False) assert 'id' in res.json, res.json url = '/api/2/entities/%s' % res.json['id'] res = self.client.get(url, headers=headers) assert res.status_code == 200, res props = res.json.get('properties') assert 'subdirectory' in props['fileName'], res.json
def load_fixtures(self): self.admin = self.create_user(foreign_id='admin', is_admin=True) self.private_coll = self.create_collection( foreign_id='test_private', label="Private Collection", category='grey', casefile=False, creator=self.admin ) self._banana = Entity.create({ 'schema': 'Person', 'properties': { 'name': ['Banana'], } }, self.private_coll) user = Role.by_foreign_id(Role.SYSTEM_USER) Permission.grant(self.private_coll, user, True, False) self.public_coll = self.create_collection( foreign_id='test_public', label="Public Collection", category='news', casefile=False, creator=self.admin ) self._kwazulu = Entity.create({ 'schema': 'Company', 'properties': { 'name': ['KwaZulu'], 'alias': ['kwazulu'] } }, self.public_coll) visitor = Role.by_foreign_id(Role.SYSTEM_GUEST) Permission.grant(self.public_coll, visitor, True, False) db.session.commit() drop_aggregator(self.public_coll) stage = get_stage(self.public_coll, OP_PROCESS) process_collection(stage, self.public_coll, ingest=False, sync=True) aggregator = get_aggregator(self.private_coll) aggregator.delete() stage = get_stage(self.private_coll, OP_PROCESS) for sample in read_entities(self.get_fixture_path('samples.ijson')): aggregator.put(sample, fragment='sample') index_aggregate(stage, self.private_coll, sync=True) aggregator.close() process_collection(stage, self.private_coll, ingest=False, sync=True)
def handle(self, task): stage = task.stage payload = task.payload collection = Collection.by_foreign_id(task.job.dataset.name) if collection is None: log.error("Collection not found: %s", task.job.dataset) return sync = task.context.get('sync', False) if stage.stage == OP_INDEX: index_aggregate(stage, collection, sync=sync, **payload) if stage.stage == OP_LOAD_MAPPING: load_mapping(stage, collection, **payload) if stage.stage == OP_FLUSH_MAPPING: flush_mapping(stage, collection, sync=sync, **payload) if stage.stage == OP_PROCESS: if payload.pop('reset', False): reset_collection(collection, sync=True) process_collection(stage, collection, sync=sync, **payload) if stage.stage == OP_XREF: xref_collection(stage, collection) if stage.stage == OP_XREF_ITEM: xref_item(stage, collection, **payload) log.info("Task [%s]: %s (done)", task.job.dataset, stage.stage)
def process(foreign_id): """Re-process documents in the given collection.""" collection = Collection.by_foreign_id(foreign_id) if collection is None: raise ValueError("No such collection: %r" % foreign_id) process_collection(collection.id)
def process(foreign_id, sync=False): """Process documents and database entities and index them.""" collection = get_collection(foreign_id) stage = get_stage(collection, OP_PROCESS) process_collection(stage, collection, sync=sync)