def test_directory_with_file(self):
_, headers = self.login(is_admin=True)
meta = {
'file_name': 'directory',
'foreign_id': 'directory',
'schema': 'Folder',
'collection_id': self.col.id,
}
data = {'meta': json.dumps(meta)}
res = self.client.post(self.url, data=data, headers=headers)
assert res.status_code == 201, res
assert 'id' in res.json, res.json
directory = res.json['id']
meta = {
'file_name': 'subdirectory',
'foreign_id': 'subdirectory',
'parent': {
'id': directory
},
'collection_id': self.col.id,
}
data = {'meta': json.dumps(meta)}
res = self.client.post(self.url, data=data, headers=headers)
assert res.status_code == 201, res
stage = get_stage(self.col, OP_PROCESS)
process_collection(stage, self.col, ingest=False)
assert 'id' in res.json, res.json
url = '/api/2/entities/%s' % res.json['id']
res = self.client.get(url, headers=headers)
assert res.status_code == 200, res
props = res.json.get('properties')
assert 'subdirectory' in props['fileName'], res.json
def load_fixtures(self):
self.admin = self.create_user(foreign_id='admin', is_admin=True)
self.private_coll = self.create_collection(
foreign_id='test_private',
label="Private Collection",
category='grey',
casefile=False,
creator=self.admin
)
self._banana = Entity.create({
'schema': 'Person',
'properties': {
'name': ['Banana'],
}
}, self.private_coll)
user = Role.by_foreign_id(Role.SYSTEM_USER)
Permission.grant(self.private_coll, user, True, False)
self.public_coll = self.create_collection(
foreign_id='test_public',
label="Public Collection",
category='news',
casefile=False,
creator=self.admin
)
self._kwazulu = Entity.create({
'schema': 'Company',
'properties': {
'name': ['KwaZulu'],
'alias': ['kwazulu']
}
}, self.public_coll)
visitor = Role.by_foreign_id(Role.SYSTEM_GUEST)
Permission.grant(self.public_coll, visitor, True, False)
db.session.commit()
drop_aggregator(self.public_coll)
stage = get_stage(self.public_coll, OP_PROCESS)
process_collection(stage, self.public_coll, ingest=False, sync=True)
aggregator = get_aggregator(self.private_coll)
aggregator.delete()
stage = get_stage(self.private_coll, OP_PROCESS)
for sample in read_entities(self.get_fixture_path('samples.ijson')):
aggregator.put(sample, fragment='sample')
index_aggregate(stage, self.private_coll, sync=True)
aggregator.close()
process_collection(stage, self.private_coll, ingest=False, sync=True)
def handle(self, task):
stage = task.stage
payload = task.payload
collection = Collection.by_foreign_id(task.job.dataset.name)
if collection is None:
log.error("Collection not found: %s", task.job.dataset)
return
sync = task.context.get('sync', False)
if stage.stage == OP_INDEX:
index_aggregate(stage, collection, sync=sync, **payload)
if stage.stage == OP_LOAD_MAPPING:
load_mapping(stage, collection, **payload)
if stage.stage == OP_FLUSH_MAPPING:
flush_mapping(stage, collection, sync=sync, **payload)
if stage.stage == OP_PROCESS:
if payload.pop('reset', False):
reset_collection(collection, sync=True)
process_collection(stage, collection, sync=sync, **payload)
if stage.stage == OP_XREF:
xref_collection(stage, collection)
if stage.stage == OP_XREF_ITEM:
xref_item(stage, collection, **payload)
log.info("Task [%s]: %s (done)", task.job.dataset, stage.stage)
def process(foreign_id):
"""Re-process documents in the given collection."""
collection = Collection.by_foreign_id(foreign_id)
if collection is None:
raise ValueError("No such collection: %r" % foreign_id)
process_collection(collection.id)
def process(foreign_id, sync=False):
"""Process documents and database entities and index them."""
collection = get_collection(foreign_id)
stage = get_stage(collection, OP_PROCESS)
process_collection(stage, collection, sync=sync)
