def view(collection_id):
"""
---
get:
summary: Get a collection
description: Return the collection with id `collection_id`
parameters:
- description: The collection ID.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/CollectionFull'
tags:
- Collection
"""
data = get_index_collection(collection_id)
cobj = get_db_collection(collection_id)
if get_flag("refresh", False):
update_collection_stats(collection_id, ["schema"])
data.update({
"statistics": get_collection_stats(cobj.id),
"status": get_status(cobj),
"shallow": False,
})
return CollectionSerializer.jsonify(data)
def status(foreign_id=None):
"""Get the queue status (pending and finished tasks.)"""
if foreign_id is not None:
collection = get_collection(foreign_id)
status = get_status(collection)
status = {"datasets": {foreign_id: status}}
else:
status = get_active_dataset_status()
headers = ["Collection", "Job", "Stage", "Pending", "Running", "Finished"]
rows = []
for foreign_id, dataset in status.get("datasets").items():
rows.append(
[
foreign_id,
"",
"",
dataset["pending"],
dataset["running"],
dataset["finished"],
]
)
for job in dataset.get("jobs"):
for stage in job.get("stages"):
rows.append(
[
foreign_id,
stage["job_id"],
stage["stage"],
stage["pending"],
stage["running"],
stage["finished"],
]
)
print(tabulate(rows, headers))
def test_upload_csv_doc(self):
_, headers = self.login(is_admin=True)
meta = {
'countries': ['de', 'usa'],
'languages': ['en'],
'mime_type': 'text/csv',
'source_url': 'http://pudo.org/experts.csv'
}
csv_path = self.get_fixture_path('experts.csv')
data = {
'meta': json.dumps(meta),
'foo': open(csv_path, 'rb'),
}
res = self.client.post(self.url, data=data, headers=headers)
assert res.status_code == 201, (res, res.data)
assert 'id' in res.json, res.json
db_id, _ = res.json.get('id').split('.', 1)
doc = Document.by_id(db_id)
assert doc.schema == Document.SCHEMA, doc.schema
assert doc.meta['countries'] == ['de', 'us'], doc.meta
assert doc.meta['languages'] == ['eng'], doc.meta
status = get_status(self.col)
assert status.get('pending') == 1, status
job = status.get('jobs')[0]
assert job.get('pending') == 1, job
stage = job.get('stages')[0]
assert stage.get('stage') == OP_INGEST, stage
assert stage.get('pending') == 1, stage
def status(collection_id):
"""
---
get:
summary: Check processing status of a collection
description: >
Return the task queue status for the collection with id `collection_id`
parameters:
- description: The collection ID.
in: path
name: collection_id
required: true
schema:
minimum: 1
type: integer
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/CollectionStatus'
tags:
- Collection
"""
collection = get_db_collection(collection_id, request.authz.READ)
request.rate_limit = None
return jsonify(get_status(collection))
def test_upload_csv_doc(self):
_, headers = self.login(is_admin=True)
meta = {
'countries': ['de', 'us'],
'languages': ['en'],
'mime_type': 'text/csv',
'source_url': 'http://pudo.org/experts.csv'
}
csv_path = self.get_fixture_path('experts.csv')
data = {
'meta': json.dumps(meta),
'foo': open(csv_path, 'rb'),
}
res = self.client.post(self.url, data=data, headers=headers)
assert res.status_code == 201, (res, res.data)
assert 'id' in res.json, res.json
doc = Document.by_id(res.json.get('id'))
assert doc.schema == Document.SCHEMA, doc.schema
status = get_status(self.col)
assert status.get('pending') == 1, status
op = status.get('operations')[0]
assert op.get('pending') == 1, op
assert op.get('operation') == OP_INGEST, op
def test_upload_csv_doc(self):
_, headers = self.login(is_admin=True)
meta = {
"countries": ["de", "usa"],
"languages": ["en"],
"mime_type": "text/csv",
"source_url": "http://pudo.org/experts.csv",
}
csv_path = self.get_fixture_path("experts.csv")
data = {
"meta": json.dumps(meta),
"foo": open(csv_path, "rb"),
}
res = self.client.post(self.url, data=data, headers=headers)
assert res.status_code == 201, (res, res.data)
assert "id" in res.json, res.json
db_id, _ = res.json.get("id").split(".", 1)
doc = Document.by_id(db_id)
assert doc.schema == Document.SCHEMA, doc.schema
assert doc.meta["countries"] == ["de", "us"], doc.meta
assert doc.meta["languages"] == ["eng"], doc.meta
status = get_status(self.col)
assert status.get("pending") == 1, status
job = status.get("jobs")[0]
assert job.get("pending") == 1, job
stage = job.get("stages")[0]
assert stage.get("stage") == OP_INGEST, stage
assert stage.get("pending") == 1, stage
def status(foreign_id=None):
"""Get the queue status (pending and finished tasks.)"""
if foreign_id is not None:
collection = get_collection(foreign_id)
status = get_status(collection)
else:
status = get_active_collection_status()
pprint(status)
def get_deep_collection(collection):
mappings = Mapping.by_collection(collection.id).count()
entitysets = EntitySet.type_counts(collection_id=collection.id)
return {
"statistics": index.get_collection_stats(collection.id),
"counts": {"mappings": mappings, "entitysets": entitysets},
"status": get_status(collection),
"shallow": False,
}
def cancel(collection_id):
collection = get_db_collection(collection_id, request.authz.WRITE)
cancel_queue(collection)
return jsonify(get_status(collection))
def status(collection_id):
collection = get_db_collection(collection_id, request.authz.READ)
return jsonify(get_status(collection))
def status(foreign_id):
"""Get the queue status (pending and finished tasks.)"""
collection = get_collection(foreign_id)
status = get_status(collection)
pprint(status)
