def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs",
"test", "config.ini"))
c = MongoClient(settings.GetConfigValue("ServiceStockageAnnotations",
"MONGO_HOST"),
int(
settings.GetConfigValue(
"ServiceStockageAnnotations", "MongoPort")),
connect=False)
c.drop_database(
settings.GetConfigValue("ServiceStockageAnnotations", "MongoDb"))
c.close()
self.d = AnnotationManager()
self.d.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection"))
self.d.addStorageCollection(
1,
settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection"))
self.d.addStorageCollection(
2,
settings.GetConfigValue("ServiceStockageAnnotations",
"BatchAnnotationCollection"))
self.d.connect()
def search_annotations_grouped():
"""
Search manual annotations (storageType 1) and group them by timeline.
The body of the request is a JSON query passed to
https://docs.mongodb.com/manual/reference/method/db.collection.aggregate/
Limit is mandatory when skip is specified.
:return: The text index fields and an array of annotation matches grouped by timeline (annotationSetId).
Each match contains the annotation and score matching the query, sorted descending by score.
Groups are also sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.get_json(force=True).get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory."}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
if limit is None and skip is not None:
return json.dumps({"error": "Limit is mandatory when skip is specified."}), 400
results = man.grouped_search_annotations(query, skip=skip, limit=limit)
indexed_fields = man.get_text_index_fields()
return jsonify({"results": results, "indexedFields": indexed_fields})
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def search_annotations():
"""
Search manual annotations (storageType 1)
The body of the request is a JSON query passed to https://docs.mongodb.com/manual/reference/method/db.collection.find/
:return: JSON Array of results containing the annotation and score matching the query, sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.json.get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory"}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
results = man.search_annotations(query, skip=skip, limit=limit)
return jsonify(results)
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def createDocumentAnnotation(document_id):
"""
:route: **/document/<document_id>/annotation**
:param document_id: The id of the document for which we want to access the annotation
:POST Creates an annotation.:
:Request:
:preconditions:
Here are minimum annotations contents:
::
{
@context: Complex object containing JSON_LD info.
}
Other custom fields which were created would be returned too.
The annotation using this method is created in HumanStorage.
:Response JSON:
Here are minimum annotations contents which will be after creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
try:
man.connect()
if request.method == 'POST':
logger.logUnknownDebug("Create Annotation",
" For document Id: {0}".format(document_id))
docId = man.createAnnotation(request.json, document_id)
return jsonify({"id": docId}), 201
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def search_annotations():
"""
Search manual annotations (storageType 1)
The body of the request is a JSON query passed to https://docs.mongodb.com/manual/reference/method/db.collection.find/
:return: JSON Array of results containing the annotation and score matching the query, sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.json.get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory"}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
results = man.search_annotations(query, skip=skip, limit=limit)
return jsonify(results)
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def createDocumentAnnotation(document_id):
"""
:route: **/document/<document_id>/annotation**
:param document_id: The id of the document for which we want to access the annotation
:POST Creates an annotation.:
:Request:
:preconditions:
Here are minimum annotations contents:
::
{
@context: Complex object containing JSON_LD info.
}
Other custom fields which were created would be returned too.
The annotation using this method is created in HumanStorage.
:Response JSON:
Here are minimum annotations contents which will be after creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
try:
man.connect()
if request.method == 'POST':
logger.logUnknownDebug("Create Annotation", " For document Id: {0}".format(document_id))
docId = man.createAnnotation(request.json, document_id)
return jsonify({"id": docId}), 201
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request", "", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def search_annotations_grouped():
"""
Search manual annotations (storageType 1) and group them by timeline.
The body of the request is a JSON query passed to
https://docs.mongodb.com/manual/reference/method/db.collection.aggregate/
Limit is mandatory when skip is specified.
:return: The text index fields and an array of annotation matches grouped by timeline (annotationSetId).
Each match contains the annotation and score matching the query, sorted descending by score.
Groups are also sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.get_json(force=True).get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory."}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
if limit is None and skip is not None:
return json.dumps(
{"error": "Limit is mandatory when skip is specified."}), 400
results = man.grouped_search_annotations(query, skip=skip, limit=limit)
indexed_fields = man.get_text_index_fields()
return jsonify({"results": results, "indexedFields": indexed_fields})
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs", "test", "config.ini"))
c = MongoClient(settings.GetConfigValue("ServiceStockageAnnotations", "MONGO_HOST"),
int(settings.GetConfigValue("ServiceStockageAnnotations", "MongoPort")),
connect=False)
c.drop_database(settings.GetConfigValue("ServiceStockageAnnotations", "MongoDb"))
c.close()
self.d = AnnotationManager()
self.d.setCollection(settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection"))
self.d.addStorageCollection(1,
settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection"))
self.d.addStorageCollection(2,
settings.GetConfigValue("ServiceStockageAnnotations", "BatchAnnotationCollection"))
self.d.connect()
class TestAnnotationsManager(unittest.TestCase):
d = None
def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs",
"test", "config.ini"))
c = MongoClient(settings.GetConfigValue("ServiceStockageAnnotations",
"MONGO_HOST"),
int(
settings.GetConfigValue(
"ServiceStockageAnnotations", "MongoPort")),
connect=False)
c.drop_database(
settings.GetConfigValue("ServiceStockageAnnotations", "MongoDb"))
c.close()
self.d = AnnotationManager()
self.d.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection"))
self.d.addStorageCollection(
1,
settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection"))
self.d.addStorageCollection(
2,
settings.GetConfigValue("ServiceStockageAnnotations",
"BatchAnnotationCollection"))
self.d.connect()
def test_connect(self):
self.assertEqual(self.d.isConnected(), True)
def l(self, strContent):
# shortcut to load json
return json.loads(strContent)
def test_createAnnotationsS(self):
jsonBatch = self.l(
'{"common":{"@context":"test"},"data":[{"a":1},{"b":2}]}')
id = self.d.createMongoDocument(
self.l('{"@context":"testing_context"}'))
self.assertRaises(AnnotationException,
lambda: self.d.createAnnotationS(
jsonBatch, "yolo", 1, 1)) # bad id
self.assertRaises(AnnotationException,
lambda: self.d.createAnnotationS(
jsonBatch, id, 1, 3)) # bad storage
self.assertRaises(AnnotationException,
lambda: self.d.createAnnotationS(
jsonBatch, id, 1, 0)) # bad storage
self.assertRaises(AnnotationException,
lambda: self.d.createAnnotationS(
jsonBatch, id, 2, 1)) # Bad format
self.assertEqual(0, self.d.createAnnotationS(self.l("{}"), id))
jsonInvalidBatch = self.l(
'{"data":[{"a":1,"@context":"test"},{"b":2}]}')
self.assertRaises(
AnnotationException,
lambda: self.d.createAnnotationS(jsonInvalidBatch, id, 1, 1))
jsonBatch = self.l(
'{"data":[{"a":1,"@context":"test"},{"b":2,"@context":"test"}]}')
resDifferentBatchFormat = self.d.createAnnotationS(jsonBatch, id, 0, 2)
self.assertEqual(
2, resDifferentBatchFormat,
"Expected to get 2, but got {0} created with batch format 0".
format(resDifferentBatchFormat))
def test_getAnnotationsS(self):
id = self.d.createMongoDocument(
self.l('{"@context":"testing_context"}'))
id2 = self.d.createMongoDocument(
self.l('{"@context":"testing_context"}'))
jsonBatch = self.l(
'{"common":{"@context":"test"},"data":[{"a":1,"b":1},{"a":2,"b":1}]}'
)
jsonBatch2 = self.l(
'{"common":{"@context":"test"},"data":[{"a":1,"c":1},{"a":2,"c":1}]}'
)
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id, 1, 1))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch2, id2, 1, 1))
#
res = self.d.getAnnotationS([id], {}, 0, 1)
self.assertEqual(2, len(res["data"]))
res = self.d.getAnnotationS([id], {"a": 1}, 0, 1)
self.assertEqual(
1, len(res["data"]),
"Filter a:1 for one docs should return 1 result. Results: {0}".
format(str(res)))
res = self.d.getAnnotationS([id, id2], {"a": 1}, 0, 1)
self.assertEqual(
2, len(res["data"]),
"Filter a:1 for two docs should return 2 result. Results: {0}".
format(str(res)))
largeJsonBatch = self.l(
'{"common":{"@context":"test"},"data":[{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3}]}'
)
self.assertEqual(27, self.d.createAnnotationS(largeJsonBatch, id2, 1,
1))
res = self.d.getAnnotationS([id, id2], {"e": {"$gt": 0}}, 0, 1)
self.assertEqual(
27, len(res["data"]),
"Filter e:1 for largebatch should return 27 result. Returned: {0}".
format(len(res["data"])))
# testing by getting
# Change for large batch
largeJsonBatch1 = self.l(
'{"common":{"@context":"test","k":1},"data":[{"a":1,"b":1},{"a":2,"b":1},{"a":3,"b":1}]}'
)
largeJsonBatch2 = self.l(
'{"common":{"@context":"test","k":2},"data":[{"a":4,"c":1},{"a":5,"c":1}]}'
)
largeJsonBatch3 = self.l(
'{"common":{"@context":"test","k":2},"data":[{"a":1,"c":1},{"a":2,"c":1},{"a":3,"c":1},{"a":4,"c":1}]}'
)
# id, 2 batches total 5 anno
# id2, 1 batch total 4 anno
self.assertEqual(3, self.d.createAnnotationS(largeJsonBatch1, id, 1,
2))
self.assertEqual(2, self.d.createAnnotationS(largeJsonBatch2, id, 1,
2))
self.assertEqual(4, self.d.createAnnotationS(largeJsonBatch3, id2, 1,
2))
# get all annotations in both documents
res = self.d.getAnnotationS([id, id2], {}, 0, 2)
self.assertEqual(9, len(res["data"]))
# get all annotations of doc with id
res = self.d.getAnnotationS([id], {}, 0, 2)
self.assertEqual(5, len(res["data"]))
res = self.d.getAnnotationS([id], {"k": 2}, 0, 2)
self.assertEqual(2, len(res["data"]))
# get all batch annotations with key k = 2 for both documents
res = self.d.getAnnotationS([id, id2], {"k": 2}, 0, 2)
self.assertEqual(6, len(res["data"]))
def test_deleteAnnotationsS(self):
id = self.d.createMongoDocument(
self.l('{"@context":"testing_context"}'))
jsonBatch = self.l(
'{"common":{"@context":"test"},"data":[{"a":1},{"b":2}]}')
jsonLargeBatch1 = self.l(
'{"common":{"@context":"test","batch":1},"data":[{"a":1},{"b":2}]}'
)
jsonLargeBatch2 = self.l(
'{"common":{"@context":"test","batch":2},"data":[{"a":1},{"b":2}]}'
)
self.assertEqual(0, self.d.deleteAnnotationS([id], {}))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id))
self.assertEqual(2, self.d.deleteAnnotationS([id], {}))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id, 1, 1))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch1, id, 1,
2))
res = self.d.deleteAnnotationS([id], {}, 1)
self.assertEqual(
2, res,
"Should only delete 2 since we created 2 per storage type, but we got {0}"
.format(res))
res = self.d.deleteAnnotationS([id], {}, 2)
self.assertEqual(
1, res, "Should delete 1 since we only have one batch".format(res))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch1, id, 1,
2))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch2, id, 1,
2))
res = self.d.deleteAnnotationS([id], {"batch": 1}, 2)
self.assertEqual(
1, res,
"Should delete 1 (only the first batch of 2 batches) we got {0}".
format(res))
def documentAnnotation(document_id, annotation_id):
"""
:route: **/document/<document_id>/annotation/<annotation_id>**
Get/Update/Delete an annotation:
:param document_id: The id of the document for which we want to access the
annotation
:param annotation_id: The id of the annotation we want to access
:GET Returns an annotation:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response json:
Here are minimum annotations contents which will be after
creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:PUT Updates an annotation:
Updates are made by changing the content of the old annotation with the new one
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE deletes an annotation.:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 204
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
if (not _getStorageTypeFromId(AnnotationManager.HUMAN_STORAGE)):
raise (StorageRestExceptions(4))
if request.method == 'GET':
logger.logUnknownDebug("Get Annotation",
" For document Id: {0}".format(document_id))
doc = man.getAnnotation(annotation_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != annotation_id:
raise (StorageRestExceptions(1))
else:
logger.logUnknownDebug(
"Update Annotation",
" For document Id: {0}".format(document_id))
man.updateAnnotation(doc, annotation_id)
return jsonify({})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Annotation",
" For document Id: {0}".format(document_id))
man.deleteAnnotation(annotation_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def documentAnnotationS(document_id):
"""
:route: **/document/<document_id>/annotations**
In case storageType = 2, annotations will be stored in batches. All operations impact
each batch (Example a PUT operation will replace all annotations in a batch).
By default each document have 1 annotation batch. To have multiple batches,
post/put batch of annotations using batchFormat = 1, and use common section to
to identify uniquely a batch. Fields "doc_id_batch", "file_fs_id_batch" are reserved,
thus will be ignored if added in common section.
:param document_id: The id of the document from which we want to access
multiple annotations
:POST Create annotations in batch.:
:Request:
:preconditions:
All must be valid annotations.
If one annotation fails, it will return an error message and
fail all create.
:params supported:
| batchFormat = 0,1
| storageType = 1,2
:params default:
| batchFormat = 1
| storageType = 1
:Response json:
| returns {"nInserted":nbAnnotationsInserted}
:http status code:
| OK: 200
| Error: See Error Codes
:PUT Updates annotations related for the document.:
:Request:
When we update we replace old contents with new ones. jsonSelect
:params supported:
| jsonSelect (only contains contents in the "common" fields for storageType = 2.)
| storageType = 2
:params default:
| storageType = 2
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE Deletes annotations related for the document.:
:Request:
:params supported:
| jsonSelect
| storageType = 1,2
:params default:
| storageType = 1
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:GET Returns annotations for the current document.:
:Request:
:params supported:
| jsonSelect
| storageType = 0,1,2
| batchFormat = 0
:params default:
| batchFormat = 0
| storageType = 0
| jsonSelect = {}
:Response json:
An array of annotations check batch format for how they will be
formatted.
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
bac = settings.GetConfigValue("ServiceStockageAnnotations",
"BatchAnnotationCollection")
man.addStorageCollection(AnnotationManager.BATCH_STORAGE, bac)
man.connect()
jsonSelect = request.args.get('jsonSelect')
storageType = request.args.get('storageType')
batchFormat = request.args.get('batchFormat')
# Note for batch operations all ids are replaced in man
if request.method == 'GET':
try:
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 0
else:
batchFormat = int(batchFormat)
except Exception as e:
raise (StorageRestExceptions(5))
batch = man.getAnnotationS([document_id], jsonSelect, batchFormat,
storageType)
return jsonify(batch)
elif request.method == 'PUT':
logger.logUnknownDebug("Update Annotations",
" For document Id: {0}".format(document_id))
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
elif request.method == 'POST':
jsonBatch = request.json
try:
if not storageType:
storageType = 1
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 1
else:
batchFormat = int(batchFormat)
logger.logUnknownDebug(
"Create Annotations",
" For document Id: {0} StorageType :{1},BatchFormat:{2}, jsonBatch: {3}"
.format(document_id, str(storageType), str(batchFormat),
str(jsonBatch)))
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsCreated = man.createAnnotationS(
jsonBatch, document_id, batchFormat, storageType)
return jsonify({"nCreated": nbAnnotationsCreated})
elif request.method == 'DELETE':
# Whenever it is true or false we don't care, if there is no
# exception
try:
logger.logUnknownDebug(
"Delete Annotations",
" For document Id: {0}".format(document_id))
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsDeleted = man.deleteAnnotationS([document_id],
jsonSelect,
storageType)
logger.logUnknownDebug(
"Delete Annotations",
" Number of deleted annotations {0} For document Id: {1}".
format(str(nbAnnotationsDeleted), document_id))
return jsonify({"nDeleted": nbAnnotationsDeleted}), 200
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def documentAnnotation(document_id, annotation_id):
"""
:route: **/document/<document_id>/annotation/<annotation_id>**
Get/Update/Delete an annotation:
:param document_id: The id of the document for which we want to access the
annotation
:param annotation_id: The id of the annotation we want to access
:GET Returns an annotation:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response json:
Here are minimum annotations contents which will be after
creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:PUT Updates an annotation:
Updates are made by changing the content of the old annotation with the new one
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE deletes an annotation.:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 204
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
if (not _getStorageTypeFromId(AnnotationManager.HUMAN_STORAGE)):
raise (StorageRestExceptions(4))
if request.method == 'GET':
logger.logUnknownDebug("Get Annotation", " For document Id: {0}".format(document_id))
doc = man.getAnnotation(annotation_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != annotation_id:
raise (StorageRestExceptions(1))
else:
logger.logUnknownDebug("Update Annotation", " For document Id: {0}".format(document_id))
man.updateAnnotation(doc, annotation_id)
return jsonify({})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Annotation", " For document Id: {0}".format(document_id))
man.deleteAnnotation(annotation_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request", "", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def documentAnnotationS(document_id):
"""
:route: **/document/<document_id>/annotations**
In case storageType = 2, annotations will be stored in batches. All operations impact
each batch (Example a PUT operation will replace all annotations in a batch).
By default each document have 1 annotation batch. To have multiple batches,
post/put batch of annotations using batchFormat = 1, and use common section to
to identify uniquely a batch. Fields "doc_id_batch", "file_fs_id_batch" are reserved,
thus will be ignored if added in common section.
:param document_id: The id of the document from which we want to access
multiple annotations
:POST Create annotations in batch.:
:Request:
:preconditions:
All must be valid annotations.
If one annotation fails, it will return an error message and
fail all create.
:params supported:
| batchFormat = 0,1
| storageType = 1,2
:params default:
| batchFormat = 1
| storageType = 1
:Response json:
| returns {"nInserted":nbAnnotationsInserted}
:http status code:
| OK: 200
| Error: See Error Codes
:PUT Updates annotations related for the document.:
:Request:
When we update we replace old contents with new ones. jsonSelect
:params supported:
| jsonSelect (only contains contents in the "common" fields for storageType = 2.)
| storageType = 2
:params default:
| storageType = 2
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE Deletes annotations related for the document.:
:Request:
:params supported:
| jsonSelect
| storageType = 1,2
:params default:
| storageType = 1
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:GET Returns annotations for the current document.:
:Request:
:params supported:
| jsonSelect
| storageType = 0,1,2
| batchFormat = 0
:params default:
| batchFormat = 0
| storageType = 0
| jsonSelect = {}
:Response json:
An array of annotations check batch format for how they will be
formatted.
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
bac = settings.GetConfigValue("ServiceStockageAnnotations",
"BatchAnnotationCollection")
man.addStorageCollection(AnnotationManager.BATCH_STORAGE, bac)
man.connect()
jsonSelect = request.args.get('jsonSelect')
storageType = request.args.get('storageType')
batchFormat = request.args.get('batchFormat')
# Note for batch operations all ids are replaced in man
if request.method == 'GET':
try:
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 0
else:
batchFormat = int(batchFormat)
except Exception as e:
raise (StorageRestExceptions(5))
batch = man.getAnnotationS([document_id], jsonSelect, batchFormat,
storageType)
return jsonify(batch)
elif request.method == 'PUT':
logger.logUnknownDebug("Update Annotations", " For document Id: {0}".format(document_id))
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request", "", "")
elif request.method == 'POST':
jsonBatch = request.json
try:
if not storageType:
storageType = 1
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 1
else:
batchFormat = int(batchFormat)
logger.logUnknownDebug("Create Annotations",
" For document Id: {0} StorageType :{1},BatchFormat:{2}, jsonBatch: {3}".format(
document_id, str(storageType), str(batchFormat), str(jsonBatch)))
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsCreated = man.createAnnotationS(jsonBatch,
document_id,
batchFormat,
storageType)
return jsonify({"nCreated": nbAnnotationsCreated})
elif request.method == 'DELETE':
# Whenever it is true or false we don't care, if there is no
# exception
try:
logger.logUnknownDebug("Delete Annotations", " For document Id: {0}".format(document_id))
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsDeleted = man.deleteAnnotationS([document_id],
jsonSelect,
storageType)
logger.logUnknownDebug("Delete Annotations",
" Number of deleted annotations {0} For document Id: {1}".format(
str(nbAnnotationsDeleted), document_id))
return jsonify({"nDeleted": nbAnnotationsDeleted}), 200
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request", "", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
class TestAnnotationsManager(unittest.TestCase):
d = None
def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs", "test", "config.ini"))
c = MongoClient(settings.GetConfigValue("ServiceStockageAnnotations", "MONGO_HOST"),
int(settings.GetConfigValue("ServiceStockageAnnotations", "MongoPort")),
connect=False)
c.drop_database(settings.GetConfigValue("ServiceStockageAnnotations", "MongoDb"))
c.close()
self.d = AnnotationManager()
self.d.setCollection(settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection"))
self.d.addStorageCollection(1,
settings.GetConfigValue("ServiceStockageAnnotations", "HumanAnnotationCollection"))
self.d.addStorageCollection(2,
settings.GetConfigValue("ServiceStockageAnnotations", "BatchAnnotationCollection"))
self.d.connect()
def test_connect(self):
self.assertEqual(self.d.isConnected(), True)
def l(self, strContent):
# shortcut to load json
return json.loads(strContent)
def test_createAnnotationsS(self):
jsonBatch = self.l('{"common":{"@context":"test"},"data":[{"a":1},{"b":2}]}')
id = self.d.createMongoDocument(self.l('{"@context":"testing_context"}'))
self.assertRaises(AnnotationException, lambda: self.d.createAnnotationS(jsonBatch, "yolo", 1, 1)) # bad id
self.assertRaises(AnnotationException, lambda: self.d.createAnnotationS(jsonBatch, id, 1, 3)) # bad storage
self.assertRaises(AnnotationException, lambda: self.d.createAnnotationS(jsonBatch, id, 1, 0)) # bad storage
self.assertRaises(AnnotationException, lambda: self.d.createAnnotationS(jsonBatch, id, 2, 1)) # Bad format
self.assertEqual(0, self.d.createAnnotationS(self.l("{}"), id))
jsonInvalidBatch = self.l('{"data":[{"a":1,"@context":"test"},{"b":2}]}')
self.assertRaises(AnnotationException, lambda: self.d.createAnnotationS(jsonInvalidBatch, id, 1, 1))
jsonBatch = self.l('{"data":[{"a":1,"@context":"test"},{"b":2,"@context":"test"}]}')
resDifferentBatchFormat = self.d.createAnnotationS(jsonBatch, id, 0, 2)
self.assertEqual(2, resDifferentBatchFormat,
"Expected to get 2, but got {0} created with batch format 0".format(resDifferentBatchFormat))
def test_getAnnotationsS(self):
id = self.d.createMongoDocument(self.l('{"@context":"testing_context"}'))
id2 = self.d.createMongoDocument(self.l('{"@context":"testing_context"}'))
jsonBatch = self.l('{"common":{"@context":"test"},"data":[{"a":1,"b":1},{"a":2,"b":1}]}')
jsonBatch2 = self.l('{"common":{"@context":"test"},"data":[{"a":1,"c":1},{"a":2,"c":1}]}')
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id, 1, 1))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch2, id2, 1, 1))
#
res = self.d.getAnnotationS([id], {}, 0, 1)
self.assertEqual(2, len(res["data"]))
res = self.d.getAnnotationS([id], {"a": 1}, 0, 1)
self.assertEqual(1, len(res["data"]),
"Filter a:1 for one docs should return 1 result. Results: {0}".format(str(res)))
res = self.d.getAnnotationS([id, id2], {"a": 1}, 0, 1)
self.assertEqual(2, len(res["data"]),
"Filter a:1 for two docs should return 2 result. Results: {0}".format(str(res)))
largeJsonBatch = self.l(
'{"common":{"@context":"test"},"data":[{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3},{"e":1},{"e":2},{"e":3}]}')
self.assertEqual(27, self.d.createAnnotationS(largeJsonBatch, id2, 1, 1))
res = self.d.getAnnotationS([id, id2], {"e": {"$gt": 0}}, 0, 1)
self.assertEqual(27, len(res["data"]),
"Filter e:1 for largebatch should return 27 result. Returned: {0}".format(len(res["data"])))
# testing by getting
# Change for large batch
largeJsonBatch1 = self.l(
'{"common":{"@context":"test","k":1},"data":[{"a":1,"b":1},{"a":2,"b":1},{"a":3,"b":1}]}')
largeJsonBatch2 = self.l('{"common":{"@context":"test","k":2},"data":[{"a":4,"c":1},{"a":5,"c":1}]}')
largeJsonBatch3 = self.l(
'{"common":{"@context":"test","k":2},"data":[{"a":1,"c":1},{"a":2,"c":1},{"a":3,"c":1},{"a":4,"c":1}]}')
# id, 2 batches total 5 anno
# id2, 1 batch total 4 anno
self.assertEqual(3, self.d.createAnnotationS(largeJsonBatch1, id, 1, 2))
self.assertEqual(2, self.d.createAnnotationS(largeJsonBatch2, id, 1, 2))
self.assertEqual(4, self.d.createAnnotationS(largeJsonBatch3, id2, 1, 2))
# get all annotations in both documents
res = self.d.getAnnotationS([id, id2], {}, 0, 2)
self.assertEqual(9, len(res["data"]))
# get all annotations of doc with id
res = self.d.getAnnotationS([id], {}, 0, 2)
self.assertEqual(5, len(res["data"]))
res = self.d.getAnnotationS([id], {"k": 2}, 0, 2)
self.assertEqual(2, len(res["data"]))
# get all batch annotations with key k = 2 for both documents
res = self.d.getAnnotationS([id, id2], {"k": 2}, 0, 2)
self.assertEqual(6, len(res["data"]))
def test_deleteAnnotationsS(self):
id = self.d.createMongoDocument(self.l('{"@context":"testing_context"}'))
jsonBatch = self.l('{"common":{"@context":"test"},"data":[{"a":1},{"b":2}]}')
jsonLargeBatch1 = self.l('{"common":{"@context":"test","batch":1},"data":[{"a":1},{"b":2}]}')
jsonLargeBatch2 = self.l('{"common":{"@context":"test","batch":2},"data":[{"a":1},{"b":2}]}')
self.assertEqual(0, self.d.deleteAnnotationS([id], {}))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id))
self.assertEqual(2, self.d.deleteAnnotationS([id], {}))
self.assertEqual(2, self.d.createAnnotationS(jsonBatch, id, 1, 1))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch1, id, 1, 2))
res = self.d.deleteAnnotationS([id], {}, 1)
self.assertEqual(2, res, "Should only delete 2 since we created 2 per storage type, but we got {0}".format(res))
res = self.d.deleteAnnotationS([id], {}, 2)
self.assertEqual(1, res, "Should delete 1 since we only have one batch".format(res))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch1, id, 1, 2))
self.assertEqual(2, self.d.createAnnotationS(jsonLargeBatch2, id, 1, 2))
res = self.d.deleteAnnotationS([id], {"batch": 1}, 2)
self.assertEqual(1, res, "Should delete 1 (only the first batch of 2 batches) we got {0}".format(res))