def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs",
"test", "config.ini"))
host = settings.GetConfigValue("ServiceStockageAnnotations",
"MONGO_HOST")
port = int(
settings.GetConfigValue("ServiceStockageAnnotations", "MongoPort"))
c = MongoClient(host, port, connect=False)
c.admin.command("ismaster")
dbname = settings.GetConfigValue("ServiceStockageAnnotations",
"MongoDb")
# Force connection test
# https://api.mongodb.com/python/current/migrate-to-pymongo3.html#mongoclient-connects-asynchronously
c.admin.command("ismaster")
c.drop_database(dbname)
c.close()
self.d = StorageManager()
self.d.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"documentCollection"))
self.d.connect()
def info():
"""
Required by CANARIE
"""
canarie_config_section = "annotation_storage"
service_info_categories = [
'name', 'synopsis', 'version', 'institution', 'releaseTime',
'supportEmail', 'category', 'researchSubject'
]
service_info = list()
for category in service_info_categories:
service_info.append(
(category, settings.GetConfigValue(canarie_config_section,
category)))
service_info.append(
('tags', settings.GetConfigValue(canarie_config_section,
'tags').split(',')))
service_info = collections.OrderedDict(service_info)
if request_wants_json():
return jsonify(service_info)
return render_template('default.html', Title="Info", Tags=service_info)
def connect(self):
"""
Connects to MongoDB and verifies connection.
"""
try:
host = settings.GetConfigValue("ServiceStockageAnnotations",
"MONGO_HOST")
port = int(
settings.GetConfigValue("ServiceStockageAnnotations",
"MongoPort"))
self.client = MongoClient(host, port, connect=False)
db = settings.GetConfigValue("ServiceStockageAnnotations",
"MongoDb")
self.mongoDb = db
# Force connection test
# https://api.mongodb.com/python/current/migrate-to-pymongo3.html#mongoclient-connects-asynchronously
self.client.admin.command("ismaster")
self.m_connected = True
return True
except pymongo.errors.ConnectionFailure:
logger.logError(StorageException(1))
except Exception as e:
logger.logUnknownError("Annotation Storage Create Document", "", e)
self.m_connected = False
return False
def get_message(self):
msg_id = self.__error_code_dict[self.error_code][1]
if self.__details:
try:
msg = settings.GetConfigValue('canarie_status_msg_details',
msg_id)
msg = msg.format(**self.__details)
return msg
except configparser.Error:
pass
return settings.GetConfigValue('canarie_status_msg', msg_id)
def get_server_restart_time():
"""
Obtain the server status provided by Apache if properly configured.
The configuration file must contain the following section:
::
# Enable the module mod_status
# See : http://httpd.apache.org/docs/2.2/mod/mod_status.html
<Location /server-status>
SetHandler server-status
Order deny,allow
#Deny access for everyone outside
Deny from all
#Allow access from IP 10* (private address inside local network)
Allow from 10
#Allow access from 132.217* (public IP range own by the CRIM)
Allow from 132.217
</Location>
# Keep track of extended status information for each request
ExtendedStatus On
"""
conn = http.client.HTTPConnection(settings.GetConfigValue(
'Server', 'Name'))
conn.request("GET", "/server-status")
response = conn.getresponse()
if response.status == http.HTTPStatus.OK:
body = response.read()
restart_time_match = re.search(
'<dt>Restart Time: [a-zA-Z]*, '
'([^ ]* [0-9:]{8}) (.*)</dt>', body)
if restart_time_match:
datetime_str = restart_time_match.group(1)
timezone_str = restart_time_match.group(2)
# EDT is not in the pytz package so use this time zone
# which gives the correct +4hours offset to the UTC
timezone_str = re.sub('EDT', 'Etc/GMT+4', timezone_str)
local = pytz.timezone(timezone_str)
localtime_unaware = datetime.datetime.strptime(
datetime_str, '%d-%b-%Y %H:%M:%S')
local_time = local.localize(localtime_unaware, is_dst=None)
utc_time = local_time.astimezone(pytz.utc)
return utc_time
logging.error("Failed to retrieve restart time of server {0}".format(
settings.GetConfigValue('Server', 'Name')))
return None
def search_annotations():
"""
Search manual annotations (storageType 1)
The body of the request is a JSON query passed to https://docs.mongodb.com/manual/reference/method/db.collection.find/
:return: JSON Array of results containing the annotation and score matching the query, sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.json.get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory"}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
results = man.search_annotations(query, skip=skip, limit=limit)
return jsonify(results)
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def createAnnotationSchema():
"""
:route: **/annotationSchema**
Used to store Annotation schemas
Same implementation as for the document.
"""
man = DocumentManager()
try:
man.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"SchemaCollection"))
man.connect()
if request.method == 'POST':
docId = man.createMongoDocument(request.json)
logger.logUnknownDebug("Create Schema", "Id: {0}".format(docId))
return jsonify({"id": docId}), 201
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def get_canarie_api_response(service_name, template_path, canarie_api_request):
"""
Provide a valid HTML response for the CANARIE API request based on the
service_route.
:param:
:service_route: Route name of the service coming from the URL e.g.:
['diarisation', 'STT', etc.]
:canarie_api_request: The request specified in the URL
:returns: A valid html response
"""
# The service configuration should return either :
# - A valid URL (in which case a redirection is performed)
# - A relative template file from which an html page is rendered
# - A comma separated list corresponding to the response tuple
# (response, status)
cfg_val = settings.GetConfigValue(service_name, canarie_api_request)
if cfg_val.find('http') == 0:
return redirect(cfg_val)
elif os.path.isfile(os.path.join(template_path, cfg_val)):
return render_template(cfg_val)
elif len(cfg_val.split(',')) == 2:
return make_response(*(cfg_val.split(',')))
else:
return make_error_response(error.Error.BAD_SERVICE_CONFIGURATION)
def createDocumentAnnotation(document_id):
"""
:route: **/document/<document_id>/annotation**
:param document_id: The id of the document for which we want to access the annotation
:POST Creates an annotation.:
:Request:
:preconditions:
Here are minimum annotations contents:
::
{
@context: Complex object containing JSON_LD info.
}
Other custom fields which were created would be returned too.
The annotation using this method is created in HumanStorage.
:Response JSON:
Here are minimum annotations contents which will be after creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
try:
man.connect()
if request.method == 'POST':
logger.logUnknownDebug("Create Annotation",
" For document Id: {0}".format(document_id))
docId = man.createAnnotation(request.json, document_id)
return jsonify({"id": docId}), 201
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def setUp(self):
settings.Settings.Instance().LoadConfig(
os.path.join(os.path.dirname(__file__), "..", "..", "configs",
"test", "config.ini"))
c = MongoClient(
settings.GetConfigValue("ServiceStockageAnnotations",
"MONGO_HOST"),
int(
settings.GetConfigValue("ServiceStockageAnnotations",
"MongoPort")))
c.drop_database(
settings.GetConfigValue("ServiceStockageAnnotations", "MongoDb"))
c.close()
self.d = StorageManager()
self.d.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"documentCollection"))
self.d.connect()
def annotationSchema(schema_id):
"""
:route: **/annotationSchema/<schema_id>**
Used to store annotation schemas.
Same implementation as for the document.
"""
man = StorageManager()
try:
man.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"SchemaCollection"))
man.connect()
if request.method == 'GET':
logger.logUnknownDebug("Get Schema", "Id: {0}".format(schema_id))
doc = man.getMongoDocument(schema_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != schema_id:
raise (StorageRestExceptions(1))
else:
logger.logUnknownDebug("Update Schema",
"Id: {0}".format(schema_id))
docId = man.updateMongoDocument(request.json)
if (docId is None):
raise (StorageRestExceptions(3))
return jsonify({"id": docId})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Schema",
"Id: {0}".format(schema_id))
man.deleteMongoDocument(schema_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def createDocument():
"""
:route: **/document**
:POST creates a new document:
:Request:
::
Preconditions: Here are the minimum required elements by the document.
{
@context: context describing the format of the document
}
All the other parameters will be saved as is.
Erases "_id", "id" fields if they exists.
:Response: Same as in, plus creates a field "id" to identify the document
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = DocumentManager()
try:
man.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"documentCollection"))
man.connect()
if request.method == 'POST':
docId = man.createMongoDocument(request.json)
logger.logUnknownDebug("Create Document", "Id: {0}".format(docId))
return jsonify({"id": docId}), 201
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def search_annotations_grouped():
"""
Search manual annotations (storageType 1) and group them by timeline.
The body of the request is a JSON query passed to
https://docs.mongodb.com/manual/reference/method/db.collection.aggregate/
Limit is mandatory when skip is specified.
:return: The text index fields and an array of annotation matches grouped by timeline (annotationSetId).
Each match contains the annotation and score matching the query, sorted descending by score.
Groups are also sorted descending by score.
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
query = request.get_json(force=True).get('query')
if query is None:
return json.dumps({"error": "body with query is mandatory."}), 400
skip = request.json.get('skip')
limit = request.json.get('limit')
if limit is None and skip is not None:
return json.dumps(
{"error": "Limit is mandatory when skip is specified."}), 400
results = man.grouped_search_annotations(query, skip=skip, limit=limit)
indexed_fields = man.get_text_index_fields()
return jsonify({"results": results, "indexedFields": indexed_fields})
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def documentAnnotation(document_id, annotation_id):
"""
:route: **/document/<document_id>/annotation/<annotation_id>**
Get/Update/Delete an annotation:
:param document_id: The id of the document for which we want to access the
annotation
:param annotation_id: The id of the annotation we want to access
:GET Returns an annotation:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response json:
Here are minimum annotations contents which will be after
creation:
::
{
doc_id: to describe the id of the document containing the
annotation. Equals to strDocId.
@context: a field linking the context of the document.
id: a unique id identifying the annotation.
}
:PUT Updates an annotation:
Updates are made by changing the content of the old annotation with the new one
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE deletes an annotation.:
:Request:
:precondtions:
Can only get annotations from HumanStorage.
:Response:
:http status code:
| OK: 204
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
man.connect()
if (not _getStorageTypeFromId(AnnotationManager.HUMAN_STORAGE)):
raise (StorageRestExceptions(4))
if request.method == 'GET':
logger.logUnknownDebug("Get Annotation",
" For document Id: {0}".format(document_id))
doc = man.getAnnotation(annotation_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != annotation_id:
raise (StorageRestExceptions(1))
else:
logger.logUnknownDebug(
"Update Annotation",
" For document Id: {0}".format(document_id))
man.updateAnnotation(doc, annotation_id)
return jsonify({})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Annotation",
" For document Id: {0}".format(document_id))
man.deleteAnnotation(annotation_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def documentAnnotationS(document_id):
"""
:route: **/document/<document_id>/annotations**
In case storageType = 2, annotations will be stored in batches. All operations impact
each batch (Example a PUT operation will replace all annotations in a batch).
By default each document have 1 annotation batch. To have multiple batches,
post/put batch of annotations using batchFormat = 1, and use common section to
to identify uniquely a batch. Fields "doc_id_batch", "file_fs_id_batch" are reserved,
thus will be ignored if added in common section.
:param document_id: The id of the document from which we want to access
multiple annotations
:POST Create annotations in batch.:
:Request:
:preconditions:
All must be valid annotations.
If one annotation fails, it will return an error message and
fail all create.
:params supported:
| batchFormat = 0,1
| storageType = 1,2
:params default:
| batchFormat = 1
| storageType = 1
:Response json:
| returns {"nInserted":nbAnnotationsInserted}
:http status code:
| OK: 200
| Error: See Error Codes
:PUT Updates annotations related for the document.:
:Request:
When we update we replace old contents with new ones. jsonSelect
:params supported:
| jsonSelect (only contains contents in the "common" fields for storageType = 2.)
| storageType = 2
:params default:
| storageType = 2
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE Deletes annotations related for the document.:
:Request:
:params supported:
| jsonSelect
| storageType = 1,2
:params default:
| storageType = 1
| jsonSelect = {}
:Response json:
| Returns number of annotations deleted
| {"nDeleted":nbAnnotationsDeleted}
:http status code:
| OK: 200
| Error: See Error Codes
:GET Returns annotations for the current document.:
:Request:
:params supported:
| jsonSelect
| storageType = 0,1,2
| batchFormat = 0
:params default:
| batchFormat = 0
| storageType = 0
| jsonSelect = {}
:Response json:
An array of annotations check batch format for how they will be
formatted.
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = AnnotationManager()
try:
hac = settings.GetConfigValue("ServiceStockageAnnotations",
"HumanAnnotationCollection")
man.addStorageCollection(AnnotationManager.HUMAN_STORAGE, hac)
bac = settings.GetConfigValue("ServiceStockageAnnotations",
"BatchAnnotationCollection")
man.addStorageCollection(AnnotationManager.BATCH_STORAGE, bac)
man.connect()
jsonSelect = request.args.get('jsonSelect')
storageType = request.args.get('storageType')
batchFormat = request.args.get('batchFormat')
# Note for batch operations all ids are replaced in man
if request.method == 'GET':
try:
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 0
else:
batchFormat = int(batchFormat)
except Exception as e:
raise (StorageRestExceptions(5))
batch = man.getAnnotationS([document_id], jsonSelect, batchFormat,
storageType)
return jsonify(batch)
elif request.method == 'PUT':
logger.logUnknownDebug("Update Annotations",
" For document Id: {0}".format(document_id))
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
elif request.method == 'POST':
jsonBatch = request.json
try:
if not storageType:
storageType = 1
else:
storageType = int(storageType)
if not batchFormat:
batchFormat = 1
else:
batchFormat = int(batchFormat)
logger.logUnknownDebug(
"Create Annotations",
" For document Id: {0} StorageType :{1},BatchFormat:{2}, jsonBatch: {3}"
.format(document_id, str(storageType), str(batchFormat),
str(jsonBatch)))
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsCreated = man.createAnnotationS(
jsonBatch, document_id, batchFormat, storageType)
return jsonify({"nCreated": nbAnnotationsCreated})
elif request.method == 'DELETE':
# Whenever it is true or false we don't care, if there is no
# exception
try:
logger.logUnknownDebug(
"Delete Annotations",
" For document Id: {0}".format(document_id))
if not jsonSelect:
jsonSelect = {}
else:
jsonSelect = json.loads(jsonSelect)
if not storageType:
storageType = 0
else:
storageType = int(storageType)
except Exception as e:
raise (StorageRestExceptions(5))
nbAnnotationsDeleted = man.deleteAnnotationS([document_id],
jsonSelect,
storageType)
logger.logUnknownDebug(
"Delete Annotations",
" Number of deleted annotations {0} For document Id: {1}".
format(str(nbAnnotationsDeleted), document_id))
return jsonify({"nDeleted": nbAnnotationsDeleted}), 200
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()
def document(document_id):
"""
:route: **/document/<document_id>**
Get/Put/Delete for the documents.
:param document_id: The id of the document we want to access
:GET returns a document:
:Response JSON:
Here are minimum document contents:
::
{
id: Id of the document = document_id
@context: Complex object containing JSON_LD
info.
}
Other custom fields which were created would be returned too.
:http status code:
| OK: 200
| Error: See Error Codes
:PUT updates a document by replacing whole document contents:
:Request:
The document must exists and contains the following contents at minimum:
::
{
id: Id of the document = document_id
@context: Complex object containing JSON_LD
info.
}
Other custom fields which were created would be saved too.
Erases "_id" field.
:Response:
:http status code:
| OK: 200
| Error: See Error Codes
:DELETE deletes the document. If the document not found do nothing.:
:Response JSON: {}
:http status code:
| OK: 200
| Error: See Error Codes
"""
man = DocumentManager()
try:
man.setCollection(
settings.GetConfigValue("ServiceStockageAnnotations",
"documentCollection"))
man.connect()
if request.method == 'GET':
logger.logUnknownDebug("Get Document",
"Id: {0}".format(document_id))
doc = man.getMongoDocument(document_id)
_convStorageIdToDocId(doc)
if (doc is None):
raise (StorageRestExceptions(2))
else:
return jsonify(doc)
elif request.method == 'PUT':
logger.logUnknownDebug("Update Document",
"Document {0}".format(document_id))
doc = request.json
_convDocIdToStorageId(doc)
if '_id' in doc and doc["_id"] != document_id:
raise (StorageRestExceptions(1))
else:
docId = man.updateMongoDocument(doc)
return jsonify({"id": docId})
elif request.method == 'DELETE':
logger.logUnknownDebug("Delete Document",
"Id: {0}".format(document_id))
man.deleteMongoDocument(document_id)
# Whenever it is true or false we don't care, if there is no
# exception
return jsonify({}), 204
else:
return error_response(http.HTTPStatus.BAD_REQUEST, "Bad Request",
"", "")
except Exception as e:
return _processCommonException(e)
finally:
man.disconnect()