def acceptsuggestion(lexicon, _id):
try:
ans = savesuggestion(lexicon, _id)
return jsonify(ans)
except (esExceptions.RequestError, esExceptions.TransportError) as e:
update.handle_update_error(e, {"id": _id}, helpers.get_user(), 'accept')
raise eh.KarpElasticSearchError("Error during update. Document not saved.",
debug_msg=str(e))
except Exception as e:
update.handle_update_error(e, {"id": _id}, helpers.get_user(), 'accept')
raise eh.KarpGeneralError(str(e))
def acceptmodified(lexicon, _id):
try:
request.get_data()
data = loads(request.data)
modified_doc = data
ans = savesuggestion(lexicon, _id,
status='accepted_modified',
source=modified_doc)
return jsonify(ans)
except (esExceptions.RequestError, esExceptions.TransportError) as e:
logging.exception(e)
update.handle_update_error(e, {"id": _id, "data": data},
helpers.get_user(), 'accept modified')
raise eh.KarpElasticSearchError("Error during update. Document not saved.",
debug_msg=str(e))
except Exception as e:
logging.exception(e)
update.handle_update_error(e, {"id": _id, "data": data},
helpers.get_user(), 'accept modified')
raise eh.KarpGeneralError(str(e))
def rejectsuggestion(lexicon, _id):
from dbhandler.dbhandler import dbselect
try:
origin = dbselect(lexicon, suggestion=True, _id=_id, max_hits=1)[0]
except Exception as e:
# if error occurs here, the suggestion is not in sql
raise eh.KarpDbError('Rejection not found',
'Rejection not found: %s' % str(e))
auth, permitted = validate_user()
set_lexicon = origin["doc"]["lexiconName"]
helpers.check_lexiconName(lexicon, set_lexicon, 'rejectsuggestion', _id)
if lexicon not in permitted:
raise eh.KarpAuthenticationError('You are not allowed to update lexicon %s'
% lexicon)
try:
origin = dbselect(lexicon, suggestion=True, _id=_id, max_hits=1)[0]
# delete from suggestion index
# the user log in is checked in delete_entry
# delete_entry raises exception if ES fails
sugg_index, typ = configM.get_lexicon_suggindex(lexicon)
ans = update.delete_entry(lexicon, _id, sql=False, live=False,
suggestion=True)
request.get_data()
data = loads(request.data)
message = data.get('message')
# mark as rejected
ok, err = update.modify_db(_id, lexicon, message, "rejected")
ans['sugg_db_loaded'] = ok
if not ok:
logging.debug(err)
update.send_notification(origin['user'], message, _id, "rejected")
return jsonify(ans)
except (esExceptions.RequestError, esExceptions.TransportError) as e:
update.handle_update_error(e, {"id": _id}, helpers.get_user(), 'reject')
raise eh.KarpElasticSearchError("Error during update. Document not saved.",
debug_msg=str(e))
except Exception as e:
update.handle_update_error(e, {"id": _id}, helpers.get_user(), 'reject')
raise eh.KarpGeneralError(str(e))
def add_multi_doc(lexicon, index=''):
import src.dbhandler.dbhandler as db
data = helpers.read_data()
documents = data.get('doc', '') or data.get('_source')
message = data['message']
es, index, typ = helpers.get_update_index(lexicon)
# validate that the user may update the lexica
authdict, permitted = auth.validate_user()
if lexicon not in permitted:
errstr = 'You are not allowed to modify the lexicon %s'
raise eh.KarpAuthenticationError(errstr % lexicon, status_code=403)
user = helpers.get_user()
try:
bulk, sql_bulk, ids = [], [], []
ok = 0
for doc in documents:
doc['lexiconName'] = lexicon
validate.validate_json(doc, lexicon)
date = datetime.datetime.now()
auto_update_document(doc, lexicon, 'add', user, date)
bulk.append({'_index': index, '_type': typ, '_source': doc})
for index, res in enumerate(eshelpers.streaming_bulk(es, bulk)):
_id = res[1].get('create').get('_id')
source = bulk[index].get('_source')
if isinstance(source, dict):
source = dumps(source)
sql_bulk.append((_id, source, user, 'multi add - %s' % message,
lexicon, 'imported'))
ids.append(_id)
ok += 1
except (esExceptions.RequestError, esExceptions.TransportError) as e:
handle_update_error(e, data, user, 'add')
raise eh.KarpElasticSearchError("Error during update. Message: %s.\n" %
str(e))
db_loaded, db_error = db.update_bulk(lexicon, sql_bulk)
if db_error:
logging.debug(db_error)
jsonans = {
'es_loaded': ok,
'sql_loaded': db_loaded,
'suggestion': False,
'ids': ids
}
return jsonify(jsonans)
def add_child(lexicon, parentid, suggestion=False):
es, index, typ = helpers.get_update_index(lexicon, suggestion=suggestion)
parent = es.get(index=index, id=parentid).get("_source")
data = helpers.read_data()
data_doc = data.get('doc', '') or data.get('_source')
msg = data["message"]
user = helpers.get_user()
# add child to parent
autoupdate_child(data_doc, parent, lexicon, user, '')
parent_doc = {"doc": parent, "user": user, "message": msg}
logging.debug('save parent %s', parent_doc)
logging.debug('save child %s', parent_doc)
parres = update_doc(lexicon, parentid, data=parent_doc, live=False)
# add child to lexicon
childres = add_doc(lexicon, index=index, data=data, live=False)
return jsonify({"parent": parres, "child": childres})
def checkuserhistory():
""" Shows the updates a user has made """
try:
auth, permitted = validate_user()
user = helpers.get_user()
except AttributeError:
raise eh.KarpGeneralError('No user name provided', 'checkuserhistory')
try:
size = helpers.get_size(default=10, settings={'allowed': permitted})
from src.dbhandler.dbhandler import dbselect
updates = []
for lexicon in permitted:
# add updates from lexicons that are kept in sql
if configM.get_lexicon_sql(lexicon):
updates.extend(dbselect(lexicon, user=user, max_hits=size))
return jsonify({'updates': updates})
except Exception as e:
logging.exception(e)
raise eh.KarpGeneralError(str(e))
def add_doc(lexicon,
index='',
_id=None,
suggestion=False,
data=None,
live=True):
""" Adds an entry to the index 'index' with type 'typ' in ES and sql.
The post data must contain a field called 'doc' with the information to
be sent.
The fields 'version' and 'message' are optional.
"""
if data is None or not data:
data = helpers.read_data()
data_doc = data.get('doc', '') or data.get('_source')
message = data['message']
version = data.get('version', -1)
es, index, typ = helpers.get_update_index(lexicon, suggestion=suggestion)
lexiconName = lexicon or data_doc.get("lexiconName", '')
helpers.check_lexiconName(lexicon, lexiconName, 'add', _id)
# lexiconOrder = data_doc.get("lexiconOrder", None)
if not lexiconName:
raise eh.KarpParsingError("The field lexiconName is empty, "
"although it is required.")
if suggestion:
orgin_id = _id or True # save as reference in db
_id = '' # add as new to the suggestion index
status = 'waiting'
user = data['user']
else:
# validate that the user may update the lexica
authdict, permitted = auth.validate_user()
if lexiconName not in permitted:
errstr = 'You are not allowed to modify the lexicon %s'
raise eh.KarpAuthenticationError(errstr % lexiconName,
status_code=403)
orgin_id = '' # not a suggestion
user = helpers.get_user()
status = 'added'
try:
# TODO validate data_doc, but this is so far sb specific!
validate.validate_json(data_doc, lexicon)
date = datetime.datetime.now()
auto_update_document(data_doc, lexiconName, 'add', user, date)
ans = es.index(index=index, doc_type=typ, body=data_doc, id=_id)
_id = ans.get('_id')
db_loaded, db_error = update_db(_id,
data_doc,
user,
message,
lexiconName,
version=version,
suggestion=orgin_id,
status=status,
date=date)
except (esExceptions.RequestError, esExceptions.TransportError) as e:
handle_update_error(e, data, user, 'add')
raise eh.KarpElasticSearchError("Error during update. Message: %s.\n" %
str(e))
except Exception as e:
raise eh.KarpGeneralError(str(e))
jsonans = {
'es_loaded': 1,
'sql_loaded': db_loaded,
'es_ans': ans,
'suggestion': suggestion,
'id': _id
}
if db_error:
logging.debug(db_error)
if live:
return jsonify(jsonans)
else:
return jsonans
def delete_entry(lexicon, _id, sql=False, live=True, suggestion=False):
# delete by id
try:
query = request.query_string
parsed = parse_qs(query)
msg = parsed.get('message', ['removed'])[0]
es, index, typ = helpers.get_update_index(lexicon,
suggestion=suggestion)
ans_entry = es.get(index=index, doc_type=typ, id=_id)
lexiconName = ans_entry['_source']['lexiconName']
helpers.check_lexiconName(lexicon, lexiconName, _id, 'delete')
authdict, permitted = auth.validate_user()
if lexiconName not in permitted:
raise eh.KarpAuthenticationError('You are not allowed to modify '
'the lexicon %s, only %s' %
(lexiconName, permitted))
# doc_type must be set
ans = es.delete(index=index, doc_type=typ, id=_id)
db_loaded, db_error = 0, ''
if sql:
# logging.debug("Delete " + msg)
logging.debug('delete from sql.\nmsg %s\nans_entry %s' %
(msg, ans_entry))
db_loaded, db_error = update_db(_id,
ans_entry['_source'],
helpers.get_user(),
msg,
lexiconName,
status='removed')
logging.debug('updated db %s %s' % (db_loaded, db_error))
if db_error:
raise eh.KarpDbError(db_error)
except eshelpers.BulkIndexError as e:
# BulkIndexException is thrown for other parse errors
# This exception has errors instead of error
handle_update_error(e, {"id": _id}, helpers.get_user(), 'delete')
err = [er['create']['error'] for er in e.errors]
raise eh.KarpElasticSearchError("Error during deletion %s.\n" %
'\n'.join(err))
except (esExceptions.TransportError, esExceptions.RequestError) as e:
# elasticsearch-py throws errors (TransportError)
# for invalid (empty) objects
handle_update_error(e, {"id": _id}, helpers.get_user(), 'delete')
err = [e.error]
raise eh.KarpElasticSearchError('Error during deletion. '
'Message: %s.\n' % '\n'.join(err))
except esExceptions.TransportError as e:
# elasticsearch-py throws errors (TransportError) for objects not found
handle_update_error(e, {"id": _id}, helpers.get_user(), 'delete')
err = [e.error]
raise eh.KarpElasticSearchError('Error during deletion. Object not '
'found. Message: %s.\n' %
'\n'.join(err))
except Exception as e:
handle_update_error(e, {"id": _id}, helpers.get_user(), 'delete')
err = ['Oops, an unpredicted error', str(e), 'Document not deleted']
raise eh.KarpGeneralError('Document not deleted',
debug_msg=' '.join(err))
jsonans = {'es_loaded': 1, 'sql_loaded': db_loaded, 'es_ans': ans}
if db_error:
logging.debug(db_error)
if live:
return jsonify(jsonans)
else:
return jsonans
def update_doc(lexicon, _id, data=None, live=True):
""" Updates a posted document in the index 'index' with type 'typ'.
The document must contain a field called 'doc' with
the information to be sent.
The fields 'version' and 'message' are optional.
"""
# send user name and password,
# {'doc' : es_doc, 'version' : last version, 'message' : update message}
authdict, permitted = auth.validate_user()
if data is None:
data = helpers.read_data()
try:
index, typ = configM.get_lexicon_index(lexicon)
es = configM.elastic(lexicon=lexicon)
origin = es.get(index=index, id=_id)
except Exception as e:
logging.warning("Looking for entry at the wrong place:")
logging.exception(e)
msg = "The entry %s in lexicon %s was not found" % (_id, lexicon)
raise eh.KarpElasticSearchError(msg,
debug_msg=msg + " in index " + index)
lexiconName = origin['_source']['lexiconName']
helpers.check_lexiconName(lexicon, lexiconName, _id, 'update')
data_doc = data.get('doc') or data.get('_source')
version = data.get('version')
msg = data["message"]
if lexicon not in permitted:
raise eh.KarpAuthenticationError('You are not allowed to modify the '
'lexicon %s, only %s' %
(lexicon, permitted),
status_code=403)
# TODO validate data_doc, but this is so far sb specific!
validate.validate_json(data_doc, lexicon)
date = datetime.datetime.now()
user = helpers.get_user()
auto_update_document(data_doc, lexicon, 'update', user, date)
try:
if version is not None and version != -1:
ans = es.index(index=index,
doc_type=typ,
id=_id,
version=version,
body=data_doc,
op_type='index')
else:
ans = es.index(index=index,
doc_type=typ,
id=_id,
body=data_doc,
op_type='index')
except (esExceptions.RequestError, esExceptions.TransportError) as e:
# Transport error might be version conflict
logging.exception(e)
logging.debug('index: %s, type: %s, id: %s' % (index, typ, _id))
handle_update_error(e, {"id": _id, "data": data}, user, 'update')
raise eh.KarpElasticSearchError("Error during update. Message: %s.\n" %
str(e))
except Exception as e:
handle_update_error(e, {"id": _id, "data": data}, user, 'update')
raise eh.KarpElasticSearchError("Unexpected error during update.")
db_loaded, db_error = update_db(_id,
data_doc,
user,
msg,
lexiconName,
status='changed',
date=date)
jsonans = {'es_loaded': 1, 'sql_loaded': db_loaded, 'es_ans': ans}
if db_error:
logging.debug(db_error)
if live:
return jsonify(jsonans)
else:
return jsonans