def comparejson(lexicon, _id, fromdate='', todate=''):
from src.dbhandler.dbhandler import dbselect
import src.server.translator.jsondiff as jsondiff
auth, permitted = validate_user()
if lexicon not in permitted:
raise eh.KarpAuthenticationError('You are not allowed to update')
try:
if not todate:
import datetime
todate = datetime.datetime.now()
tojson = dbselect(lexicon, max_hits=1, to_date=todate, _id=_id)[0]
else:
tojson = dbselect(lexicon, exact_date=todate, _id=_id)[0]
# TODO catch the error here and print it to the log.
# It is probably not really sql that raises the exception
except Exception:
raise eh.KarpDbError('Could not find any entry from %s' % todate)
try:
if not fromdate:
jsons = dbselect(lexicon, max_hits=2, to_date=todate, _id=_id)
fromjson = {'doc': {}} if len(jsons) == 1 else jsons[1]
else:
fromjson = dbselect(lexicon, exact_date=fromdate, _id=_id)[0]
fromjson = fromjson['doc']
tojson = tojson['doc']
# TODO catch the error here and print it to the log.
# It is probably not really sql that raises the exception
except Exception:
raise eh.KarpDbError('Could not find any entry from %s' % fromdate)
return jsonify({'diff': jsondiff.compare(fromjson, tojson)})
def printlatestversion(lexicon, debug=True, with_id=False):
to_keep = {}
engine, db_entry = db.get_engine(lexicon, echo=False)
count = 0
for entry in db.dbselect(lexicon, engine=engine, db_entry=db_entry,
max_hits=-1):
_id = entry['id']
count += 1
if _id: # don't add things without id, they should not be here at all
if _id in to_keep:
last = to_keep[_id]['date']
if last < entry['date']:
to_keep[_id] = entry
else:
to_keep[_id] = entry
if debug:
print >> sys.stderr, 'count', count
if with_id:
out = [{'_id': i, '_source': val['doc']} for i, val in to_keep.items()
if val['status'] != 'removed']
print json.dumps(out)
else:
print '\n'.join((json.dumps(val['doc']) for val in to_keep.values()
if val['status'] != 'removed'))
def recover_add(index, suffix, lexicon):
# TODO test this
""" Recovers the data to ES, uses SQL as the trusted base version.
Find the last version of every SQL entry and send this to ES.
Adds the specified lexicons to an existing index
"""
import src.server.translator.bulkify as bulk
es = configM.elastic(index)
print 'Save %s to %s' % (lexicon, index)
to_keep = {}
for lex in lexicon:
engine, db_entry = db.get_engine(lex, echo=False)
for entry in db.dbselect(lex, engine=engine, db_entry=db_entry,
max_hits=-1):
_id = entry['id']
if _id: # don't add things without id, they are errors
if _id in to_keep:
last = to_keep[_id]['date']
if last < entry['date']:
to_keep[_id] = entry
else:
to_keep[_id] = entry
print len(to_keep), 'entries to keep'
data = bulk.bulkify_sql(to_keep, bulk_info={'index': index})
ok, err = helpers.bulk(es, data)
if err:
msg = "Error during upload. %s documents successfully uploaded. \
Message: %s.\n"
raise Exception(msg % (ok, '\n'.join(err)))
print 'recovery done'
def checkhistory(lexicon, lid):
""" Shows the update log of an entry """
from src.dbhandler.dbhandler import dbselect
auth, permitted = validate_user(mode="read")
settings = {'allowed': permitted}
size = helpers.get_size(default=10, settings=settings)
return jsonify({'updates': dbselect(lexicon, _id=lid, max_hits=size)})
def export(lexicon):
# TODO can user with only read permissions export all the lexicon?
# (eg saol)
auth, permitted = validate_user(mode="read")
if lexicon not in permitted:
raise eh.KarpAuthenticationError('You are not allowed to search the '
'lexicon %s' % lexicon)
settings = parser.make_settings(permitted, {
"size": -1,
"resource": lexicon
})
query = request.query_string
parsed = parser.parse_qs(query)
parser.parse_extra(parsed, settings)
date = settings.get('date', '')
mode = settings.get('mode', '')
if date:
from dateutil.parser import parserinfo, parse
from datetime import datetime
# parse the date as inclusive (including the whole selected day)
date = parse(date,
parserinfo(yearfirst=True),
default=datetime(1999, 01, 01, 23, 59))
to_keep = {}
engine, db_entry = db.get_engine(lexicon, echo=False)
logging.debug('exporting entries from %s ' % lexicon)
for entry in db.dbselect(lexicon,
engine=engine,
db_entry=db_entry,
max_hits=-1,
to_date=date):
_id = entry['id']
if _id in to_keep:
last = to_keep[_id]['date']
if last < entry['date']:
to_keep[_id] = entry
else:
to_keep[_id] = entry
ans = [
val['doc'] for val in to_keep.values() if val['status'] != 'removed'
]
ans = ans[:settings['size']]
logging.debug('exporting %s entries' % len(ans))
if settings.get('format', ''):
toformat = settings.get('format')
index, typ = configM.get_mode_index(mode)
msg = 'Unkown %s %s for mode %s' % ('format', toformat, mode)
format_posts = configM.extra_src(mode, 'exportformat',
helpers.notdefined(msg))
lmf, err = format_posts(ans, lexicon, mode, toformat)
return Response(lmf, mimetype='text/xml')
else:
return jsonify({lexicon: ans})
def recover(alias, suffix, lexicon, create_new=True):
# TODO test this
""" Recovers the data to ES, uses SQL as the trusted base version.
Find the last version of every SQL entry and send this to ES.
"""
import src.server.translator.bulkify as bulk
to_keep = {}
# if not lexicon:
# lexicon = conf.keys()
mapping = get_mapping(alias)
index = make_indexname(alias, suffix)
typ = configM.get_mode_type(alias)
print 'Save %s to %s' % (lexicon or 'all', index)
es = configM.elastic(alias)
if create_new:
# Create the index
ans = es.indices.create(index=index, body=mapping)
print ans
for lex in lexicon:
engine, db_entry = db.get_engine(lex, echo=False)
for entry in db.dbselect(lex, engine=engine, db_entry=db_entry, max_hits=-1):
_id = entry['id']
if _id: # don't add things without id, they are errors
if _id in to_keep:
last = to_keep[_id]['date']
if last < entry['date']:
to_keep[_id] = entry
else:
to_keep[_id] = entry
print len(to_keep), 'entries to keep'
data = bulk.bulkify_sql(to_keep, bulk_info={'index': index, 'type': typ})
try:
ok, err = helpers.bulk(es, data)
except:
print data
if err:
msg = "Error during upload. %s documents successfully uploaded. \
Message: %s.\n"
raise Exception(msg % (ok, '\n'.join(err)))
print 'recovery done'
def checkuserhistory():
""" Shows the updates a user has made """
try:
auth, permitted = validate_user()
user = helpers.get_user()
except AttributeError:
raise eh.KarpGeneralError('No user name provided', 'checkuserhistory')
try:
size = helpers.get_size(default=10, settings={'allowed': permitted})
from src.dbhandler.dbhandler import dbselect
updates = []
for lexicon in permitted:
# add updates from lexicons that are kept in sql
if configM.get_lexicon_sql(lexicon):
updates.extend(dbselect(lexicon, user=user, max_hits=size))
return jsonify({'updates': updates})
except Exception as e:
logging.exception(e)
raise eh.KarpGeneralError(str(e))
def checklexiconhistory(lexicon, date):
""" Shows the updates on one lexicon """
try:
auth, permitted = validate_user()
if lexicon not in permitted:
raise eh.KarpAuthenticationError(
'You are not allowed to update lexicon %s' % lexicon)
settings = {"allowed": permitted}
helpers.get_querysettings(settings)
size = settings.get('size', 10)
status = settings.get('status', ['added', 'changed', 'removed'])
from src.dbhandler.dbhandler import dbselect
return jsonify({
'resource':
lexicon,
'updates':
dbselect(lexicon, status=status, from_date=date, max_hits=size)
})
except Exception as e:
raise eh.KarpGeneralError(str(e))