def _report_timings(dbname, start, msg=None):
delta = datetime.now() - start
strdelta = str(delta).replace('0:00:0', '') # take out zero min & hour
queries = normdb.get_query_count(dbname)
normdb.reset_query_count(dbname)
Messager.info("Processed " + str(queries) + " queries in " + strdelta +
(msg if msg is not None else ""))
def get_configs(directory, filename, defaultstr, minconf, sections):
if (directory, filename) not in get_configs.__cache:
configstr, source = __read_first_in_directory_tree(directory, filename)
if configstr is None:
# didn't get one; try default dir and fall back to the default
configstr = __read_or_default(filename, defaultstr)
if configstr == defaultstr:
Messager.info(
"Project configuration: no configuration file (%s) found, using default."
% filename, 5)
source = "[default]"
else:
source = filename
# try to parse what was found, fall back to minimal config
try:
configs = __parse_configs(configstr, source, sections)
except:
Messager.warning(
"Project configuration: Falling back to minimal default. Configuration is likely wrong.",
5)
configs = minconf
get_configs.__cache[(directory, filename)] = configs
return get_configs.__cache[(directory, filename)]
def login(user, password):
if not _is_authenticated(user, password):
raise InvalidAuthError
get_session()['user'] = user
Messager.info('Hello!')
return {}
def login(user, password):
if not _is_authenticated(user, password):
raise InvalidAuthError
get_session()['user'] = user
Messager.info('Hello!')
return {}
def login(user, password):
if not _is_authenticated(user, password):
raise InvalidAuthError
get_session()['user'] = user
# Messager.info('Hello!')
Messager.info('Hello, your ID is ' + user) ##JESSY
return {}
def json_response(self, response=None):
if response is None:
response = {}
# debugging
if DEBUG:
msg_str = ''
if self.__added:
msg_str += ('Added the following line(s):\n'
+ '\n'.join([unicode(a).rstrip() for a in self.__added]))
if self.__changed:
changed_strs = []
for before, after in self.__changed:
changed_strs.append('\t%s\n\tInto:\n\t%s' % (unicode(before).rstrip(), unicode(after).rstrip()))
msg_str += ('Changed the following line(s):\n'
+ '\n'.join([unicode(a).rstrip() for a in changed_strs]))
if self.__deleted:
msg_str += ('Deleted the following line(s):\n'
+ '\n'.join([unicode(a).rstrip() for a in self.__deleted]))
if msg_str:
Messager.info(msg_str, duration=3*len(self))
else:
Messager.info('No changes made')
# highlighting
response['edited'] = []
# TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes
for a in self.__added:
try:
response['edited'].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
for b,a in self.__changed:
# can't mark "before" since it's stopped existing
try:
response['edited'].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
# unique, preserve order
seen = set()
uniqued = []
for i in response['edited']:
s = str(i)
if s not in seen:
uniqued.append(i)
seen.add(s)
response['edited'] = uniqued
#added deleted by sander naert
response['deleted'] = []
for a in self.__deleted:
try:
response['deleted'].append(a.id)
except AttributeError:
pass
return response
def logout():
try:
del get_session()['user']
except KeyError:
# Already deleted, let it slide
pass
# TODO: Really send this message?
Messager.info('Bye!')
return {}
def logout():
try:
del get_session()['user']
except KeyError:
# Already deleted, let it slide
pass
# TODO: Really send this message?
Messager.info('Bye!')
return {}
def json_response(self, response=None):
if response is None:
response = {}
# debugging
if DEBUG:
msg_str = ''
if self.__added:
msg_str += (
'Added the following line(s):\n' +
'\n'.join([unicode(a).rstrip() for a in self.__added]))
if self.__changed:
changed_strs = []
for before, after in self.__changed:
changed_strs.append(
'\t%s\n\tInto:\n\t%s' %
(unicode(before).rstrip(), unicode(after).rstrip()))
msg_str += (
'Changed the following line(s):\n' +
'\n'.join([unicode(a).rstrip() for a in changed_strs]))
if self.__deleted:
msg_str += (
'Deleted the following line(s):\n' +
'\n'.join([unicode(a).rstrip() for a in self.__deleted]))
if msg_str:
Messager.info(msg_str, duration=3 * len(self))
else:
Messager.info('No changes made')
# highlighting
response['edited'] = []
# TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes
for a in self.__added:
try:
response['edited'].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
for b, a in self.__changed:
# can't mark "before" since it's stopped existing
try:
response['edited'].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
# unique, preserve order
seen = set()
uniqued = []
for i in response['edited']:
s = str(i)
if s not in seen:
uniqued.append(i)
seen.add(s)
response['edited'] = uniqued
return response
def change_password(user, oldpassword, newpassword):
if isfile('users/' + user):
with open('users/' + user) as infile:
encrypter_password = infile.read()
if encrypter_password == _password_hash(oldpassword):
with open('users/' + user, 'w') as outfile:
outfile.write(_password_hash(newpassword))
Messager.info("Password changed!")
else:
Messager.error('Wrong password!')
else:
Messager.error("You have to login first!")
return {}
def compcode(compcode, collection, document):
## We want to write the compcode and user somewhere
try:
user = get_session()['user']
except KeyError:
Messager.warning('Not logged in??')
user = '******'
with open('/afs/inf.ed.ac.uk/web/securepages/clai/web/brat/work/userlog.txt', 'a') as f:
f.write("COMPLETION, %s, %s, %s, %s\n" % (str(datetime.now()), user, compcode, collection))
Messager.info('Thank you! Task completion has been logged!')
return {}
def save_document(text, docid, collection=None):
directory = collection
real_dir = real_directory(directory)
assert_allowed_to_read(real_dir)
json_dic = {}
file = text
try:
user = get_session().get('user')
except KeyError:
user = None
if user is None:
user = '******'
else:
db = DBlite()
names_complete = db.comlete_Ann_files(directory, file, user)
print("names_ING", names_complete, file=sys.stderr)
json_dic = {"message": "保存完成！", "user": user, "rontom": text, 'collection': directory}
Messager.info("保存完成")
return json_dic
def register(user, password):
# import json
# with open('users.txt') as infile:
# USER_PASSWORD = json.loads(infile.read())
# if user in USER_PASSWORD:
# Messager.info("Registration Failed! The username \"%s\" already exists!!" % user)
# else:
# USER_PASSWORD[user] = password
# with open('users.txt', 'w') as outfile:
# outfile.write(json.dumps(USER_PASSWORD))
# ######################
# if user != 'admin':
# import os
# mydatadir = path_join(DATA_DIR, user)
# if not os.path.isdir(mydatadir):
# os.makedirs(mydatadir)
# myrobots = path_join(mydatadir, 'acl.conf')
# if not os.path.isfile(myrobots):
# with open(myrobots, 'w') as myrr:
# myrr.write("User-agent: " + user + "\nDisallow:\n\nUser-agent: admin\nDisallow:\n\nUser-agent: *\nDisallow:/\n")
# ######################
# Messager.info("Registration Complete! The username is \"%s\"" % user)
# return {}
usernames = listdir('users/')
if user in usernames:
Messager.info("Registration Failed! The username \"%s\" already exists!!" % user)
else:
with open('users/' + user, 'w') as outfile:
outfile.write(_password_hash(password))
if user != 'admin':
mydatadir = path_join(DATA_DIR, user)
if not isdir(mydatadir):
makedirs(mydatadir)
myrobots = path_join(mydatadir, 'acl.conf')
if not isfile(myrobots):
with open(myrobots, 'w') as myrr:
myrr.write("User-agent: " + user + "\nDisallow:\n\nUser-agent: admin\nDisallow:\n\nUser-agent: *\nDisallow:/\n")
Messager.info("Registration Complete! The username is \"%s\"" % user)
return {}
def label(label_array, collection, document):
file_path = u'./data' + collection + document
file_path += u'.lbl'
f = open(file_path, 'w')
with open("label.json") as json_file:
json_data = json.load(json_file)
# Messager.info("Instance labeled at "+str(json_data[1]))
for i in range(json_data["num"]):
if (label_array[i] == 'T'):
f.write(str(json_data[str(i)]) + ":1" + " ")
else:
f.write(str(json_data[str(i)]) + ":0" + " ")
# f.write("\t" + str(datetime.now()) + "\n");
f.close()
#f.write(label1+" "+label2+" "+label3 + " "+str(datetime.now())+"\n")
# f.close()
Messager.info("Instance labeled at " + file_path)
return {}
def get_configs(directory, filename, defaultstr, minconf, sections):
if (directory, filename) not in get_configs.__cache:
configstr, source = __read_first_in_directory_tree(directory, filename)
if configstr is None:
# didn't get one; try default dir and fall back to the default
configstr = __read_or_default(filename, defaultstr)
if configstr == defaultstr:
Messager.info("Project configuration: no configuration file (%s) found, using default." % filename, 5)
source = "[default]"
else:
source = filename
# try to parse what was found, fall back to minimal config
try:
configs = __parse_configs(configstr, source, sections)
except:
Messager.warning("Project configuration: Falling back to minimal default. Configuration is likely wrong.", 5)
configs = minconf
get_configs.__cache[(directory, filename)] = configs
return get_configs.__cache[(directory, filename)]
def json_response(self, response=None):
if response is None:
response = {}
# debugging
msg_str = ""
if self.__added:
msg_str += "Added the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in self.__added])
if self.__changed:
changed_strs = []
for before, after in self.__changed:
changed_strs.append("\t%s\n\tInto:\n\t%s" % (unicode(before).rstrip(), unicode(after).rstrip()))
msg_str += "Changed the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in changed_strs])
if self.__deleted:
msg_str += "Deleted the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in self.__deleted])
if msg_str:
Messager.info(msg_str, duration=3 * len(self))
else:
Messager.info("No changes made")
# highlighting
response["edited"] = []
# TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes
for a in self.__added:
try:
response["edited"].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
for b, a in self.__changed:
# can't mark "before" since it's stopped existing
try:
response["edited"].append(a.reference_id())
except AttributeError:
pass # not all implement reference_id()
return response
def dumpy(collection, document):
file_path = u'./data' + collection
outputpath = u'./data/output' + collection
# f = open(file_path)
# Messager.error("hdhhd")
# Messager.error(file_path)
# for i, line in enumerate(f):
# Messager.error(line)
# f.close()
if not os.path.exists(outputpath):
os.makedirs(outputpath)
Messager.info("Traing start:")
# run(file_path, outputpath)
call(['python', 'server/src/learn/main.py', file_path, outputpath],
stdout=sys.stderr,
stderr=sys.stderr)
Messager.info("Back-end model training finished")
outputlbl = outputpath + "/" + document + ".lbl"
outputann = outputpath + "/" + document + ".ann"
for filename in listdir(outputpath):
lines = ""
if filename.endswith(".ann"):
with open(file_path + filename, "r") as original_f:
for line in original_f:
vec = line.split("\t")
if (not ("EXP" in vec[0]) and not ("S" in vec[0])):
lines += line
if filename.endswith(".ann"):
with open(outputpath + "/" + filename, "r") as f:
with open(file_path + filename, "w") as f1:
f1.write(lines)
for line in f:
f1.write(line)
# original_f = open(file_path + document + ".ann", 'a');
# with open(outputann, "r") as f:
# for line in f:
# original_f.write(line)
# original_f.close();
Messager.info("back-end model trained")
return {}
def logout():
get_session().invalidate()
# TODO: Really send this message?
Messager.info('Bye!')
return {}
def delete_document(collection, document):
#Messager.info(collection + ' ' + document)
directory = collection
if directory is None:
dir_path = DATA_DIR
else:
#XXX: These "security" measures can surely be fooled
if (directory.count('../') or directory == '..'):
raise InvalidDirError(directory)
dir_path = real_directory(directory)
#Messager.info('dir_path: ' + dir_path)
# Is the directory a directory and are we allowed to write?
if not isdir(dir_path):
raise InvalidDirError(dir_path)
if not access(dir_path, W_OK):
raise NoWritePermissionError(dir_path)
############################
from session import get_session
try:
username = get_session()['user']
except KeyError:
username = None
if username != 'admin':
if (not username) or username + '/' not in dir_path:
raise NoWritePermissionError(dir_path)
############################
base_path = join_path(dir_path, document)
txt_path = base_path + '.' + TEXT_FILE_SUFFIX
ann_path = base_path + '.' + JOINED_ANN_FILE_SUFF
#Messager.info('txt_path: ' + txt_path)
#Messager.info('ann_path: ' + ann_path)
if isdir(base_path):
import shutil
try:
shutil.rmtree(base_path)
except Exception as e:
Messager.error(e, -1)
# Delete the file
for path in (txt_path, ann_path):
#Messager.error("Removing " + path, -1)
path = abspath(path)
#Messager.error("Removing " + path, -1)
if isfile(path):
try:
os.remove(path)
except Exception as e:
Messager.error(e, -1)
#Messager.info(path)
#Messager.error("Document deletion not supported in this version.")
Messager.info("Document removed!")
return {}
def _norm_search_name_attr(database, name, attr,
matched, score_by_id, score_by_str,
best_score=0, exactmatch=False,
threshold=simstringdb.DEFAULT_THRESHOLD):
# helper for norm_search, searches for matches where given name
# appears either in full or as an approximate substring of a full
# name (if exactmatch is False) in given DB. If attr is not None,
# requires its value to appear as an attribute of the entry with
# the matched name. Updates matched, score_by_id, and
# score_by_str, returns best_score.
# If there are no strict substring matches for a given attribute
# in the simstring DB, we can be sure that no query can succeed,
# and can fail early.
# TODO: this would be more effective (as would some other things)
# if the attributes were in a separate simstring DB from the
# names.
if attr is not None:
utfattr = attr.encode('UTF-8')
normattr = string_norm_form(utfattr)
if not simstringdb.ssdb_supstring_exists(normattr, database, 1.0):
# debugging
#Messager.info('Early norm search fail on "%s"' % attr)
return best_score
if exactmatch:
# only candidate string is given name
strs = [name]
ss_norm_score = {string_norm_form(name): 1.0}
else:
# expand to substrings using simstring
# simstring requires UTF-8
utfname = name.encode('UTF-8')
normname = string_norm_form(utfname)
str_scores = simstringdb.ssdb_supstring_lookup(normname, database,
threshold, True)
strs = [s[0] for s in str_scores]
ss_norm_score = dict(str_scores)
# TODO: recreate this older filter; watch out for which name to use!
# # filter to strings not already considered
# strs = [s for s in strs if (normname, s) not in score_by_str]
# look up IDs
if attr is None:
id_names = normdb.ids_by_names(database, strs, False, True)
else:
id_names = normdb.ids_by_names_attr(database, strs, attr, False, True)
# sort by simstring (n-gram overlap) score to prioritize likely
# good hits.
# TODO: this doesn't seem to be having a very significant effect.
# consider removing as unnecessary complication (ss_norm_score also).
id_name_scores = [(i, n, ss_norm_score[string_norm_form(n)])
for i, n in id_names]
id_name_scores.sort(key=lambda a: a[2], reverse=True)
id_names = [(i, n) for i, n, s in id_name_scores]
# update matches and scores
for i, n in id_names:
if n not in matched:
matched[n] = set()
matched[n].add(i)
max_cost = MAX_SCORE - best_score + MAX_DIFF_TO_BEST_SCORE + 1
if (name, n) not in score_by_str:
# TODO: decide whether to use normalized or unnormalized strings
# for scoring here.
#score_by_str[(name, n)] = _norm_score(name, n, max_cost)
score_by_str[(name, n)] = _norm_score(
string_norm_form(name), string_norm_form(n), max_cost)
score = score_by_str[(name, n)]
best_score = max(score, best_score)
score_by_id[i] = max(score_by_id.get(i, -1),
score_by_str[(name, n)])
# stop if max count reached
if len(score_by_id) > MAX_SEARCH_RESULT_NUMBER:
Messager.info(
'Note: more than %d search results, only retrieving top matches' %
MAX_SEARCH_RESULT_NUMBER)
break
return best_score
def logout():
get_session().invalidate()
# TODO: Really send this message?
Messager.info('Bye!')
return {}
