def _report_timings(dbname, start, msg=None): delta = datetime.now() - start strdelta = str(delta).replace('0:00:0', '') # take out zero min & hour queries = normdb.get_query_count(dbname) normdb.reset_query_count(dbname) Messager.info("Processed " + str(queries) + " queries in " + strdelta + (msg if msg is not None else ""))
def get_configs(directory, filename, defaultstr, minconf, sections): if (directory, filename) not in get_configs.__cache: configstr, source = __read_first_in_directory_tree(directory, filename) if configstr is None: # didn't get one; try default dir and fall back to the default configstr = __read_or_default(filename, defaultstr) if configstr == defaultstr: Messager.info( "Project configuration: no configuration file (%s) found, using default." % filename, 5) source = "[default]" else: source = filename # try to parse what was found, fall back to minimal config try: configs = __parse_configs(configstr, source, sections) except: Messager.warning( "Project configuration: Falling back to minimal default. Configuration is likely wrong.", 5) configs = minconf get_configs.__cache[(directory, filename)] = configs return get_configs.__cache[(directory, filename)]
def login(user, password): if not _is_authenticated(user, password): raise InvalidAuthError get_session()['user'] = user Messager.info('Hello!') return {}
def login(user, password): if not _is_authenticated(user, password): raise InvalidAuthError get_session()['user'] = user # Messager.info('Hello!') Messager.info('Hello, your ID is ' + user) ##JESSY return {}
def json_response(self, response=None): if response is None: response = {} # debugging if DEBUG: msg_str = '' if self.__added: msg_str += ('Added the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in self.__added])) if self.__changed: changed_strs = [] for before, after in self.__changed: changed_strs.append('\t%s\n\tInto:\n\t%s' % (unicode(before).rstrip(), unicode(after).rstrip())) msg_str += ('Changed the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in changed_strs])) if self.__deleted: msg_str += ('Deleted the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in self.__deleted])) if msg_str: Messager.info(msg_str, duration=3*len(self)) else: Messager.info('No changes made') # highlighting response['edited'] = [] # TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes for a in self.__added: try: response['edited'].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() for b,a in self.__changed: # can't mark "before" since it's stopped existing try: response['edited'].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() # unique, preserve order seen = set() uniqued = [] for i in response['edited']: s = str(i) if s not in seen: uniqued.append(i) seen.add(s) response['edited'] = uniqued #added deleted by sander naert response['deleted'] = [] for a in self.__deleted: try: response['deleted'].append(a.id) except AttributeError: pass return response
def logout(): try: del get_session()['user'] except KeyError: # Already deleted, let it slide pass # TODO: Really send this message? Messager.info('Bye!') return {}
def json_response(self, response=None): if response is None: response = {} # debugging if DEBUG: msg_str = '' if self.__added: msg_str += ( 'Added the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in self.__added])) if self.__changed: changed_strs = [] for before, after in self.__changed: changed_strs.append( '\t%s\n\tInto:\n\t%s' % (unicode(before).rstrip(), unicode(after).rstrip())) msg_str += ( 'Changed the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in changed_strs])) if self.__deleted: msg_str += ( 'Deleted the following line(s):\n' + '\n'.join([unicode(a).rstrip() for a in self.__deleted])) if msg_str: Messager.info(msg_str, duration=3 * len(self)) else: Messager.info('No changes made') # highlighting response['edited'] = [] # TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes for a in self.__added: try: response['edited'].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() for b, a in self.__changed: # can't mark "before" since it's stopped existing try: response['edited'].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() # unique, preserve order seen = set() uniqued = [] for i in response['edited']: s = str(i) if s not in seen: uniqued.append(i) seen.add(s) response['edited'] = uniqued return response
def change_password(user, oldpassword, newpassword): if isfile('users/' + user): with open('users/' + user) as infile: encrypter_password = infile.read() if encrypter_password == _password_hash(oldpassword): with open('users/' + user, 'w') as outfile: outfile.write(_password_hash(newpassword)) Messager.info("Password changed!") else: Messager.error('Wrong password!') else: Messager.error("You have to login first!") return {}
def compcode(compcode, collection, document): ## We want to write the compcode and user somewhere try: user = get_session()['user'] except KeyError: Messager.warning('Not logged in??') user = '******' with open('/afs/inf.ed.ac.uk/web/securepages/clai/web/brat/work/userlog.txt', 'a') as f: f.write("COMPLETION, %s, %s, %s, %s\n" % (str(datetime.now()), user, compcode, collection)) Messager.info('Thank you! Task completion has been logged!') return {}
def save_document(text, docid, collection=None): directory = collection real_dir = real_directory(directory) assert_allowed_to_read(real_dir) json_dic = {} file = text try: user = get_session().get('user') except KeyError: user = None if user is None: user = '******' else: db = DBlite() names_complete = db.comlete_Ann_files(directory, file, user) print("names_ING", names_complete, file=sys.stderr) json_dic = {"message": "保存完成!", "user": user, "rontom": text, 'collection': directory} Messager.info("保存完成") return json_dic
def register(user, password): # import json # with open('users.txt') as infile: # USER_PASSWORD = json.loads(infile.read()) # if user in USER_PASSWORD: # Messager.info("Registration Failed! The username \"%s\" already exists!!" % user) # else: # USER_PASSWORD[user] = password # with open('users.txt', 'w') as outfile: # outfile.write(json.dumps(USER_PASSWORD)) # ###################### # if user != 'admin': # import os # mydatadir = path_join(DATA_DIR, user) # if not os.path.isdir(mydatadir): # os.makedirs(mydatadir) # myrobots = path_join(mydatadir, 'acl.conf') # if not os.path.isfile(myrobots): # with open(myrobots, 'w') as myrr: # myrr.write("User-agent: " + user + "\nDisallow:\n\nUser-agent: admin\nDisallow:\n\nUser-agent: *\nDisallow:/\n") # ###################### # Messager.info("Registration Complete! The username is \"%s\"" % user) # return {} usernames = listdir('users/') if user in usernames: Messager.info("Registration Failed! The username \"%s\" already exists!!" % user) else: with open('users/' + user, 'w') as outfile: outfile.write(_password_hash(password)) if user != 'admin': mydatadir = path_join(DATA_DIR, user) if not isdir(mydatadir): makedirs(mydatadir) myrobots = path_join(mydatadir, 'acl.conf') if not isfile(myrobots): with open(myrobots, 'w') as myrr: myrr.write("User-agent: " + user + "\nDisallow:\n\nUser-agent: admin\nDisallow:\n\nUser-agent: *\nDisallow:/\n") Messager.info("Registration Complete! The username is \"%s\"" % user) return {}
def label(label_array, collection, document): file_path = u'./data' + collection + document file_path += u'.lbl' f = open(file_path, 'w') with open("label.json") as json_file: json_data = json.load(json_file) # Messager.info("Instance labeled at "+str(json_data[1])) for i in range(json_data["num"]): if (label_array[i] == 'T'): f.write(str(json_data[str(i)]) + ":1" + " ") else: f.write(str(json_data[str(i)]) + ":0" + " ") # f.write("\t" + str(datetime.now()) + "\n"); f.close() #f.write(label1+" "+label2+" "+label3 + " "+str(datetime.now())+"\n") # f.close() Messager.info("Instance labeled at " + file_path) return {}
def get_configs(directory, filename, defaultstr, minconf, sections): if (directory, filename) not in get_configs.__cache: configstr, source = __read_first_in_directory_tree(directory, filename) if configstr is None: # didn't get one; try default dir and fall back to the default configstr = __read_or_default(filename, defaultstr) if configstr == defaultstr: Messager.info("Project configuration: no configuration file (%s) found, using default." % filename, 5) source = "[default]" else: source = filename # try to parse what was found, fall back to minimal config try: configs = __parse_configs(configstr, source, sections) except: Messager.warning("Project configuration: Falling back to minimal default. Configuration is likely wrong.", 5) configs = minconf get_configs.__cache[(directory, filename)] = configs return get_configs.__cache[(directory, filename)]
def json_response(self, response=None): if response is None: response = {} # debugging msg_str = "" if self.__added: msg_str += "Added the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in self.__added]) if self.__changed: changed_strs = [] for before, after in self.__changed: changed_strs.append("\t%s\n\tInto:\n\t%s" % (unicode(before).rstrip(), unicode(after).rstrip())) msg_str += "Changed the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in changed_strs]) if self.__deleted: msg_str += "Deleted the following line(s):\n" + "\n".join([unicode(a).rstrip() for a in self.__deleted]) if msg_str: Messager.info(msg_str, duration=3 * len(self)) else: Messager.info("No changes made") # highlighting response["edited"] = [] # TODO: implement cleanly, e.g. add a highlightid() method to Annotation classes for a in self.__added: try: response["edited"].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() for b, a in self.__changed: # can't mark "before" since it's stopped existing try: response["edited"].append(a.reference_id()) except AttributeError: pass # not all implement reference_id() return response
def dumpy(collection, document): file_path = u'./data' + collection outputpath = u'./data/output' + collection # f = open(file_path) # Messager.error("hdhhd") # Messager.error(file_path) # for i, line in enumerate(f): # Messager.error(line) # f.close() if not os.path.exists(outputpath): os.makedirs(outputpath) Messager.info("Traing start:") # run(file_path, outputpath) call(['python', 'server/src/learn/main.py', file_path, outputpath], stdout=sys.stderr, stderr=sys.stderr) Messager.info("Back-end model training finished") outputlbl = outputpath + "/" + document + ".lbl" outputann = outputpath + "/" + document + ".ann" for filename in listdir(outputpath): lines = "" if filename.endswith(".ann"): with open(file_path + filename, "r") as original_f: for line in original_f: vec = line.split("\t") if (not ("EXP" in vec[0]) and not ("S" in vec[0])): lines += line if filename.endswith(".ann"): with open(outputpath + "/" + filename, "r") as f: with open(file_path + filename, "w") as f1: f1.write(lines) for line in f: f1.write(line) # original_f = open(file_path + document + ".ann", 'a'); # with open(outputann, "r") as f: # for line in f: # original_f.write(line) # original_f.close(); Messager.info("back-end model trained") return {}
def logout(): get_session().invalidate() # TODO: Really send this message? Messager.info('Bye!') return {}
def delete_document(collection, document): #Messager.info(collection + ' ' + document) directory = collection if directory is None: dir_path = DATA_DIR else: #XXX: These "security" measures can surely be fooled if (directory.count('../') or directory == '..'): raise InvalidDirError(directory) dir_path = real_directory(directory) #Messager.info('dir_path: ' + dir_path) # Is the directory a directory and are we allowed to write? if not isdir(dir_path): raise InvalidDirError(dir_path) if not access(dir_path, W_OK): raise NoWritePermissionError(dir_path) ############################ from session import get_session try: username = get_session()['user'] except KeyError: username = None if username != 'admin': if (not username) or username + '/' not in dir_path: raise NoWritePermissionError(dir_path) ############################ base_path = join_path(dir_path, document) txt_path = base_path + '.' + TEXT_FILE_SUFFIX ann_path = base_path + '.' + JOINED_ANN_FILE_SUFF #Messager.info('txt_path: ' + txt_path) #Messager.info('ann_path: ' + ann_path) if isdir(base_path): import shutil try: shutil.rmtree(base_path) except Exception as e: Messager.error(e, -1) # Delete the file for path in (txt_path, ann_path): #Messager.error("Removing " + path, -1) path = abspath(path) #Messager.error("Removing " + path, -1) if isfile(path): try: os.remove(path) except Exception as e: Messager.error(e, -1) #Messager.info(path) #Messager.error("Document deletion not supported in this version.") Messager.info("Document removed!") return {}
def _norm_search_name_attr(database, name, attr, matched, score_by_id, score_by_str, best_score=0, exactmatch=False, threshold=simstringdb.DEFAULT_THRESHOLD): # helper for norm_search, searches for matches where given name # appears either in full or as an approximate substring of a full # name (if exactmatch is False) in given DB. If attr is not None, # requires its value to appear as an attribute of the entry with # the matched name. Updates matched, score_by_id, and # score_by_str, returns best_score. # If there are no strict substring matches for a given attribute # in the simstring DB, we can be sure that no query can succeed, # and can fail early. # TODO: this would be more effective (as would some other things) # if the attributes were in a separate simstring DB from the # names. if attr is not None: utfattr = attr.encode('UTF-8') normattr = string_norm_form(utfattr) if not simstringdb.ssdb_supstring_exists(normattr, database, 1.0): # debugging #Messager.info('Early norm search fail on "%s"' % attr) return best_score if exactmatch: # only candidate string is given name strs = [name] ss_norm_score = {string_norm_form(name): 1.0} else: # expand to substrings using simstring # simstring requires UTF-8 utfname = name.encode('UTF-8') normname = string_norm_form(utfname) str_scores = simstringdb.ssdb_supstring_lookup(normname, database, threshold, True) strs = [s[0] for s in str_scores] ss_norm_score = dict(str_scores) # TODO: recreate this older filter; watch out for which name to use! # # filter to strings not already considered # strs = [s for s in strs if (normname, s) not in score_by_str] # look up IDs if attr is None: id_names = normdb.ids_by_names(database, strs, False, True) else: id_names = normdb.ids_by_names_attr(database, strs, attr, False, True) # sort by simstring (n-gram overlap) score to prioritize likely # good hits. # TODO: this doesn't seem to be having a very significant effect. # consider removing as unnecessary complication (ss_norm_score also). id_name_scores = [(i, n, ss_norm_score[string_norm_form(n)]) for i, n in id_names] id_name_scores.sort(key=lambda a: a[2], reverse=True) id_names = [(i, n) for i, n, s in id_name_scores] # update matches and scores for i, n in id_names: if n not in matched: matched[n] = set() matched[n].add(i) max_cost = MAX_SCORE - best_score + MAX_DIFF_TO_BEST_SCORE + 1 if (name, n) not in score_by_str: # TODO: decide whether to use normalized or unnormalized strings # for scoring here. #score_by_str[(name, n)] = _norm_score(name, n, max_cost) score_by_str[(name, n)] = _norm_score( string_norm_form(name), string_norm_form(n), max_cost) score = score_by_str[(name, n)] best_score = max(score, best_score) score_by_id[i] = max(score_by_id.get(i, -1), score_by_str[(name, n)]) # stop if max count reached if len(score_by_id) > MAX_SEARCH_RESULT_NUMBER: Messager.info( 'Note: more than %d search results, only retrieving top matches' % MAX_SEARCH_RESULT_NUMBER) break return best_score