def complete_mining(entity_type=Idea, filename='graph.txt', root='./'): occur_filename = root + "graph-" + filename edge_filename = root + "edge-" + filename sql_filename = root + "sql-" + filename print "processing articles..." process_articles(entity_type, occur_filename) print "running apriori miner..." dm.apriori(occur_filename, edge_filename) print "processing edges..." edges = dm.process_edges(occur_filename, edge_filename) ents = dm.calculate_node_entropy(edges) edges = dm.calculate_edge_weight(edges, ents) print "creating sql files..." with open(sql_filename, 'w') as f: for edge, props in edges.iteritems(): ante, cons = edge row = "%s::%s" % edge row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props f.write(row) print "updating term entropy..." for term_id, entropy in ents.iteritems(): term = Session.query(Idea).get(term_id) if term: term.entropy = entropy Session.flush() Session.commit()
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = ["ISSN", "noesisInclude", "URL", "source", "abbr", "language", "student", "active"] params = request.params.mixed() if '_method' in params: del params['_method'] if 'name' in params: name = params['name'] del params['name'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) journal = Journal(name, **params) Session.add(journal) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='journal', action='view', id=journal.ID) return "Moved temporarily"
def _evaluate(self, evaltype, id, id2=None, uid=None, username=None, degree=-1, maxdegree=4): """ Function to submit an evaluation. Takes a POST request containing the consequesnt id and all or none of: generality, relatedness, hyperrank, hyporank. """ if not h.auth.is_logged_in(): abort(401) id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.environ.get('REMOTE_USER', username) print "grabbing eval for", username, uid if request.environ.get('REMOTE_USER', False): evaluation = self._get_evaluation(id, id2, None, username) else: evaluation = self._get_anon_evaluation(id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0')) # Populate proper generality, relatedness, hyperrank and hyporank values # Attempt to convert to integers, if unable, throw HTTP 400 try: setattr(evaluation, evaltype, int(request.params.get('degree', getattr(evaluation, evaltype)))) except TypeError: abort(400) # Create and commit evaluation Session.flush() Session.commit() # Issue an HTTP success response.status_int = 200 return "OK"
def _thinker_evaluate(self, evaltype=None, id=None, id2=None, uid=None, username=None, degree=1, maxdegree=1): """ Private method to handle generic evaluations. See ``teacher_of`` and ``has_influenced`` for use. """ id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.params.get('username', username) evaluation = self._get_evaluation(evaltype, id, id2, uid, username) try: evaluation.degree = int(request.params.get('degree', degree)) except TypeError: abort(400) # Create and commit evaluation Session.flush() # Issue an HTTP success response.status_int = 200 return "OK"
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = ["sep_dir", "wiki"] params = request.params.mixed() if '_method' in params: del params['_method'] if 'name' in params: name = params['name'] del params['name'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) thinker = Thinker(name, **params) Session.add(thinker) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='thinker', action='view', id=thinker.ID) return "Moved temporarily"
def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None): if not h.auth.is_logged_in(): abort(401) id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.params.get('username', username) evaluation = self._get_evaluation(id, id2, uid, username, autoCreate=False) if not evaluation: abort(404) current_uid = h.get_user(request.environ['REMOTE_USER']).ID if evaluation.uid != current_uid or not h.auth.is_admin(): abort(401) setattr(evaluation, evaltype, -1) # Delete evaluation if this eliminates both settings, new db schema # will eliminate this need #if evaluation.generality == -1 and evaluation.relatedness == -1: # h.delete_obj(evaluation) Session.flush() Session.commit() response.status_int = 200 return "OK"
def submit_changes(self): ''' This function validates the submitted profile edit form and commits the changes. Restricted to ``POST`` requests. If successful, redirects to the result action to prevent resubmission. ''' if not h.auth.is_logged_in(): abort(401) c.user = h.get_user(request.environ['REMOTE_USER']) if self.form_result['password'] != '': c.user.set_password(self.form_result['password']) # TODO: Enable area editing #c.user.first_area_id=self.form_result['first_area'], #user.first_area_level=self.form_result['first_area_level'], #if self.form_result['second_area']: # c.user.second_area_id=self.form_result['second_area'], # c.user.second_area_level=self.form_result['second_area_level'] c.user.fullname = self.form_result['fullname'] Session.flush() Session.commit() h.redirect(h.url(controller='account', action='profile', message='edited'))
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = ["sep_dir", "wiki"] params = request.params.mixed() if '_method' in params: del params['_method'] if 'name' in params: name = params['name'] del params['name'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) thinker = Thinker(name, **params) Session.add(thinker) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='thinker', action='view', id=thinker.ID) return "Moved temporarily"
def complete_mining(entity_type=Idea, filename='graph.txt', root='./'): occur_filename = root + "graph-" + filename edge_filename = root + "edge-" + filename sql_filename = root + "sql-" + filename print "processing articles..." process_articles(entity_type, occur_filename) print "running apriori miner..." dm.apriori(occur_filename, edge_filename) print "processing edges..." edges = dm.process_edges(occur_filename, edge_filename) ents = dm.calculate_node_entropy(edges) edges = dm.calculate_edge_weight(edges, ents) print "creating sql files..." with open(sql_filename, 'w') as f: for edge, props in edges.iteritems(): ante,cons = edge row = "%s::%s" % edge row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props f.write(row) print "updating term entropy..." for term_id, entropy in ents.iteritems(): term = Session.query(Idea).get(term_id) if term: term.entropy = entropy Session.flush() Session.commit()
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = ["sep_dir", "searchstring", "searchpattern"] params = request.params.mixed() if '_method' in params: del params['_method'] if 'label' in params: label = params['label'] del params['label'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) idea = Idea(label) Session.add(idea) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='idea', action='view', id=idea.ID) return "Moved temporarily"
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = ["sep_dir", "searchstring", "searchpattern"] params = request.params.mixed() if '_method' in params: del params['_method'] if 'label' in params: label = params['label'] del params['label'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) idea = Idea(label) Session.add(idea) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='idea', action='view', id=idea.ID) return "Moved temporarily"
def create(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) valid_params = [ "ISSN", "noesisInclude", "URL", "source", "abbr", "language", "student", "active" ] params = request.params.mixed() if '_method' in params: del params['_method'] if 'name' in params: name = params['name'] del params['name'] else: abort(400) for k in params.keys(): if k not in valid_params: abort(400) journal = Journal(name, **params) Session.add(journal) Session.flush() # Issue an HTTP success response.status_int = 302 response.headers['location'] = h.url(controller='journal', action='view', id=journal.ID) return "Moved temporarily"
def _evaluate(self, evaltype, id, id2=None, uid=None, username=None, degree=-1, maxdegree=4): """ Function to submit an evaluation. Takes a POST request containing the consequesnt id and all or none of: generality, relatedness, hyperrank, hyporank. """ if not h.auth.is_logged_in(): abort(401) id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.environ.get('REMOTE_USER', username) print "grabbing eval for", username, uid if request.environ.get('REMOTE_USER', False): evaluation = self._get_evaluation(id, id2, None, username) else: evaluation = self._get_anon_evaluation(id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0')) # Populate proper generality, relatedness, hyperrank and hyporank values # Attempt to convert to integers, if unable, throw HTTP 400 try: setattr(evaluation, evaltype, int(request.params.get('degree', getattr(evaluation, evaltype)))) except TypeError: abort(400) # Create and commit evaluation Session.flush() Session.commit() # Issue an HTTP success response.status_int = 200 return "OK"
def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None): if not h.auth.is_logged_in(): abort(401) id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.params.get('username', username) evaluation = self._get_evaluation(id, id2, uid, username, autoCreate=False) if not evaluation: abort(404) current_uid = h.get_user(request.environ['REMOTE_USER']).ID if evaluation.uid != current_uid or not h.auth.is_admin(): abort(401) setattr(evaluation, evaltype, -1) # Delete evaluation if this eliminates both settings, new db schema # will eliminate this need if evaluation.generality == -1 and evaluation.relatedness == -1: h.delete_obj(evaluation) else: # save change in evaluation Session.flush() Session.commit() response.status_int = 200 return "OK"
def delete_obj(obj): """ Deletes any arbitrary object from the SQLAlchemy Session and cascades deletes to evaluations. :param obj: object to delete """ Session.delete(obj) Session.flush()
def delete_obj(obj): """ Deletes any arbitrary object from the SQLAlchemy Session and cascades deletes to evaluations. :param obj: object to delete """ Session.delete(obj) Session.flush()
def update_obj(obj, attributes, params): """ Updates any arbitrary object. Takes a list of attributes and a dictionary of update parameters. Checks if each key is in the list of approved attributes and then attempts to set it. If the object does not have that key, throw an HTTP 400 Bad Request :param obj: object to update :param attributes: list of approved attributes :param params: dictionary of update parameters """ for key in params.keys(): if key in attributes: try: set_attribute(obj, key, params[key]) except: abort(400) Session.flush()
def update_obj(obj, attributes, params): """ Updates any arbitrary object. Takes a list of attributes and a dictionary of update parameters. Checks if each key is in the list of approved attributes and then attempts to set it. If the object does not have that key, throw an HTTP 400 Bad Request :param obj: object to update :param attributes: list of approved attributes :param params: dictionary of update parameters """ for key in params.keys(): if key in attributes: try: set_attribute(obj, key, params[key]) except: abort(400) Session.flush()
def _thinker_evaluate(self, evaltype=None, id=None, id2=None, uid=None, username=None, degree=1, maxdegree=1): """ Private method to handle generic evaluations. See ``teacher_of`` and ``has_influenced`` for use. """ id2 = request.params.get('id2', id2) uid = request.params.get('uid', uid) username = request.params.get('username', username) evaluation = self._get_evaluation(evaltype, id, id2, uid, username) try: evaluation.degree = int(request.params.get('degree', degree)) except TypeError: abort(400) # Create and commit evaluation Session.flush() # Issue an HTTP success response.status_int = 200 return "OK"
def getentries(db_dir): #to do feb 10: do something more elegant than just delete existing and rewrite...perhaps only write new? #first, clear out the current table #Session.query(SEPEntry).delete() #Session.flush() #Session.commit() entries = defaultdict(lambda: { 'title': '', 'published': False, 'status': '' }) pars = HTMLParser.HTMLParser() #get published entries published = open('/Users/inpho/SitesOld/dev/entries-published.txt') status = open(os.path.join(db_dir, 'entrystatus.txt')) entrylist = open(os.path.join(db_dir, 'entries.txt')) #set up entries dict for line in entrylist: line = line.split('::') sep_dir = pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[0])) title = re.sub("\\\\\'", "'", pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[1]))) entries[sep_dir]['title'] = title for line in published: line = re.sub('\\n', '', line) if entries[line]['title']: entries[line]['published'] = True else: print "uh-oh, " + line + "doesn't appear to be in dict object" for line in status: line = line.split('::') if entries[line[0]]['title']: entries[line[0]]['status'] = line[1] #print "status = " + line[1] + ' for ' + line[0] else: print "uh-oh, " + line[0] + "doesn't appear to be in dict object" for key in entries.keys(): #so, what I should really do here is figure out whether the entry already has a place in the table; if so, update its existing stats; if not, #insert a new one; #also need to check if old entries in sepentries table are no longer in file... entry_q = Session.query(SEPEntry) o = SEPEntry.title.like(entries[key]['title']) entry_q = entry_q.filter(o) # if only 1 result, go ahead and view that idea if entry_q.count() == 0: entry_add = SEPEntry(entries[key]['title'], key, entries[key]['published'], entries[key]['status']) Session.add(entry_add) elif entry_q.count() == 1: #replace data from most recent from SEPMirror entry = entry_q.first() entry.title = entries[key]['title'] entry.published = entries[key]['published'] entry.status = entries[key]['status'] #need to really add something here to delete entries no longer in the DB... entry_q2 = Session.query(SEPEntry) for entry in entry_q2: if not entries[entry.sep_dir]['title']: print str( entry.title ) + " appears to have been deleted from SEPMirror; deleting from InPhO database." Session.delete(entry) Session.flush() Session.commit()
def process(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 q = request.params.get('label', None) label = q searchpattern = request.params.get('searchpattern', None) searchstring = request.params.get('searchstring', None) sep_dir = request.params.get('sep_dir', None) ioru = request.params.get('ioru', 'u') print "***" print "sep_dir is " print sep_dir print "and label is " print q print "***" values = dict(request.params) c.found = values.get('found', False) c.custom = values.get('custom', False) c.new = values.get('new', False) #ADD ACTION -- add a new entity (id/label cannot already exist) if action=='Add': #lowercase label and check to see whether it is valid #if so commit idea with standard pluralizations as searchpattern and give user a message that add was successful q = q.lower() #check to see whether label already exists idea_q = Session.query(Idea) # Check for query o = Idea.label.like(q) idea_q = idea_q.filter(o) # if 0 results, proceed to add idea if idea_q.count() == 0: print "***action add, idea q count == 0" #if no exact match for label, create new object with that label to add idea_add = Idea(q) #add searchstring = label idea_add.searchstring = q #add pluralizations to existing searchpatterns idea_add.searchpatterns = idea_add.pluralize() #setup the search string list c.idea = idea_add c.search_string_list = c.idea.setup_SSL() #setup sep_dir if present if sep_dir: c.idea.sep_dir = sep_dir #commit #take user to disambiguation page if the label contains an 'and' #otherwise, commit idea Session.add(idea_add) Session.flush() Session.commit() c.found = True something = 1 else: #already a match; give user error message that they should edit the preexisting page, and send them back to the submit idea #for edit page print "****action add, idea_q.count() not equal 0" c.message = "Idea with that name already exists in database; please edit the existing entry (returned below) or try a new label." c.idea = idea_q.first() c.search_string_list = c.idea.setup_SSL() return render('admin/idea-edit.html') c.message = "Idea added successfully." return render('admin/idea-edit.html') #Modify action -- edit an existing entry, id must exist elif action == 'Modify': #retrieve entity corresponding to id c.idea = h.fetch_obj(Idea, values['ID']) changed = False #generate searchpattern from searchstring if not already provided; current value stored in searchpattern, generate from searchstring if null #update parameters with form values if c.idea.sep_dir != values['sep_dir']: c.idea.sep_dir = values['sep_dir'] changed = True if c.idea.label != label: c.idea.label = label changed = True if not c.custom: if c.idea.searchstring != s.convertSS(searchstring, ioru)[0]: c.idea.searchstring = s.convertSS(searchstring, ioru)[0] #add default pluralizations to searchpatterns for label in c.idea.pluralize(): if label not in c.idea.searchpatterns: c.idea.searchpatterns.append(label) changed = True else: searchpatterns = [] for k, v in values.items(): key = "" if k.startswith('searchpatterns'): varname, num = k.split('.') key = 'delsearchpattern.%s'%(num) keyval = request.params.get(key, False) if not keyval: searchpatterns.append(v) if values['newsearchpattern']: searchpatterns.append(values['newsearchpattern']) changed = True #do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed if c.idea.searchpatterns != searchpatterns: c.idea.searchpatterns = searchpatterns changed = True if c.idea.searchstring != searchstring: c.idea.searchstring = searchstring #add default pluralizations for new searchstring as well for label in c.idea.pluralize(): if label not in c.idea.searchpatterns: c.idea.searchpatterns.append(label) changed = True #make sure that searchpattern and searchstring are not shared by another entity #print "searchpatterns is " #print searchpatterns #print "c.idea.searchpatterns is" #print c.idea.searchpatterns #commit changes Session.flush() Session.commit() #return success message if changed == True: c.message = "Idea modified successfully." else: c.message = "No change required; idea not modified." c.found = True c.search_string_list = c.idea.setup_SSL() #prepare form return render('admin/idea-edit.html') #Delete action -- Delete an existing entity (id must exist) elif action=='Delete': c.idea = h.fetch_obj(Idea, values['ID']) c.message = "Idea # " + values['ID'] + " ("+ c.idea.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.idea) Session.flush() Session.commit() return render('admin/idea-edit.html') #use custom searchstring action elif action=='Use Custom Searchstring/Searchpattern': #toggle the c.custom to True and re-render form with htmlfill c.custom = True c.idea = h.fetch_obj(Idea, values['ID']) c.found = True c.message = 'Enter custom searchstring for ' + c.idea.label + "." return render('admin/idea-edit.html') #use standard searchstring action elif action=='Use Standard Searchstring/Searchpattern': #toggle the c.custom to False and re-render form with htmlfill c.custom = False c.idea = h.fetch_obj(Idea, values['ID']) c.found = True c.search_string_list = c.idea.setup_SSL() c.message = 'Now displaying standard searchstring options for ' + c.idea.label + "." return render('admin/idea-edit.html') else: raise Exception('I don\'t know how you got here, but you shouldn\'t be here...please start at the main page. Invalid action %s'%action)
def fuzzymatchall(SEPEntrieslist): #takes outputs from addlist() and saves all fuzzy match IDs to SEPEntry.fuzzymatch with verdicts (percent of words matched) #now change so that it only updates ones that don't currently have a fuzzymatchlist #clear out fuzzymatch table--otherwise old fuzzies will accumulate, and nobody wants that delquery = Session.query(Fuzzymatch) delquery.delete() Session.flush() Session.commit() for SEPEntry in SEPEntrieslist: print "working on " + SEPEntry.title.encode('utf-8') + "\n" entities = Session.query(Entity) #exclude journals and nodes from fuzzy matching entities = entities.filter(Entity.typeID != 2) entities = entities.filter(Entity.typeID != 4) #reset fuzzymatches for that entry #SEPEntry.fuzzymatches = "" ##string1 = string1.decode('utf8') for entity in entities: php = PHP("set_include_path('/usr/lib/php/');") php = PHP("require 'fuzzymatch.php';") #php = PHP() #print "testing " + entity.label.encode('utf8') + " against " + string1.encode('utf8') + "\n" code = '$string1 = utf8_decode("' + SEPEntry.title.encode( 'utf8') + '");' #code = code + "$string2 = '" + entity.label.encode('latin-1', 'replace') + "';" #code = code + "print $string1; print $string2;" #print code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");' code = code + '$string2 = utf8_decode("' + entity.label.encode( 'utf8') + '");' code = code + """print fuzzy_match($string1, $string2, 2);""" verdict = php.get_raw(code) #print "verdict is " + verdict + "\n" verdict = verdict.split(',') if float(verdict[0]) >= .20: #print entity.label + " is a match!\n" #entity.matchvalue = verdict #string = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict #if len(string) < 400: # SEPEntry.fuzzymatches = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict #else: # print "sorry, too many matches! Can't add " + str(entity.ID) + " to fuzzy matches; over 400 chars." fmatch = Fuzzymatch(entity.ID) fmatch.sep_dir = SEPEntry.sep_dir fmatch.strength = verdict[0] fmatch.edits = verdict[1] SEPEntry.fmatches.append(fmatch) Session.flush() Session.commit()
def process(self, id=None): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False label = request.params.get('label', None) id = request.params.get('ID', id) sep_dir = request.params.get('sep_dir', None) URL = request.params.get('URL', None) language = request.params.get('language', None) queries = [request.params.get('queries', None)] openAccess = request.params.get('openAccess', None) active = request.params.get('active', None) student = request.params.get('student', None) ISSN = request.params.get('ISSN', None) action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 values = dict(request.params) #abbrs = [request.params.get('abbrs', None)] abbrs = [] queries = [] for k, v in values.items(): key = "" if k.startswith('abbrs'): varname, num = k.split('.') key = 'delabbr.%s' % (num) keyval = request.params.get(key, False) if not keyval: abbrs.append(v) elif k.startswith('queries'): varname, num = k.split('.') key = 'delquer.%s' % (num) keyval = request.params.get(key, False) if not keyval: queries.append(v) if action == "Add": journal_add = Journal() journal_add.label = label #setup search string and search pattern journalname = journal_add.label journalname_q = Session.query(Entity) o = Entity.searchpattern.like('( ' + journalname + ' )') journalname_q = journalname_q.filter(o).order_by( func.length(Entity.label)) if journalname_q.count() == 0: journal_add.searchpattern = "( " + journalname + " )" journal_add.searchstring = journalname else: journal_add.searchpattern = "( " + label + " )" journal_add.searchcstring = label #reset old journal pattern to whole name too to avoid conflict oldjournal = h.fetch_obj(Journal, journalname_q.first().ID) oldjournal.searchpattern = "( " + oldjournal.label + " )" oldjournal.searchstring = oldjournal.label Session.add(oldjournal) if sep_dir: journal_add.sep_dir = sep_dir c.journal = journal_add Session.add(journal_add) Session.flush() Session.commit() c.found = True c.message = "Journal " + c.journal.label + " added successfully." return render('admin/journal-edit.html') elif action == "Modify": c.journal = h.fetch_obj(Journal, id) c.found = True changed = False #set values from form if c.journal.label != label: c.journal.label = label changed = True if c.journal.sep_dir != sep_dir: c.journal.sep_dir = sep_dir changed = True if c.journal.URL != URL: c.journal.URL = URL changed = True if c.journal.language != language: c.journal.language = language changed = True if c.journal.abbrs != abbrs: c.journal.abbrs = abbrs changed = True if c.journal.queries != queries: c.journal.queries = queries changed = True if c.journal.openAccess != openAccess: c.journal.openAccess = openAccess changed = True if c.journal.active != active: c.journal.active = active changed = True if c.journal.student != student: c.journal.student = student changed = True if c.journal.ISSN != ISSN: c.journal.ISSN = ISSN changed = True if values['newabbr']: c.journal.abbrs.append(values['newabbr']) changed = True if values['newquery']: c.journal.queries.append(values['newquery']) changed = True #commit changes Session.flush() Session.commit() if changed: c.message = "Journal " + c.journal.label + " modified successfully." else: c.message = "No change required; Journal " + c.journal.label + " not modified." return render('admin/journal-edit.html') elif action == "Delete": c.journal = h.fetch_obj(Journal, values['ID']) c.message = "Journal # " + values[ 'ID'] + " (" + c.journal.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.journal) Session.flush() Session.commit() c.found = False return render('admin/journal-edit.html')
def fuzzymatchall(SEPEntrieslist): #takes outputs from addlist() and saves all fuzzy match IDs to SEPEntry.fuzzymatch with verdicts (percent of words matched) #now change so that it only updates ones that don't currently have a fuzzymatchlist #clear out fuzzymatch table--otherwise old fuzzies will accumulate, and nobody wants that delquery = Session.query(Fuzzymatch) delquery.delete() Session.flush() Session.commit() for SEPEntry in SEPEntrieslist: print "working on " + SEPEntry.title.encode('utf-8') + "\n" entities = Session.query(Entity) #exclude journals and nodes from fuzzy matching entities = entities.filter(Entity.typeID != 2) entities = entities.filter(Entity.typeID != 4) #reset fuzzymatches for that entry #SEPEntry.fuzzymatches = "" ##string1 = string1.decode('utf8') for entity in entities: php = PHP("set_include_path('/usr/lib/php/');") php = PHP("require 'fuzzymatch.php';") #php = PHP() #print "testing " + entity.label.encode('utf8') + " against " + string1.encode('utf8') + "\n" code = '$string1 = utf8_decode("' + SEPEntry.title.encode('utf8') + '");' #code = code + "$string2 = '" + entity.label.encode('latin-1', 'replace') + "';" #code = code + "print $string1; print $string2;" #print code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");' code = code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");' code = code + """print fuzzy_match($string1, $string2, 2);""" verdict = php.get_raw(code) #print "verdict is " + verdict + "\n" verdict = verdict.split(',') if float(verdict[0])>=.20: #print entity.label + " is a match!\n" #entity.matchvalue = verdict #string = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict #if len(string) < 400: # SEPEntry.fuzzymatches = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict #else: # print "sorry, too many matches! Can't add " + str(entity.ID) + " to fuzzy matches; over 400 chars." fmatch = Fuzzymatch(entity.ID) fmatch.sep_dir = SEPEntry.sep_dir fmatch.strength = verdict[0] fmatch.edits = verdict[1] SEPEntry.fmatches.append(fmatch) Session.flush() Session.commit()
def getentries(db_dir): #to do feb 10: do something more elegant than just delete existing and rewrite...perhaps only write new? #first, clear out the current table #Session.query(SEPEntry).delete() #Session.flush() #Session.commit() entries = defaultdict(lambda: {'title' : '', 'published' : False, 'status' : ''}) pars = HTMLParser.HTMLParser() #get published entries published = open('/Users/inpho/SitesOld/dev/entries-published.txt') status = open(os.path.join(db_dir , 'entrystatus.txt')) entrylist = open(os.path.join(db_dir, 'entries.txt')) #set up entries dict for line in entrylist: line = line.split('::') sep_dir = pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[0])) title = re.sub("\\\\\'", "'", pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[1]))) entries[sep_dir]['title'] = title for line in published: line = re.sub('\\n', '', line) if entries[line]['title']: entries[line]['published'] = True else: print "uh-oh, " + line + "doesn't appear to be in dict object" for line in status: line = line.split('::') if entries[line[0]]['title']: entries[line[0]]['status'] = line[1] #print "status = " + line[1] + ' for ' + line[0] else: print "uh-oh, " + line[0] + "doesn't appear to be in dict object" for key in entries.keys(): #so, what I should really do here is figure out whether the entry already has a place in the table; if so, update its existing stats; if not, #insert a new one; #also need to check if old entries in sepentries table are no longer in file... entry_q = Session.query(SEPEntry) o = SEPEntry.title.like(entries[key]['title']) entry_q = entry_q.filter(o) # if only 1 result, go ahead and view that idea if entry_q.count() == 0: entry_add = SEPEntry(entries[key]['title'], key, entries[key]['published'], entries[key]['status']) Session.add(entry_add) elif entry_q.count() == 1: #replace data from most recent from SEPMirror entry = entry_q.first() entry.title = entries[key]['title'] entry.published = entries[key]['published'] entry.status = entries[key]['status'] #need to really add something here to delete entries no longer in the DB... entry_q2 = Session.query(SEPEntry) for entry in entry_q2: if not entries[entry.sep_dir]['title']: print str(entry.title) + " appears to have been deleted from SEPMirror; deleting from InPhO database." Session.delete(entry) Session.flush() Session.commit()
def process(self): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 q = request.params.get('label', None) label = q searchpattern = request.params.get('searchpattern', None) searchstring = request.params.get('searchstring', None) sep_dir = request.params.get('sep_dir', None) ioru = request.params.get('ioru', 'u') print "***" print "sep_dir is " print sep_dir print "and label is " print q print "***" values = dict(request.params) c.found = values.get('found', False) c.custom = values.get('custom', False) c.new = values.get('new', False) #ADD ACTION -- add a new entity (id/label cannot already exist) if action=='Add': #lowercase label and check to see whether it is valid #if so commit idea with standard pluralizations as searchpattern and give user a message that add was successful q = q.lower() #check to see whether label already exists idea_q = Session.query(Idea) # Check for query o = Idea.label.like(q) idea_q = idea_q.filter(o) # if 0 results, proceed to add idea if idea_q.count() == 0: print "***action add, idea q count == 0" #if no exact match for label, create new object with that label to add idea_add = Idea(q) #add searchstring = label idea_add.searchstring = q #add pluralizations to existing searchpatterns idea_add.searchpatterns = idea_add.pluralize() #setup the search string list c.idea = idea_add c.search_string_list = c.idea.setup_SSL() #setup sep_dir if present if sep_dir: c.idea.sep_dir = sep_dir #commit #take user to disambiguation page if the label contains an 'and' #otherwise, commit idea Session.add(idea_add) Session.flush() Session.commit() c.found = True something = 1 else: #already a match; give user error message that they should edit the preexisting page, and send them back to the submit idea #for edit page print "****action add, idea_q.count() not equal 0" c.message = "Idea with that name already exists in database; please edit the existing entry (returned below) or try a new label." c.idea = idea_q.first() c.search_string_list = c.idea.setup_SSL() return render('admin/idea-edit.html') c.message = "Idea added successfully." return render('admin/idea-edit.html') #Modify action -- edit an existing entry, id must exist elif action == 'Modify': #retrieve entity corresponding to id c.idea = h.fetch_obj(Idea, values['ID']) changed = False #generate searchpattern from searchstring if not already provided; current value stored in searchpattern, generate from searchstring if null #update parameters with form values if c.idea.sep_dir != values['sep_dir']: c.idea.sep_dir = values['sep_dir'] changed = True if c.idea.label != label: c.idea.label = label changed = True if not c.custom: if c.idea.searchstring != s.convertSS(searchstring, ioru)[0]: c.idea.searchstring = s.convertSS(searchstring, ioru)[0] #add default pluralizations to searchpatterns for label in c.idea.pluralize(): if label not in c.idea.searchpatterns: c.idea.searchpatterns.append(label) changed = True else: searchpatterns = [] for k, v in values.items(): key = "" if k.startswith('searchpatterns'): varname, num = k.split('.') key = 'delsearchpattern.%s'%(num) keyval = request.params.get(key, False) if not keyval: searchpatterns.append(v) if values['newsearchpattern']: searchpatterns.append(values['newsearchpattern']) changed = True #do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed if c.idea.searchpatterns != searchpatterns: c.idea.searchpatterns = searchpatterns changed = True if c.idea.searchstring != searchstring: c.idea.searchstring = searchstring #add default pluralizations for new searchstring as well for label in c.idea.pluralize(): if label not in c.idea.searchpatterns: c.idea.searchpatterns.append(label) changed = True #make sure that searchpattern and searchstring are not shared by another entity #print "searchpatterns is " #print searchpatterns #print "c.idea.searchpatterns is" #print c.idea.searchpatterns #commit changes Session.flush() Session.commit() #return success message if changed == True: c.message = "Idea modified successfully." else: c.message = "No change required; idea not modified." c.found = True c.search_string_list = c.idea.setup_SSL() #prepare form return render('admin/idea-edit.html') #Delete action -- Delete an existing entity (id must exist) elif action=='Delete': c.idea = h.fetch_obj(Idea, values['ID']) c.message = "Idea # " + values['ID'] + " ("+ c.idea.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.idea) Session.flush() Session.commit() return render('admin/idea-edit.html') #use custom searchstring action elif action=='Use Custom Searchstring/Searchpattern': #toggle the c.custom to True and re-render form with htmlfill c.custom = True c.idea = h.fetch_obj(Idea, values['ID']) c.found = True c.message = 'Enter custom searchstring for ' + c.idea.label + "." return render('admin/idea-edit.html') #use standard searchstring action elif action=='Use Standard Searchstring/Searchpattern': #toggle the c.custom to False and re-render form with htmlfill c.custom = False c.idea = h.fetch_obj(Idea, values['ID']) c.found = True c.search_string_list = c.idea.setup_SSL() c.message = 'Now displaying standard searchstring options for ' + c.idea.label + "." return render('admin/idea-edit.html') else: raise Exception('I don\'t know how you got here, but you shouldn\'t be here...please start at the main page. Invalid action %s'%action)
def process(self, id=None): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False label = request.params.get('label', None) id = request.params.get('ID', id) sep_dir = request.params.get('sep_dir', None) action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 values = dict(request.params) if action=="Add": thinker_add = Thinker(label) thinker_add.label = label #setup search string and search pattern lastname = thinker_add.label.split(' ').pop() lastname_q = Session.query(Entity) o = Entity.searchstring.like(lastname) lastname_q = lastname_q.filter(o).order_by(func.length(Entity.label)) if lastname_q.count() == 0: #if there's no match currently to last name, can use last name alone as searchpattern/searchstring thinker_add.searchpatterns.append(lastname) thinker_add.searchstring = lastname else: #otherwise, we need to use the whole name for both, and reset the other pattern to full name too thinker_add.searchpatterns.append(label) thinker_add.searchstring = label #reset old thinker pattern to whole name too to avoid conflict oldthinker = h.fetch_obj(Thinker, lastname_q.first().ID) oldthinker.searchpatterns = [oldthinker.label] oldthinker.searchstring = oldthinker.label Session.add(oldthinker) if sep_dir: thinker_add.sep_dir = sep_dir c.thinker = thinker_add Session.add(thinker_add) Session.flush() Session.commit() c.found = True c.message = "Thinker " + c.thinker.label + " added successfully." return render ('admin/thinker-edit.html') elif action=="Modify": c.thinker = h.fetch_obj(Thinker, id) c.found = True changed = False searchpatterns = [] for k, v in values.items(): key = "" if k.startswith('searchpatterns'): varname, num = k.split('.') key = 'delsearchpattern.%s'%(num) keyval = request.params.get(key, False) if not keyval: searchpatterns.append(v) if values['newsearchpattern']: searchpatterns.append(values['newsearchpattern']) changed = True #do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed if c.thinker.searchpatterns != searchpatterns: c.thinker.searchpatterns = searchpatterns changed = True #set values from form if c.thinker.name != values['name']: c.thinker.name = values['name'] changed = True if c.thinker.label != values['label']: c.thinker.name = values['label'] changed = True if c.thinker.searchstring != values['searchstring']: c.thinker.searchstring = values['searchstring'] changed = True if c.thinker.sep_dir != values['sep_dir']: c.thinker.sep_dir = values['sep_dir'] changed = True if c.thinker.wiki != values['wiki']: c.thinker.wiki = values['wiki'] changed = True if c.thinker.birth_day != values['birth_day']: c.thinker.birth_day = values['birth_day'] changed = True c.thinker.birth_day = values['birth_day'] if c.thinker.death_day != values['death_day']: c.thinker.death_day = values['death_day'] changed = True if c.thinker.birth_month != values['birth_month']: c.thinker.birth_month = values['birth_month'] changed = True if c.thinker.death_month != values['death_month']: c.thinker.death_month = values['death_month'] changed = True if not (c.thinker.birth_year == values['birth_year'] + " " + values['bornbc']): if c.thinker.birth_year != values['birth_year']: c.thinker.birth_year = values['birth_year'] changed = True if c.thinker.birth_year and values['bornbc'] and not re.search("(BC)|(AD)",c.thinker.birth_year): c.thinker.birth_year = c.thinker.birth_year + " " + values['bornbc'] changed = True if not (c.thinker.death_year == values['death_year'] + " " + values['diedbc']): if c.thinker.death_year != values['death_year']: c.thinker.death_year = values['death_year'] changed = True if c.thinker.death_year and values['diedbc']and not re.search("(BC)|(AD)",c.thinker.death_year): c.thinker.death_year = c.thinker.death_year + " " + values['diedbc'] changed = True #commit changes Session.flush() Session.commit() if changed: c.message = "Thinker " + c.thinker.label + " modified successfully." else: c.message = "No changes detected. Thinker " + c.thinker.label + " not modified." return render ('admin/thinker-edit.html') elif action == "Delete": c.thinker = h.fetch_obj(Thinker, values['ID']) c.message = "Thinker # " + values['ID'] + " ("+ c.thinker.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.thinker) Session.flush() Session.commit() c.found = False return render('admin/thinker-edit.html')
def process(self, id=None): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False label = request.params.get('label', None) id = request.params.get('ID', id) sep_dir = request.params.get('sep_dir', None) action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 values = dict(request.params) if action == "Add": thinker_add = Thinker(label) thinker_add.label = label #setup search string and search pattern lastname = thinker_add.label.split(' ').pop() lastname_q = Session.query(Entity) o = Entity.searchstring.like(lastname) lastname_q = lastname_q.filter(o).order_by( func.length(Entity.label)) if lastname_q.count() == 0: #if there's no match currently to last name, can use last name alone as searchpattern/searchstring thinker_add.searchpatterns.append(lastname) thinker_add.searchstring = lastname else: #otherwise, we need to use the whole name for both, and reset the other pattern to full name too thinker_add.searchpatterns.append(label) thinker_add.searchstring = label #reset old thinker pattern to whole name too to avoid conflict oldthinker = h.fetch_obj(Thinker, lastname_q.first().ID) oldthinker.searchpatterns = [oldthinker.label] oldthinker.searchstring = oldthinker.label Session.add(oldthinker) if sep_dir: thinker_add.sep_dir = sep_dir c.thinker = thinker_add Session.add(thinker_add) Session.flush() Session.commit() c.found = True c.message = "Thinker " + c.thinker.label + " added successfully." return render('admin/thinker-edit.html') elif action == "Modify": c.thinker = h.fetch_obj(Thinker, id) c.found = True changed = False searchpatterns = [] for k, v in values.items(): key = "" if k.startswith('searchpatterns'): varname, num = k.split('.') key = 'delsearchpattern.%s' % (num) keyval = request.params.get(key, False) if not keyval: searchpatterns.append(v) if values['newsearchpattern']: searchpatterns.append(values['newsearchpattern']) changed = True #do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed if c.thinker.searchpatterns != searchpatterns: c.thinker.searchpatterns = searchpatterns changed = True #set values from form if c.thinker.name != values['name']: c.thinker.name = values['name'] changed = True if c.thinker.label != values['label']: c.thinker.name = values['label'] changed = True if c.thinker.searchstring != values['searchstring']: c.thinker.searchstring = values['searchstring'] changed = True if c.thinker.sep_dir != values['sep_dir']: c.thinker.sep_dir = values['sep_dir'] changed = True if c.thinker.wiki != values['wiki']: c.thinker.wiki = values['wiki'] changed = True if c.thinker.birth_day != values['birth_day']: c.thinker.birth_day = values['birth_day'] changed = True c.thinker.birth_day = values['birth_day'] if c.thinker.death_day != values['death_day']: c.thinker.death_day = values['death_day'] changed = True if c.thinker.birth_month != values['birth_month']: c.thinker.birth_month = values['birth_month'] changed = True if c.thinker.death_month != values['death_month']: c.thinker.death_month = values['death_month'] changed = True if not (c.thinker.birth_year == values['birth_year'] + " " + values['bornbc']): if c.thinker.birth_year != values['birth_year']: c.thinker.birth_year = values['birth_year'] changed = True if c.thinker.birth_year and values['bornbc'] and not re.search( "(BC)|(AD)", c.thinker.birth_year): c.thinker.birth_year = c.thinker.birth_year + " " + values[ 'bornbc'] changed = True if not (c.thinker.death_year == values['death_year'] + " " + values['diedbc']): if c.thinker.death_year != values['death_year']: c.thinker.death_year = values['death_year'] changed = True if c.thinker.death_year and values['diedbc'] and not re.search( "(BC)|(AD)", c.thinker.death_year): c.thinker.death_year = c.thinker.death_year + " " + values[ 'diedbc'] changed = True #commit changes Session.flush() Session.commit() if changed: c.message = "Thinker " + c.thinker.label + " modified successfully." else: c.message = "No changes detected. Thinker " + c.thinker.label + " not modified." return render('admin/thinker-edit.html') elif action == "Delete": c.thinker = h.fetch_obj(Thinker, values['ID']) c.message = "Thinker # " + values[ 'ID'] + " (" + c.thinker.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.thinker) Session.flush() Session.commit() c.found = False return render('admin/thinker-edit.html')
def process(self, id=None): if not h.auth.is_logged_in(): abort(401) if not h.auth.is_admin(): abort(403) c.sepdirnew = False c.alreadysepdir = False label = request.params.get('label', None) id = request.params.get('ID', id) sep_dir = request.params.get('sep_dir', None) URL = request.params.get('URL', None) language = request.params.get('language', None) queries = [request.params.get('queries', None)] openAccess = request.params.get('openAccess', None) active = request.params.get('active', None) student = request.params.get('student', None) ISSN = request.params.get('ISSN', None) action = request.params.get('action', None) action2 = request.params.get('action2', None) if action2: action = action2 values = dict(request.params) #abbrs = [request.params.get('abbrs', None)] abbrs = [] queries = [] for k, v in values.items(): key = "" if k.startswith('abbrs'): varname, num = k.split('.') key = 'delabbr.%s'%(num) keyval = request.params.get(key, False) if not keyval: abbrs.append(v) elif k.startswith('queries'): varname, num = k.split('.') key = 'delquer.%s'%(num) keyval = request.params.get(key, False) if not keyval: queries.append(v) if action=="Add": journal_add = Journal() journal_add.label = label #setup search string and search pattern journalname = journal_add.label journalname_q = Session.query(Entity) o = Entity.searchpattern.like('( '+ journalname + ' )') journalname_q = journalname_q.filter(o).order_by(func.length(Entity.label)) if journalname_q.count() == 0: journal_add.searchpattern = "( " + journalname + " )" journal_add.searchstring = journalname else: journal_add.searchpattern = "( " + label + " )" journal_add.searchcstring = label #reset old journal pattern to whole name too to avoid conflict oldjournal = h.fetch_obj(Journal, journalname_q.first().ID) oldjournal.searchpattern = "( " + oldjournal.label + " )" oldjournal.searchstring = oldjournal.label Session.add(oldjournal) if sep_dir: journal_add.sep_dir = sep_dir c.journal = journal_add Session.add(journal_add) Session.flush() Session.commit() c.found = True c.message = "Journal " + c.journal.label + " added successfully." return render ('admin/journal-edit.html') elif action=="Modify": c.journal = h.fetch_obj(Journal, id) c.found = True changed = False #set values from form if c.journal.label != label: c.journal.label = label changed = True if c.journal.sep_dir != sep_dir: c.journal.sep_dir = sep_dir changed = True if c.journal.URL != URL: c.journal.URL = URL changed = True if c.journal.language != language: c.journal.language = language changed = True if c.journal.abbrs != abbrs: c.journal.abbrs = abbrs changed = True if c.journal.queries != queries: c.journal.queries = queries changed = True if c.journal.openAccess != openAccess: c.journal.openAccess = openAccess changed = True if c.journal.active != active: c.journal.active = active changed = True if c.journal.student != student: c.journal.student = student changed = True if c.journal.ISSN != ISSN: c.journal.ISSN = ISSN changed = True if values['newabbr']: c.journal.abbrs.append(values['newabbr']) changed = True if values['newquery']: c.journal.queries.append(values['newquery']) changed = True #commit changes Session.flush() Session.commit() if changed: c.message = "Journal " + c.journal.label + " modified successfully." else: c.message = "No change required; Journal " + c.journal.label + " not modified." return render ('admin/journal-edit.html') elif action == "Delete": c.journal = h.fetch_obj(Journal, values['ID']) c.message = "Journal # " + values['ID'] + " ("+ c.journal.label + ") deleted; please search for a new entity label on the left." h.delete_obj(c.journal) Session.flush() Session.commit() c.found = False return render('admin/journal-edit.html')