def _evaluate(self, evaltype, id, id2=None, uid=None, username=None, degree=-1, maxdegree=4):
"""
Function to submit an evaluation. Takes a POST request containing the consequesnt id and
all or none of: generality, relatedness, hyperrank, hyporank.
"""
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.environ.get('REMOTE_USER', username)
print "grabbing eval for", username, uid
if request.environ.get('REMOTE_USER', False):
evaluation = self._get_evaluation(id, id2, None, username)
else:
evaluation = self._get_anon_evaluation(id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0'))
# Populate proper generality, relatedness, hyperrank and hyporank values
# Attempt to convert to integers, if unable, throw HTTP 400
try:
setattr(evaluation, evaltype,
int(request.params.get('degree', getattr(evaluation, evaltype))))
except TypeError:
abort(400)
# Create and commit evaluation
Session.flush()
Session.commit()
# Issue an HTTP success
response.status_int = 200
return "OK"
def submit_changes(self):
'''
This function validates the submitted profile edit form and commits the
changes. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
if not h.auth.is_logged_in():
abort(401)
c.user = h.get_user(request.environ['REMOTE_USER'])
if self.form_result['password'] != '':
c.user.set_password(self.form_result['password'])
# TODO: Enable area editing
#c.user.first_area_id=self.form_result['first_area'],
#user.first_area_level=self.form_result['first_area_level'],
#if self.form_result['second_area']:
# c.user.second_area_id=self.form_result['second_area'],
# c.user.second_area_level=self.form_result['second_area_level']
c.user.fullname = self.form_result['fullname']
Session.flush()
Session.commit()
h.redirect(h.url(controller='account', action='profile', message='edited'))
def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None):
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(id, id2, uid, username, autoCreate=False)
if not evaluation:
abort(404)
current_uid = h.get_user(request.environ['REMOTE_USER']).ID
if evaluation.uid != current_uid or not h.auth.is_admin():
abort(401)
setattr(evaluation, evaltype, -1)
# Delete evaluation if this eliminates both settings, new db schema
# will eliminate this need
#if evaluation.generality == -1 and evaluation.relatedness == -1:
# h.delete_obj(evaluation)
Session.flush()
Session.commit()
response.status_int = 200
return "OK"
def _reset(self, username=None):
username = username or request.environ.get('REMOTE_USER', False)
if not username:
abort(401)
try:
user = h.get_user(username)
except:
abort(400)
new_password = user.reset_password()
msg = Message("*****@*****.**", user.email,
"InPhO password reset")
msg.plain = """
%(name)s, your password at the Indiana Philosophy Ontology (InPhO) has been changed to:
Username: %(uname)s
Password: %(passwd)s
The Indiana Philosophy Ontology (InPhO) Team
[email protected]
""" % {'passwd' : new_password,
'uname' : user.username,
'name' : user.fullname or user.username or ''}
msg.send()
Session.commit()
h.redirect(h.url(controller='account', action='reset_result'))
def complete_mining(entity_type=Idea, filename='graph.txt', root='./'):
occur_filename = root + "graph-" + filename
edge_filename = root + "edge-" + filename
sql_filename = root + "sql-" + filename
print "processing articles..."
process_articles(entity_type, occur_filename)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante, cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
def complete_mining(entity_type=Idea, filename='graph.txt', root='./'):
occur_filename = root + "graph-" + filename
edge_filename = root + "edge-" + filename
sql_filename = root + "sql-" + filename
print "processing articles..."
process_articles(entity_type, occur_filename)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante,cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
def _evaluate(self, evaltype, id, id2=None, uid=None, username=None, degree=-1, maxdegree=4):
"""
Function to submit an evaluation. Takes a POST request containing the consequesnt id and
all or none of: generality, relatedness, hyperrank, hyporank.
"""
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.environ.get('REMOTE_USER', username)
print "grabbing eval for", username, uid
if request.environ.get('REMOTE_USER', False):
evaluation = self._get_evaluation(id, id2, None, username)
else:
evaluation = self._get_anon_evaluation(id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0'))
# Populate proper generality, relatedness, hyperrank and hyporank values
# Attempt to convert to integers, if unable, throw HTTP 400
try:
setattr(evaluation, evaltype,
int(request.params.get('degree', getattr(evaluation, evaltype))))
except TypeError:
abort(400)
# Create and commit evaluation
Session.flush()
Session.commit()
# Issue an HTTP success
response.status_int = 200
return "OK"
def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None):
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(id, id2, uid, username, autoCreate=False)
if not evaluation:
abort(404)
current_uid = h.get_user(request.environ['REMOTE_USER']).ID
if evaluation.uid != current_uid or not h.auth.is_admin():
abort(401)
setattr(evaluation, evaltype, -1)
# Delete evaluation if this eliminates both settings, new db schema
# will eliminate this need
if evaluation.generality == -1 and evaluation.relatedness == -1:
h.delete_obj(evaluation)
else:
# save change in evaluation
Session.flush()
Session.commit()
response.status_int = 200
return "OK"
def submit(self):
'''
This function validates the submitted registration form and creates a
new user. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
user = User(
self.form_result['username'],
self.form_result['password'],
email=self.form_result['email'],
first_area_id=self.form_result['first_area'],
first_area_level=self.form_result['first_area_level'],
second_area_id=self.form_result['second_area'],
second_area_level=self.form_result['second_area_level']
)
Session.add(user)
Session.commit()
msg = Message("*****@*****.**", self.form_result['email'],
"InPhO registration")
msg.plain = """%s, thank you for registering with the Indiana Philosophy
Ontology Project (InPhO). You can access your """ % self.form_result['username']
msg.send()
h.redirect(h.url(controller='account', action='result'))
def submit(self):
'''
This function validates the submitted registration form and creates a
new user. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
user = User(self.form_result['username'],
self.form_result['password'],
email=self.form_result['email'],
first_area_id=self.form_result['first_area'],
first_area_level=self.form_result['first_area_level'],
second_area_id=self.form_result['second_area'],
second_area_level=self.form_result['second_area_level'])
Session.add(user)
Session.commit()
msg = Message("*****@*****.**", self.form_result['email'],
"InPhO registration")
msg.plain = """%s, thank you for registering with the Indiana Philosophy
Ontology Project (InPhO). You can access your """ % self.form_result[
'username']
msg.send()
h.redirect(h.url(controller='account', action='result'))
def _delete_unary(self, type, id, id2=None):
thinker = h.fetch_obj(Thinker, id)
id2 = request.params.get('id2', id2)
obj = h.fetch_obj(unary_vars[type]['object'], id2)
if obj in getattr(thinker, unary_vars[type]['property']):
getattr(thinker, unary_vars[type]['property']).remove(obj)
Session.commit()
response.status_int = 200
return "OK"
def _delete_unary(self, type, id, id2=None):
thinker = h.fetch_obj(Thinker, id)
id2 = request.params.get('id2', id2)
obj = h.fetch_obj(unary_vars[type]['object'], id2)
if obj in getattr(thinker, unary_vars[type]['property']):
getattr(thinker, unary_vars[type]['property']).remove(obj)
Session.commit()
response.status_int = 200
return "OK"
def submit(self):
'''
This function validates the submitted registration form and creates a
new user. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
user = User(
self.form_result['username'],
fullname=self.form_result['fullname'],
email=self.form_result['email'],
first_area_id=self.form_result['first_area'],
first_area_level=self.form_result['first_area_level'],
second_area_id=self.form_result['second_area'],
second_area_level=self.form_result['second_area_level']
)
Session.add(user)
password = user.reset_password()
Session.commit()
msg = Message("*****@*****.**", self.form_result['email'],
"InPhO registration")
msg.plain = """Dear %(name)s,
Thank you for registering with the Indiana Philosophy Ontology Project (InPhO).
You can sign in at https://inpho.cogs.indiana.edu/signin with the following
information:
Username: %(uname)s
Password: %(passwd)s
You may change your password at https://inpho.cogs.indiana.edu/account/edit .
The Indiana Philosophy Ontology Project (InPhO) Team
[email protected]
""" % {'passwd' : password,
'uname' : user.username,
'name' : user.fullname or user.username or ''}
msg.send()
h.redirect(h.url(controller='account', action='result'))
def getentries(db_dir):
#to do feb 10: do something more elegant than just delete existing and rewrite...perhaps only write new?
#first, clear out the current table
#Session.query(SEPEntry).delete()
#Session.flush()
#Session.commit()
entries = defaultdict(lambda: {
'title': '',
'published': False,
'status': ''
})
pars = HTMLParser.HTMLParser()
#get published entries
published = open('/Users/inpho/SitesOld/dev/entries-published.txt')
status = open(os.path.join(db_dir, 'entrystatus.txt'))
entrylist = open(os.path.join(db_dir, 'entries.txt'))
#set up entries dict
for line in entrylist:
line = line.split('::')
sep_dir = pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[0]))
title = re.sub("\\\\\'", "'",
pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[1])))
entries[sep_dir]['title'] = title
for line in published:
line = re.sub('\\n', '', line)
if entries[line]['title']:
entries[line]['published'] = True
else:
print "uh-oh, " + line + "doesn't appear to be in dict object"
for line in status:
line = line.split('::')
if entries[line[0]]['title']:
entries[line[0]]['status'] = line[1]
#print "status = " + line[1] + ' for ' + line[0]
else:
print "uh-oh, " + line[0] + "doesn't appear to be in dict object"
for key in entries.keys():
#so, what I should really do here is figure out whether the entry already has a place in the table; if so, update its existing stats; if not,
#insert a new one;
#also need to check if old entries in sepentries table are no longer in file...
entry_q = Session.query(SEPEntry)
o = SEPEntry.title.like(entries[key]['title'])
entry_q = entry_q.filter(o)
# if only 1 result, go ahead and view that idea
if entry_q.count() == 0:
entry_add = SEPEntry(entries[key]['title'], key,
entries[key]['published'],
entries[key]['status'])
Session.add(entry_add)
elif entry_q.count() == 1:
#replace data from most recent from SEPMirror
entry = entry_q.first()
entry.title = entries[key]['title']
entry.published = entries[key]['published']
entry.status = entries[key]['status']
#need to really add something here to delete entries no longer in the DB...
entry_q2 = Session.query(SEPEntry)
for entry in entry_q2:
if not entries[entry.sep_dir]['title']:
print str(
entry.title
) + " appears to have been deleted from SEPMirror; deleting from InPhO database."
Session.delete(entry)
Session.flush()
Session.commit()
def process(self):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
q = request.params.get('label', None)
label = q
searchpattern = request.params.get('searchpattern', None)
searchstring = request.params.get('searchstring', None)
sep_dir = request.params.get('sep_dir', None)
ioru = request.params.get('ioru', 'u')
print "***"
print "sep_dir is "
print sep_dir
print "and label is "
print q
print "***"
values = dict(request.params)
c.found = values.get('found', False)
c.custom = values.get('custom', False)
c.new = values.get('new', False)
#ADD ACTION -- add a new entity (id/label cannot already exist)
if action=='Add':
#lowercase label and check to see whether it is valid
#if so commit idea with standard pluralizations as searchpattern and give user a message that add was successful
q = q.lower()
#check to see whether label already exists
idea_q = Session.query(Idea)
# Check for query
o = Idea.label.like(q)
idea_q = idea_q.filter(o)
# if 0 results, proceed to add idea
if idea_q.count() == 0:
print "***action add, idea q count == 0"
#if no exact match for label, create new object with that label to add
idea_add = Idea(q)
#add searchstring = label
idea_add.searchstring = q
#add pluralizations to existing searchpatterns
idea_add.searchpatterns = idea_add.pluralize()
#setup the search string list
c.idea = idea_add
c.search_string_list = c.idea.setup_SSL()
#setup sep_dir if present
if sep_dir:
c.idea.sep_dir = sep_dir
#commit
#take user to disambiguation page if the label contains an 'and'
#otherwise, commit idea
Session.add(idea_add)
Session.flush()
Session.commit()
c.found = True
something = 1
else:
#already a match; give user error message that they should edit the preexisting page, and send them back to the submit idea
#for edit page
print "****action add, idea_q.count() not equal 0"
c.message = "Idea with that name already exists in database; please edit the existing entry (returned below) or try a new label."
c.idea = idea_q.first()
c.search_string_list = c.idea.setup_SSL()
return render('admin/idea-edit.html')
c.message = "Idea added successfully."
return render('admin/idea-edit.html')
#Modify action -- edit an existing entry, id must exist
elif action == 'Modify':
#retrieve entity corresponding to id
c.idea = h.fetch_obj(Idea, values['ID'])
changed = False
#generate searchpattern from searchstring if not already provided; current value stored in searchpattern, generate from searchstring if null
#update parameters with form values
if c.idea.sep_dir != values['sep_dir']:
c.idea.sep_dir = values['sep_dir']
changed = True
if c.idea.label != label:
c.idea.label = label
changed = True
if not c.custom:
if c.idea.searchstring != s.convertSS(searchstring, ioru)[0]:
c.idea.searchstring = s.convertSS(searchstring, ioru)[0]
#add default pluralizations to searchpatterns
for label in c.idea.pluralize():
if label not in c.idea.searchpatterns:
c.idea.searchpatterns.append(label)
changed = True
else:
searchpatterns = []
for k, v in values.items():
key = ""
if k.startswith('searchpatterns'):
varname, num = k.split('.')
key = 'delsearchpattern.%s'%(num)
keyval = request.params.get(key, False)
if not keyval:
searchpatterns.append(v)
if values['newsearchpattern']:
searchpatterns.append(values['newsearchpattern'])
changed = True
#do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed
if c.idea.searchpatterns != searchpatterns:
c.idea.searchpatterns = searchpatterns
changed = True
if c.idea.searchstring != searchstring:
c.idea.searchstring = searchstring
#add default pluralizations for new searchstring as well
for label in c.idea.pluralize():
if label not in c.idea.searchpatterns:
c.idea.searchpatterns.append(label)
changed = True
#make sure that searchpattern and searchstring are not shared by another entity
#print "searchpatterns is "
#print searchpatterns
#print "c.idea.searchpatterns is"
#print c.idea.searchpatterns
#commit changes
Session.flush()
Session.commit()
#return success message
if changed == True:
c.message = "Idea modified successfully."
else:
c.message = "No change required; idea not modified."
c.found = True
c.search_string_list = c.idea.setup_SSL()
#prepare form
return render('admin/idea-edit.html')
#Delete action -- Delete an existing entity (id must exist)
elif action=='Delete':
c.idea = h.fetch_obj(Idea, values['ID'])
c.message = "Idea # " + values['ID'] + " ("+ c.idea.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.idea)
Session.flush()
Session.commit()
return render('admin/idea-edit.html')
#use custom searchstring action
elif action=='Use Custom Searchstring/Searchpattern':
#toggle the c.custom to True and re-render form with htmlfill
c.custom = True
c.idea = h.fetch_obj(Idea, values['ID'])
c.found = True
c.message = 'Enter custom searchstring for ' + c.idea.label + "."
return render('admin/idea-edit.html')
#use standard searchstring action
elif action=='Use Standard Searchstring/Searchpattern':
#toggle the c.custom to False and re-render form with htmlfill
c.custom = False
c.idea = h.fetch_obj(Idea, values['ID'])
c.found = True
c.search_string_list = c.idea.setup_SSL()
c.message = 'Now displaying standard searchstring options for ' + c.idea.label + "."
return render('admin/idea-edit.html')
else:
raise Exception('I don\'t know how you got here, but you shouldn\'t be here...please start at the main page. Invalid action %s'%action)
def fuzzymatchall(SEPEntrieslist):
#takes outputs from addlist() and saves all fuzzy match IDs to SEPEntry.fuzzymatch with verdicts (percent of words matched)
#now change so that it only updates ones that don't currently have a fuzzymatchlist
#clear out fuzzymatch table--otherwise old fuzzies will accumulate, and nobody wants that
delquery = Session.query(Fuzzymatch)
delquery.delete()
Session.flush()
Session.commit()
for SEPEntry in SEPEntrieslist:
print "working on " + SEPEntry.title.encode('utf-8') + "\n"
entities = Session.query(Entity)
#exclude journals and nodes from fuzzy matching
entities = entities.filter(Entity.typeID != 2)
entities = entities.filter(Entity.typeID != 4)
#reset fuzzymatches for that entry
#SEPEntry.fuzzymatches = ""
##string1 = string1.decode('utf8')
for entity in entities:
php = PHP("set_include_path('/usr/lib/php/');")
php = PHP("require 'fuzzymatch.php';")
#php = PHP()
#print "testing " + entity.label.encode('utf8') + " against " + string1.encode('utf8') + "\n"
code = '$string1 = utf8_decode("' + SEPEntry.title.encode(
'utf8') + '");'
#code = code + "$string2 = '" + entity.label.encode('latin-1', 'replace') + "';"
#code = code + "print $string1; print $string2;"
#print code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");'
code = code + '$string2 = utf8_decode("' + entity.label.encode(
'utf8') + '");'
code = code + """print fuzzy_match($string1, $string2, 2);"""
verdict = php.get_raw(code)
#print "verdict is " + verdict + "\n"
verdict = verdict.split(',')
if float(verdict[0]) >= .20:
#print entity.label + " is a match!\n"
#entity.matchvalue = verdict
#string = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict
#if len(string) < 400:
# SEPEntry.fuzzymatches = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict
#else:
# print "sorry, too many matches! Can't add " + str(entity.ID) + " to fuzzy matches; over 400 chars."
fmatch = Fuzzymatch(entity.ID)
fmatch.sep_dir = SEPEntry.sep_dir
fmatch.strength = verdict[0]
fmatch.edits = verdict[1]
SEPEntry.fmatches.append(fmatch)
Session.flush()
Session.commit()
def process(self, id=None):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
label = request.params.get('label', None)
id = request.params.get('ID', id)
sep_dir = request.params.get('sep_dir', None)
URL = request.params.get('URL', None)
language = request.params.get('language', None)
queries = [request.params.get('queries', None)]
openAccess = request.params.get('openAccess', None)
active = request.params.get('active', None)
student = request.params.get('student', None)
ISSN = request.params.get('ISSN', None)
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
values = dict(request.params)
#abbrs = [request.params.get('abbrs', None)]
abbrs = []
queries = []
for k, v in values.items():
key = ""
if k.startswith('abbrs'):
varname, num = k.split('.')
key = 'delabbr.%s' % (num)
keyval = request.params.get(key, False)
if not keyval:
abbrs.append(v)
elif k.startswith('queries'):
varname, num = k.split('.')
key = 'delquer.%s' % (num)
keyval = request.params.get(key, False)
if not keyval:
queries.append(v)
if action == "Add":
journal_add = Journal()
journal_add.label = label
#setup search string and search pattern
journalname = journal_add.label
journalname_q = Session.query(Entity)
o = Entity.searchpattern.like('( ' + journalname + ' )')
journalname_q = journalname_q.filter(o).order_by(
func.length(Entity.label))
if journalname_q.count() == 0:
journal_add.searchpattern = "( " + journalname + " )"
journal_add.searchstring = journalname
else:
journal_add.searchpattern = "( " + label + " )"
journal_add.searchcstring = label
#reset old journal pattern to whole name too to avoid conflict
oldjournal = h.fetch_obj(Journal, journalname_q.first().ID)
oldjournal.searchpattern = "( " + oldjournal.label + " )"
oldjournal.searchstring = oldjournal.label
Session.add(oldjournal)
if sep_dir:
journal_add.sep_dir = sep_dir
c.journal = journal_add
Session.add(journal_add)
Session.flush()
Session.commit()
c.found = True
c.message = "Journal " + c.journal.label + " added successfully."
return render('admin/journal-edit.html')
elif action == "Modify":
c.journal = h.fetch_obj(Journal, id)
c.found = True
changed = False
#set values from form
if c.journal.label != label:
c.journal.label = label
changed = True
if c.journal.sep_dir != sep_dir:
c.journal.sep_dir = sep_dir
changed = True
if c.journal.URL != URL:
c.journal.URL = URL
changed = True
if c.journal.language != language:
c.journal.language = language
changed = True
if c.journal.abbrs != abbrs:
c.journal.abbrs = abbrs
changed = True
if c.journal.queries != queries:
c.journal.queries = queries
changed = True
if c.journal.openAccess != openAccess:
c.journal.openAccess = openAccess
changed = True
if c.journal.active != active:
c.journal.active = active
changed = True
if c.journal.student != student:
c.journal.student = student
changed = True
if c.journal.ISSN != ISSN:
c.journal.ISSN = ISSN
changed = True
if values['newabbr']:
c.journal.abbrs.append(values['newabbr'])
changed = True
if values['newquery']:
c.journal.queries.append(values['newquery'])
changed = True
#commit changes
Session.flush()
Session.commit()
if changed:
c.message = "Journal " + c.journal.label + " modified successfully."
else:
c.message = "No change required; Journal " + c.journal.label + " not modified."
return render('admin/journal-edit.html')
elif action == "Delete":
c.journal = h.fetch_obj(Journal, values['ID'])
c.message = "Journal # " + values[
'ID'] + " (" + c.journal.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.journal)
Session.flush()
Session.commit()
c.found = False
return render('admin/journal-edit.html')
def getentries(db_dir):
#to do feb 10: do something more elegant than just delete existing and rewrite...perhaps only write new?
#first, clear out the current table
#Session.query(SEPEntry).delete()
#Session.flush()
#Session.commit()
entries = defaultdict(lambda: {'title' : '', 'published' : False, 'status' : ''})
pars = HTMLParser.HTMLParser()
#get published entries
published = open('/Users/inpho/SitesOld/dev/entries-published.txt')
status = open(os.path.join(db_dir , 'entrystatus.txt'))
entrylist = open(os.path.join(db_dir, 'entries.txt'))
#set up entries dict
for line in entrylist:
line = line.split('::')
sep_dir = pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[0]))
title = re.sub("\\\\\'", "'", pars.unescape(re.sub('<(/)?[a-zA-Z]*>', '', line[1])))
entries[sep_dir]['title'] = title
for line in published:
line = re.sub('\\n', '', line)
if entries[line]['title']:
entries[line]['published'] = True
else:
print "uh-oh, " + line + "doesn't appear to be in dict object"
for line in status:
line = line.split('::')
if entries[line[0]]['title']:
entries[line[0]]['status'] = line[1]
#print "status = " + line[1] + ' for ' + line[0]
else:
print "uh-oh, " + line[0] + "doesn't appear to be in dict object"
for key in entries.keys():
#so, what I should really do here is figure out whether the entry already has a place in the table; if so, update its existing stats; if not,
#insert a new one;
#also need to check if old entries in sepentries table are no longer in file...
entry_q = Session.query(SEPEntry)
o = SEPEntry.title.like(entries[key]['title'])
entry_q = entry_q.filter(o)
# if only 1 result, go ahead and view that idea
if entry_q.count() == 0:
entry_add = SEPEntry(entries[key]['title'], key, entries[key]['published'], entries[key]['status'])
Session.add(entry_add)
elif entry_q.count() == 1:
#replace data from most recent from SEPMirror
entry = entry_q.first()
entry.title = entries[key]['title']
entry.published = entries[key]['published']
entry.status = entries[key]['status']
#need to really add something here to delete entries no longer in the DB...
entry_q2 = Session.query(SEPEntry)
for entry in entry_q2:
if not entries[entry.sep_dir]['title']:
print str(entry.title) + " appears to have been deleted from SEPMirror; deleting from InPhO database."
Session.delete(entry)
Session.flush()
Session.commit()
def fuzzymatchall(SEPEntrieslist):
#takes outputs from addlist() and saves all fuzzy match IDs to SEPEntry.fuzzymatch with verdicts (percent of words matched)
#now change so that it only updates ones that don't currently have a fuzzymatchlist
#clear out fuzzymatch table--otherwise old fuzzies will accumulate, and nobody wants that
delquery = Session.query(Fuzzymatch)
delquery.delete()
Session.flush()
Session.commit()
for SEPEntry in SEPEntrieslist:
print "working on " + SEPEntry.title.encode('utf-8') + "\n"
entities = Session.query(Entity)
#exclude journals and nodes from fuzzy matching
entities = entities.filter(Entity.typeID != 2)
entities = entities.filter(Entity.typeID != 4)
#reset fuzzymatches for that entry
#SEPEntry.fuzzymatches = ""
##string1 = string1.decode('utf8')
for entity in entities:
php = PHP("set_include_path('/usr/lib/php/');")
php = PHP("require 'fuzzymatch.php';")
#php = PHP()
#print "testing " + entity.label.encode('utf8') + " against " + string1.encode('utf8') + "\n"
code = '$string1 = utf8_decode("' + SEPEntry.title.encode('utf8') + '");'
#code = code + "$string2 = '" + entity.label.encode('latin-1', 'replace') + "';"
#code = code + "print $string1; print $string2;"
#print code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");'
code = code + '$string2 = utf8_decode("' + entity.label.encode('utf8') + '");'
code = code + """print fuzzy_match($string1, $string2, 2);"""
verdict = php.get_raw(code)
#print "verdict is " + verdict + "\n"
verdict = verdict.split(',')
if float(verdict[0])>=.20:
#print entity.label + " is a match!\n"
#entity.matchvalue = verdict
#string = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict
#if len(string) < 400:
# SEPEntry.fuzzymatches = SEPEntry.fuzzymatches + "|" + str(entity.ID) + "," + verdict
#else:
# print "sorry, too many matches! Can't add " + str(entity.ID) + " to fuzzy matches; over 400 chars."
fmatch = Fuzzymatch(entity.ID)
fmatch.sep_dir = SEPEntry.sep_dir
fmatch.strength = verdict[0]
fmatch.edits = verdict[1]
SEPEntry.fmatches.append(fmatch)
Session.flush()
Session.commit()
def process(self):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
q = request.params.get('label', None)
label = q
searchpattern = request.params.get('searchpattern', None)
searchstring = request.params.get('searchstring', None)
sep_dir = request.params.get('sep_dir', None)
ioru = request.params.get('ioru', 'u')
print "***"
print "sep_dir is "
print sep_dir
print "and label is "
print q
print "***"
values = dict(request.params)
c.found = values.get('found', False)
c.custom = values.get('custom', False)
c.new = values.get('new', False)
#ADD ACTION -- add a new entity (id/label cannot already exist)
if action=='Add':
#lowercase label and check to see whether it is valid
#if so commit idea with standard pluralizations as searchpattern and give user a message that add was successful
q = q.lower()
#check to see whether label already exists
idea_q = Session.query(Idea)
# Check for query
o = Idea.label.like(q)
idea_q = idea_q.filter(o)
# if 0 results, proceed to add idea
if idea_q.count() == 0:
print "***action add, idea q count == 0"
#if no exact match for label, create new object with that label to add
idea_add = Idea(q)
#add searchstring = label
idea_add.searchstring = q
#add pluralizations to existing searchpatterns
idea_add.searchpatterns = idea_add.pluralize()
#setup the search string list
c.idea = idea_add
c.search_string_list = c.idea.setup_SSL()
#setup sep_dir if present
if sep_dir:
c.idea.sep_dir = sep_dir
#commit
#take user to disambiguation page if the label contains an 'and'
#otherwise, commit idea
Session.add(idea_add)
Session.flush()
Session.commit()
c.found = True
something = 1
else:
#already a match; give user error message that they should edit the preexisting page, and send them back to the submit idea
#for edit page
print "****action add, idea_q.count() not equal 0"
c.message = "Idea with that name already exists in database; please edit the existing entry (returned below) or try a new label."
c.idea = idea_q.first()
c.search_string_list = c.idea.setup_SSL()
return render('admin/idea-edit.html')
c.message = "Idea added successfully."
return render('admin/idea-edit.html')
#Modify action -- edit an existing entry, id must exist
elif action == 'Modify':
#retrieve entity corresponding to id
c.idea = h.fetch_obj(Idea, values['ID'])
changed = False
#generate searchpattern from searchstring if not already provided; current value stored in searchpattern, generate from searchstring if null
#update parameters with form values
if c.idea.sep_dir != values['sep_dir']:
c.idea.sep_dir = values['sep_dir']
changed = True
if c.idea.label != label:
c.idea.label = label
changed = True
if not c.custom:
if c.idea.searchstring != s.convertSS(searchstring, ioru)[0]:
c.idea.searchstring = s.convertSS(searchstring, ioru)[0]
#add default pluralizations to searchpatterns
for label in c.idea.pluralize():
if label not in c.idea.searchpatterns:
c.idea.searchpatterns.append(label)
changed = True
else:
searchpatterns = []
for k, v in values.items():
key = ""
if k.startswith('searchpatterns'):
varname, num = k.split('.')
key = 'delsearchpattern.%s'%(num)
keyval = request.params.get(key, False)
if not keyval:
searchpatterns.append(v)
if values['newsearchpattern']:
searchpatterns.append(values['newsearchpattern'])
changed = True
#do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed
if c.idea.searchpatterns != searchpatterns:
c.idea.searchpatterns = searchpatterns
changed = True
if c.idea.searchstring != searchstring:
c.idea.searchstring = searchstring
#add default pluralizations for new searchstring as well
for label in c.idea.pluralize():
if label not in c.idea.searchpatterns:
c.idea.searchpatterns.append(label)
changed = True
#make sure that searchpattern and searchstring are not shared by another entity
#print "searchpatterns is "
#print searchpatterns
#print "c.idea.searchpatterns is"
#print c.idea.searchpatterns
#commit changes
Session.flush()
Session.commit()
#return success message
if changed == True:
c.message = "Idea modified successfully."
else:
c.message = "No change required; idea not modified."
c.found = True
c.search_string_list = c.idea.setup_SSL()
#prepare form
return render('admin/idea-edit.html')
#Delete action -- Delete an existing entity (id must exist)
elif action=='Delete':
c.idea = h.fetch_obj(Idea, values['ID'])
c.message = "Idea # " + values['ID'] + " ("+ c.idea.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.idea)
Session.flush()
Session.commit()
return render('admin/idea-edit.html')
#use custom searchstring action
elif action=='Use Custom Searchstring/Searchpattern':
#toggle the c.custom to True and re-render form with htmlfill
c.custom = True
c.idea = h.fetch_obj(Idea, values['ID'])
c.found = True
c.message = 'Enter custom searchstring for ' + c.idea.label + "."
return render('admin/idea-edit.html')
#use standard searchstring action
elif action=='Use Standard Searchstring/Searchpattern':
#toggle the c.custom to False and re-render form with htmlfill
c.custom = False
c.idea = h.fetch_obj(Idea, values['ID'])
c.found = True
c.search_string_list = c.idea.setup_SSL()
c.message = 'Now displaying standard searchstring options for ' + c.idea.label + "."
return render('admin/idea-edit.html')
else:
raise Exception('I don\'t know how you got here, but you shouldn\'t be here...please start at the main page. Invalid action %s'%action)
def process(self, id=None):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
label = request.params.get('label', None)
id = request.params.get('ID', id)
sep_dir = request.params.get('sep_dir', None)
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
values = dict(request.params)
if action=="Add":
thinker_add = Thinker(label)
thinker_add.label = label
#setup search string and search pattern
lastname = thinker_add.label.split(' ').pop()
lastname_q = Session.query(Entity)
o = Entity.searchstring.like(lastname)
lastname_q = lastname_q.filter(o).order_by(func.length(Entity.label))
if lastname_q.count() == 0:
#if there's no match currently to last name, can use last name alone as searchpattern/searchstring
thinker_add.searchpatterns.append(lastname)
thinker_add.searchstring = lastname
else:
#otherwise, we need to use the whole name for both, and reset the other pattern to full name too
thinker_add.searchpatterns.append(label)
thinker_add.searchstring = label
#reset old thinker pattern to whole name too to avoid conflict
oldthinker = h.fetch_obj(Thinker, lastname_q.first().ID)
oldthinker.searchpatterns = [oldthinker.label]
oldthinker.searchstring = oldthinker.label
Session.add(oldthinker)
if sep_dir:
thinker_add.sep_dir = sep_dir
c.thinker = thinker_add
Session.add(thinker_add)
Session.flush()
Session.commit()
c.found = True
c.message = "Thinker " + c.thinker.label + " added successfully."
return render ('admin/thinker-edit.html')
elif action=="Modify":
c.thinker = h.fetch_obj(Thinker, id)
c.found = True
changed = False
searchpatterns = []
for k, v in values.items():
key = ""
if k.startswith('searchpatterns'):
varname, num = k.split('.')
key = 'delsearchpattern.%s'%(num)
keyval = request.params.get(key, False)
if not keyval:
searchpatterns.append(v)
if values['newsearchpattern']:
searchpatterns.append(values['newsearchpattern'])
changed = True
#do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed
if c.thinker.searchpatterns != searchpatterns:
c.thinker.searchpatterns = searchpatterns
changed = True
#set values from form
if c.thinker.name != values['name']:
c.thinker.name = values['name']
changed = True
if c.thinker.label != values['label']:
c.thinker.name = values['label']
changed = True
if c.thinker.searchstring != values['searchstring']:
c.thinker.searchstring = values['searchstring']
changed = True
if c.thinker.sep_dir != values['sep_dir']:
c.thinker.sep_dir = values['sep_dir']
changed = True
if c.thinker.wiki != values['wiki']:
c.thinker.wiki = values['wiki']
changed = True
if c.thinker.birth_day != values['birth_day']:
c.thinker.birth_day = values['birth_day']
changed = True
c.thinker.birth_day = values['birth_day']
if c.thinker.death_day != values['death_day']:
c.thinker.death_day = values['death_day']
changed = True
if c.thinker.birth_month != values['birth_month']:
c.thinker.birth_month = values['birth_month']
changed = True
if c.thinker.death_month != values['death_month']:
c.thinker.death_month = values['death_month']
changed = True
if not (c.thinker.birth_year == values['birth_year'] + " " + values['bornbc']):
if c.thinker.birth_year != values['birth_year']:
c.thinker.birth_year = values['birth_year']
changed = True
if c.thinker.birth_year and values['bornbc'] and not re.search("(BC)|(AD)",c.thinker.birth_year):
c.thinker.birth_year = c.thinker.birth_year + " " + values['bornbc']
changed = True
if not (c.thinker.death_year == values['death_year'] + " " + values['diedbc']):
if c.thinker.death_year != values['death_year']:
c.thinker.death_year = values['death_year']
changed = True
if c.thinker.death_year and values['diedbc']and not re.search("(BC)|(AD)",c.thinker.death_year):
c.thinker.death_year = c.thinker.death_year + " " + values['diedbc']
changed = True
#commit changes
Session.flush()
Session.commit()
if changed:
c.message = "Thinker " + c.thinker.label + " modified successfully."
else:
c.message = "No changes detected. Thinker " + c.thinker.label + " not modified."
return render ('admin/thinker-edit.html')
elif action == "Delete":
c.thinker = h.fetch_obj(Thinker, values['ID'])
c.message = "Thinker # " + values['ID'] + " ("+ c.thinker.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.thinker)
Session.flush()
Session.commit()
c.found = False
return render('admin/thinker-edit.html')
def process(self, id=None):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
label = request.params.get('label', None)
id = request.params.get('ID', id)
sep_dir = request.params.get('sep_dir', None)
URL = request.params.get('URL', None)
language = request.params.get('language', None)
queries = [request.params.get('queries', None)]
openAccess = request.params.get('openAccess', None)
active = request.params.get('active', None)
student = request.params.get('student', None)
ISSN = request.params.get('ISSN', None)
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
values = dict(request.params)
#abbrs = [request.params.get('abbrs', None)]
abbrs = []
queries = []
for k, v in values.items():
key = ""
if k.startswith('abbrs'):
varname, num = k.split('.')
key = 'delabbr.%s'%(num)
keyval = request.params.get(key, False)
if not keyval:
abbrs.append(v)
elif k.startswith('queries'):
varname, num = k.split('.')
key = 'delquer.%s'%(num)
keyval = request.params.get(key, False)
if not keyval:
queries.append(v)
if action=="Add":
journal_add = Journal()
journal_add.label = label
#setup search string and search pattern
journalname = journal_add.label
journalname_q = Session.query(Entity)
o = Entity.searchpattern.like('( '+ journalname + ' )')
journalname_q = journalname_q.filter(o).order_by(func.length(Entity.label))
if journalname_q.count() == 0:
journal_add.searchpattern = "( " + journalname + " )"
journal_add.searchstring = journalname
else:
journal_add.searchpattern = "( " + label + " )"
journal_add.searchcstring = label
#reset old journal pattern to whole name too to avoid conflict
oldjournal = h.fetch_obj(Journal, journalname_q.first().ID)
oldjournal.searchpattern = "( " + oldjournal.label + " )"
oldjournal.searchstring = oldjournal.label
Session.add(oldjournal)
if sep_dir:
journal_add.sep_dir = sep_dir
c.journal = journal_add
Session.add(journal_add)
Session.flush()
Session.commit()
c.found = True
c.message = "Journal " + c.journal.label + " added successfully."
return render ('admin/journal-edit.html')
elif action=="Modify":
c.journal = h.fetch_obj(Journal, id)
c.found = True
changed = False
#set values from form
if c.journal.label != label:
c.journal.label = label
changed = True
if c.journal.sep_dir != sep_dir:
c.journal.sep_dir = sep_dir
changed = True
if c.journal.URL != URL:
c.journal.URL = URL
changed = True
if c.journal.language != language:
c.journal.language = language
changed = True
if c.journal.abbrs != abbrs:
c.journal.abbrs = abbrs
changed = True
if c.journal.queries != queries:
c.journal.queries = queries
changed = True
if c.journal.openAccess != openAccess:
c.journal.openAccess = openAccess
changed = True
if c.journal.active != active:
c.journal.active = active
changed = True
if c.journal.student != student:
c.journal.student = student
changed = True
if c.journal.ISSN != ISSN:
c.journal.ISSN = ISSN
changed = True
if values['newabbr']:
c.journal.abbrs.append(values['newabbr'])
changed = True
if values['newquery']:
c.journal.queries.append(values['newquery'])
changed = True
#commit changes
Session.flush()
Session.commit()
if changed:
c.message = "Journal " + c.journal.label + " modified successfully."
else:
c.message = "No change required; Journal " + c.journal.label + " not modified."
return render ('admin/journal-edit.html')
elif action == "Delete":
c.journal = h.fetch_obj(Journal, values['ID'])
c.message = "Journal # " + values['ID'] + " ("+ c.journal.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.journal)
Session.flush()
Session.commit()
c.found = False
return render('admin/journal-edit.html')
def process(self, id=None):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
c.sepdirnew = False
c.alreadysepdir = False
label = request.params.get('label', None)
id = request.params.get('ID', id)
sep_dir = request.params.get('sep_dir', None)
action = request.params.get('action', None)
action2 = request.params.get('action2', None)
if action2:
action = action2
values = dict(request.params)
if action == "Add":
thinker_add = Thinker(label)
thinker_add.label = label
#setup search string and search pattern
lastname = thinker_add.label.split(' ').pop()
lastname_q = Session.query(Entity)
o = Entity.searchstring.like(lastname)
lastname_q = lastname_q.filter(o).order_by(
func.length(Entity.label))
if lastname_q.count() == 0:
#if there's no match currently to last name, can use last name alone as searchpattern/searchstring
thinker_add.searchpatterns.append(lastname)
thinker_add.searchstring = lastname
else:
#otherwise, we need to use the whole name for both, and reset the other pattern to full name too
thinker_add.searchpatterns.append(label)
thinker_add.searchstring = label
#reset old thinker pattern to whole name too to avoid conflict
oldthinker = h.fetch_obj(Thinker, lastname_q.first().ID)
oldthinker.searchpatterns = [oldthinker.label]
oldthinker.searchstring = oldthinker.label
Session.add(oldthinker)
if sep_dir:
thinker_add.sep_dir = sep_dir
c.thinker = thinker_add
Session.add(thinker_add)
Session.flush()
Session.commit()
c.found = True
c.message = "Thinker " + c.thinker.label + " added successfully."
return render('admin/thinker-edit.html')
elif action == "Modify":
c.thinker = h.fetch_obj(Thinker, id)
c.found = True
changed = False
searchpatterns = []
for k, v in values.items():
key = ""
if k.startswith('searchpatterns'):
varname, num = k.split('.')
key = 'delsearchpattern.%s' % (num)
keyval = request.params.get(key, False)
if not keyval:
searchpatterns.append(v)
if values['newsearchpattern']:
searchpatterns.append(values['newsearchpattern'])
changed = True
#do manually edited searchpatterns first, so we don't write over them with the new default ones if the searchstring has been changed
if c.thinker.searchpatterns != searchpatterns:
c.thinker.searchpatterns = searchpatterns
changed = True
#set values from form
if c.thinker.name != values['name']:
c.thinker.name = values['name']
changed = True
if c.thinker.label != values['label']:
c.thinker.name = values['label']
changed = True
if c.thinker.searchstring != values['searchstring']:
c.thinker.searchstring = values['searchstring']
changed = True
if c.thinker.sep_dir != values['sep_dir']:
c.thinker.sep_dir = values['sep_dir']
changed = True
if c.thinker.wiki != values['wiki']:
c.thinker.wiki = values['wiki']
changed = True
if c.thinker.birth_day != values['birth_day']:
c.thinker.birth_day = values['birth_day']
changed = True
c.thinker.birth_day = values['birth_day']
if c.thinker.death_day != values['death_day']:
c.thinker.death_day = values['death_day']
changed = True
if c.thinker.birth_month != values['birth_month']:
c.thinker.birth_month = values['birth_month']
changed = True
if c.thinker.death_month != values['death_month']:
c.thinker.death_month = values['death_month']
changed = True
if not (c.thinker.birth_year
== values['birth_year'] + " " + values['bornbc']):
if c.thinker.birth_year != values['birth_year']:
c.thinker.birth_year = values['birth_year']
changed = True
if c.thinker.birth_year and values['bornbc'] and not re.search(
"(BC)|(AD)", c.thinker.birth_year):
c.thinker.birth_year = c.thinker.birth_year + " " + values[
'bornbc']
changed = True
if not (c.thinker.death_year
== values['death_year'] + " " + values['diedbc']):
if c.thinker.death_year != values['death_year']:
c.thinker.death_year = values['death_year']
changed = True
if c.thinker.death_year and values['diedbc'] and not re.search(
"(BC)|(AD)", c.thinker.death_year):
c.thinker.death_year = c.thinker.death_year + " " + values[
'diedbc']
changed = True
#commit changes
Session.flush()
Session.commit()
if changed:
c.message = "Thinker " + c.thinker.label + " modified successfully."
else:
c.message = "No changes detected. Thinker " + c.thinker.label + " not modified."
return render('admin/thinker-edit.html')
elif action == "Delete":
c.thinker = h.fetch_obj(Thinker, values['ID'])
c.message = "Thinker # " + values[
'ID'] + " (" + c.thinker.label + ") deleted; please search for a new entity label on the left."
h.delete_obj(c.thinker)
Session.flush()
Session.commit()
c.found = False
return render('admin/thinker-edit.html')