def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None):
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(id,
id2,
uid,
username,
autoCreate=False)
if not evaluation:
abort(404)
current_uid = h.get_user(request.environ['REMOTE_USER']).ID
if evaluation.uid != current_uid or not h.auth.is_admin():
abort(401)
setattr(evaluation, evaltype, -1)
# Delete evaluation if this eliminates both settings, new db schema
# will eliminate this need
#if evaluation.generality == -1 and evaluation.relatedness == -1:
# h.delete_obj(evaluation)
Session.flush()
Session.commit()
response.status_int = 200
return "OK"
def _delete_evaluation(self, evaltype, id, id2, uid=None, username=None):
if not h.auth.is_logged_in():
abort(401)
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(id, id2, uid, username, autoCreate=False)
if not evaluation:
abort(404)
current_uid = h.get_user(request.environ['REMOTE_USER']).ID
if evaluation.uid != current_uid or not h.auth.is_admin():
abort(401)
setattr(evaluation, evaltype, -1)
# Delete evaluation if this eliminates both settings, new db schema
# will eliminate this need
#if evaluation.generality == -1 and evaluation.relatedness == -1:
# h.delete_obj(evaluation)
Session.flush()
Session.commit()
response.status_int = 200
return "OK"
def submit_changes(self):
'''
This function validates the submitted profile edit form and commits the
changes. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
if not h.auth.is_logged_in():
abort(401)
c.user = h.get_user(request.environ['REMOTE_USER'])
if self.form_result['password'] != '':
c.user.set_password(self.form_result['password'])
# TODO: Enable area editing
#c.user.first_area_id=self.form_result['first_area'],
#user.first_area_level=self.form_result['first_area_level'],
#if self.form_result['second_area']:
# c.user.second_area_id=self.form_result['second_area'],
# c.user.second_area_level=self.form_result['second_area_level']
c.user.fullname = self.form_result['fullname']
Session.flush()
Session.commit()
h.redirect(
h.url(controller='account', action='profile', message='edited'))
def create(self):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
valid_params = ["sep_dir", "wiki"]
params = request.params.mixed()
if '_method' in params:
del params['_method']
if 'label' in params:
label = params['label']
del params['label']
else:
abort(400)
for k in params.keys():
if k not in valid_params:
abort(400)
school_of_thought = SchoolOfThought(name, **params)
Session.add(school_of_thought)
Session.flush()
# Issue an HTTP success
response.status_int = 302
response.headers['location'] = h.url(controller='school_of_thought',
action='view',
id=school_of_thought.ID)
return "Moved temporarily"
def submit_changes(self):
'''
This function validates the submitted profile edit form and commits the
changes. Restricted to ``POST`` requests. If successful, redirects to
the result action to prevent resubmission.
'''
if not h.auth.is_logged_in():
abort(401)
c.user = h.get_user(request.environ['REMOTE_USER'])
if self.form_result['password'] != '':
c.user.set_password(self.form_result['password'])
# TODO: Enable area editing
#c.user.first_area_id=self.form_result['first_area'],
#user.first_area_level=self.form_result['first_area_level'],
#if self.form_result['second_area']:
# c.user.second_area_id=self.form_result['second_area'],
# c.user.second_area_level=self.form_result['second_area_level']
c.user.fullname = self.form_result['fullname']
Session.flush()
Session.commit()
h.redirect(h.url(controller='account', action='profile', message='edited'))
def _thinker_evaluate(self,
evaltype=None,
id=None,
id2=None,
uid=None,
username=None,
degree=1,
maxdegree=1):
"""
Private method to handle generic evaluations. See ``teacher_of`` and ``has_influenced``
for use.
"""
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(evaltype, id, id2, uid, username)
try:
evaluation.degree = int(request.params.get('degree', degree))
except TypeError:
abort(400)
# Create and commit evaluation
Session.flush()
# Issue an HTTP success
response.status_int = 200
return "OK"
def create(self):
if not h.auth.is_logged_in():
abort(401)
if not h.auth.is_admin():
abort(403)
valid_params = ["sep_dir", "wiki"]
params = request.params.mixed()
if '_method' in params:
del params['_method']
if 'label' in params:
label = params['label']
del params['label']
else:
abort(400)
for k in params.keys():
if k not in valid_params:
abort(400)
school_of_thought = SchoolOfThought(name, **params)
Session.add(school_of_thought)
Session.flush()
# Issue an HTTP success
response.status_int = 302
response.headers['location'] = h.url(controller='school_of_thought',
action='view', id=school_of_thought.ID)
return "Moved temporarily"
def _evaluate(self,
evaltype,
id,
id2=None,
uid=None,
username=None,
degree=-1,
maxdegree=4,
errors=0):
"""
Function to submit an evaluation. Takes a POST request containing the consequesnt id and
all or none of: generality, relatedness, hyperrank, hyporank.
"""
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
try:
username = h.auth.get_username_from_cookie(
request.params.get('cookieAuth', ''))
except ValueError:
# invalid IP, abort
username = None
print "grabbing eval for", username, uid
if request.environ.get('REMOTE_USER', False):
username = request.environ.get('REMOTE_USER', username)
evaluation = self._get_evaluation(id, id2, None, username)
elif username:
evaluation = self._get_evaluation(id, id2, None, username)
else:
evaluation = self._get_anon_evaluation(
id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0'))
# Populate proper generality, relatedness, hyperrank and hyporank values
evaluation.time = time.time()
# Attempt to convert to integers, if unable, throw HTTP 400
try:
setattr(
evaluation, evaltype,
int(request.params.get('degree', getattr(evaluation,
evaltype))))
except TypeError:
abort(400)
# Create and commit evaluation
try:
Session.flush()
Session.commit()
except IntegrityError:
Session.rollback()
if not errors:
self._evaluate(evaltype, id, id2, username, degree, maxdegree,
errors + 1)
# Issue an HTTP success
response.status_int = 200
return "OK"
def complete_mining(entity_type=Idea,
filename='graph.txt',
root='./',
corpus_root='corpus/',
update_entropy=False,
update_occurrences=False,
update_db=False):
occur_filename = os.path.abspath(root + "occurrences.txt")
graph_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
doc_terms = doc_terms_list()
if update_occurrences:
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "filtering occurrences..."
filter_apriori_input(occur_filename, graph_filename, entity_type,
doc_terms)
print "running apriori miner..."
dm.apriori(graph_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(graph_filename, edge_filename, occur_filename,
doc_terms)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante, cons = edge
row = "%s::%s" % edge
row += ("::%(confidence)s::%(jweight)s::%(weight)s"
"::%(occurs_in)s\n" % props)
f.write(row)
if update_entropy:
print "updating term entropy..."
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
if update_db:
print "updating the database..."
update_graph(entity_type, sql_filename)
def delete_obj(obj):
"""
Deletes any arbitrary object from the SQLAlchemy Session and cascades deletes to evaluations.
:param obj: object to delete
"""
Session.delete(obj)
Session.flush()
Session.commit()
def delete_obj(obj):
"""
Deletes any arbitrary object from the SQLAlchemy Session and cascades deletes to evaluations.
:param obj: object to delete
"""
Session.delete(obj)
Session.flush()
Session.commit()
def complete_mining(entity_type=Idea, filename='graph.txt', root='./',
corpus_root='corpus/', update_entropy=False,
update_occurrences=False, update_db=False):
occur_filename = os.path.abspath(root + "occurrences.txt")
graph_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
doc_terms = doc_terms_list()
if update_occurrences:
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "filtering occurrences..."
filter_apriori_input(
occur_filename, graph_filename, entity_type, doc_terms)
print "running apriori miner..."
dm.apriori(graph_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(
graph_filename, edge_filename, occur_filename, doc_terms)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante,cons = edge
row = "%s::%s" % edge
row += ("::%(confidence)s::%(jweight)s::%(weight)s"
"::%(occurs_in)s\n" % props)
f.write(row)
if update_entropy:
print "updating term entropy..."
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
if update_db:
print "updating the database..."
update_graph(entity_type, sql_filename)
def _evaluate(self, evaltype, id, id2=None, uid=None, username=None,
degree=-1, maxdegree=4, errors=0):
"""
Function to submit an evaluation. Takes a POST request containing the consequesnt id and
all or none of: generality, relatedness, hyperrank, hyporank.
"""
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
try:
username = h.auth.get_username_from_cookie(request.params.get('cookieAuth', ''))
except ValueError:
# invalid IP, abort
username = None
print "grabbing eval for", username, uid
if request.environ.get('REMOTE_USER', False):
username = request.environ.get('REMOTE_USER', username)
evaluation = self._get_evaluation(id, id2, None, username)
elif username:
evaluation = self._get_evaluation(id, id2, None, username)
else:
evaluation = self._get_anon_evaluation(id, id2, request.environ.get('REMOTE_ADDR', '0.0.0.0'))
# Populate proper generality, relatedness, hyperrank and hyporank values
evaluation.time = time.time()
# Attempt to convert to integers, if unable, throw HTTP 400
try:
setattr(evaluation, evaltype,
int(request.params.get('degree', getattr(evaluation, evaltype))))
except TypeError:
abort(400)
# Create and commit evaluation
try:
Session.flush()
Session.commit()
except IntegrityError:
Session.rollback()
if not errors:
self._evaluate(evaltype, id, id2, username,
degree, maxdegree, errors+1)
# Issue an HTTP success
response.status_int = 200
return "OK"
def update_obj(obj, attributes, params):
"""
Updates any arbitrary object. Takes a list of attributes and a dictionary of update
parameters. Checks if each key is in the list of approved attributes and then attempts
to set it. If the object does not have that key, throw an HTTP 400 Bad Request
:param obj: object to update
:param attributes: list of approved attributes
:param params: dictionary of update parameters
"""
for key in params.keys():
if key in attributes:
try:
set_attribute(obj, key, params[key])
except:
abort(400)
Session.flush()
Session.commit()
def update_obj(obj, attributes, params):
"""
Updates any arbitrary object. Takes a list of attributes and a dictionary of update
parameters. Checks if each key is in the list of approved attributes and then attempts
to set it. If the object does not have that key, throw an HTTP 400 Bad Request
:param obj: object to update
:param attributes: list of approved attributes
:param params: dictionary of update parameters
"""
for key in params.keys():
if key in attributes:
try:
set_attribute(obj, key, params[key])
except:
abort(400)
Session.flush()
Session.commit()
def _thinker_evaluate(self, evaltype=None, id=None, id2=None,
uid=None, username=None,
degree=1, maxdegree=1):
"""
Private method to handle generic evaluations. See ``teacher_of`` and ``has_influenced``
for use.
"""
id2 = request.params.get('id2', id2)
uid = request.params.get('uid', uid)
username = request.params.get('username', username)
evaluation = self._get_evaluation(evaltype, id, id2, uid, username)
try:
evaluation.degree = int(request.params.get('degree', degree))
except TypeError:
abort(400)
# Create and commit evaluation
Session.flush()
# Issue an HTTP success
response.status_int = 200
return "OK"
def complete_mining(entity_type=Idea, filename='graph.txt', root='./',
corpus_root='corpus/', update_entropy=False):
occur_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante,cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
if update_entropy:
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
DELETE FROM %(table)s;
""" % {'filename' : sql_filename, 'table' : table })
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight);
SET foreign_key_checks=1;
""" % {'filename' : sql_filename, 'table' : table })
Session.close()
# to see if it is still valid. If the status code is a 302 redirect,
# update the URL in the database. If it's accessible (status code ==
# 200 or 30x), update the last_accessed field. If it doesn't open at all,
# raise an error. If it's been inaccessible for four weeks, raise an
# error.
journal_list = Session.query(Journal).all()
for journal in journal_list:
try:
f = urllib.urlopen(journal.URL)
status = f.getcode()
if (status == 302):
journal.URL = f.geturl() # UNTESTED!!
if (status <= 307):
journal.last_accessed = time.time()
except:
errormsg = "As of {0}, the journal {1} had a bad URL: {2}"
print >> sys.stderr, errormsg.format(time.strftime("%Y-%m-%d %H:%M:%S"), journal.name, journal.URL)
# "magic number" 2419200 == four weeks in seconds
if not journal.last_accessed or (time.time() - journal.last_accessed > 2419200):
errormsg = "As of {0}, the journal {1} has been inaccessible for four weeks."
print >> sys.stderr, errormsg.format(time.strftime("%Y-%m-%d %H:%M:%S"), journal.name)
# write to the database
Session.commit()
Session.flush()
print "Succesfully checked {0} journal URLS.".format(len(journal_list))
def complete_mining(entity_type=Idea,
filename='graph.txt',
root='./',
corpus_root='corpus/',
update_entropy=False):
occur_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante, cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
if update_entropy:
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
DELETE FROM %(table)s;
""" % {
'filename': sql_filename,
'table': table
})
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight);
SET foreign_key_checks=1;
""" % {
'filename': sql_filename,
'table': table
})
Session.close()
