def update_graph(entity_type, sql_filename):
"""
Performs a complete update of the database graph information, adding
jweight, entropy and occurrence data from the sql file generated by
complete_mining. This will remove ALL previous graph data.
"""
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
TRUNCATE TABLE %(table)s;
""" % {'filename' : sql_filename, 'table' : table })
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOCK TABLES %(table)s WRITE;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight, occurs_in);
UNLOCK TABLES;
SET foreign_key_checks=1;
""" % {'filename' : sql_filename, 'table' : table })
Session.close()
def update_graph(entity_type, sql_filename):
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
TRUNCATE TABLE %(table)s;
""" % {'filename' : sql_filename, 'table' : table })
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOCK TABLES %(table)s WRITE;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight, occurs_in);
UNLOCK TABLES;
SET foreign_key_checks=1;
""" % {'filename' : sql_filename, 'table' : table })
Session.close()
def update_graph(entity_type, sql_filename):
"""
Performs a complete update of the database graph information, adding
jweight, entropy and occurrence data from the sql file generated by
complete_mining. This will remove ALL previous graph data.
"""
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
TRUNCATE TABLE %(table)s;
""" % {
'filename': sql_filename,
'table': table
})
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOCK TABLES %(table)s WRITE;
LOAD DATA LOCAL INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight, occurs_in);
UNLOCK TABLES;
SET foreign_key_checks=1;
""" % {
'filename': sql_filename,
'table': table
})
Session.close()
def complete_mining(entity_type=Idea,
filename='graph.txt',
root='./',
corpus_root='corpus/',
update_entropy=False):
occur_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante, cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
if update_entropy:
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
DELETE FROM %(table)s;
""" % {
'filename': sql_filename,
'table': table
})
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight);
SET foreign_key_checks=1;
""" % {
'filename': sql_filename,
'table': table
})
Session.close()
def complete_mining(entity_type=Idea, filename='graph.txt', root='./',
corpus_root='corpus/', update_entropy=False):
occur_filename = os.path.abspath(root + "graph-" + filename)
edge_filename = os.path.abspath(root + "edge-" + filename)
sql_filename = os.path.abspath(root + "sql-" + filename)
print "processing articles..."
process_articles(entity_type, occur_filename, corpus_root=corpus_root)
print "running apriori miner..."
dm.apriori(occur_filename, edge_filename)
print "processing edges..."
edges = dm.process_edges(occur_filename, edge_filename)
ents = dm.calculate_node_entropy(edges)
edges = dm.calculate_edge_weight(edges, ents)
print "creating sql files..."
with open(sql_filename, 'w') as f:
for edge, props in edges.iteritems():
ante,cons = edge
row = "%s::%s" % edge
row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props
f.write(row)
print "updating term entropy..."
if update_entropy:
for term_id, entropy in ents.iteritems():
term = Session.query(Idea).get(term_id)
if term:
term.entropy = entropy
Session.flush()
Session.commit()
Session.close()
# Import SQL statements
if entity_type == Idea:
table = "idea_graph_edges"
elif entity_type == Thinker:
table = "thinker_graph_edges"
else:
table = "idea_thinker_graph_edges"
connection = Session.connection()
print "deleting old graph information ..."
connection.execute("""
DELETE FROM %(table)s;
""" % {'filename' : sql_filename, 'table' : table })
print "inserting new graph information"
connection.execute("""
SET foreign_key_checks=0;
LOAD DATA INFILE '%(filename)s'
INTO TABLE %(table)s
FIELDS TERMINATED BY '::'
(ante_id, cons_id, confidence, jweight, weight);
SET foreign_key_checks=1;
""" % {'filename' : sql_filename, 'table' : table })
Session.close()
