def update_graph(entity_type, sql_filename): """ Performs a complete update of the database graph information, adding jweight, entropy and occurrence data from the sql file generated by complete_mining. This will remove ALL previous graph data. """ # Import SQL statements if entity_type == Idea: table = "idea_graph_edges" elif entity_type == Thinker: table = "thinker_graph_edges" else: table = "idea_thinker_graph_edges" connection = Session.connection() print "deleting old graph information ..." connection.execute(""" TRUNCATE TABLE %(table)s; """ % {'filename' : sql_filename, 'table' : table }) print "inserting new graph information" connection.execute(""" SET foreign_key_checks=0; LOCK TABLES %(table)s WRITE; LOAD DATA INFILE '%(filename)s' INTO TABLE %(table)s FIELDS TERMINATED BY '::' (ante_id, cons_id, confidence, jweight, weight, occurs_in); UNLOCK TABLES; SET foreign_key_checks=1; """ % {'filename' : sql_filename, 'table' : table }) Session.close()
def update_graph(entity_type, sql_filename): # Import SQL statements if entity_type == Idea: table = "idea_graph_edges" elif entity_type == Thinker: table = "thinker_graph_edges" else: table = "idea_thinker_graph_edges" connection = Session.connection() print "deleting old graph information ..." connection.execute(""" TRUNCATE TABLE %(table)s; """ % {'filename' : sql_filename, 'table' : table }) print "inserting new graph information" connection.execute(""" SET foreign_key_checks=0; LOCK TABLES %(table)s WRITE; LOAD DATA INFILE '%(filename)s' INTO TABLE %(table)s FIELDS TERMINATED BY '::' (ante_id, cons_id, confidence, jweight, weight, occurs_in); UNLOCK TABLES; SET foreign_key_checks=1; """ % {'filename' : sql_filename, 'table' : table }) Session.close()
def update_graph(entity_type, sql_filename): """ Performs a complete update of the database graph information, adding jweight, entropy and occurrence data from the sql file generated by complete_mining. This will remove ALL previous graph data. """ # Import SQL statements if entity_type == Idea: table = "idea_graph_edges" elif entity_type == Thinker: table = "thinker_graph_edges" else: table = "idea_thinker_graph_edges" connection = Session.connection() print "deleting old graph information ..." connection.execute(""" TRUNCATE TABLE %(table)s; """ % { 'filename': sql_filename, 'table': table }) print "inserting new graph information" connection.execute(""" SET foreign_key_checks=0; LOCK TABLES %(table)s WRITE; LOAD DATA LOCAL INFILE '%(filename)s' INTO TABLE %(table)s FIELDS TERMINATED BY '::' (ante_id, cons_id, confidence, jweight, weight, occurs_in); UNLOCK TABLES; SET foreign_key_checks=1; """ % { 'filename': sql_filename, 'table': table }) Session.close()
def complete_mining(entity_type=Idea, filename='graph.txt', root='./', corpus_root='corpus/', update_entropy=False): occur_filename = os.path.abspath(root + "graph-" + filename) edge_filename = os.path.abspath(root + "edge-" + filename) sql_filename = os.path.abspath(root + "sql-" + filename) print "processing articles..." process_articles(entity_type, occur_filename, corpus_root=corpus_root) print "running apriori miner..." dm.apriori(occur_filename, edge_filename) print "processing edges..." edges = dm.process_edges(occur_filename, edge_filename) ents = dm.calculate_node_entropy(edges) edges = dm.calculate_edge_weight(edges, ents) print "creating sql files..." with open(sql_filename, 'w') as f: for edge, props in edges.iteritems(): ante, cons = edge row = "%s::%s" % edge row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props f.write(row) print "updating term entropy..." if update_entropy: for term_id, entropy in ents.iteritems(): term = Session.query(Idea).get(term_id) if term: term.entropy = entropy Session.flush() Session.commit() Session.close() # Import SQL statements if entity_type == Idea: table = "idea_graph_edges" elif entity_type == Thinker: table = "thinker_graph_edges" else: table = "idea_thinker_graph_edges" connection = Session.connection() print "deleting old graph information ..." connection.execute(""" DELETE FROM %(table)s; """ % { 'filename': sql_filename, 'table': table }) print "inserting new graph information" connection.execute(""" SET foreign_key_checks=0; LOAD DATA INFILE '%(filename)s' INTO TABLE %(table)s FIELDS TERMINATED BY '::' (ante_id, cons_id, confidence, jweight, weight); SET foreign_key_checks=1; """ % { 'filename': sql_filename, 'table': table }) Session.close()
def complete_mining(entity_type=Idea, filename='graph.txt', root='./', corpus_root='corpus/', update_entropy=False): occur_filename = os.path.abspath(root + "graph-" + filename) edge_filename = os.path.abspath(root + "edge-" + filename) sql_filename = os.path.abspath(root + "sql-" + filename) print "processing articles..." process_articles(entity_type, occur_filename, corpus_root=corpus_root) print "running apriori miner..." dm.apriori(occur_filename, edge_filename) print "processing edges..." edges = dm.process_edges(occur_filename, edge_filename) ents = dm.calculate_node_entropy(edges) edges = dm.calculate_edge_weight(edges, ents) print "creating sql files..." with open(sql_filename, 'w') as f: for edge, props in edges.iteritems(): ante,cons = edge row = "%s::%s" % edge row += "::%(confidence)s::%(jweight)s::%(weight)s\n" % props f.write(row) print "updating term entropy..." if update_entropy: for term_id, entropy in ents.iteritems(): term = Session.query(Idea).get(term_id) if term: term.entropy = entropy Session.flush() Session.commit() Session.close() # Import SQL statements if entity_type == Idea: table = "idea_graph_edges" elif entity_type == Thinker: table = "thinker_graph_edges" else: table = "idea_thinker_graph_edges" connection = Session.connection() print "deleting old graph information ..." connection.execute(""" DELETE FROM %(table)s; """ % {'filename' : sql_filename, 'table' : table }) print "inserting new graph information" connection.execute(""" SET foreign_key_checks=0; LOAD DATA INFILE '%(filename)s' INTO TABLE %(table)s FIELDS TERMINATED BY '::' (ante_id, cons_id, confidence, jweight, weight); SET foreign_key_checks=1; """ % {'filename' : sql_filename, 'table' : table }) Session.close()