def submit_predictions(self, curs, schema_instance, prediction_pair2instance, cluster_id2properties):
sys.stderr.write("Submitting predictions...\n")
MpiPredictionFilter_instance = MpiPredictionFilter()
MpiPredictionFilter_instance.createGeneTable(curs, schema_instance.p_gene_table)
no_of_total_genes = get_no_of_total_genes(curs)
go_no2gene_no_set = get_go_no2gene_no_set(curs)
counter = 0
for prediction_pair, p_attr_instance in prediction_pair2instance.iteritems():
#1st fill those empty items
properties = cluster_id2properties[p_attr_instance.mcl_id]
vertex_set = properties[2]
p_attr_instance.p_value_cut_off = cal_hg_p_value(p_attr_instance.gene_no, p_attr_instance.go_no,\
vertex_set, no_of_total_genes, go_no2gene_no_set, r)
p_attr_instance.avg_p_value = p_attr_instance.p_value_cut_off
p_attr_instance.connectivity_cut_off = properties[0]
p_attr_instance.cluster_size_cut_off = len(vertex_set)
p_attr_instance.unknown_cut_off = properties[1]
MpiPredictionFilter_instance.submit_to_p_gene_table(curs, schema_instance.p_gene_table, p_attr_instance)
counter += 1
if self.report and counter%2000==0:
sys.stderr.write("%s%s"%('\x08'*20, counter))
if self.report:
sys.stderr.write("%s%s"%('\x08'*20, counter))
sys.stderr.write("Done.\n")
def get_known_data(self, curs, fname, filter_type, is_correct_type, need_cal_hg_p_value): schema_instance = form_schema_tables(fname) no_of_total_genes = get_no_of_total_genes(curs) go_no2gene_no_set = get_go_no2gene_no_set(curs) prediction_ls, all_data, known_data = self.data_fetch(curs, schema_instance, filter_type, is_correct_type, \ no_of_total_genes, go_no2gene_no_set, need_cal_hg_p_value) del prediction_ls, all_data return known_data
def get_data(self, curs, fname, filter_type, is_correct_type, need_cal_hg_p_value): """ 11-19-05 data_fetch() of rpart_prediction.py changed return unknown_data """ schema_instance = form_schema_tables(fname) no_of_total_genes = get_no_of_total_genes(curs) go_no2gene_no_set = get_go_no2gene_no_set(curs) unknown_prediction_ls, known_prediction_ls, unknown_data, known_data = self.data_fetch(curs, schema_instance, \ filter_type, is_correct_type, no_of_total_genes, go_no2gene_no_set, need_cal_hg_p_value) del unknown_prediction_ls, known_prediction_ls return unknown_data, known_data
def run(self):
"""
11-09-05
11-09-05 add rpart_cp
11-10-05 add need_cal_hg_p_value
--db_connect()
--form_schema_tables()
--form_schema_tables()
--get_no_of_total_genes()
--get_go_no2gene_no_set()
--data_fetch()
--get_vertex_list()
--cal_hg_p_value()
--rpart_fit_and_predict()
--MpiPredictionFilter_instance....()
--record_data()
"""
(conn, curs) = db_connect(self.hostname, self.dbname, self.schema)
old_schema_instance = form_schema_tables(self.fname1)
new_schema_instance = form_schema_tables(self.fname2)
no_of_total_genes = get_no_of_total_genes(curs)
go_no2gene_no_set = get_go_no2gene_no_set(curs)
prediction_ls, all_data, known_data = self.data_fetch(curs, old_schema_instance, self.filter_type, self.is_correct_type, \
no_of_total_genes, go_no2gene_no_set, need_cal_hg_p_value)
"""
testing_acc_ls, training_acc_ls = self.rpart_validation(known_data, self.no_of_buckets, self.rpart_cp, \
self.loss_matrix, self.prior_prob)
print testing_acc_ls
print training_acc_ls
"""
pred, pred_training = self.rpart_fit_and_predict(all_data, known_data, self.rpart_cp, self.loss_matrix, self.prior_prob)
MpiPredictionFilter_instance = MpiPredictionFilter()
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.splat_table, new_schema_instance.splat_table)
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.mcl_table, new_schema_instance.mcl_table)
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.pattern_table, new_schema_instance.pattern_table)
MpiPredictionFilter_instance.createGeneTable(curs, new_schema_instance.p_gene_table)
self.record_data(curs, MpiPredictionFilter_instance, prediction_ls, pred, new_schema_instance)
if self.commit:
curs.execute("end")
def run(self):
"""
11-09-05
11-09-05 add rpart_cp
11-10-05 add need_cal_hg_p_value
11-23-05
rpart_fit_and_predict() is split
2006-12-05
add need_output_data_for_R flag
--db_connect()
--form_schema_tables()
--form_schema_tables()
--get_no_of_total_genes()
--get_go_no2gene_no_set()
--data_fetch()
--get_vertex_list()
--cal_hg_p_value()
--output_data_for_R()
--rpart_fit()
--rpart_predict()
--rpart_predict()
--MpiPredictionFilter_instance....()
--record_data()
"""
(conn, curs) = db_connect(self.hostname, self.dbname, self.schema)
old_schema_instance = form_schema_tables(self.fname1)
new_schema_instance = form_schema_tables(self.fname2)
no_of_total_genes = get_no_of_total_genes(curs)
go_no2gene_no_set = get_go_no2gene_no_set(curs)
unknown_prediction_ls, known_prediction_ls, unknown_data, known_data = self.data_fetch(
curs,
old_schema_instance,
self.filter_type,
self.is_correct_type,
no_of_total_genes,
go_no2gene_no_set,
need_cal_hg_p_value,
)
if self.need_output_data_for_R: # 2006-12-05
self.output_data_for_R(known_data, "%s.known" % self.fname1)
self.output_data_for_R(unknown_data, "%s.unknown" % self.fname1)
"""
testing_acc_ls, training_acc_ls = self.rpart_validation(known_data, self.training_perc, self.rpart_cp, \
self.loss_matrix, self.prior_prob)
print testing_acc_ls
print training_acc_ls
"""
fit_model = self.fit_function_dict[self.type](known_data, self.parameter_list_dict[self.type], self.bit_string)
known_pred = self.predict_function_dict[self.type](fit_model, known_data)
unknown_pred = self.predict_function_dict[self.type](fit_model, unknown_data)
if self.debug:
if self.type == 2:
# randomForest's model has its own oob prediction
fit_model_py = fit_model.as_py(BASIC_CONVERSION)
print self.cal_accuracy(known_data, fit_model_py["predicted"], pred_type=1)
print self.cal_accuracy(known_data, known_pred, pred_type=self.type)
print self.cal_accuracy(unknown_data, unknown_pred, pred_type=self.type)
if self.commit:
MpiPredictionFilter_instance = MpiPredictionFilter()
MpiPredictionFilter_instance.view_from_table(
curs, old_schema_instance.splat_table, new_schema_instance.splat_table
)
MpiPredictionFilter_instance.view_from_table(
curs, old_schema_instance.mcl_table, new_schema_instance.mcl_table
)
MpiPredictionFilter_instance.view_from_table(
curs, old_schema_instance.pattern_table, new_schema_instance.pattern_table
)
MpiPredictionFilter_instance.createGeneTable(curs, new_schema_instance.p_gene_table)
self.record_data(
curs,
MpiPredictionFilter_instance,
unknown_prediction_ls,
unknown_pred,
new_schema_instance,
pred_type=self.type,
)
if (
self.type == 2
): # 2006-10-31 randomForest's model has its own oob prediction, but use rpart's way of storing prediction
fit_model_py = fit_model.as_py(BASIC_CONVERSION)
known_pred = fit_model_py["predicted"]
self.record_data(
curs,
MpiPredictionFilter_instance,
known_prediction_ls,
known_pred,
new_schema_instance,
pred_type=1,
)
else:
self.record_data(
curs,
MpiPredictionFilter_instance,
known_prediction_ls,
known_pred,
new_schema_instance,
pred_type=self.type,
)
curs.execute("end")
def run(self):
"""
09-05-05
10-23-05
create views from old schema
result goes to the new schema's p_gene_table
(input_node)
--db_connect()
--form_schema_tables()
--form_schema_tables()
--get_gene_no2go_no_set()
--get_go_no2depth()
(pass data to computing_node)
(computing_node)
(take data from other nodes, 0 and size-1)
(judge_node)
--gene_stat()
--db_connect()
--gene_p_map_redundancy()
(output_node)
--db_connect()
--form_schema_tables()
--form_schema_tables()
--MpiPredictionFilter()
--MpiPredictionFilter_instance.createGeneTable()
--get_go_no2edge_counter_list()(if necessary)
(pass go_no2edge_counter_list to computing_node)
(input_node)
--fetch_cluster_block()
(computing_node)
--get_no_of_unknown_genes()
--node_fire_handler()
--cleanup_handler()
--judge_node()
--gene_stat_instance.(match functions)
--output_node()
--output_node_handler()
--MpiPredictionFilter_instance.submit_to_p_gene_table()
"""
communicator = MPI.world.duplicate()
node_rank = communicator.rank
if node_rank == 0:
(conn, curs) = db_connect(self.hostname, self.dbname, self.schema)
"""
#01-02-06
old_schema_instance = form_schema_tables(self.input_fname)
new_schema_instance = form_schema_tables(self.jnput_fname)
"""
gene_no2go_no = get_gene_no2go_no_set(curs)
gene_no2go_no_pickle = cPickle.dumps(gene_no2go_no, -1) #-1 means use the highest protocol
go_no2depth = get_go_no2depth(curs)
go_no2depth_pickle = cPickle.dumps(go_no2depth, -1)
go_no2gene_no_set = get_go_no2gene_no_set(curs)
go_no2gene_no_set_pickle = cPickle.dumps(go_no2gene_no_set, -1)
for node in range(1, communicator.size-2): #send it to the computing_node
communicator.send(gene_no2go_no_pickle, node, 0)
communicator.send(go_no2depth_pickle, node, 0)
communicator.send(go_no2gene_no_set_pickle, node, 0)
elif node_rank<=communicator.size-3: #WATCH: last 2 nodes are not here.
data, source, tag = communicator.receiveString(0, 0)
gene_no2go_no = cPickle.loads(data) #take the data
data, source, tag = communicator.receiveString(0, 0)
go_no2depth = cPickle.loads(data)
data, source, tag = communicator.receiveString(0, 0)
go_no2gene_no_set = cPickle.loads(data)
data, source, tag = communicator.receiveString(communicator.size-1, 0) #from the last node
go_no2edge_counter_list = cPickle.loads(data)
#choose a functor for recurrence_array
functor_dict = {0: None,
1: lambda x: int(x>=self.recurrence_x),
2: lambda x: math.pow(x, self.recurrence_x)}
functor = functor_dict[self.recurrence_x_type]
elif node_rank == communicator.size-2: #judge node
gene_stat_instance = gene_stat(depth_cut_off=self.depth)
(conn, curs) = db_connect(self.hostname, self.dbname, self.schema)
gene_stat_instance.dstruc_loadin(curs)
from gene_p_map_redundancy import gene_p_map_redundancy
node_distance_class = gene_p_map_redundancy()
elif node_rank==communicator.size-1: #establish connection before pursuing
(conn, curs) = db_connect(self.hostname, self.dbname, self.schema)
"""
#01-02-06, input and output are all directed to files
old_schema_instance = form_schema_tables(self.input_fname)
new_schema_instance = form_schema_tables(self.jnput_fname)
MpiPredictionFilter_instance = MpiPredictionFilter()
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.splat_table, new_schema_instance.splat_table)
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.mcl_table, new_schema_instance.mcl_table)
MpiPredictionFilter_instance.view_from_table(curs, old_schema_instance.pattern_table, new_schema_instance.pattern_table)
if self.new_table:
MpiPredictionFilter_instance.createGeneTable(curs, new_schema_instance.p_gene_table)
"""
if self.go_no2edge_counter_list_fname:
go_no2edge_counter_list = cPickle.load(open(self.go_no2edge_counter_list_fname,'r'))
else:
if self.eg_d_type==2:
go_no2edge_counter_list = None
else:
gene_no2go_no = get_gene_no2go_no_set(curs)
go_no2edge_counter_list = get_go_no2edge_counter_list(curs, gene_no2go_no, self.edge_type2index)
go_no2edge_counter_list_pickle = cPickle.dumps(go_no2edge_counter_list, -1)
for node in range(1, communicator.size-2): #send it to the computing_node
communicator.send(go_no2edge_counter_list_pickle, node, 0)
mpi_synchronize(communicator)
free_computing_nodes = range(1,communicator.size-2) #exclude the last node
if node_rank == 0:
"""
curs.execute("DECLARE crs CURSOR FOR SELECT id, vertex_set, edge_set, no_of_edges,\
connectivity, unknown_gene_ratio, recurrence_array, d_matrix from %s"%(old_schema_instance.pattern_table))
"""
self.counter = 0 #01-02-06 counter is used as id
reader = csv.reader(open(self.input_fname, 'r'), delimiter='\t')
parameter_list = [reader]
input_node(communicator, parameter_list, free_computing_nodes, self.message_size, \
self.report, input_handler=self.input_handler)
del reader
elif node_rank in free_computing_nodes:
no_of_unknown_genes = get_no_of_unknown_genes(gene_no2go_no)
GradientScorePrediction_instance = GradientScorePrediction(gene_no2go_no, go_no2gene_no_set, go_no2depth, \
go_no2edge_counter_list, no_of_unknown_genes, self.depth, self.min_layer1_associated_genes, \
self.min_layer1_ratio, self.min_layer2_associated_genes, self.min_layer2_ratio, self.exponent, \
self.score_list, self.max_layer, self.norm_exp, self.eg_d_type, self.debug)
parameter_list = [GradientScorePrediction_instance, functor]
computing_node(communicator, parameter_list, self.node_fire_handler, self.cleanup_handler, self.report)
elif node_rank == communicator.size-2:
self.judge_node(communicator, curs, gene_stat_instance, node_distance_class)
elif node_rank==communicator.size-1:
#01-02-06 output goes to plain file, not database
writer = csv.writer(open(self.jnput_fname, 'w'), delimiter='\t')
parameter_list = [writer]
output_node(communicator, free_computing_nodes, parameter_list, self.output_node_handler, self.report)
del writer
