def compute_bbox_set_agreement(example_boxes, gold_boxes): nExB = len(example_boxes) nGtB = len(gold_boxes) if nExB == 0: if nGtB == 0: return 1 else: return 0 if nGtB == 0: print "WARNING: new object" return 0 A = cvxmod.zeros(rows=nExB, cols=nGtB) for iBox, ex in enumerate(example_boxes): for jBox, gt in enumerate(gold_boxes): A[iBox, jBox] = ex.overlap_score(gt) S = [] S2 = [] for iBox, ex in enumerate(example_boxes): S_tmp = [0] * (iBox) * nGtB + [1] * nGtB + [0] * (nExB - iBox - 1) * nGtB S.append(S_tmp) for jBox in range(0, nGtB): S2_tmp = [0] * nExB * nGtB for j2 in range(0, nExB): S2_tmp[j2 * nGtB + jBox] = 1 S2.append(S2_tmp) S = cvxmod.transpose(cvxmod.matrix(S, size=(nExB * nGtB, nExB))) S2 = cvxmod.transpose(cvxmod.matrix(S2, size=(nExB * nGtB, nGtB))) A2 = cvxmod.matrix(A, (1, nExB * nGtB)) x = cvxmod.optvar('x', rows=nExB * nGtB, cols=1) p = cvxmod.problem(cvxmod.maximize(A2 * x)) p.constr.append(x <= 1) p.constr.append(x >= 0) p.constr.append(S * x <= 1) p.constr.append(S2 * x <= 1) p.solve(True) overlap = cvxmod.value(p) / max(nExB, nGtB) assert (overlap < 1.0001) return overlap
def _train(self, train_data, param): """ training procedure using training examples and labels @param train_data: Data relevant to SVM training @type train_data: dict<str, list<instances> > @param param: Parameters for the training procedure @type param: ParameterSvm """ # split for training weak_learners and boosting (train_weak, train_boosting) = split_data(train_data, 4) # merge data sets data = PreparedMultitaskData(train_weak, shuffle=True) # create shogun label lab = shogun_factory.create_labels(data.labels) ################################################## # define pockets ################################################## pockets = [0]*9 pockets[0] = [1, 5, 6, 7, 8, 31, 32, 33, 34] pockets[1] = [1, 2, 3, 4, 6, 7, 8, 9, 11, 21, 31] pockets[2] = [11, 20, 21, 22, 29, 31] pockets[3] = [8, 30, 31, 32] pockets[4] = [10, 11, 30] pockets[5] = [10, 11, 12, 13, 20, 29] pockets[6] = [10, 12, 20, 22, 26, 27, 28, 29] pockets[7] = [12, 14, 15, 26] pockets[8] = [13, 15, 16, 17, 18, 19, 20, 23, 24, 25, 26] pockets = [] for i in xrange(35): pockets.append([i]) #new_pockets = [] # merge neighboring pockets #for i in range(8): # new_pockets.append(list(set(pockets[i]).union(set(pockets[i+1])))) #pockets = new_pockets ######################################################## print "creating a kernel:" ######################################################## # init seq handler pseudoseqs = SequencesHandler() classifiers = [] for pocket in pockets: print "creating normalizer" #import pdb #pdb.set_trace() normalizer = MultitaskKernelNormalizer(data.task_vector_nums) print "processing pocket", pocket # set similarity for task_name_lhs in data.get_task_names(): for task_name_rhs in data.get_task_names(): similarity = 0.0 for pseudo_seq_pos in pocket: similarity += float(pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pseudo_seq_pos-1)) # normalize similarity = similarity / float(len(pocket)) print "pocket %s (%s, %s) = %f" % (str(pocket), task_name_lhs, task_name_rhs, similarity) normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity) print "creating empty kernel" kernel = shogun_factory.create_kernel(data.examples, param) print "setting normalizer" kernel.set_normalizer(normalizer) print "training SVM for pocket", pocket svm = self._train_single_svm(param, kernel, lab) classifiers.append(svm) print "done obtaining weak learners" # save additional info #self.additional_information["svm_objective"] = svm.get_objective() #self.additional_information["svm num sv"] = svm.get_num_support_vectors() #self.additional_information["post_weights"] = combined_kernel.get_subkernel_weights() #print self.additional_information ################################################## # combine weak learners for each task ################################################## # set constants some = 0.9 import cvxmod # wrap up predictors svms = {} # use a reference to the same svm several times for task_name in train_boosting.keys(): instances = train_boosting[task_name] N = len(instances) F = len(pockets) examples = [inst.example for inst in instances] labels = [inst.label for inst in instances] # dim = (F x N) out = cvxmod.zeros((N,F)) for i in xrange(F): svm = classifiers[i] tmp_out = self._predict_weak(svm, examples, data.name_to_id(task_name)) out[:,i] = numpy.sign(tmp_out) #out[:,i] = tmp_out #TODO: fix helper.save("/tmp/out_sparse", (out,labels)) pdb.set_trace() weights = solve_boosting(out, labels, some, solver="mosek") svms[task_name] = (data.name_to_id(task_name), svm) return svms
def solve_boosting(out, labels, nu, solver): ''' solve boosting formulation used by gelher and novozin @param out: matrix (N,F) of predictions (for each f_i) for all examples @param y: vector (N,1) label for each example @param p: regularization constant ''' N = out.size[0] F = out.size[1] assert(N==len(labels)) norm_fact = 1.0 / (nu * float(N)) print norm_fact label_matrix = cvxmod.zeros((N,N)) # avoid point-wise product for i in xrange(N): label_matrix[i,i] = labels[i] #### parameters f = cvxmod.param("f", N, F) y = cvxmod.param("y", N, N, symm=True) norm = cvxmod.param("norm", 1) #### varibales # rho rho = cvxmod.optvar("rho", 1) # dim = (N x 1) chi = cvxmod.optvar("chi", N) # dim = (F x 1) beta = cvxmod.optvar("beta", F) #objective = -rho + cvxmod.sum(chi) * norm_fact + square(norm2(beta)) objective = -rho + cvxmod.sum(chi) * norm_fact print objective # create problem p = cvxmod.problem(cvxmod.minimize(objective)) # create contraint for probability simplex #p.constr.append(beta |cvxmod.In| probsimp(F)) p.constr.append(cvxmod.sum(beta)==1.0) #p.constr.append(square(norm2(beta)) <= 1.0) p.constr.append(beta >= 0.0) # y f beta y f*beta y*f*beta # (N x N) (N x F) (F x 1) --> (N x N) (N x 1) --> (N x 1) p.constr.append(y * (f * beta) + chi >= rho) ###### set values f.value = out y.value = label_matrix norm.value = norm_fact p.solve(lpsolver=solver) weights = numpy.array(cvxmod.value(beta)) #print weights cvxmod.printval(chi) cvxmod.printval(beta) cvxmod.printval(rho) return p
def _train(self, train_data, param): """ training procedure using training examples and labels @param train_data: Data relevant to SVM training @type train_data: dict<str, list<instances> > @param param: Parameters for the training procedure @type param: ParameterSvm """ for task_id in train_data.keys(): print "task_id:", task_id # split data for training weak_learners and boosting (train_weak, train_boosting) = split_data(train_data, 4) # train on first part of dataset (evaluate on other) prepared_data_weak = PreparedMultitaskData(train_weak, shuffle=False) classifiers = self._inner_train(prepared_data_weak, param) # train on entire dataset prepared_data_final = PreparedMultitaskData(train_data, shuffle=False) final_classifiers = self._inner_train(prepared_data_final, param) print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" print "done training weak learners" ##################################################### # perform boosting and wrap things up ##################################################### # wrap up predictors for later use predictors = {} for task_name in train_boosting.keys(): instances = train_boosting[task_name] N = len(instances) F = len(classifiers) examples = [inst.example for inst in instances] labels = [inst.label for inst in instances] # dim = (F x N) out = cvxmod.zeros((N,F)) for i in xrange(F): svm = classifiers[i] tmp_out = self._predict_weak(svm, examples, prepared_data_weak.name_to_id(task_name), param) if param.flags["signum"]: out[:,i] = numpy.sign(tmp_out) else: out[:,i] = tmp_out if param.flags["boosting"] == "ones": weights = numpy.ones(F)/float(F) if param.flags["boosting"] == "L1": weights = solve_boosting(out, labels, param.transform, solver="glpk") if param.flags["boosting"] == "L2": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=False) if param.flags["boosting"] == "L2_reg": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=True) predictors[task_name] = (final_classifiers, weights, prepared_data_final.name_to_id(task_name), param) assert prepared_data_final.name_to_id(task_name)==prepared_data_weak.name_to_id(task_name), "name mappings don't match" ##################################################### # Some sanity checks ##################################################### # make sure we have the same keys (potentiall in a different order) sym_diff_keys = set(train_weak.keys()).symmetric_difference(set(predictors.keys())) assert len(sym_diff_keys)==0, "symmetric difference between keys non-empty: " + str(sym_diff_keys) return predictors
def solve_svm(out, labels, nu, solver): ''' solve boosting formulation used by gelher and nowozin @param out: matrix (N,F) of predictions (for each f_i) for all examples @param labels: vector (N,1) label for each example @param nu: regularization constant @param solver: which solver to use. options: 'mosek', 'glpk' ''' # get dimension N = out.size[0] F = out.size[1] assert N == len(labels), str(N) + " " + str(len(labels)) norm_fact = 1.0 / (nu * float(N)) print "normalization factor %f" % (norm_fact) # avoid point-wise product label_matrix = cvxmod.zeros((N, N)) for i in xrange(N): label_matrix[i, i] = labels[i] #### parameters f = cvxmod.param("f", N, F) y = cvxmod.param("y", N, N, symm=True) norm = cvxmod.param("norm", 1) #### varibales # rho rho = cvxmod.optvar("rho", 1) # dim = (N x 1) chi = cvxmod.optvar("chi", N) # dim = (F x 1) beta = cvxmod.optvar("beta", F) #objective = -rho + cvxmod.sum(chi) * norm_fact + square(norm2(beta)) objective = -rho + cvxmod.sum(chi) * norm_fact print objective # create problem p = cvxmod.problem(cvxmod.minimize(objective)) # create contraints for probability simplex #p.constr.append(beta |cvxmod.In| probsimp(F)) p.constr.append(cvxmod.sum(beta) == 1.0) p.constr.append(beta >= 0.0) p.constr.append(chi >= 0.0) # attempt to perform non-sparse boosting #p.constr.append(square(norm2(beta)) <= 1.0) # y f beta y f*beta y*f*beta # (N x N) (N x F) (F x 1) --> (N x N) (N x 1) --> (N x 1) p.constr.append(y * (f * beta) + chi >= rho) # set values for parameters f.value = out y.value = label_matrix norm.value = norm_fact print "solving problem" print "=============================================" print p print "=============================================" # start solver p.solve(lpsolver=solver) # print variables cvxmod.printval(chi) cvxmod.printval(beta) cvxmod.printval(rho) return numpy.array(cvxmod.value(beta))
def _train(self, train_data, param): """ training procedure using training examples and labels @param train_data: Data relevant to SVM training @type train_data: dict<str, list<instances> > @param param: Parameters for the training procedure @type param: ParameterSvm """ for task_id in train_data.keys(): print "task_id:", task_id # split data for training weak_learners and boosting (train_weak, train_boosting) = split_data(train_data, 4) # train on first part of dataset (evaluate on other) prepared_data_weak = PreparedMultitaskData(train_weak, shuffle=False) classifiers = self._inner_train(prepared_data_weak, param) # train on entire dataset prepared_data_final = PreparedMultitaskData(train_data, shuffle=False) final_classifiers = self._inner_train(prepared_data_final, param) print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" print "done training weak learners" ##################################################### # perform boosting and wrap things up ##################################################### # wrap up predictors for later use predictors = {} for task_name in train_boosting.keys(): instances = train_boosting[task_name] N = len(instances) F = len(classifiers) examples = [inst.example for inst in instances] labels = [inst.label for inst in instances] # dim = (F x N) out = cvxmod.zeros((N,F)) for i in xrange(F): svm = classifiers[i] tmp_out = self._predict_weak(svm, examples, prepared_data_weak.name_to_id(task_name)) if param.flags["signum"]: out[:,i] = numpy.sign(tmp_out) else: out[:,i] = tmp_out if param.flags["boosting"] == "ones": weights = numpy.ones(F)/float(F) if param.flags["boosting"] == "L1": weights = solve_boosting(out, labels, param.transform, solver="glpk") if param.flags["boosting"] == "L2": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=False) if param.flags["boosting"] == "L2_reg": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=True) predictors[task_name] = (final_classifiers, weights, prepared_data_final.name_to_id(task_name)) assert prepared_data_final.name_to_id(task_name)==prepared_data_weak.name_to_id(task_name), "name mappings don't match" ##################################################### # Some sanity checks ##################################################### # make sure we have the same keys (potentiall in a different order) sym_diff_keys = set(train_weak.keys()).symmetric_difference(set(predictors.keys())) assert len(sym_diff_keys)==0, "symmetric difference between keys non-empty: " + str(sym_diff_keys) return predictors
def _train(self, train_data, param): """ training procedure using training examples and labels @param train_data: Data relevant to SVM training @type train_data: dict<str, list<instances> > @param param: Parameters for the training procedure @type param: ParameterSvm """ # split data for training weak_learners and boosting (train_weak, train_boosting) = split_data(train_data, 4) for task_id in train_data.keys(): print "task_id:", task_id root = param.taxonomy.data # train on first part of dataset (evaluate on other) (classifiers, classifier_at_node) = self._inner_train(train_weak, param) # train on entire dataset (final_classifiers, final_classifier_at_node) = self._inner_train(train_data, param) ### print "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" print "done training weak learners" ##################################################### # perform boosting and wrap things up ##################################################### # wrap up predictors for later use predictors = {} for task_name in train_boosting.keys(): instances = train_boosting[task_name] # get ids of predecessor nodes node_names = [node.name for node in root.get_node(task_name).get_path_root()] node_names.append(task_name) print "node: %s --> %s" % (task_name, str(node_names)) N = len(instances) if param.flags["use_all_nodes"]: # use classifiers only from parent nodes F = len(classifiers) tmp_classifiers = classifiers tmp_final_classifiers = final_classifiers else: # use classifiers from all leaves F = len(node_names) tmp_classifiers = [] tmp_final_classifiers = [] examples = [inst.example for inst in instances] labels = [inst.label for inst in instances] # dim = (F x N) out = cvxmod.zeros((N,F)) for i in xrange(F): if param.flags["use_all_nodes"]: svm = classifiers[i] else: svm = classifier_at_node[node_names[i]] tmp_classifiers.append(svm) final_svm = final_classifier_at_node[node_names[i]] tmp_final_classifiers.append(final_svm) tmp_out = self._predict_weak(svm, examples, task_name) if param.flags["signum"]: out[:,i] = numpy.sign(tmp_out) else: out[:,i] = tmp_out if param.flags["boosting"] == "ones": weights = numpy.ones(F)/float(F) if param.flags["boosting"] == "L1": weights = solve_boosting(out, labels, param.transform, solver="glpk") if param.flags["boosting"] == "L2": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=False) if param.flags["boosting"] == "L2_reg": weights = solve_nu_svm(out, labels, param.transform, solver="glpk", reg=True) predictors[task_name] = (tmp_final_classifiers, weights) ##################################################### # Some sanity checks ##################################################### # make sure we have the same keys (potentiall in a different order) sym_diff_keys = set(train_weak.keys()).symmetric_difference(set(predictors.keys())) assert len(sym_diff_keys)==0, "symmetric difference between keys non-empty: " + str(sym_diff_keys) # save graph plot mypath = "/fml/ag-raetsch/share/projects/multitask/graphs/" filename = mypath + "graph_" + str(param.id) root.plot(filename)#, plot_cost=True, plot_B=True) return predictors
def _train(self, train_data, param): """ training procedure using training examples and labels @param train_data: Data relevant to SVM training @type train_data: dict<str, list<instances> > @param param: Parameters for the training procedure @type param: ParameterSvm """ # split for training weak_learners and boosting (train_weak, train_boosting) = split_data(train_data, 4) # merge data sets data = PreparedMultitaskData(train_weak, shuffle=True) # create shogun label lab = shogun_factory.create_labels(data.labels) ######################################################## print "creating a kernel:" ######################################################## # init seq handler pseudoseqs = SequencesHandler() classifiers = [] for pocket in pockets: print "creating normalizer" #import pdb #pdb.set_trace() normalizer = MultitaskKernelNormalizer(data.task_vector_nums) print "processing pocket", pocket # set similarity for task_name_lhs in data.get_task_names(): for task_name_rhs in data.get_task_names(): similarity = 0.0 for pseudo_seq_pos in pocket: similarity += float(pseudoseqs.get_similarity(task_name_lhs, task_name_rhs, pseudo_seq_pos-1)) # normalize similarity = similarity / float(len(pocket)) print "pocket %s (%s, %s) = %f" % (str(pocket), task_name_lhs, task_name_rhs, similarity) normalizer.set_task_similarity(data.name_to_id(task_name_lhs), data.name_to_id(task_name_rhs), similarity) print "creating empty kernel" kernel = shogun_factory.create_kernel(data.examples, param) print "setting normalizer" kernel.set_normalizer(normalizer) print "training SVM for pocket", pocket svm = self._train_single_svm(param, kernel, lab) classifiers.append(svm) print "done obtaining weak learners" # save additional info #self.additional_information["svm_objective"] = svm.get_objective() #self.additional_information["svm num sv"] = svm.get_num_support_vectors() #self.additional_information["post_weights"] = combined_kernel.get_subkernel_weights() #print self.additional_information ################################################## # combine weak learners for each task ################################################## # set constants some = 0.9 import cvxmod # wrap up predictors svms = {} # use a reference to the same svm several times for task_name in train_boosting.keys(): instances = train_boosting[task_name] N = len(instances) F = len(pockets) examples = [inst.example for inst in instances] labels = [inst.label for inst in instances] # dim = (F x N) out = cvxmod.zeros((N,F)) for i in xrange(F): svm = classifiers[i] tmp_out = self._predict_weak(svm, examples, data.name_to_id(task_name)) out[:,i] = numpy.sign(tmp_out) #out[:,i] = tmp_out #TODO: fix helper.save("/tmp/out_sparse", (out,labels)) pdb.set_trace() weights = solve_boosting(out, labels, some, solver="mosek") svms[task_name] = (data.name_to_id(task_name), svm) return svms
def solve_boosting(out, labels, nu, solver): ''' solve boosting formulation used by gelher and novozin @param out: matrix (N,F) of predictions (for each f_i) for all examples @param y: vector (N,1) label for each example @param p: regularization constant ''' N = out.size[0] F = out.size[1] assert(N==len(labels)) norm_fact = 1.0 / (nu * float(N)) print norm_fact label_matrix = cvxmod.zeros((N,N)) # avoid point-wise product for i in xrange(N): label_matrix[i,i] = labels[i] #### parameters f = cvxmod.param("f", N, F) y = cvxmod.param("y", N, N, symm=True) norm = cvxmod.param("norm", 1) #### varibales # rho rho = cvxmod.optvar("rho", 1) # dim = (N x 1) chi = cvxmod.optvar("chi", N) # dim = (F x 1) beta = cvxmod.optvar("beta", F) #objective = -rho + cvxmod.sum(chi) * norm_fact + square(norm2(beta)) objective = -rho + cvxmod.sum(chi) * norm_fact print objective # create problem p = cvxmod.problem(cvxmod.minimize(objective)) # create contraint for probability simplex #p.constr.append(beta |cvxmod.In| probsimp(F)) p.constr.append(cvxmod.sum(beta)==1.0) #p.constr.append(square(norm2(beta)) <= 1.0) p.constr.append(beta >= 0.0) # y f beta y f*beta y*f*beta # (N x N) (N x F) (F x 1) --> (N x N) (N x 1) --> (N x 1) p.constr.append(y * (f * beta) + chi >= rho) ###### set values f.value = out y.value = label_matrix norm.value = norm_fact p.solve(lpsolver=solver) weights = numpy.array(cvxmod.value(beta)) #print weights cvxmod.printval(chi) cvxmod.printval(beta) cvxmod.printval(rho) return p