def modelselection_grid_search_kernel(): num_subsets=3 num_vectors=20 dim_vectors=3 # create some (non-sense) data matrix=rand(dim_vectors, num_vectors) # create num_feautres 2-dimensional vectors features=RealFeatures() features.set_feature_matrix(matrix) # create labels, two classes labels=BinaryLabels(num_vectors) for i in range(num_vectors): labels.set_label(i, 1 if i%2==0 else -1) # create svm classifier=LibSVM() # splitting strategy splitting_strategy=StratifiedCrossValidationSplitting(labels, num_subsets) # accuracy evaluation evaluation_criterion=ContingencyTableEvaluation(ACCURACY) # cross validation class for evaluation in model selection cross=CrossValidation(classifier, features, labels, splitting_strategy, evaluation_criterion) cross.set_num_runs(1) # print all parameter available for modelselection # Dont worry if yours is not included, simply write to the mailing list classifier.print_modsel_params() # model parameter selection param_tree=create_param_tree() param_tree.print_tree() grid_search=GridSearchModelSelection(param_tree, cross) print_state=True best_combination=grid_search.select_model(print_state) print("best parameter(s):") best_combination.print_tree() best_combination.apply_to_machine(classifier) # larger number of runs to have tighter confidence intervals cross.set_num_runs(10) cross.set_conf_int_alpha(0.01) result=cross.evaluate() print("result: ") result.print_result() return 0
def serialization_svmlight_modular(num, dist, width, C): from shogun.IO import MSG_DEBUG from shogun.Features import RealFeatures, BinaryLabels, DNA, Alphabet from shogun.Kernel import WeightedDegreeStringKernel, GaussianKernel from shogun.Classifier import SVMLight from numpy import concatenate, ones from numpy.random import randn, seed import sys import types import random import bz2 try: import cPickle as pickle except ImportError: import pickle as pickle import inspect def save(filename, myobj): """ save object to file using pickle @param filename: name of destination file @type filename: str @param myobj: object to save (has to be pickleable) @type myobj: obj """ try: f = bz2.BZ2File(filename, 'wb') except IOError as details: sys.stderr.write('File ' + filename + ' cannot be written\n') sys.stderr.write(details) return pickle.dump(myobj, f, protocol=2) f.close() def load(filename): """ Load from filename using pickle @param filename: name of file to load from @type filename: str """ try: f = bz2.BZ2File(filename, 'rb') except IOError as details: sys.stderr.write('File ' + filename + ' cannot be read\n') sys.stderr.write(details) return myobj = pickle.load(f) f.close() return myobj ################################################## # set up toy data and svm traindata_real = concatenate((randn(2, num) - dist, randn(2, num) + dist), axis=1) testdata_real = concatenate((randn(2, num) - dist, randn(2, num) + dist), axis=1) trainlab = concatenate((-ones(num), ones(num))) testlab = concatenate((-ones(num), ones(num))) feats_train = RealFeatures(traindata_real) feats_test = RealFeatures(testdata_real) kernel = GaussianKernel(feats_train, feats_train, width) #kernel.io.set_loglevel(MSG_DEBUG) labels = BinaryLabels(trainlab) svm = SVMLight(C, kernel, labels) svm.train() #svm.io.set_loglevel(MSG_DEBUG) ################################################## # serialize to file fn = "serialized_svm.bz2" print("serializing SVM to file", fn) save(fn, svm) ################################################## # unserialize and sanity check print("unserializing SVM") svm2 = load(fn) print("comparing objectives") svm2.train() print("objective before serialization:", svm.get_objective()) print("objective after serialization:", svm2.get_objective()) print("comparing predictions") out = svm.apply(feats_test).get_labels() out2 = svm2.apply(feats_test).get_labels() # assert outputs are close for i in xrange(len(out)): assert abs(out[i] - out2[i] < 0.000001) print("all checks passed.") return True
elif ctype == 'kernel': feats = util.get_features(indata, 'kernel_') else: feats = util.get_features(indata, prefix) machine = _get_machine(indata, prefix, feats) try: fun = eval(indata[prefix + 'name']) except NameError, e: print "%s is disabled/unavailable!" % indata[prefix + 'name'] return False # cannot refactor into function, because labels is unrefed otherwise if indata.has_key(prefix + 'labels'): labels = BinaryLabels(double(indata[prefix + 'labels'])) if ctype == 'kernel': classifier = fun(indata[prefix + 'C'], machine, labels) elif ctype == 'linear': classifier = fun(indata[prefix + 'C'], feats['train'], labels) elif ctype == 'knn': classifier = fun(indata[prefix + 'k'], machine, labels) elif ctype == 'lda': classifier = fun(indata[prefix + 'gamma'], feats['train'], labels) elif ctype == 'perceptron': classifier = fun(feats['train'], labels) elif ctype == 'wdsvmocas': classifier = fun(indata[prefix + 'C'], indata[prefix + 'degree'], indata[prefix + 'degree'], feats['train'], labels) else: return False
def serialization_svmlight_modular(num, dist, width, C): from shogun.IO import MSG_DEBUG from shogun.Features import RealFeatures, BinaryLabels, DNA, Alphabet from shogun.Kernel import WeightedDegreeStringKernel, GaussianKernel from shogun.Classifier import SVMLight from numpy import concatenate, ones from numpy.random import randn, seed import sys import types import random import bz2 try: import cPickle as pickle except ImportError: import pickle as pickle import inspect def save(filename, myobj): """ save object to file using pickle @param filename: name of destination file @type filename: str @param myobj: object to save (has to be pickleable) @type myobj: obj """ try: f = bz2.BZ2File(filename, 'wb') except IOError as details: sys.stderr.write('File ' + filename + ' cannot be written\n') sys.stderr.write(details) return pickle.dump(myobj, f, protocol=2) f.close() def load(filename): """ Load from filename using pickle @param filename: name of file to load from @type filename: str """ try: f = bz2.BZ2File(filename, 'rb') except IOError as details: sys.stderr.write('File ' + filename + ' cannot be read\n') sys.stderr.write(details) return myobj = pickle.load(f) f.close() return myobj ################################################## seed(17) traindata_real = concatenate((randn(2, num) - dist, randn(2, num) + dist), axis=1) testdata_real = concatenate((randn(2, num) - dist, randn(2, num) + dist), axis=1) trainlab = concatenate((-ones(num), ones(num))) testlab = concatenate((-ones(num), ones(num))) feats_train = RealFeatures(traindata_real) feats_test = RealFeatures(testdata_real) kernel = GaussianKernel(feats_train, feats_train, width) #kernel.io.set_loglevel(MSG_DEBUG) labels = BinaryLabels(trainlab) svm = SVMLight(C, kernel, labels) svm.train() #svm.io.set_loglevel(MSG_DEBUG) ################################################## #print("labels:") #print(pickle.dumps(labels)) # #print("features") #print(pickle.dumps(feats_train)) # #print("kernel") #print(pickle.dumps(kernel)) # #print("svm") #print(pickle.dumps(svm)) # #print("#################################") fn = "serialized_svm.bz2" #print("serializing SVM to file", fn) save(fn, svm) #print("#################################") #print("unserializing SVM") svm2 = load(fn) #print("#################################") #print("comparing training") svm2.train() #print("objective before serialization:", svm.get_objective()) #print("objective after serialization:", svm2.get_objective()) return svm, svm.get_objective(), svm2, svm2.get_objective()
def classifier_non_separable_svm(n=100, distance=5, seed=None, C1=1, C2=100): ''' n is the number of examples per class and m is the number of examples per class that gets its label swapped to force non-linear separability C1 and C2 are the two regularization values used ''' from shogun.Features import RealFeatures, BinaryLabels # 2D data _DIM = 2 # To get the nice message that the perceptron has converged dummy = BinaryLabels() np.random.seed(seed) # Produce some (probably) linearly separable training data by hand # Two Gaussians at a far enough distance X = np.array(2 * np.random.randn(_DIM, n)) + distance Y = np.array(1.5 * np.random.randn(_DIM, n)) - distance label_train_twoclass = np.hstack((np.ones(n), -np.ones(n))) # The last 5 points of the positive class are closer to the negative examples X[:, -3:-1] = np.random.randn(_DIM, 2) - 0.5 * distance / 2 fm_train_real = np.hstack((X, Y)) # add a feature with all ones to learn implicitily a bias fm_train_real = np.vstack((fm_train_real, np.ones(2 * n))) feats_train = RealFeatures(fm_train_real) labels = BinaryLabels(label_train_twoclass) # Find limits for visualization x_min = min(np.min(X[0, :]), np.min(Y[0, :])) x_max = max(np.max(X[0, :]), np.max(Y[0, :])) # Train first SVM and plot its hyperplane svm1 = train_svm(feats_train, labels, C1) plot_hyperplane(svm1, x_min, x_max, 'g') # Train second SVM and plot its hyperplane svm2 = train_svm(feats_train, labels, C2) plot_hyperplane(svm2, x_min, x_max, 'y') # Plot the two-class data plt.scatter(X[0, :], X[1, :], s=40, marker='o', facecolors='none', edgecolors='b') plt.scatter(Y[0, :], Y[1, :], s=40, marker='s', facecolors='none', edgecolors='r') # Customize the plot y_min = min(np.min(X[1, :]), np.min(Y[1, :])) y_max = max(np.max(X[1, :]), np.max(Y[1, :])) plt.axis([x_min - 1, x_max + 1, y_min - 1, y_max + 1]) plt.title('SVM trade-off') plt.xlabel('x') plt.ylabel('y') plt.show() return svm1, svm2
def classifier_non_separable_svm(n=100, m=10, distance=5, seed=None): ''' n is the number of examples per class and m is the number of examples per class that gets its label swapped to force non-linear separability ''' from shogun.Features import RealFeatures, BinaryLabels from shogun.Classifier import LibLinear, L2R_L2LOSS_SVC # 2D data _DIM = 2 # To get the nice message that the perceptron has converged dummy = BinaryLabels() np.random.seed(seed) # Produce some (probably) linearly separable training data by hand # Two Gaussians at a far enough distance X = np.array(np.random.randn(_DIM, n)) + distance Y = np.array(np.random.randn(_DIM, n)) # The last five points of each class are swapped to force non-linear separable data label_train_twoclass = np.hstack( (np.ones(n - m), -np.ones(m), -np.ones(n - m), np.ones(m))) fm_train_real = np.hstack((X, Y)) feats_train = RealFeatures(fm_train_real) labels = BinaryLabels(label_train_twoclass) # Train linear SVM C = 1 epsilon = 1e-3 svm = LibLinear(C, feats_train, labels) svm.set_liblinear_solver_type(L2R_L2LOSS_SVC) svm.set_epsilon(epsilon) svm.set_bias_enabled(True) svm.train() # Get hyperplane parameters b = svm.get_bias() w = svm.get_w() # Find limits for visualization x_min = min(np.min(X[0, :]), np.min(Y[0, :])) x_max = max(np.max(X[0, :]), np.max(Y[0, :])) y_min = min(np.min(X[1, :]), np.min(Y[1, :])) y_max = max(np.max(X[1, :]), np.max(Y[1, :])) hx = np.linspace(x_min - 1, x_max + 1) hy = -w[1] / w[0] * hx plt.plot(hx, -1 / w[1] * (w[0] * hx + b), 'k', linewidth=2.0) # Plot the two-class data pos_idxs = label_train_twoclass == +1 plt.scatter(fm_train_real[0, pos_idxs], fm_train_real[1, pos_idxs], s=40, marker='o', facecolors='none', edgecolors='b') neg_idxs = label_train_twoclass == -1 plt.scatter(fm_train_real[0, neg_idxs], fm_train_real[1, neg_idxs], s=40, marker='s', facecolors='none', edgecolors='r') # Customize the plot plt.axis([x_min - 1, x_max + 1, y_min - 1, y_max + 1]) plt.title('SVM with non-linearly separable data') plt.xlabel('x') plt.ylabel('y') plt.show() return svm
def mkl_binclass_modular (fm_train_real=traindat,fm_test_real=testdat,fm_label_twoclass = label_traindat): ################################## # set up and train # create some poly train/test matrix tfeats = RealFeatures(fm_train_real) tkernel = PolyKernel(10,3) tkernel.init(tfeats, tfeats) K_train = tkernel.get_kernel_matrix() pfeats = RealFeatures(fm_test_real) tkernel.init(tfeats, pfeats) K_test = tkernel.get_kernel_matrix() # create combined train features feats_train = CombinedFeatures() feats_train.append_feature_obj(RealFeatures(fm_train_real)) # and corresponding combined kernel kernel = CombinedKernel() kernel.append_kernel(CustomKernel(K_train)) kernel.append_kernel(PolyKernel(10,2)) kernel.init(feats_train, feats_train) # train mkl labels = BinaryLabels(fm_label_twoclass) mkl = MKLClassification() # which norm to use for MKL mkl.set_mkl_norm(1) #2,3 # set cost (neg, pos) mkl.set_C(1, 1) # set kernel and labels mkl.set_kernel(kernel) mkl.set_labels(labels) # train mkl.train() #w=kernel.get_subkernel_weights() #kernel.set_subkernel_weights(w) ################################## # test # create combined test features feats_pred = CombinedFeatures() feats_pred.append_feature_obj(RealFeatures(fm_test_real)) # and corresponding combined kernel kernel = CombinedKernel() kernel.append_kernel(CustomKernel(K_test)) kernel.append_kernel(PolyKernel(10, 2)) kernel.init(feats_train, feats_pred) # and classify mkl.set_kernel(kernel) mkl.apply() return mkl.apply(),kernel
def modelselection_grid_search_liblinear_modular(traindat=traindat, label_traindat=label_traindat ): from shogun.Evaluation import CrossValidation, CrossValidationResult from shogun.Evaluation import ContingencyTableEvaluation, ACCURACY from shogun.Evaluation import StratifiedCrossValidationSplitting from shogun.ModelSelection import GridSearchModelSelection from shogun.ModelSelection import ModelSelectionParameters, R_EXP from shogun.ModelSelection import ParameterCombination from shogun.Features import BinaryLabels from shogun.Features import RealFeatures from shogun.Classifier import LibLinear, L2R_L2LOSS_SVC # build parameter tree to select C1 and C2 param_tree_root = ModelSelectionParameters() c1 = ModelSelectionParameters("C1") param_tree_root.append_child(c1) c1.build_values(-2.0, 2.0, R_EXP) c2 = ModelSelectionParameters("C2") param_tree_root.append_child(c2) c2.build_values(-2.0, 2.0, R_EXP) # training data features = RealFeatures(traindat) labels = BinaryLabels(label_traindat) # classifier classifier = LibLinear(L2R_L2LOSS_SVC) # print all parameter available for modelselection # Dont worry if yours is not included but, write to the mailing list #classifier.print_modsel_params() # splitting strategy for cross-validation splitting_strategy = StratifiedCrossValidationSplitting(labels, 10) # evaluation method evaluation_criterium = ContingencyTableEvaluation(ACCURACY) # cross-validation instance cross_validation = CrossValidation(classifier, features, labels, splitting_strategy, evaluation_criterium) cross_validation.set_autolock(False) # model selection instance model_selection = GridSearchModelSelection(param_tree_root, cross_validation) # perform model selection with selected methods #print "performing model selection of" #param_tree_root.print_tree() best_parameters = model_selection.select_model() # print best parameters #print "best parameters:" #best_parameters.print_tree() # apply them and print result best_parameters.apply_to_machine(classifier) result = cross_validation.evaluate()
def solve(self, C, all_xt, all_lt, task_indicator, M, L): """ wrap shogun solver with same interface as others """ xt = numpy.array(all_xt) lt = numpy.array(all_lt) tt = numpy.array(task_indicator, dtype=numpy.int32) tsm = numpy.array(M) laplacian = numpy.array(L) print "task_sim:", tsm num_tasks = L.shape[0] # sanity checks assert len(xt) == len(lt) == len(tt) assert M.shape == L.shape assert num_tasks == len(set(tt)) # set up shogun objects if type(xt[0]) == str or type(xt[0]) == numpy.string_: feat = create_hashed_features_wdk(xt, 8) else: feat = RealFeatures(xt.T) lab = BinaryLabels(lt) # set up machinery svm = LibLinearMTL() svm.io.set_loglevel(MSG_DEBUG) svm.set_epsilon(self.eps) svm.set_C(C, C) svm.set_bias_enabled(False) # set MTL stuff svm.set_task_indicator_lhs(tt) svm.set_task_indicator_rhs(tt) svm.set_num_tasks(num_tasks) svm.set_use_cache(False) #print "setting sparse matrix!" tsm_sp = csc_matrix(tsm) svm.set_task_similarity_matrix(tsm_sp) #svm.set_task_similarity_matrix(tsm) svm.set_graph_laplacian(laplacian) # invoke training svm.set_labels(lab) # how often do we like to compute objective etc svm.set_record_interval(self.record_interval) svm.set_min_interval(self.min_interval) svm.set_max_iterations(10000000) # start training start_time = time.time() svm.train(feat) if self.record_variables: self.final_train_time = time.time() - start_time print "total training time:", self.final_train_time, "seconds" self.primal_objectives = svm.get_primal_objectives() self.dual_objectives = svm.get_dual_objectives() self.train_times = svm.get_training_times() print "computing objectives one last time" self.final_primal_obj = svm.compute_primal_obj() self.final_dual_obj = svm.compute_dual_obj() print "obj primal", self.final_primal_obj print "obj dual", self.final_dual_obj print "actual duality gap:", self.final_primal_obj - self.final_dual_obj #print "V", svm.get_V() self.V = svm.get_V() self.W = svm.get_W() self.alphas = svm.get_alphas() # get model parameters #V = svm.get_W().T if self.sanity_check: print "comparing to python implementation" #dual_obj_python = compute_dual_objective(alphas, xt, lt, task_indicator, M) #print "dual obj python", dual_obj_python #print "dual obj C++", dual_obj #print alphas #W = alphas_to_w(alphas, xt, lt, task_indicator, M) #print W #primal_obj = compute_primal_objective(W.reshape(W.shape[0] * W.shape[1]), C, xt, lt, task_indicator, L) #print "python primal", primal_obj # compare dual obj #return objectives#, train_times return True
def solve(self, C, all_xt, all_lt, task_indicator, M, L): """ implementation using multitask kernel """ xt = numpy.array(all_xt) lt = numpy.array(all_lt) tt = numpy.array(task_indicator, dtype=numpy.int32) tsm = numpy.array(M) print "task_sim:", tsm num_tasks = L.shape[0] # sanity checks assert len(xt) == len(lt) == len(tt) assert M.shape == L.shape assert num_tasks == len(set(tt)) # set up shogun objects if type(xt[0]) == numpy.string_: feat = StringCharFeatures(DNA) xt = [str(a) for a in xt] feat.set_features(xt) base_kernel = WeightedDegreeStringKernel(feat, feat, 8) else: feat = RealFeatures(xt.T) base_kernel = LinearKernel(feat, feat) lab = BinaryLabels(lt) # set up normalizer normalizer = MultitaskKernelNormalizer(tt.tolist()) for i in xrange(num_tasks): for j in xrange(num_tasks): normalizer.set_task_similarity(i, j, M[i, j]) print "num of unique tasks: ", normalizer.get_num_unique_tasks( task_indicator) # set up kernel base_kernel.set_cache_size(4000) base_kernel.set_normalizer(normalizer) base_kernel.init_normalizer() # set up svm svm = SVMLight() #LibSVM() svm.set_epsilon(self.eps) #SET THREADS TO 1 #print "reducing num threads to one" #segfaults #svm.parallel.set_num_threads(1) #print "using one thread" # how often do we like to compute objective etc svm.set_record_interval(self.record_interval) svm.set_min_interval(self.min_interval) #svm.set_target_objective(target_obj) svm.set_linadd_enabled(False) svm.set_batch_computation_enabled(False) #svm.set_shrinking_enabled(False) svm.io.set_loglevel(MSG_DEBUG) svm.set_C(C, C) svm.set_bias_enabled(False) # prepare for training svm.set_labels(lab) svm.set_kernel(base_kernel) # train svm svm.train() if self.record_variables: print "recording variables" self.dual_objectives = [-obj for obj in svm.get_dual_objectives()] self.train_times = svm.get_training_times() # get model parameters sv_idx = svm.get_support_vectors() sparse_alphas = svm.get_alphas() assert len(sv_idx) == len(sparse_alphas) # compute dense alpha (remove label) self.alphas = numpy.zeros(len(xt)) for id_sparse, id_dense in enumerate(sv_idx): self.alphas[id_dense] = sparse_alphas[id_sparse] * lt[id_dense] # print alphas W = alphas_to_w(self.alphas, xt, lt, task_indicator, M) self.W = W # self.final_primal_obj = compute_primal_objective( W.reshape(W.shape[0] * W.shape[1]), C, all_xt, all_lt, task_indicator, L) print "MTK duality gap:", self.dual_objectives[ -1] - self.final_primal_obj return True
trainData = trainData.reshape(-1, 10000) f.close() f = open(os.path.dirname(__file__) + '../data/arcene_train.label') trainLabel = np.fromfile(f, dtype=np.int32, sep=' ') f.close() # Load test data. f = open(os.path.dirname(__file__) + '../data/arcene_test.data') testData = np.fromfile(f, dtype=np.float64, sep=' ') testData = testData.reshape(-1, 10000) f.close() f = open(os.path.dirname(__file__) + '../data/arcene_test.label') testLabel = np.fromfile(f, dtype=np.float64, sep=' ') f.close() # Construct a KNN classifier with a neighborhood size of 9. feat = RealFeatures(trainData.T) distance = EuclideanDistance(feat, feat) labels = BinaryLabels(trainLabel.astype(np.float64)) testFeat = RealFeatures(testData.T) knn = KNN(9, distance, labels) knn.train() # Predict the classification. output = knn.apply(testFeat).get_labels() # Validate the classification. print output == testLabel