def fit(self, bags, y): """ @param bags : a sequence of n bags; each bag is an m-by-k array-like object containing m instances with k features @param y : an array-like object of length n containing -1/+1 labels """ self._bags = map(np.asmatrix, bags) bs = BagSplitter(self._bags, np.asmatrix(y).reshape((-1, 1))) self._X = bs.instances Ln = bs.L_n Lp = bs.L_p Xp = bs.X_p m = Ln + Xp if self.scale_C: C = self.C / float(len(self._bags)) else: C = self.C K = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)(self._X, self._X) new_classes = np.matrix(np.vstack([-np.ones((Ln, 1)), np.ones((Xp, 1))])) self._y = new_classes D = spdiag(new_classes) setup = list(self._setup_svm(new_classes, new_classes, C))[1:] setup[0] = np.matrix([0]) qp = IterativeQP(*setup) c = cvxmat(np.hstack([np.zeros(Lp + 1), np.ones(Xp + Ln)])) b = cvxmat(np.ones((Xp, 1))) A = spz(Xp, Lp + 1 + Xp + Ln) for row, (i, j) in enumerate(slices(bs.pos_groups)): A[row, i:j] = 1.0 bottom_left = sparse(t([[-spI(Lp), spz(Lp)], [spz(m, Lp), spz(m)]])) bottom_right = sparse([spz(Lp, m), -spI(m)]) inst_cons = sparse(t([[spz(Xp, Lp), -spo(Xp)], [spz(Ln, Lp), spo(Ln)]])) G = sparse(t([[inst_cons, -spI(m)], [bottom_left, bottom_right]])) h = cvxmat(np.vstack([-np.ones((Xp, 1)), np.zeros((Ln + Lp + m, 1))])) def to_V(upsilon): bot = np.zeros((Xp, Lp)) for row, (i, j) in enumerate(slices(bs.pos_groups)): bot[row, i:j] = upsilon.flat[i:j] return sp.bmat([[sp.eye(Ln, Ln), None], [None, sp.coo_matrix(bot)]]) class MICACCCP(CCCP): def bailout(cself, alphas, upsilon, svm): return svm def iterate(cself, alphas, upsilon, svm): V = to_V(upsilon) cself.mention('Update QP...') qp.update_H(D * V * K * V.T * D) cself.mention('Solve QP...') alphas, obj = qp.solve(self.verbose) svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p, verbose=self.verbose, sv_cutoff=self.sv_cutoff) svm._X = self._X svm._y = self._y svm._V = V svm._alphas = alphas svm._objective = obj svm._compute_separator(K) svm._K = K cself.mention('Update LP...') for row, (i, j) in enumerate(slices(bs.pos_groups)): G[row, i:j] = cvxmat(-svm._dotprods[Ln + i: Ln + j].T) h[Xp: Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln])) cself.mention('Solve LP...') sol, _ = linprog(c, G, h, A, b, verbose=self.verbose) new_upsilon = sol[:Lp] if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)): return None, svm return {'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm}, None best_obj = float('inf') best_svm = None for rr in range(self.restarts + 1): if rr == 0: if self.verbose: print 'Non-random start...' upsilon0 = np.matrix(np.vstack([np.ones((size, 1)) / float(size) for size in bs.pos_groups])) else: if self.verbose: print 'Random restart %d of %d...' % (rr, self.restarts) upsilon0 = np.matrix(np.vstack([rand_convex(size).T for size in bs.pos_groups])) cccp = MICACCCP(verbose=self.verbose, alphas=None, upsilon=upsilon0, svm=None, max_iters=self.max_iters) svm = cccp.solve() if svm is not None: obj = float(svm._objective) if obj < best_obj: best_svm = svm best_obj = obj if best_svm is not None: self._V = best_svm._V self._alphas = best_svm._alphas self._objective = best_svm._objective self._compute_separator(best_svm._K) self._bag_predictions = self.predict(self._bags)
def fit(self, bags, y): """ @param bags : a sequence of n bags; each bag is an m-by-k array-like object containing m instances with k features @param y : an array-like object of length n containing -1/+1 labels """ def transform(mx): """ Transform into np.matrix if array/list ignore scipy.sparse matrix """ if issparse(mx): return mx.todense() return np.asmatrix(mx) self._bags = [transform(bag) for bag in bags] y = np.asmatrix(y).reshape((-1, 1)) bs = BagSplitter(self._bags, y) best_obj = float('inf') best_svm = None for rr in range(self.restarts + 1): if rr == 0: if self.verbose: print 'Non-random start...' pos_bag_avgs = np.vstack([np.average(bag, axis=0) for bag in bs.pos_bags]) else: if self.verbose: print 'Random restart %d of %d...' % (rr, self.restarts) pos_bag_avgs = np.vstack([rand_convex(len(bag)) * bag for bag in bs.pos_bags]) intial_instances = np.vstack([bs.neg_instances, pos_bag_avgs]) classes = np.vstack([-np.ones((bs.L_n, 1)), np.ones((bs.X_p, 1))]) # Setup SVM and QP if self.scale_C: C = self.C / float(len(intial_instances)) else: C = self.C setup = self._setup_svm(intial_instances, classes, C) K = setup[0] qp = IterativeQP(*setup[1:]) # Fix Gx <= h neg_cons = spzeros(bs.X_n, bs.L_n) for b, (l, u) in enumerate(slices(bs.neg_groups)): neg_cons[b, l:u] = 1.0 pos_cons = speye(bs.X_p) bot_left = spzeros(bs.X_p, bs.L_n) top_right = spzeros(bs.X_n, bs.X_p) half_cons = sparse([[neg_cons, bot_left], [top_right, pos_cons]]) qp.G = sparse([-speye(bs.X_p + bs.L_n), half_cons]) qp.h = cvxmat(np.vstack([np.zeros((bs.X_p + bs.L_n, 1)), C * np.ones((bs.X_p + bs.X_n, 1))])) # Precompute kernel for all positive instances kernel = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p) K_all = kernel(bs.instances, bs.instances) neg_selectors = np.array(range(bs.L_n)) class MISVMCCCP(CCCP): def bailout(cself, svm, selectors, instances, K): return svm def iterate(cself, svm, selectors, instances, K): cself.mention('Training SVM...') alphas, obj = qp.solve(cself.verbose) # Construct SVM from solution svm = SVM(kernel=self.kernel, gamma=self.gamma, p=self.p, verbose=self.verbose, sv_cutoff=self.sv_cutoff) svm._X = instances svm._y = classes svm._alphas = alphas svm._objective = obj svm._compute_separator(K) svm._K = K cself.mention('Recomputing classes...') p_confs = svm.predict(bs.pos_instances) pos_selectors = bs.L_n + np.array([l + np.argmax(p_confs[l:u]) for l, u in slices(bs.pos_groups)]) new_selectors = np.hstack([neg_selectors, pos_selectors]) if selectors is None: sel_diff = len(new_selectors) else: sel_diff = np.nonzero(new_selectors - selectors)[0].size cself.mention('Selector differences: %d' % sel_diff) if sel_diff == 0: return None, svm elif sel_diff > 5: # Clear results to avoid a # bad starting point in # the next iteration qp.clear_results() cself.mention('Updating QP...') indices = (new_selectors,) K = K_all[indices].T[indices].T D = spdiag(classes) qp.update_H(D * K * D) return {'svm': svm, 'selectors': new_selectors, 'instances': bs.instances[indices], 'K': K}, None cccp = MISVMCCCP(verbose=self.verbose, svm=None, selectors=None, instances=intial_instances, K=K, max_iters=self.max_iters) svm = cccp.solve() if svm is not None: obj = float(svm._objective) if obj < best_obj: best_svm = svm best_obj = obj if best_svm is not None: self._X = best_svm._X self._y = best_svm._y self._alphas = best_svm._alphas self._objective = best_svm._objective self._compute_separator(best_svm._K)
def fit(self, bags, y): """ @param bags : a sequence of n bags; each bag is an m-by-k array-like object containing m instances with k features @param y : an array-like object of length n containing -1/+1 labels """ def transform(mx): """ Transform into np.matrix if array/list ignore scipy.sparse matrix """ if issparse(mx): return mx.todense() return np.asmatrix(mx) self._bags = [transform(bag) for bag in bags] y = np.asmatrix(y).reshape((-1, 1)) bs = BagSplitter(self._bags, y) best_obj = float('inf') best_svm = None for rr in range(self.restarts + 1): if rr == 0: if self.verbose: print('Non-random start...') pos_bag_avgs = np.vstack([np.average(bag, axis=0) for bag in bs.pos_bags]) else: if self.verbose: print('Random restart %d of %d...' % (rr, self.restarts)) pos_bag_avgs = np.vstack([rand_convex(len(bag)) * bag for bag in bs.pos_bags]) intial_instances = np.vstack([bs.neg_instances, pos_bag_avgs]) classes = np.vstack([-np.ones((bs.L_n, 1)), np.ones((bs.X_p, 1))]) # Setup SVM and QP if self.scale_C: C = self.C / float(len(intial_instances)) else: C = self.C setup = self._setup_svm(intial_instances, classes, C) K = setup[0] qp = IterativeQP(*setup[1:]) # Fix Gx <= h neg_cons = spzeros(bs.X_n, bs.L_n) for b, (l, u) in enumerate(slices(bs.neg_groups)): neg_cons[b, l:u] = 1.0 pos_cons = speye(bs.X_p) bot_left = spzeros(bs.X_p, bs.L_n) top_right = spzeros(bs.X_n, bs.X_p) half_cons = sparse([[neg_cons, bot_left], [top_right, pos_cons]]) qp.G = sparse([-speye(bs.X_p + bs.L_n), half_cons]) qp.h = cvxmat(np.vstack([np.zeros((bs.X_p + bs.L_n, 1)), C * np.ones((bs.X_p + bs.X_n, 1))])) # Precompute kernel for all positive instances kernel = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p) K_all = kernel(bs.instances, bs.instances) neg_selectors = np.array(range(bs.L_n)) class MISVMCCCP(CCCP): def bailout(cself, svm, selectors, instances, K): return svm def iterate(cself, svm, selectors, instances, K): cself.mention('Training SVM...') alphas, obj = qp.solve(cself.verbose) # Construct SVM from solution svm = SVM(kernel=self.kernel, gamma=self.gamma, p=self.p, verbose=self.verbose, sv_cutoff=self.sv_cutoff) svm._X = instances svm._y = classes svm._alphas = alphas svm._objective = obj svm._compute_separator(K) svm._K = K cself.mention('Recomputing classes...') p_confs = svm.predict(bs.pos_instances) pos_selectors = bs.L_n + np.array([l + np.argmax(p_confs[l:u]) for l, u in slices(bs.pos_groups)]) new_selectors = np.hstack([neg_selectors, pos_selectors]) if selectors is None: sel_diff = len(new_selectors) else: sel_diff = np.nonzero(new_selectors - selectors)[0].size cself.mention('Selector differences: %d' % sel_diff) if sel_diff == 0: return None, svm elif sel_diff > 5: # Clear results to avoid a # bad starting point in # the next iteration qp.clear_results() cself.mention('Updating QP...') indices = (new_selectors,) K = K_all[indices].T[indices].T D = spdiag(classes) qp.update_H(D * K * D) return {'svm': svm, 'selectors': new_selectors, 'instances': bs.instances[indices], 'K': K}, None cccp = MISVMCCCP(verbose=self.verbose, svm=None, selectors=None, instances=intial_instances, K=K, max_iters=self.max_iters) svm = cccp.solve() if svm is not None: obj = float(svm._objective) if obj < best_obj: best_svm = svm best_obj = obj if best_svm is not None: self._X = best_svm._X self._y = best_svm._y self._alphas = best_svm._alphas self._objective = best_svm._objective self._compute_separator(best_svm._K)
def fit(self, bags, y): """ @param bags : a sequence of n bags; each bag is an m-by-k array-like object containing m instances with k features @param y : an array-like object of length n containing -1/+1 labels """ self._bags = map(np.asmatrix, bags) bs = BagSplitter(self._bags, np.asmatrix(y).reshape((-1, 1))) self._X = bs.instances Ln = bs.L_n Lp = bs.L_p Xp = bs.X_p m = Ln + Xp if self.scale_C: C = self.C / float(len(self._bags)) else: C = self.C K = kernel_by_name(self.kernel, gamma=self.gamma, p=self.p)(self._X, self._X) new_classes = np.matrix( np.vstack([-np.ones((Ln, 1)), np.ones((Xp, 1))])) self._y = new_classes D = spdiag(new_classes) setup = list(self._setup_svm(new_classes, new_classes, C))[1:] setup[0] = np.matrix([0]) qp = IterativeQP(*setup) c = cvxmat(np.hstack([np.zeros(Lp + 1), np.ones(Xp + Ln)])) b = cvxmat(np.ones((Xp, 1))) A = spz(Xp, Lp + 1 + Xp + Ln) for row, (i, j) in enumerate(slices(bs.pos_groups)): A[row, i:j] = 1.0 bottom_left = sparse(t([[-spI(Lp), spz(Lp)], [spz(m, Lp), spz(m)]])) bottom_right = sparse([spz(Lp, m), -spI(m)]) inst_cons = sparse(t([[spz(Xp, Lp), -spo(Xp)], [spz(Ln, Lp), spo(Ln)]])) G = sparse(t([[inst_cons, -spI(m)], [bottom_left, bottom_right]])) h = cvxmat(np.vstack([-np.ones((Xp, 1)), np.zeros((Ln + Lp + m, 1))])) def to_V(upsilon): bot = np.zeros((Xp, Lp)) for row, (i, j) in enumerate(slices(bs.pos_groups)): bot[row, i:j] = upsilon.flat[i:j] return sp.bmat([[sp.eye(Ln, Ln), None], [None, sp.coo_matrix(bot)]]) class MICACCCP(CCCP): def bailout(cself, alphas, upsilon, svm): return svm def iterate(cself, alphas, upsilon, svm): V = to_V(upsilon) cself.mention('Update QP...') qp.update_H(D * V * K * V.T * D) cself.mention('Solve QP...') alphas, obj = qp.solve(self.verbose) svm = MICA(kernel=self.kernel, gamma=self.gamma, p=self.p, verbose=self.verbose, sv_cutoff=self.sv_cutoff) svm._X = self._X svm._y = self._y svm._V = V svm._alphas = alphas svm._objective = obj svm._compute_separator(K) svm._K = K cself.mention('Update LP...') for row, (i, j) in enumerate(slices(bs.pos_groups)): G[row, i:j] = cvxmat(-svm._dotprods[Ln + i:Ln + j].T) h[Xp:Xp + Ln] = cvxmat(-(1 + svm._dotprods[:Ln])) cself.mention('Solve LP...') sol, _ = linprog(c, G, h, A, b, verbose=self.verbose) new_upsilon = sol[:Lp] if cself.check_tolerance(np.linalg.norm(upsilon - new_upsilon)): return None, svm return { 'alphas': alphas, 'upsilon': new_upsilon, 'svm': svm }, None best_obj = float('inf') best_svm = None for rr in range(self.restarts + 1): if rr == 0: if self.verbose: print 'Non-random start...' upsilon0 = np.matrix( np.vstack([ np.ones((size, 1)) / float(size) for size in bs.pos_groups ])) else: if self.verbose: print 'Random restart %d of %d...' % (rr, self.restarts) upsilon0 = np.matrix( np.vstack([rand_convex(size).T for size in bs.pos_groups])) cccp = MICACCCP(verbose=self.verbose, alphas=None, upsilon=upsilon0, svm=None, max_iters=self.max_iters) svm = cccp.solve() if svm is not None: obj = float(svm._objective) if obj < best_obj: best_svm = svm best_obj = obj if best_svm is not None: self._V = best_svm._V self._alphas = best_svm._alphas self._objective = best_svm._objective self._compute_separator(best_svm._K) self._bag_predictions = self.predict(self._bags)