def test_logistic(self) -> None: """Test domain for logistic. """ expr = cp.logistic(self.a) self.a.value = 2 self.assertAlmostEqual(expr.grad[self.a], np.exp(2) / (1 + np.exp(2))) self.a.value = 3 self.assertAlmostEqual(expr.grad[self.a], np.exp(3) / (1 + np.exp(3))) self.a.value = -1 self.assertAlmostEqual(expr.grad[self.a], np.exp(-1) / (1 + np.exp(-1))) expr = cp.logistic(self.x) self.x.value = [3, 4] val = np.zeros((2, 2)) + np.diag(np.exp([3, 4]) / (1 + np.exp([3, 4]))) self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), val) expr = cp.logistic(self.x) self.x.value = [-1e-9, 4] val = np.zeros( (2, 2)) + np.diag(np.exp([-1e-9, 4]) / (1 + np.exp([-1e-9, 4]))) self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), val) expr = cp.logistic(self.A) self.A.value = [[1, 2], [3, 4]] val = np.zeros((4, 4)) + np.diag( np.exp([1, 2, 3, 4]) / (1 + np.exp([1, 2, 3, 4]))) self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), val)
def switch(self): if self.phi_name == 'logistic': self.cvx_phi = lambda z: cvx.logistic(-z) # / math.log(2, math.e) elif self.phi_name == 'hinge': self.cvx_phi = lambda z: cvx.pos(1 - z) elif self.phi_name == 'squared': self.cvx_phi = lambda z: cvx.square(-z) elif self.phi_name == 'exponential': self.cvx_phi = lambda z: cvx.exp(-z) else: logger.error('%s is not include' % self.phi_name) logger.info('Logistic is the default setting') self.cvx_phi = lambda z: cvx.logistic(-z) # / math.log(2, math.e) if self.kappa_name == 'logistic': self.cvx_kappa = lambda z: cvx.logistic(z) # / math.log(2, math.e) self.psi_kappa = lambda mu: ((1 + mu) * math.log(1 + mu) + (1 - mu) * math.log(3 - mu)) / 2 elif self.kappa_name == 'hinge': self.cvx_kappa = lambda z: cvx.pos(1 + z) self.psi_kappa = lambda mu: mu elif self.kappa_name == 'squared': self.cvx_kappa = lambda z: cvx.square(1 + z) self.psi_kappa = lambda mu: mu**2 elif self.kappa_name == 'exponential': self.cvx_kappa = lambda z: cvx.exp(z) self.psi_kappa = lambda mu: 1 - math.sqrt(1 - mu**2) else: logger.error('%s is not include' % self.kappa_name) logger.info('hinge is the default setting') self.cvx_kappa = lambda z: cvx.pos(1 + z) self.psi_kappa = lambda mu: mu if self.delta_name == 'logistic': self.cvx_delta = lambda z: 1 - cvx.logistic( -z) # / math.log(2, math.e) self.psi_delta = lambda mu: ((1 + mu) * math.log(1 + mu) + (1 - mu) * math.log(1 - mu)) / 2 elif self.delta_name == 'hinge': self.cvx_delta = lambda z: 1 - cvx.pos(1 - z) self.psi_delta = lambda mu: mu elif self.delta_name == 'squared': self.cvx_delta = lambda z: 1 - cvx.square(1 - z) self.psi_delta = lambda mu: mu**2 elif self.delta_name == 'exponential': self.cvx_delta = lambda z: 1 - cvx.exp(-z) self.psi_delta = lambda mu: 1 - math.sqrt(1 - mu**2) else: logger.error('%s is not include' % self.delta_name) logger.info('hinge is the default setting') self.cvx_delta = lambda z: cvx.pos(1 + z) self.psi_delta = lambda mu: mu
def switch(self): if self.phi_name == 'logistic': self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2.0, math.e) elif self.phi_name == 'hinge': self.cvx_phi = lambda z: cvx.pos(1.0 - z) elif self.phi_name == 'squared': self.cvx_phi = lambda z: cvx.square(-z) elif self.phi_name == 'exponential': self.cvx_phi = lambda z: cvx.exp(-z) else: logger.error('%s is not include' % self.phi_name) logger.info('Logistic is the default setting') self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2, math.e)
def _train(self, data, label, delta): pos_idx = label == 1 neg_idx = label == -1 pos_num = sum(pos_idx) P_data_size = 0 for i in range(self.s, self.chunk_count - 1): P_data_size += sum(self.stored_label[i] == 1) if pos_num + P_data_size > delta: self.s += 1 P_data = array([]).reshape(-1, self.fea_num) ts = array([]) for i in range(self.s, self.chunk_count): P_data = r_[P_data, self.stored_data[i][self.stored_label[i] == 1]] ts = r_[ts, i * ones(sum(self.stored_label[i] == 1))] N_data = data[neg_idx] pos_num = P_data.shape[0] neg_num = N_data.shape[0] rand_idx = random.permutation(pos_num) P_data = P_data[rand_idx] ts = ts[rand_idx].astype(int) N_data = N_data[random.permutation(neg_num)] pos_train_num = int(self.train_ratio * pos_num) neg_train_num = int(self.train_ratio * neg_num) train_data = r_[P_data[:pos_train_num], N_data[:neg_train_num]] train_label = r_[ones(pos_train_num), -ones(neg_train_num)] train_ts = ts[:pos_train_num] hold_data = r_[P_data[pos_train_num:], N_data[neg_train_num:]] hold_label = r_[ones(pos_num - pos_train_num), -ones(neg_num - neg_train_num)] hold_pred = zeros([hold_label.size, self.fea_group_num]) self.ensemble = list() for fea_comb_i in range(self.fea_group_num): self._learnH(train_data[:, self.fea_comb[fea_comb_i]], train_label, train_ts) hold_pred = self._predict_base(hold_data) # solve convex optimization problem c = ones_like(hold_label) c[hold_label == 1] = sum(hold_label == -1) / sum(hold_label == 1) w = cvxpy.Variable(self.fea_group_num) obj = cvxpy.Minimize( c.T * cvxpy.logistic(-cvxpy.mul_elemwise(hold_label, hold_pred * w))) constraints = [cvxpy.sum_entries(w) == 1, w >= 0] prob = cvxpy.Problem(obj, constraints) try: prob.solve() except (cvxpy.error.SolverError): prob.solve(solver=cvxpy.CVXOPT) self.wd = array(w.value).squeeze()
def Log_SVM(absDataOrigin, absLabels, cmpDataOrigin, cmpLabels, absWeight, lamda): # This function combined both logistic regression for absolute labels and SVM for comparison labels. # Parameter: # ------------ # absDataOrigin : N by d numpy matrix where N the number of absolute label data and d is the dimension of data # abslabels : (N,) numpy array, +1 means positive label and -1 represents negative labels # lamda : weight on L1 penalty. Large lamda would have more zeros in beta. # Return: # ------------ # beta : the logistic regression model parameter # const : the logistic regression global constant. cmpWeight = 1.0 - absWeight absN, d = np.shape(absDataOrigin) cmpN, _ = np.shape(cmpDataOrigin) beta = cp.Variable(d) const = cp.Variable(1) objective = absWeight*cp.sum_entries(cp.logistic(cp.mul_elemwise(absLabels, absDataOrigin*beta+const))\ -cp.mul_elemwise(absLabels, absDataOrigin*beta+const))+ \ cmpWeight * cp.sum_entries(cp.pos(1 - cp.mul_elemwise(cmpLabels, cmpDataOrigin * beta + const))) \ + lamda * cp.norm(beta, 1) prob = cp.Problem(cp.Minimize(objective)) prob.solve(solver=cp.SCS) return beta.value, const.value
def optimize_logistic_regression(X, Y): beta = cp.Variable(n) lambd = cp.Parameter(nonneg=True) log_likelihood = cp.sum(cp.multiply(Y, X @ beta) - cp.logistic(X @ beta)) problem = cp.Problem( cp.Maximize(log_likelihood / n - lambd * cp.norm(beta, 1))) return problem
def logreg_cvxpy(X, y, alpha, idx_train, idx_val): alpha = float(alpha) assert np.all(np.unique(y) == np.array([-1, 1])) Xtrain, Xtest, ytrain, ytest = map( torch.from_numpy, [X[idx_train, :], X[idx_val], y[idx_train], y[idx_val]]) n_samples_train, n_features = Xtrain.shape # set up variables and parameters beta_cp = cp.Variable(n_features) alpha_cp = cp.Parameter(nonneg=True) # set up objective loss = cp.sum(cp.logistic(cp.multiply(-ytrain, Xtrain @ beta_cp))) / n_samples_train reg = alpha_cp * cp.norm(beta_cp, 1) objective = loss + reg # define problem problem = cp.Problem(cp.Minimize(objective)) assert problem.is_dpp() # solve problem layer = CvxpyLayer(problem, parameters=[alpha_cp], variables=[beta_cp]) alpha_th = torch.tensor(alpha, requires_grad=True) beta_, = layer(alpha_th) # get test loss and it's gradient test_loss = torch.mean(torch.log(1 + torch.exp(-ytest * (Xtest @ beta_)))) test_loss.backward() val = test_loss.detach().numpy() grad = np.array(alpha_th.grad) return val, grad
def log_likelihood(self, x, y, w): y = np.array(y).flatten() log_likelihood = cp.sum( cp.multiply(y, x @ w) - cp.logistic(x @ w) ) return log_likelihood
def cvxpy_logistic_loss_l1(w, X, y, lam=None, num_points=None): """CVXPY implementation of L1 regularized logistic loss. Args: w (np.ndarray): 1D, the weight matrix with shape (n_features,). X (np.ndarray): 2D, the features with shape (n_samples, n_features) y (np.ndarray): 1D, the true labels with shape (n_samples,). lam (float): regularization parameter. num_points (int): number of points in X (corresponds to the first dimension of X "n", but some methods pass a different value for scaling). Returns: (float): the loss. """ if num_points is None: num_points = X.shape[0] if lam is None: lam = 1.0 yz = cvxpy.multiply(-y, X * w) logistic_loss = cvxpy.sum(cvxpy.logistic(yz)) l1_reg = (float(lam) / 2.0) * cvxpy.norm1(w) out = logistic_loss + l1_reg return out / num_points
def update_g(self): for i in range(self.num_nodes): gt = cvx.Variable(self.dim) loss = cvx.logistic(-cvx.mul_elemwise(self.y[i], self.X[i]*gt+self.b[i]))*self.temp.node[i]['pos_node_prob'] loss += cvx.norm(self.W - gt + self.h[i,:])**2*self.Rho/2 problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) self.g[i,:] = gt.value.ravel()
def choose_phi(self, phi_name): if phi_name == 'logistic': self.cvx_phi = lambda z: cvx.logistic(-z) elif phi_name == 'hinge': self.cvx_phi = lambda z: cvx.pos(1.0 - z) elif phi_name == 'exponential': self.cvx_phi = lambda z: cvx.exp(-z) else: print("Your surrogate function doesn't exist.")
def switch(self): if self.phi_name == 'logistic': self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2, math.e) else: self.cvx_phi = lambda z: cvx.pos(1 - z) if self.kappa_name == 'Zafar_hinge': self.cvx_kappa = lambda z: cvx.pos(z) else: self.cvx_kappa = lambda z: z
def __init__(self, theta_shape, X, y, y_orig, init_lam=1, per_target_model=False): self.X = X self.y = y self.y_orig = y_orig self.per_target_model = per_target_model self.theta_intercept = cp.Variable() self.theta = cp.Variable(theta_shape[0], theta_shape[1]) theta_norm = cp.norm(self.theta, 1) self.lam = cp.Parameter(sign="positive", value=init_lam) # This is the log denominator of the probability of mutating (-log(1 + exp(-theta))) log_ll = -cp.sum_entries( cp.logistic(-(X * (self.theta[:, 0:1] + self.theta_intercept)))) # If no mutation happened, then we also need the log numerator of probability of not mutating # since exp(-theta)/(1 + exp(-theta)) is prob not mutate no_mutate_X = X[y == 0, :] no_mutate_numerator = -(no_mutate_X * (self.theta[:, 0:1] + self.theta_intercept)) log_ll = log_ll + cp.sum_entries(no_mutate_numerator) if per_target_model: # If per target, need the substitution probabilities too for orig_i in range(NUM_NUCLEOTIDES): for i in range(NUM_NUCLEOTIDES): if orig_i == i: continue # Find the elements that mutated to y and mutated from y_orig mutate_X_targ = X[(y == (i + 1)) & (y_orig == (orig_i + 1)), :] # Create the 3 column theta excluding the column corresponding to y_orig theta_3col = [] for j in range(NUM_NUCLEOTIDES): if j != orig_i: theta_3col.append(self.theta[:, j + 1] + self.theta_intercept) theta_3col = cp.hstack(theta_3col) target_ll = ( # log of numerator in softmax -(mutate_X_targ * (self.theta[:, i + 1] + self.theta_intercept)) # log of denominator in softmax - cp.log_sum_exp(-(mutate_X_targ * theta_3col), axis=1)) log_ll += cp.sum_entries(target_ll) self.problem = cp.Problem(cp.Maximize(log_ll - self.lam * theta_norm))
def set_objective(self, X, y, lmbd): self.X, self.y, self.lmbd = X, y, lmbd n_features = self.X.shape[1] self.beta = cp.Variable(n_features) loss = cp.sum(cp.logistic(-cp.multiply(self.y, self.X * self.beta))) self.problem = cp.Problem( cp.Minimize(loss + self.lmbd * cp.norm(self.beta, 1)))
def update_b(self): B = [] for i in range(self.num_nodes): bi = cvx.Variable(1) loss = cvx.logistic(-cvx.mul_elemwise(self.y[i], self.X[i].dot(self.g[i,:])+bi))*self.temp.node[i]['pos_node_prob'] for Id in self.temp.neighbors(i): loss = loss+(bi-self.Z[i,Id]+ self.U[i,Id])**2*self.Rho/2 problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) self.b[i] = bi.value
def __optimize__(self, full_set, labels): # Citation: https://www.cvxpy.org/examples/machine_learning/logistic_regression.html n = 2 beta = cp.Variable(n) log_likelihood = cp.sum( cp.multiply(labels, full_set @ beta) - cp.logistic(full_set @ beta)) problem = cp.Problem(cp.Maximize(log_likelihood)) problem.solve() return beta.value
def solve(self): n, m = self.n, self.m self.x = cvx.Variable(m) obj = cvx.Minimize(0) for i in range(n): obj += cvx.Minimize( cvx.logistic(self.A[i] * self.x) - self.b[i] * self.A[i] * self.x) obj += cvx.Minimize(1 / 2 * self.lam * cvx.norm(self.x)**2) self.prob = cvx.Problem(obj) self.prob.solve(verbose=True, abstol=1.0e-10, feastol=1.0e-10) print(self.prob.status, self.x.value)
def costL1(K, X, S, lam): # print(type(K)) loss = 0. for q in S: i, j, k = q # should this be Mt^T * K?? Mt = (2. * np.outer(X[i], X[j]) - 2. * np.outer(X[i], X[k]) - np.outer(X[j], X[j]) + np.outer(X[k], X[k])) score = cvx.trace(Mt.T * K) loss = loss + cvx.logistic(-score) loss = loss / len(S) + lam * cvx.norm(K, 1) # add L1 penalty return loss
def logistic_nmistregression(x, y, d): beta = cp.Variable(d) log_likelihood = cp.sum(cp.multiply(y, x @ beta) - cp.logistic(x @ beta)) problem = cp.Problem(cp.Maximize(log_likelihood / d)) problem.solve(solver=SCS, verbose=True) #beta = beta.value errort = 0 for i in range(len(y)): errort = errort + errorf((x[i] @ beta).value, y[i]) print("Logistic regression empirical error=", "={:.3f} %".format(errort / x.shape[0] * 100)) return beta.value
def solve(self): dim = self.X.shape[1] w = cvx.Variable(dim) num_nodes = nx.number_of_nodes(self.temp) b = cvx.Variable(num_nodes) loss = cvx.sum_entries(cvx.mul_elemwise(np.array(self.pos_node),cvx.logistic(-cvx.mul_elemwise(self.y, self.X*w+b)))) + self.Lambda*cvx.quad_form(b,self.P) problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) opt = problem.value self.W = w.value self.b = b.value self.value = opt
def costL12(K, X, S, lam): # compute log loss loss = 0. for q in S: i, j, k = q # should this be Mt^T * K?? Mt = (2. * np.outer(X[i], X[j]) - 2. * np.outer(X[i], X[k]) - np.outer(X[j], X[j]) + np.outer(X[k], X[k])) score = cvx.trace(Mt.T * K) loss = loss + cvx.logistic(-score) # regularize with the 1,2 norm loss = loss / len(S) + lam * cvx.mixed_norm(K, 2, 1) return loss
def solve_one_time_by_type(problem, x_train, y_train, x_test, y_test, gamma, weight, is_squared=True): fp_weight = float(weight if problem.fp else 0) fn_weight = float(weight if problem.fn else 0) protected_index = problem.protected_index x_1, y_1, x_0, y_0 = split_by_protected_value(x_train, y_train, protected_index) x_1_pos = get_positive_examples(x_1, y_1) x_1_neg = get_negative_examples(x_1, y_1) x_0_pos = get_positive_examples(x_0, y_0) x_0_neg = get_negative_examples(x_0, y_0) w = cp.Variable(x_train.shape[1]) diff_pos = (np.sum(x_1_pos, axis=0)/x_1_pos.shape[0]) - (np.sum(x_0_pos, axis=0)/x_0_pos.shape[0]) diff_neg = (np.sum(x_1_neg, axis=0)/x_1_neg.shape[0]) - (np.sum(x_0_neg, axis=0)/x_0_neg.shape[0]) log_likelihood = (y_train.T * (x_train * w) - cp.sum(cp.logistic(x_train * w))) if is_squared: fpr_diff = cp.square(diff_neg*w) fnr_diff = cp.square(diff_pos*w) else: fpr_diff = cp.abs(diff_neg*w) fnr_diff = cp.abs(diff_pos*w) w_norm_square = cp.sum_squares(w) objective = cp.Minimize(-log_likelihood + fn_weight*fnr_diff + fp_weight*fpr_diff + gamma*w_norm_square) p = cp.Problem(objective) p.solve(solver='ECOS', verbose=False, max_iters=1000) solution = { 'w': w.value, 'll': log_likelihood.value, 'fnr_diff': fnr_diff.value, 'fpr_diff': fpr_diff.value, 'objective': -log_likelihood.value + fn_weight*fnr_diff.value + fp_weight*fpr_diff.value } train_real_measures = measure_objective_results(x_train, y_train, solution['w'], problem) test_relaxed_measures = measure_relaxed_results(x_test, y_test, solution['w'], weight, problem, is_squared=is_squared) test_real_measures = measure_objective_results(x_test, y_test, solution['w'], problem) return { 'weight': weight, 'gamma': gamma, 'train_relaxed_measures': solution, 'train_real_measures': train_real_measures, 'test_relaxed_measures': test_relaxed_measures, 'test_real_measures': test_real_measures }
def addConstraint(self, updatedData): """ Adds a constraint based on the current best response of the attacker. :param updatedData: Set of manipulations chosen by the attacker at current time step :return: _ """ x = None # TODO: retrive x from updated data. y = None # TODO: retrive y from updated data. x = np.asarray(x) y = np.asarray(y) cons = self.delta l = cp.sum(cp.multiply(y, x @ self.theta) - cp.logistic(x @ self.theta)) cons -= l self.constraints.append(cons <= 0)
def fit(self, X, y): """ Fit the model according to the given training data. Parameters ---------- X : {array-like, sparse matrix} of shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. y : array-like of shape (n_samples,) Target vector relative to X. """ dim = X.shape[1] beta = cp.Variable(dim) # coeefficients b = cp.Variable(1) # intercept t = cp.Variable(1) if self.fit_intercept: log_likelihood = cp.sum( cp.multiply(y, X @ beta + b) - cp.logistic(X @ beta + b)) else: log_likelihood = cp.sum( cp.multiply(y, X @ beta) - cp.logistic(X @ beta)) cons = [] if self.reg: cons.append(cp.SOC(t, beta)) else: cons.append(t == 0) self.problem = cp.Problem( cp.Maximize(log_likelihood / dim - self.reg * t), cons) self.problem.solve(solver=cp.ECOS, abstol=1e-15, verbose=False) self.coef_ = beta.value if self.fit_intercept: self.intercept_ = b.value else: self.intercept_ = 0
def M_step(self): expected_LL = 0 for k in range(self.num_classes): w = cvx.Variable(self.dim) b = cvx.Variable(1) loss = cvx.sum_entries( cvx.mul_elemwise( np.array(self.posterior_mat[:, k]), cvx.logistic(-cvx.mul_elemwise( self.Y, self.X * w + np.ones(self.num_nodes) * b)))) problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False, solver='SCS') expected_LL -= problem.value self.W[:, k] = np.array(w.value).flatten() self.B[k] = b.value self.expected_LL = expected_LL
def train_predict(X, y, z, train_index, test_index, C1, C2=float('-inf'), ws=False): X_train, X_test = X[train_index], X[test_index] y_train, y_test = numpy.matrix(y[train_index]), y[test_index] z_train, z_test = (numpy.matrix(z[train_index]).transpose(), numpy.matrix(z[test_index]).transpose()) A = create_active_vec(X_train) w = cvxpy.Variable(216) constraints = [w >= 0] rmseerence = 2**C2 * cvxpy.sum_squares(A * w - z_train) logregression = 2**C1 * cvxpy.norm(w, 1) + cvxpy.sum_entries( cvxpy.logistic(cvxpy.diag((X_train * w) * (-y_train)))) objective = cvxpy.Minimize(logregression + rmseerence) prob = cvxpy.Problem(objective, constraints) prob.solve(solver=cvxpy.SCS, warm_start=ws) predicted_test = numpy.matrix(X[test_index]) * w.value expected_test = y_test fpr_test, tpr_test, threesholds = roc_curve(expected_test, predicted_test) predicted_train = numpy.matrix(X[train_index]) * w.value expected_train = numpy.squeeze(numpy.asarray(y_train)) fpr_train, tpr_train, threesholds = roc_curve(expected_train, predicted_train) number = 0 for i in w.value: if i > 1e-06: number += 1 C = create_active_vec(X_test) size = z_test.size norm_rmse = w.value.sum() w.value = w.value / norm_rmse vector = numpy.array((C * w.value - z_test).transpose())[0] rmse = (sum_squares(vector) / size)**0.5 return fpr_test, tpr_test, fpr_train, tpr_train, rmse, number
def fit(self, X, y, sample_weight=None): '''fit a logistic model with integer coefficient and L1 regularization. In case the optimization fails, fit lasso and round coefs. Params ------ sample_weight: np.ndarray (n,), optional weight for each individual sample ''' X, y = check_X_y(X, y) check_classification_targets(y) self.n_features_in_ = X.shape[1] self.classes_, y = np.unique( y, return_inverse=True) # deals with str inputs self.model_ = LogisticRegression() self.model_.classes_ = self.classes_ # declare the integer-valued optimization variable w = cp.Variable(X.shape[1], integer=True) # set up the minimization problem logits = -X @ w residuals = cp.multiply(1 - y, logits) - cp.logistic(logits) if sample_weight is not None: residuals = cp.multiply(sample_weight, residuals) try: celoss = -cp.sum(residuals) l1_penalty = self.alpha * cp.norm(w, 1) obj = cp.Minimize(celoss + l1_penalty) prob = cp.Problem(obj) # solve the problem using an appropriate solver prob.solve() self.model_.coef_ = np.array([w.value.astype(int)]) self.model_.intercept_ = 0 except SolverError as e: warnings.warn( "mosek solver required for mixed-integer logistic regression. " "rounding non-integer coefficients instead") m = LogisticRegression(C=1 / self.alpha) m.fit(X, y, sample_weight=sample_weight) self.model_.coef_ = np.round(m.coef_).astype(int) self.model_.intercept_ = m.intercept_ return self
def cvxpy_logistic_loss(w, X, y, num_points=None): """CVXPY implementation of logistic loss. Args: w (np.ndarray): 1D, the weight matrix with shape (n_features,). X (np.ndarray): 2D, the features with shape (n_samples, n_features) y (np.ndarray): 1D, the true labels with shape (n_samples,). num_points (int): number of points in X (first dimension of X "n_samples", but some methods pass a different value for scaling). Returns: (float): the loss. """ if num_points is None: num_points = X.shape[0] return (cvxpy.sum(cvxpy.logistic(cvxpy.multiply(-y, X * w))) / num_points)
def runCVX_LR(A, y, lam, intercept=False): (m, d) = A.shape x_cvx = cvx.Variable(d) f = (1 / m) * cvx.sum(cvx.logistic(-cvx.multiply(y, A @ x_cvx))) if intercept: # the intercept term is assumed to be the first coefficient f += lam * cvx.norm(x_cvx[1:d], 1) else: f += lam * cvx.norm(x_cvx, 1) prob = cvx.Problem(cvx.Minimize(f)) prob.solve(verbose=True) opt = prob.value xopt = x_cvx.value xopt = np.squeeze(np.array(xopt)) return opt, xopt
def train(self, x, y, gamma=0.1): # ensure labels are [0, 1] y[y == -1] = 0 assert np.unique(y).tolist() == [0, 1] # regularization parameter g = cvx.Parameter(sign="positive") g.value = gamma / x.shape[1] # define model variables w = cvx.Variable(x.shape[1]) b = cvx.Variable() # compute affine transform a = x * w - b # compute log-likelihood l = cvx.sum_entries(cvx.mul_elemwise(y, a)) - cvx.sum_entries( cvx.logistic(a)) # minimize negative log-likelihood plus l-2 regularization obj = cvx.Minimize(-l + g * cvx.sum_squares(w)) try: # form problem and solve prob = cvx.Problem(obj) prob.solve() # throw error if not optimal assert prob.status == 'optimal' # save model parameters self.w = np.array(w.value) self.b = np.array(b.value) # return success return True except: # return failure return False
def logistic_loss(theta, X, y): if not all(np.unique(y) == [-1, 1]): raise ValueError("y must have binary labels in {-1,1}") return cp.sum_entries(cp.logistic(-sp.diags([y],[0])*X*theta))
def logistic_loss(theta, X, y): return cp.sum_entries(cp.logistic(-sp.diags([y],[0])*X*theta))
prox("NORM_NUCLEAR", lambda: cp.norm(X, "nuc")), prox("SECOND_ORDER_CONE", None, C_soc_scaled), prox("SECOND_ORDER_CONE", None, C_soc_scaled_translated), prox("SECOND_ORDER_CONE", None, C_soc_translated), prox("SECOND_ORDER_CONE", None, lambda: [cp.norm(X, "fro") <= t]), prox("SECOND_ORDER_CONE", None, lambda: [cp.norm2(x) <= t]), prox("SEMIDEFINITE", None, lambda: [X >> 0]), prox("SUM_DEADZONE", f_dead_zone), prox("SUM_EXP", lambda: cp.sum_entries(cp.exp(x))), prox("SUM_HINGE", f_hinge), prox("SUM_HINGE", lambda: cp.sum_entries(cp.max_elemwise(1-x, 0))), prox("SUM_HINGE", lambda: cp.sum_entries(cp.max_elemwise(1-x, 0))), prox("SUM_INV_POS", lambda: cp.sum_entries(cp.inv_pos(x))), prox("SUM_KL_DIV", lambda: cp.sum_entries(cp.kl_div(p1,q1))), prox("SUM_LARGEST", lambda: cp.sum_largest(x, 4)), prox("SUM_LOGISTIC", lambda: cp.sum_entries(cp.logistic(x))), prox("SUM_NEG_ENTR", lambda: cp.sum_entries(-cp.entr(x))), prox("SUM_NEG_LOG", lambda: cp.sum_entries(-cp.log(x))), prox("SUM_QUANTILE", f_quantile), prox("SUM_QUANTILE", f_quantile_elemwise), prox("SUM_SQUARE", f_least_squares_matrix), prox("SUM_SQUARE", lambda: f_least_squares(20)), prox("SUM_SQUARE", lambda: f_least_squares(5)), prox("SUM_SQUARE", f_quad_form), prox("TOTAL_VARIATION_1D", lambda: cp.tv(x)), prox("ZERO", None, C_linear_equality), prox("ZERO", None, C_linear_equality_matrix_lhs), prox("ZERO", None, C_linear_equality_matrix_rhs), prox("ZERO", None, C_linear_equality_multivariate), prox("ZERO", None, C_linear_equality_multivariate2), prox("ZERO", None, lambda: C_linear_equality_graph(20)),