def test_logistic(self) -> None:
"""Test domain for logistic.
"""
expr = cp.logistic(self.a)
self.a.value = 2
self.assertAlmostEqual(expr.grad[self.a], np.exp(2) / (1 + np.exp(2)))
self.a.value = 3
self.assertAlmostEqual(expr.grad[self.a], np.exp(3) / (1 + np.exp(3)))
self.a.value = -1
self.assertAlmostEqual(expr.grad[self.a],
np.exp(-1) / (1 + np.exp(-1)))
expr = cp.logistic(self.x)
self.x.value = [3, 4]
val = np.zeros((2, 2)) + np.diag(np.exp([3, 4]) / (1 + np.exp([3, 4])))
self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), val)
expr = cp.logistic(self.x)
self.x.value = [-1e-9, 4]
val = np.zeros(
(2, 2)) + np.diag(np.exp([-1e-9, 4]) / (1 + np.exp([-1e-9, 4])))
self.assertItemsAlmostEqual(expr.grad[self.x].toarray(), val)
expr = cp.logistic(self.A)
self.A.value = [[1, 2], [3, 4]]
val = np.zeros((4, 4)) + np.diag(
np.exp([1, 2, 3, 4]) / (1 + np.exp([1, 2, 3, 4])))
self.assertItemsAlmostEqual(expr.grad[self.A].toarray(), val)
def switch(self):
if self.phi_name == 'logistic':
self.cvx_phi = lambda z: cvx.logistic(-z) # / math.log(2, math.e)
elif self.phi_name == 'hinge':
self.cvx_phi = lambda z: cvx.pos(1 - z)
elif self.phi_name == 'squared':
self.cvx_phi = lambda z: cvx.square(-z)
elif self.phi_name == 'exponential':
self.cvx_phi = lambda z: cvx.exp(-z)
else:
logger.error('%s is not include' % self.phi_name)
logger.info('Logistic is the default setting')
self.cvx_phi = lambda z: cvx.logistic(-z) # / math.log(2, math.e)
if self.kappa_name == 'logistic':
self.cvx_kappa = lambda z: cvx.logistic(z) # / math.log(2, math.e)
self.psi_kappa = lambda mu: ((1 + mu) * math.log(1 + mu) +
(1 - mu) * math.log(3 - mu)) / 2
elif self.kappa_name == 'hinge':
self.cvx_kappa = lambda z: cvx.pos(1 + z)
self.psi_kappa = lambda mu: mu
elif self.kappa_name == 'squared':
self.cvx_kappa = lambda z: cvx.square(1 + z)
self.psi_kappa = lambda mu: mu**2
elif self.kappa_name == 'exponential':
self.cvx_kappa = lambda z: cvx.exp(z)
self.psi_kappa = lambda mu: 1 - math.sqrt(1 - mu**2)
else:
logger.error('%s is not include' % self.kappa_name)
logger.info('hinge is the default setting')
self.cvx_kappa = lambda z: cvx.pos(1 + z)
self.psi_kappa = lambda mu: mu
if self.delta_name == 'logistic':
self.cvx_delta = lambda z: 1 - cvx.logistic(
-z) # / math.log(2, math.e)
self.psi_delta = lambda mu: ((1 + mu) * math.log(1 + mu) +
(1 - mu) * math.log(1 - mu)) / 2
elif self.delta_name == 'hinge':
self.cvx_delta = lambda z: 1 - cvx.pos(1 - z)
self.psi_delta = lambda mu: mu
elif self.delta_name == 'squared':
self.cvx_delta = lambda z: 1 - cvx.square(1 - z)
self.psi_delta = lambda mu: mu**2
elif self.delta_name == 'exponential':
self.cvx_delta = lambda z: 1 - cvx.exp(-z)
self.psi_delta = lambda mu: 1 - math.sqrt(1 - mu**2)
else:
logger.error('%s is not include' % self.delta_name)
logger.info('hinge is the default setting')
self.cvx_delta = lambda z: cvx.pos(1 + z)
self.psi_delta = lambda mu: mu
def switch(self):
if self.phi_name == 'logistic':
self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2.0, math.e)
elif self.phi_name == 'hinge':
self.cvx_phi = lambda z: cvx.pos(1.0 - z)
elif self.phi_name == 'squared':
self.cvx_phi = lambda z: cvx.square(-z)
elif self.phi_name == 'exponential':
self.cvx_phi = lambda z: cvx.exp(-z)
else:
logger.error('%s is not include' % self.phi_name)
logger.info('Logistic is the default setting')
self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2, math.e)
def _train(self, data, label, delta):
pos_idx = label == 1
neg_idx = label == -1
pos_num = sum(pos_idx)
P_data_size = 0
for i in range(self.s, self.chunk_count - 1):
P_data_size += sum(self.stored_label[i] == 1)
if pos_num + P_data_size > delta:
self.s += 1
P_data = array([]).reshape(-1, self.fea_num)
ts = array([])
for i in range(self.s, self.chunk_count):
P_data = r_[P_data, self.stored_data[i][self.stored_label[i] == 1]]
ts = r_[ts, i * ones(sum(self.stored_label[i] == 1))]
N_data = data[neg_idx]
pos_num = P_data.shape[0]
neg_num = N_data.shape[0]
rand_idx = random.permutation(pos_num)
P_data = P_data[rand_idx]
ts = ts[rand_idx].astype(int)
N_data = N_data[random.permutation(neg_num)]
pos_train_num = int(self.train_ratio * pos_num)
neg_train_num = int(self.train_ratio * neg_num)
train_data = r_[P_data[:pos_train_num], N_data[:neg_train_num]]
train_label = r_[ones(pos_train_num), -ones(neg_train_num)]
train_ts = ts[:pos_train_num]
hold_data = r_[P_data[pos_train_num:], N_data[neg_train_num:]]
hold_label = r_[ones(pos_num - pos_train_num),
-ones(neg_num - neg_train_num)]
hold_pred = zeros([hold_label.size, self.fea_group_num])
self.ensemble = list()
for fea_comb_i in range(self.fea_group_num):
self._learnH(train_data[:, self.fea_comb[fea_comb_i]], train_label,
train_ts)
hold_pred = self._predict_base(hold_data)
# solve convex optimization problem
c = ones_like(hold_label)
c[hold_label == 1] = sum(hold_label == -1) / sum(hold_label == 1)
w = cvxpy.Variable(self.fea_group_num)
obj = cvxpy.Minimize(
c.T *
cvxpy.logistic(-cvxpy.mul_elemwise(hold_label, hold_pred * w)))
constraints = [cvxpy.sum_entries(w) == 1, w >= 0]
prob = cvxpy.Problem(obj, constraints)
try:
prob.solve()
except (cvxpy.error.SolverError):
prob.solve(solver=cvxpy.CVXOPT)
self.wd = array(w.value).squeeze()
def Log_SVM(absDataOrigin, absLabels, cmpDataOrigin, cmpLabels, absWeight,
lamda):
# This function combined both logistic regression for absolute labels and SVM for comparison labels.
# Parameter:
# ------------
# absDataOrigin : N by d numpy matrix where N the number of absolute label data and d is the dimension of data
# abslabels : (N,) numpy array, +1 means positive label and -1 represents negative labels
# lamda : weight on L1 penalty. Large lamda would have more zeros in beta.
# Return:
# ------------
# beta : the logistic regression model parameter
# const : the logistic regression global constant.
cmpWeight = 1.0 - absWeight
absN, d = np.shape(absDataOrigin)
cmpN, _ = np.shape(cmpDataOrigin)
beta = cp.Variable(d)
const = cp.Variable(1)
objective = absWeight*cp.sum_entries(cp.logistic(cp.mul_elemwise(absLabels, absDataOrigin*beta+const))\
-cp.mul_elemwise(absLabels, absDataOrigin*beta+const))+ \
cmpWeight * cp.sum_entries(cp.pos(1 - cp.mul_elemwise(cmpLabels, cmpDataOrigin * beta + const))) \
+ lamda * cp.norm(beta, 1)
prob = cp.Problem(cp.Minimize(objective))
prob.solve(solver=cp.SCS)
return beta.value, const.value
def optimize_logistic_regression(X, Y):
beta = cp.Variable(n)
lambd = cp.Parameter(nonneg=True)
log_likelihood = cp.sum(cp.multiply(Y, X @ beta) - cp.logistic(X @ beta))
problem = cp.Problem(
cp.Maximize(log_likelihood / n - lambd * cp.norm(beta, 1)))
return problem
def logreg_cvxpy(X, y, alpha, idx_train, idx_val):
alpha = float(alpha)
assert np.all(np.unique(y) == np.array([-1, 1]))
Xtrain, Xtest, ytrain, ytest = map(
torch.from_numpy,
[X[idx_train, :], X[idx_val], y[idx_train], y[idx_val]])
n_samples_train, n_features = Xtrain.shape
# set up variables and parameters
beta_cp = cp.Variable(n_features)
alpha_cp = cp.Parameter(nonneg=True)
# set up objective
loss = cp.sum(cp.logistic(cp.multiply(-ytrain,
Xtrain @ beta_cp))) / n_samples_train
reg = alpha_cp * cp.norm(beta_cp, 1)
objective = loss + reg
# define problem
problem = cp.Problem(cp.Minimize(objective))
assert problem.is_dpp()
# solve problem
layer = CvxpyLayer(problem, parameters=[alpha_cp], variables=[beta_cp])
alpha_th = torch.tensor(alpha, requires_grad=True)
beta_, = layer(alpha_th)
# get test loss and it's gradient
test_loss = torch.mean(torch.log(1 + torch.exp(-ytest * (Xtest @ beta_))))
test_loss.backward()
val = test_loss.detach().numpy()
grad = np.array(alpha_th.grad)
return val, grad
def log_likelihood(self, x, y, w):
y = np.array(y).flatten()
log_likelihood = cp.sum(
cp.multiply(y, x @ w) - cp.logistic(x @ w)
)
return log_likelihood
def cvxpy_logistic_loss_l1(w, X, y, lam=None, num_points=None):
"""CVXPY implementation of L1 regularized logistic loss.
Args:
w (np.ndarray): 1D, the weight matrix with shape (n_features,).
X (np.ndarray): 2D, the features with shape (n_samples, n_features)
y (np.ndarray): 1D, the true labels with shape (n_samples,).
lam (float): regularization parameter.
num_points (int): number of points in X (corresponds to the first
dimension of X "n",
but some methods pass a different value for scaling).
Returns:
(float): the loss.
"""
if num_points is None:
num_points = X.shape[0]
if lam is None:
lam = 1.0
yz = cvxpy.multiply(-y, X * w)
logistic_loss = cvxpy.sum(cvxpy.logistic(yz))
l1_reg = (float(lam) / 2.0) * cvxpy.norm1(w)
out = logistic_loss + l1_reg
return out / num_points
def update_g(self): for i in range(self.num_nodes): gt = cvx.Variable(self.dim) loss = cvx.logistic(-cvx.mul_elemwise(self.y[i], self.X[i]*gt+self.b[i]))*self.temp.node[i]['pos_node_prob'] loss += cvx.norm(self.W - gt + self.h[i,:])**2*self.Rho/2 problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) self.g[i,:] = gt.value.ravel()
def choose_phi(self, phi_name):
if phi_name == 'logistic':
self.cvx_phi = lambda z: cvx.logistic(-z)
elif phi_name == 'hinge':
self.cvx_phi = lambda z: cvx.pos(1.0 - z)
elif phi_name == 'exponential':
self.cvx_phi = lambda z: cvx.exp(-z)
else:
print("Your surrogate function doesn't exist.")
def switch(self):
if self.phi_name == 'logistic':
self.cvx_phi = lambda z: cvx.logistic(-z) / math.log(2, math.e)
else:
self.cvx_phi = lambda z: cvx.pos(1 - z)
if self.kappa_name == 'Zafar_hinge':
self.cvx_kappa = lambda z: cvx.pos(z)
else:
self.cvx_kappa = lambda z: z
def __init__(self,
theta_shape,
X,
y,
y_orig,
init_lam=1,
per_target_model=False):
self.X = X
self.y = y
self.y_orig = y_orig
self.per_target_model = per_target_model
self.theta_intercept = cp.Variable()
self.theta = cp.Variable(theta_shape[0], theta_shape[1])
theta_norm = cp.norm(self.theta, 1)
self.lam = cp.Parameter(sign="positive", value=init_lam)
# This is the log denominator of the probability of mutating (-log(1 + exp(-theta)))
log_ll = -cp.sum_entries(
cp.logistic(-(X * (self.theta[:, 0:1] + self.theta_intercept))))
# If no mutation happened, then we also need the log numerator of probability of not mutating
# since exp(-theta)/(1 + exp(-theta)) is prob not mutate
no_mutate_X = X[y == 0, :]
no_mutate_numerator = -(no_mutate_X *
(self.theta[:, 0:1] + self.theta_intercept))
log_ll = log_ll + cp.sum_entries(no_mutate_numerator)
if per_target_model:
# If per target, need the substitution probabilities too
for orig_i in range(NUM_NUCLEOTIDES):
for i in range(NUM_NUCLEOTIDES):
if orig_i == i:
continue
# Find the elements that mutated to y and mutated from y_orig
mutate_X_targ = X[(y == (i + 1)) &
(y_orig == (orig_i + 1)), :]
# Create the 3 column theta excluding the column corresponding to y_orig
theta_3col = []
for j in range(NUM_NUCLEOTIDES):
if j != orig_i:
theta_3col.append(self.theta[:, j + 1] +
self.theta_intercept)
theta_3col = cp.hstack(theta_3col)
target_ll = (
# log of numerator in softmax
-(mutate_X_targ *
(self.theta[:, i + 1] + self.theta_intercept))
# log of denominator in softmax
-
cp.log_sum_exp(-(mutate_X_targ * theta_3col), axis=1))
log_ll += cp.sum_entries(target_ll)
self.problem = cp.Problem(cp.Maximize(log_ll - self.lam * theta_norm))
def set_objective(self, X, y, lmbd):
self.X, self.y, self.lmbd = X, y, lmbd
n_features = self.X.shape[1]
self.beta = cp.Variable(n_features)
loss = cp.sum(cp.logistic(-cp.multiply(self.y, self.X * self.beta)))
self.problem = cp.Problem(
cp.Minimize(loss + self.lmbd * cp.norm(self.beta, 1)))
def update_b(self): B = [] for i in range(self.num_nodes): bi = cvx.Variable(1) loss = cvx.logistic(-cvx.mul_elemwise(self.y[i], self.X[i].dot(self.g[i,:])+bi))*self.temp.node[i]['pos_node_prob'] for Id in self.temp.neighbors(i): loss = loss+(bi-self.Z[i,Id]+ self.U[i,Id])**2*self.Rho/2 problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) self.b[i] = bi.value
def __optimize__(self, full_set, labels):
# Citation: https://www.cvxpy.org/examples/machine_learning/logistic_regression.html
n = 2
beta = cp.Variable(n)
log_likelihood = cp.sum(
cp.multiply(labels, full_set @ beta) -
cp.logistic(full_set @ beta))
problem = cp.Problem(cp.Maximize(log_likelihood))
problem.solve()
return beta.value
def solve(self):
n, m = self.n, self.m
self.x = cvx.Variable(m)
obj = cvx.Minimize(0)
for i in range(n):
obj += cvx.Minimize(
cvx.logistic(self.A[i] * self.x) -
self.b[i] * self.A[i] * self.x)
obj += cvx.Minimize(1 / 2 * self.lam * cvx.norm(self.x)**2)
self.prob = cvx.Problem(obj)
self.prob.solve(verbose=True, abstol=1.0e-10, feastol=1.0e-10)
print(self.prob.status, self.x.value)
def costL1(K, X, S, lam):
# print(type(K))
loss = 0.
for q in S:
i, j, k = q
# should this be Mt^T * K??
Mt = (2. * np.outer(X[i], X[j]) - 2. * np.outer(X[i], X[k]) -
np.outer(X[j], X[j]) + np.outer(X[k], X[k]))
score = cvx.trace(Mt.T * K)
loss = loss + cvx.logistic(-score)
loss = loss / len(S) + lam * cvx.norm(K, 1) # add L1 penalty
return loss
def logistic_nmistregression(x, y, d):
beta = cp.Variable(d)
log_likelihood = cp.sum(cp.multiply(y, x @ beta) - cp.logistic(x @ beta))
problem = cp.Problem(cp.Maximize(log_likelihood / d))
problem.solve(solver=SCS, verbose=True)
#beta = beta.value
errort = 0
for i in range(len(y)):
errort = errort + errorf((x[i] @ beta).value, y[i])
print("Logistic regression empirical error=",
"={:.3f} %".format(errort / x.shape[0] * 100))
return beta.value
def solve(self): dim = self.X.shape[1] w = cvx.Variable(dim) num_nodes = nx.number_of_nodes(self.temp) b = cvx.Variable(num_nodes) loss = cvx.sum_entries(cvx.mul_elemwise(np.array(self.pos_node),cvx.logistic(-cvx.mul_elemwise(self.y, self.X*w+b)))) + self.Lambda*cvx.quad_form(b,self.P) problem = cvx.Problem(cvx.Minimize(loss)) problem.solve(verbose=False) opt = problem.value self.W = w.value self.b = b.value self.value = opt
def costL12(K, X, S, lam):
# compute log loss
loss = 0.
for q in S:
i, j, k = q
# should this be Mt^T * K??
Mt = (2. * np.outer(X[i], X[j]) - 2. * np.outer(X[i], X[k]) -
np.outer(X[j], X[j]) + np.outer(X[k], X[k]))
score = cvx.trace(Mt.T * K)
loss = loss + cvx.logistic(-score)
# regularize with the 1,2 norm
loss = loss / len(S) + lam * cvx.mixed_norm(K, 2, 1)
return loss
def solve_one_time_by_type(problem, x_train, y_train, x_test, y_test, gamma, weight, is_squared=True):
fp_weight = float(weight if problem.fp else 0)
fn_weight = float(weight if problem.fn else 0)
protected_index = problem.protected_index
x_1, y_1, x_0, y_0 = split_by_protected_value(x_train, y_train, protected_index)
x_1_pos = get_positive_examples(x_1, y_1)
x_1_neg = get_negative_examples(x_1, y_1)
x_0_pos = get_positive_examples(x_0, y_0)
x_0_neg = get_negative_examples(x_0, y_0)
w = cp.Variable(x_train.shape[1])
diff_pos = (np.sum(x_1_pos, axis=0)/x_1_pos.shape[0]) - (np.sum(x_0_pos, axis=0)/x_0_pos.shape[0])
diff_neg = (np.sum(x_1_neg, axis=0)/x_1_neg.shape[0]) - (np.sum(x_0_neg, axis=0)/x_0_neg.shape[0])
log_likelihood = (y_train.T * (x_train * w) - cp.sum(cp.logistic(x_train * w)))
if is_squared:
fpr_diff = cp.square(diff_neg*w)
fnr_diff = cp.square(diff_pos*w)
else:
fpr_diff = cp.abs(diff_neg*w)
fnr_diff = cp.abs(diff_pos*w)
w_norm_square = cp.sum_squares(w)
objective = cp.Minimize(-log_likelihood +
fn_weight*fnr_diff +
fp_weight*fpr_diff +
gamma*w_norm_square)
p = cp.Problem(objective)
p.solve(solver='ECOS', verbose=False, max_iters=1000)
solution = {
'w': w.value,
'll': log_likelihood.value,
'fnr_diff': fnr_diff.value,
'fpr_diff': fpr_diff.value,
'objective': -log_likelihood.value + fn_weight*fnr_diff.value + fp_weight*fpr_diff.value
}
train_real_measures = measure_objective_results(x_train, y_train, solution['w'], problem)
test_relaxed_measures = measure_relaxed_results(x_test, y_test, solution['w'], weight, problem, is_squared=is_squared)
test_real_measures = measure_objective_results(x_test, y_test, solution['w'], problem)
return {
'weight': weight,
'gamma': gamma,
'train_relaxed_measures': solution,
'train_real_measures': train_real_measures,
'test_relaxed_measures': test_relaxed_measures,
'test_real_measures': test_real_measures
}
def addConstraint(self, updatedData):
"""
Adds a constraint based on the current best response of the attacker.
:param updatedData: Set of manipulations chosen by the attacker at current time step
:return: _
"""
x = None # TODO: retrive x from updated data.
y = None # TODO: retrive y from updated data.
x = np.asarray(x)
y = np.asarray(y)
cons = self.delta
l = cp.sum(cp.multiply(y, x @ self.theta) - cp.logistic(x @ self.theta))
cons -= l
self.constraints.append(cons <= 0)
def fit(self, X, y):
"""
Fit the model according to the given training data.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
y : array-like of shape (n_samples,)
Target vector relative to X.
"""
dim = X.shape[1]
beta = cp.Variable(dim) # coeefficients
b = cp.Variable(1) # intercept
t = cp.Variable(1)
if self.fit_intercept:
log_likelihood = cp.sum(
cp.multiply(y, X @ beta + b) - cp.logistic(X @ beta + b))
else:
log_likelihood = cp.sum(
cp.multiply(y, X @ beta) - cp.logistic(X @ beta))
cons = []
if self.reg:
cons.append(cp.SOC(t, beta))
else:
cons.append(t == 0)
self.problem = cp.Problem(
cp.Maximize(log_likelihood / dim - self.reg * t), cons)
self.problem.solve(solver=cp.ECOS, abstol=1e-15, verbose=False)
self.coef_ = beta.value
if self.fit_intercept:
self.intercept_ = b.value
else:
self.intercept_ = 0
def M_step(self):
expected_LL = 0
for k in range(self.num_classes):
w = cvx.Variable(self.dim)
b = cvx.Variable(1)
loss = cvx.sum_entries(
cvx.mul_elemwise(
np.array(self.posterior_mat[:, k]),
cvx.logistic(-cvx.mul_elemwise(
self.Y, self.X * w + np.ones(self.num_nodes) * b))))
problem = cvx.Problem(cvx.Minimize(loss))
problem.solve(verbose=False, solver='SCS')
expected_LL -= problem.value
self.W[:, k] = np.array(w.value).flatten()
self.B[k] = b.value
self.expected_LL = expected_LL
def train_predict(X,
y,
z,
train_index,
test_index,
C1,
C2=float('-inf'),
ws=False):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = numpy.matrix(y[train_index]), y[test_index]
z_train, z_test = (numpy.matrix(z[train_index]).transpose(),
numpy.matrix(z[test_index]).transpose())
A = create_active_vec(X_train)
w = cvxpy.Variable(216)
constraints = [w >= 0]
rmseerence = 2**C2 * cvxpy.sum_squares(A * w - z_train)
logregression = 2**C1 * cvxpy.norm(w, 1) + cvxpy.sum_entries(
cvxpy.logistic(cvxpy.diag((X_train * w) * (-y_train))))
objective = cvxpy.Minimize(logregression + rmseerence)
prob = cvxpy.Problem(objective, constraints)
prob.solve(solver=cvxpy.SCS, warm_start=ws)
predicted_test = numpy.matrix(X[test_index]) * w.value
expected_test = y_test
fpr_test, tpr_test, threesholds = roc_curve(expected_test, predicted_test)
predicted_train = numpy.matrix(X[train_index]) * w.value
expected_train = numpy.squeeze(numpy.asarray(y_train))
fpr_train, tpr_train, threesholds = roc_curve(expected_train,
predicted_train)
number = 0
for i in w.value:
if i > 1e-06:
number += 1
C = create_active_vec(X_test)
size = z_test.size
norm_rmse = w.value.sum()
w.value = w.value / norm_rmse
vector = numpy.array((C * w.value - z_test).transpose())[0]
rmse = (sum_squares(vector) / size)**0.5
return fpr_test, tpr_test, fpr_train, tpr_train, rmse, number
def fit(self, X, y, sample_weight=None):
'''fit a logistic model with integer coefficient and L1 regularization.
In case the optimization fails, fit lasso and round coefs.
Params
------
sample_weight: np.ndarray (n,), optional
weight for each individual sample
'''
X, y = check_X_y(X, y)
check_classification_targets(y)
self.n_features_in_ = X.shape[1]
self.classes_, y = np.unique(
y, return_inverse=True) # deals with str inputs
self.model_ = LogisticRegression()
self.model_.classes_ = self.classes_
# declare the integer-valued optimization variable
w = cp.Variable(X.shape[1], integer=True)
# set up the minimization problem
logits = -X @ w
residuals = cp.multiply(1 - y, logits) - cp.logistic(logits)
if sample_weight is not None:
residuals = cp.multiply(sample_weight, residuals)
try:
celoss = -cp.sum(residuals)
l1_penalty = self.alpha * cp.norm(w, 1)
obj = cp.Minimize(celoss + l1_penalty)
prob = cp.Problem(obj)
# solve the problem using an appropriate solver
prob.solve()
self.model_.coef_ = np.array([w.value.astype(int)])
self.model_.intercept_ = 0
except SolverError as e:
warnings.warn(
"mosek solver required for mixed-integer logistic regression. "
"rounding non-integer coefficients instead")
m = LogisticRegression(C=1 / self.alpha)
m.fit(X, y, sample_weight=sample_weight)
self.model_.coef_ = np.round(m.coef_).astype(int)
self.model_.intercept_ = m.intercept_
return self
def cvxpy_logistic_loss(w, X, y, num_points=None):
"""CVXPY implementation of logistic loss.
Args:
w (np.ndarray): 1D, the weight matrix with shape (n_features,).
X (np.ndarray): 2D, the features with shape (n_samples, n_features)
y (np.ndarray): 1D, the true labels with shape (n_samples,).
num_points (int): number of points in X
(first dimension of X "n_samples",
but some methods pass a different value for scaling).
Returns:
(float): the loss.
"""
if num_points is None:
num_points = X.shape[0]
return (cvxpy.sum(cvxpy.logistic(cvxpy.multiply(-y, X * w))) /
num_points)
def runCVX_LR(A, y, lam, intercept=False):
(m, d) = A.shape
x_cvx = cvx.Variable(d)
f = (1 / m) * cvx.sum(cvx.logistic(-cvx.multiply(y, A @ x_cvx)))
if intercept:
# the intercept term is assumed to be the first coefficient
f += lam * cvx.norm(x_cvx[1:d], 1)
else:
f += lam * cvx.norm(x_cvx, 1)
prob = cvx.Problem(cvx.Minimize(f))
prob.solve(verbose=True)
opt = prob.value
xopt = x_cvx.value
xopt = np.squeeze(np.array(xopt))
return opt, xopt
def train(self, x, y, gamma=0.1):
# ensure labels are [0, 1]
y[y == -1] = 0
assert np.unique(y).tolist() == [0, 1]
# regularization parameter
g = cvx.Parameter(sign="positive")
g.value = gamma / x.shape[1]
# define model variables
w = cvx.Variable(x.shape[1])
b = cvx.Variable()
# compute affine transform
a = x * w - b
# compute log-likelihood
l = cvx.sum_entries(cvx.mul_elemwise(y, a)) - cvx.sum_entries(
cvx.logistic(a))
# minimize negative log-likelihood plus l-2 regularization
obj = cvx.Minimize(-l + g * cvx.sum_squares(w))
try:
# form problem and solve
prob = cvx.Problem(obj)
prob.solve()
# throw error if not optimal
assert prob.status == 'optimal'
# save model parameters
self.w = np.array(w.value)
self.b = np.array(b.value)
# return success
return True
except:
# return failure
return False
def logistic_loss(theta, X, y):
if not all(np.unique(y) == [-1, 1]):
raise ValueError("y must have binary labels in {-1,1}")
return cp.sum_entries(cp.logistic(-sp.diags([y],[0])*X*theta))
def logistic_loss(theta, X, y):
return cp.sum_entries(cp.logistic(-sp.diags([y],[0])*X*theta))
prox("NORM_NUCLEAR", lambda: cp.norm(X, "nuc")),
prox("SECOND_ORDER_CONE", None, C_soc_scaled),
prox("SECOND_ORDER_CONE", None, C_soc_scaled_translated),
prox("SECOND_ORDER_CONE", None, C_soc_translated),
prox("SECOND_ORDER_CONE", None, lambda: [cp.norm(X, "fro") <= t]),
prox("SECOND_ORDER_CONE", None, lambda: [cp.norm2(x) <= t]),
prox("SEMIDEFINITE", None, lambda: [X >> 0]),
prox("SUM_DEADZONE", f_dead_zone),
prox("SUM_EXP", lambda: cp.sum_entries(cp.exp(x))),
prox("SUM_HINGE", f_hinge),
prox("SUM_HINGE", lambda: cp.sum_entries(cp.max_elemwise(1-x, 0))),
prox("SUM_HINGE", lambda: cp.sum_entries(cp.max_elemwise(1-x, 0))),
prox("SUM_INV_POS", lambda: cp.sum_entries(cp.inv_pos(x))),
prox("SUM_KL_DIV", lambda: cp.sum_entries(cp.kl_div(p1,q1))),
prox("SUM_LARGEST", lambda: cp.sum_largest(x, 4)),
prox("SUM_LOGISTIC", lambda: cp.sum_entries(cp.logistic(x))),
prox("SUM_NEG_ENTR", lambda: cp.sum_entries(-cp.entr(x))),
prox("SUM_NEG_LOG", lambda: cp.sum_entries(-cp.log(x))),
prox("SUM_QUANTILE", f_quantile),
prox("SUM_QUANTILE", f_quantile_elemwise),
prox("SUM_SQUARE", f_least_squares_matrix),
prox("SUM_SQUARE", lambda: f_least_squares(20)),
prox("SUM_SQUARE", lambda: f_least_squares(5)),
prox("SUM_SQUARE", f_quad_form),
prox("TOTAL_VARIATION_1D", lambda: cp.tv(x)),
prox("ZERO", None, C_linear_equality),
prox("ZERO", None, C_linear_equality_matrix_lhs),
prox("ZERO", None, C_linear_equality_matrix_rhs),
prox("ZERO", None, C_linear_equality_multivariate),
prox("ZERO", None, C_linear_equality_multivariate2),
prox("ZERO", None, lambda: C_linear_equality_graph(20)),
