def _check_optimality_conditions(self, model_params, lambdas, opt_thres=1e-2): # sanity check function to see that cvxpy is solving to a good enough accuracy # check that the gradient is close to zero # can use this to check that our implicit derivative assumptions hold # lambdas must be an exploded lambda matrix print "check_optimality_conditions!" alpha = model_params["alpha"] beta = model_params["beta"] gamma = model_params["gamma"] u_hat, sigma_hat, v_hat = self._get_svd_mini(gamma) d_square_loss = -1.0 / self.num_train * np.multiply( self.train_vec, make_column_major_flat(self.data.observed_matrix - get_matrix_completion_fitted_values( self.data.row_features, self.data. col_features, alpha, beta, gamma))) left_grad_at_opt_gamma = (u_hat.T * make_column_major_reshape( d_square_loss, (self.data.num_rows, self.data.num_cols)) + lambdas[0] * np.sign(sigma_hat) * v_hat.T) right_grad_at_opt_gamma = (make_column_major_reshape( d_square_loss, (self.data.num_rows, self.data.num_cols)) * v_hat + lambdas[0] * u_hat * np.sign(sigma_hat)) left_grad_norm = np.linalg.norm(left_grad_at_opt_gamma) right_grad_norm = np.linalg.norm(right_grad_at_opt_gamma) print "grad_at_opt wrt gamma (should be zero)", left_grad_norm, right_grad_norm assert (left_grad_norm < opt_thres) assert (right_grad_norm < opt_thres) print "alpha", alpha grad_at_opt_alpha = [] for i in range(alpha.size): if np.abs(alpha[i]) > self.zero_thres: alpha_sign = np.sign(alpha[i]) grad_at_opt_alpha.append( (d_square_loss.T * make_column_major_flat( self.data.row_features[:, i] * self.onesT_row) + lambdas[1] * alpha_sign + lambdas[2] * alpha[i])[0, 0]) print "grad_at_opt wrt alpha (should be zero)", grad_at_opt_alpha assert (np.all(np.abs(grad_at_opt_alpha) < opt_thres)) print "beta", beta grad_at_opt_beta = [] for i in range(beta.size): if np.abs(beta[i]) > self.zero_thres: beta_sign = np.sign(beta[i]) grad_at_opt_beta.append( (d_square_loss.T * make_column_major_flat( (self.data.col_features[:, i] * self.onesT_col).T) + lambdas[3] * beta_sign + lambdas[4] * beta[i])[0, 0]) print "grad_at_opt wrt beta (should be zero)", grad_at_opt_beta assert (np.all(np.abs(grad_at_opt_beta) < opt_thres))
def get_value(self, alphas, betas, gamma, gamma_nuc_norm=None): matrix_eval = make_column_major_flat( self.observed_matrix - get_matrix_completion_groups_fitted_values( self.row_features, self.col_features, alphas, betas, gamma, )) square_loss = 0.5 / self.num_train * get_norm2( matrix_eval[self.train_idx], power=2, ) if gamma_nuc_norm is not None: nuc_norm = self.lambdas[0] * gamma_nuc_norm else: nuc_norm = self.lambdas[0] * np.linalg.norm(gamma, ord="nuc") alpha_pen = 0 for i, a in enumerate(alphas): # group lasso penalties alpha_pen += self.lambdas[1 + i] * get_norm2(a, power=1) beta_pen = 0 for i, b in enumerate(betas): # group lasso penalties beta_pen += self.lambdas[1 + self.num_row_groups + i] * get_norm2( b, power=1) return square_loss + nuc_norm + alpha_pen + beta_pen
def _create_sigma_mask(self, sigma_hat): # mask with a zero along diagonal where sigma_hat is zero. # everywhere else is a one sigma_mask = np.ones(sigma_hat.shape) for i in range(sigma_hat.shape[0]): if sigma_hat[i, i] == 0: sigma_mask[i, i] = 0 sigma_mask = make_column_major_flat(sigma_mask) return np.diag(sigma_mask.flatten())
def _get_d_square_loss(self, alphas, betas, gamma, row_features, col_features): # get first derivative of the square loss wrt X = gamma + stuff d_square_loss = -1.0 / self.num_train * np.multiply( self.train_vec, make_column_major_flat( self.data.observed_matrix - get_matrix_completion_groups_fitted_values( row_features, col_features, alphas, betas, gamma))) return d_square_loss
def _get_val_gradient(self, grad_dict, alphas, betas, gamma, row_features, col_features): # get gradient of the validation loss wrt lambda given the gradient of the # model parameters wrt lambda model_grad = grad_dict["dgamma_dlambda"] for da_dlambda, row_f in zip(grad_dict["dalphas_dlambda"], row_features): model_grad += row_f * da_dlambda * self.onesT_row for db_dlambda, col_f in zip(grad_dict["dbetas_dlambda"], col_features): model_grad += (col_f * db_dlambda * self.onesT_col).T dval_dlambda = -1.0 / self.num_val * (make_column_major_flat( self.data.observed_matrix - get_matrix_completion_groups_fitted_values( row_features, col_features, alphas, betas, gamma)))[ self.data.validate_idx].T * make_column_major_flat( model_grad)[self.data.validate_idx] return dval_dlambda
def _get_dsquare_loss(): d_square_loss = -1.0 / self.num_train * make_column_major_flat( self.observed_matrix - get_matrix_completion_fitted_values( self.row_features, self.col_features, self.alpha_curr, self.beta_curr, self.gamma_curr, )) d_square_loss[self.non_train_mask_vec] = 0 return d_square_loss
def _get_val_gradient(self, grad_dict, alpha, beta, gamma, row_features, col_features): # get gradient of the validation loss wrt lambda given the gradient of the # model parameters wrt lambda model_grad = grad_dict["dgamma_dlambda"] if alpha.size > 0: model_grad += row_features * grad_dict[ "dalpha_dlambda"] * self.onesT_row if beta.size > 0: model_grad += (col_features * grad_dict["dbeta_dlambda"] * self.onesT_col).T dval_dlambda = -1.0 / self.num_val * make_column_major_flat( self.data.observed_matrix - get_matrix_completion_fitted_values( row_features, col_features, alpha, beta, gamma, ))[self.data.validate_idx].T * make_column_major_flat(model_grad)[ self.data.validate_idx] return dval_dlambda
def get_value(self, alpha, beta, gamma, given_nuc_norm=None): matrix_eval = make_column_major_flat( self.observed_matrix - get_matrix_completion_fitted_values( self.row_features, self.col_features, alpha, beta, gamma, )) square_loss = 0.5 / self.num_train * get_norm2( matrix_eval[self.train_idx], power=2, ) if given_nuc_norm is None: nuc_norm = self.lambdas[0] * np.linalg.norm(gamma, ord="nuc") else: nuc_norm = self.lambdas[0] * given_nuc_norm alpha_norm1 = self.lambdas[1] * np.linalg.norm(alpha, ord=1) alpha_norm2 = 0.5 * self.lambdas[2] * get_norm2(alpha, power=2) beta_norm1 = self.lambdas[3] * np.linalg.norm(beta, ord=1) beta_norm2 = 0.5 * self.lambdas[4] * get_norm2(beta, power=2) return square_loss + nuc_norm + alpha_norm1 + alpha_norm2 + beta_norm1 + beta_norm2
def _get_masked(self, obs_matrix): masked_obs_vec = make_column_major_flat(obs_matrix) masked_obs_vec[self.non_train_mask_vec] = 0 masked_obs_m = make_column_major_reshape( masked_obs_vec, (self.num_rows, self.num_cols)) return masked_obs_m
def _double_check_derivative_indepth_lambda0(self, model1, model2, model0, eps): # not everything should be zero, if it is not differentiable at that point dalpha_dlambda = (model1["alpha"] - model2["alpha"]) / (eps * 2) dbeta_dlambda = (model1["beta"] - model2["beta"]) / (eps * 2) gamma1 = model1["gamma"] u1, s1, v1 = self._get_svd_mini(gamma1) gamma2 = model2["gamma"] u2, s2, v2 = self._get_svd_mini(gamma2) gamma0 = model0["gamma"] u_hat, sigma_hat, v_hat = self._get_svd_mini(gamma0) dU_dlambda = (u1 - u2) / (eps * 2) dV_dlambda = (v1 - v2) / (eps * 2) dSigma_dlambda = (s1 - s2) / (eps * 2) dgamma_dlambda = (gamma1 - gamma2) / (eps * 2) print "dalpha_dlambda0, %s" % (dalpha_dlambda) print "dBeta_dlambda0, %s" % (dbeta_dlambda) print "dU_dlambda0", dU_dlambda print "ds_dlambda0, %s" % (dSigma_dlambda) print "dgamma_dlambda0, %s" % (dgamma_dlambda) split_dgamma_dlambda = dU_dlambda * sigma_hat * v_hat.T + u_hat * dSigma_dlambda * v_hat.T + u_hat * sigma_hat * dV_dlambda.T # print "alpha1", model1["alpha"] # print 'alpha2', model2["alpha"] # print "eps", eps # print "u_hat", u_hat # print "u1", u1 # print "u2", u2 # print "v_hat", v_hat # print "v1", v1 # print "v2", v2 # print "sigma_hat", sigma_hat # print "s1", s1 # print "s2", s2 print "should be zero? dU_dlambda * u.T", u_hat.T * dU_dlambda + dU_dlambda.T * u_hat print "should be zero? dv_dlambda * v.T", dV_dlambda.T * v_hat + v_hat.T * dV_dlambda print "should be zero? dgamma_dlambda - dgamma_dlambda", split_dgamma_dlambda - dgamma_dlambda d_square_loss = 1.0 / self.num_train * self.train_vec_diag * make_column_major_flat( dgamma_dlambda + self.data.row_features * dalpha_dlambda * self.onesT_row + (self.data.col_features * dbeta_dlambda * self.onesT_col).T) dalpha_dlambda_imp = [] for i in range(dalpha_dlambda.size): dalpha_dlambda_imp.append( (d_square_loss.T * make_column_major_flat( self.data.row_features[:, i] * self.onesT_row) + self.fmodel.current_lambdas[1] * dalpha_dlambda[i])[0, 0]) print "should be zero? numerical plugin to the imp deriv eqn, dalpha_dlambda", dalpha_dlambda_imp db_dlambda_imp = [] for i in range(dbeta_dlambda.size): db_dlambda_imp.append( (d_square_loss.T * make_column_major_flat( (self.data.col_features[:, i] * self.onesT_col).T) + self.fmodel.current_lambdas[1] * dbeta_dlambda[i])[0, 0]) print "should be zero? numerical plugin to the imp deriv eqn, dbeta_dlambda_imp", db_dlambda_imp print "should be zero? numerical plugin to the imp deriv eqn, dgamma_dlambda", ( u_hat.T * make_column_major_reshape( d_square_loss, (self.data.num_rows, self.data.num_cols)) * v_hat + np.sign(sigma_hat) + u_hat.T * self.fmodel.current_lambdas[0] * dU_dlambda * np.sign(sigma_hat) + self.fmodel.current_lambdas[0] * np.sign(sigma_hat) * dV_dlambda.T * v_hat)
def _check_optimality_conditions(self, model_params, lambdas, opt_thres=1e-2): # sanity check function to see that cvxpy is solving to a good enough accuracy # check that the gradient is close to zero # can use this to check that our implicit derivative assumptions hold # lambdas must be an exploded lambda matrix print "check_optimality_conditions!" alphas = model_params["alphas"] betas = model_params["betas"] gamma = model_params["gamma"] u_hat, sigma_hat, v_hat = self._get_svd_mini(gamma) a = self.data.observed_matrix - get_matrix_completion_groups_fitted_values( self.data.row_features, self.data.col_features, alphas, betas, gamma) d_square_loss = self._get_d_square_loss( alphas, betas, gamma, self.data.row_features, self.data.col_features, ) left_grad_at_opt_gamma = (make_column_major_reshape( d_square_loss, (self.data.num_rows, self.data.num_cols)) * v_hat + lambdas[0] * u_hat * np.sign(sigma_hat)) right_grad_at_opt_gamma = (u_hat.T * make_column_major_reshape( d_square_loss, (self.data.num_rows, self.data.num_cols)) + lambdas[0] * np.sign(sigma_hat) * v_hat.T) print "left grad_at_opt wrt gamma (should be zero)", get_norm2( left_grad_at_opt_gamma) print "right grad_at_opt wrt gamma (should be zero)", get_norm2( right_grad_at_opt_gamma) # assert(get_norm2(left_grad_at_opt_gamma) < opt_thres) # assert(get_norm2(right_grad_at_opt_gamma) < opt_thres) for i, a_f_tuple in enumerate(zip(alphas, self.data.row_features)): alpha, row_f = a_f_tuple if np.linalg.norm(alpha) > 1e-5: grad_at_opt_alpha = [] for j in range(alpha.size): grad_at_opt_alpha.append( (d_square_loss.T * make_column_major_flat(row_f[:, j] * self.onesT_row) + lambdas[1 + i] * alpha[j] / np.linalg.norm(alpha, ord=None))[0, 0]) print "grad_at_opt wrt alpha (should be zero)", get_norm2( grad_at_opt_alpha) # assert(np.linalg.norm(grad_at_opt_alpha) < opt_thres) for i, b_f_tuple in enumerate(zip(betas, self.data.col_features)): beta, col_f = b_f_tuple if np.linalg.norm(beta) > 1e-5: grad_at_opt_beta = [] for j in range(beta.size): grad_at_opt_beta.append( (d_square_loss.T * make_column_major_flat( (col_f[:, j] * self.onesT_col).T) + lambdas[1 + self.settings.num_row_groups + i] * beta[j] / np.linalg.norm(beta, ord=None))[0, 0]) print "grad_at_opt wrt beta (should be zero)", get_norm2( grad_at_opt_beta)