def get_full_psd_matrix(self):
"""Function that returns the tf graph corresponding to the entire matrix M.
Returns:
matrix_h: unrolled version of tf matrix corresponding to H
matrix_m: unrolled tf matrix corresponding to M
"""
if self.matrix_m is not None:
return self.matrix_h, self.matrix_m
# Computing the matrix term
h_columns = []
for i in range(self.nn_params.num_hidden_layers + 1):
current_col_elems = []
for j in range(i):
current_col_elems.append(
tf.zeros(
[self.nn_params.sizes[j], self.nn_params.sizes[i]]))
# For the first layer, there is no relu constraint
if i == 0:
current_col_elems.append(utils.diag(self.lambda_lu[i]))
else:
current_col_elems.append(
utils.diag(self.lambda_lu[i] + self.lambda_quad[i]))
if i < self.nn_params.num_hidden_layers:
current_col_elems.append(
tf.matmul(
utils.diag(-1 * self.lambda_quad[i + 1]),
self.nn_params.weights[i],
))
for j in range(i + 2, self.nn_params.num_hidden_layers + 1):
current_col_elems.append(
tf.zeros(
[self.nn_params.sizes[j], self.nn_params.sizes[i]]))
current_column = tf.concat(current_col_elems, 0)
h_columns.append(current_column)
self.matrix_h = tf.concat(h_columns, 1)
self.matrix_h = self.matrix_h + tf.transpose(self.matrix_h)
self.matrix_m = tf.concat(
[
tf.concat(
[tf.reshape(self.nu, (1, 1)),
tf.transpose(self.vector_g)],
axis=1),
tf.concat([self.vector_g, self.matrix_h], axis=1),
],
axis=0,
)
return self.matrix_h, self.matrix_m
def project_dual(self):
"""Function that projects the input dual variables onto the feasible set.
Returns:
projected_dual: Feasible dual solution corresponding to current dual
projected_certificate: Objective value of feasible dual
"""
# TODO: consider whether we can use shallow copy of the lists without
# using tf.identity
projected_lambda_pos = [tf.identity(x) for x in self.lambda_pos]
projected_lambda_neg = [tf.identity(x) for x in self.lambda_neg]
projected_lambda_quad = [tf.identity(x) for x in self.lambda_quad]
projected_lambda_lu = [tf.identity(x) for x in self.lambda_lu]
projected_nu = tf.identity(self.nu)
# TODO: get rid of the special case for one hidden layer
# Different projection for 1 hidden layer
if self.nn_params.num_hidden_layers == 1:
# Creating equivalent PSD matrix for H by Schur complements
diag_entries = 0.5 * tf.divide(
tf.square(self.lambda_quad[self.nn_params.num_hidden_layers]),
(self.lambda_quad[self.nn_params.num_hidden_layers] +
self.lambda_lu[self.nn_params.num_hidden_layers]))
# If lambda_quad[i], lambda_lu[i] are 0, entry is NaN currently,
# but we want to set that to 0
diag_entries = tf.where(tf.is_nan(diag_entries),
tf.zeros_like(diag_entries), diag_entries)
matrix = (tf.matmul(
tf.matmul(
tf.transpose(
self.nn_params.weights[self.nn_params.num_hidden_layers
- 1]),
utils.diag(diag_entries)),
self.nn_params.weights[self.nn_params.num_hidden_layers - 1]))
new_matrix = utils.diag(
2 *
self.lambda_lu[self.nn_params.num_hidden_layers - 1]) - matrix
# Making symmetric
new_matrix = 0.5 * (new_matrix + tf.transpose(new_matrix))
eig_vals = tf.self_adjoint_eigvals(new_matrix)
min_eig = tf.reduce_min(eig_vals)
# If min_eig is positive, already feasible, so don't add
# Otherwise add to make PSD [1E-6 is for ensuring strictly PSD (useful
# while inverting)
projected_lambda_lu[0] = (projected_lambda_lu[0] +
0.5 * tf.maximum(-min_eig, 0) + 1E-6)
else:
# Minimum eigen value of H
# TODO: Write this in terms of matrix multiply
# matrix H is a submatrix of M, thus we just need to extend existing code
# for computing matrix-vector product (see get_psd_product function).
# Then use the same trick to compute smallest eigenvalue.
eig_vals = tf.self_adjoint_eigvals(self.matrix_h)
min_eig = tf.reduce_min(eig_vals)
for i in range(self.nn_params.num_hidden_layers + 1):
# Since lambda_lu appears only in diagonal terms, can subtract to
# make PSD and feasible
projected_lambda_lu[i] = (projected_lambda_lu[i] +
0.5 * tf.maximum(-min_eig, 0) + 1E-6)
# Adjusting lambda_neg wherever possible so that lambda_neg + lambda_lu
# remains close to unchanged
# projected_lambda_neg[i] = tf.maximum(0.0, projected_lambda_neg[i] +
# (0.5*min_eig - 1E-6)*
# (self.lower[i] + self.upper[i]))
projected_dual_var = {
'lambda_pos': projected_lambda_pos,
'lambda_neg': projected_lambda_neg,
'lambda_lu': projected_lambda_lu,
'lambda_quad': projected_lambda_quad,
'nu': projected_nu
}
projected_dual_object = DualFormulation(
projected_dual_var, self.nn_params, self.test_input,
self.true_class, self.adv_class, self.input_minval,
self.input_maxval, self.epsilon)
projected_certificate = projected_dual_object.compute_certificate()
return projected_certificate
