def load_AE(codec_prefix, print_summary=False):
saveFilePrefix = "models/AE_codec/" + codec_prefix + "_"
decoder_model_filename = saveFilePrefix + "decoder.json"
decoder_weight_filename = saveFilePrefix + "decoder.h5"
if not os.path.isfile(decoder_model_filename):
raise Exception("The file for decoder model does not exist:{}".format(
decoder_model_filename))
json_file = open(decoder_model_filename, 'r')
decoder = model_from_json(json_file.read(), custom_objects={"tf": tf})
json_file.close()
if not os.path.isfile(decoder_weight_filename):
raise Exception(
"The file for decoder weights does not exist:{}".format(
decoder_weight_filename))
decoder.load_weights(decoder_weight_filename)
if print_summary:
print("Decoder summaries")
decoder.summary()
return decoder
def __init__(self, sess, model, mask_mat, mode, batch_size, kappa, init_learning_rate,
binary_search_steps, max_iterations, initial_const, beta, gamma, attributes, aix360_path):
"""
Initialize PP explainer object.
Args:
sess (tensorflow.python.client.session.Session): Tensorflow session
model: KerasClassifier that contains a trained model to be explained
mask_mat (numpy.ndarry): Array containing PP masks for each class
mode (str): "PN" for pertinent negative or "PP" for pertinent positive
batch_size (int): batch size for how many instances to explain
kappa (float): Confidence parameter that controls difference between prediction of
PN (or PP) and original prediction
init_learning_rate (float): initial learning rate for gradient descent optimizer
binary_search_steps (int): Controls number of random restarts to find best PN
max_iterations (int): Max number iterations to run some version of gradient descent on
PP optimization problem from a single random initialization, i.e., total
number of iterations wll be arg_binary_search_steps * arg_max_iterations
initial_const (int): Constant used for upper/lower bounds in binary search
gamma (float): Penalty parameter encouraging addition of attributes for PP
attributes (str list): list of attributes to load attribute classifiers for
aix360_path (str): path to aix360 used to determine paths to pretrained attribute classifiers
"""
# image_size, num_channels, nun_classes = model.image_size, model.num_channels, model.num_labels
# %%change%%
image_size = model._input_shape[0]
num_channels = model._input_shape[2]
nun_classes = model._nb_classes
shape = (batch_size, image_size, image_size, num_channels)
mask_shape = (batch_size, image_size, image_size, 1)
mask_num = mask_mat.shape[0]
mask_vec_shape = (mask_num, 1, 1)
mask_mat_shape = (mask_num, image_size, image_size)
self.mask_num = mask_num
self.sess = sess
self.INIT_LEARNING_RATE = init_learning_rate
self.MAX_ITERATIONS = max_iterations
self.BINARY_SEARCH_STEPS = binary_search_steps
self.kappa = kappa
self.init_const = initial_const
self.batch_size = batch_size
self.AE = None
self.mode = mode
self.beta = beta
self.gamma = gamma
self.attributes = attributes
self.aix360_path = aix360_path
### Load attribute classifier
nn_type = "simple"
#import copy
attr_model_list=[]
for attr in self.attributes:
# load test data into memory using Image Data Generator
# print("Loading data for " + attr + " into memory")
# load json and create model
json_file_name = os.path.join(aix360_path, "models/CEM_MAF/{}_{}_model.json".format(nn_type, attr))
json_file = open(json_file_name, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
weight_file_name = os.path.join(aix360_path, "models/CEM_MAF/{}_{}_weights.h5".format(nn_type, attr))
loaded_model.load_weights(weight_file_name)
print("Loaded model for " + attr + " from disk")
attr_model_list.append(loaded_model)
print("# of attr models is",len(attr_model_list))
# print("beta:{}".format(self.beta))
# these are variables to be more efficient in sending data to tf
self.orig_img = tf.Variable(np.zeros(shape), dtype=tf.float32)
self.mask_vec = tf.Variable(np.zeros(mask_vec_shape), dtype=tf.float32)
self.mask_vec_s = tf.Variable(np.zeros(mask_vec_shape), dtype=tf.float32)
self.mask_mat = tf.constant(mask_mat, dtype=tf.float32)
# self.img_mask = tf.Variable(np.zeros(mask_shape), dtype=tf.float32)
# self.img_mask_s = tf.Variable(np.zeros(mask_shape), dtype=tf.float32, name="var_mask_s")
self.target_lab = tf.Variable(np.zeros((batch_size,nun_classes)), dtype=tf.float32)
self.const = tf.Variable(np.zeros(batch_size), dtype=tf.float32)
self.global_step = tf.Variable(0.0, trainable=False)
# and here's what we use to assign them
self.assign_orig_img = tf.placeholder(tf.float32, shape)
self.assign_mask_vec = tf.placeholder(tf.float32, mask_vec_shape)
self.assign_mask_vec_s = tf.placeholder(tf.float32, mask_vec_shape)
# self.assign_img_mask = tf.placeholder(tf.float32, mask_shape)
# self.assign_img_mask_s = tf.placeholder(tf.float32, mask_shape)
self.assign_target_lab = tf.placeholder(tf.float32, (batch_size,nun_classes))
self.assign_const = tf.placeholder(tf.float32, [batch_size])
"""Fast Iterative Soft Thresholding"""
"""--------------------------------"""
# self.zt = tf.divide(self.global_step, self.global_step+tf.cast(3, tf.float32))
# cond1 = tf.cast(tf.greater(tf.subtract(self.adv_img_mask_s, self.orig_img),self.beta), tf.float32)
# cond2 = tf.cast(tf.less_equal(tf.abs(tf.subtract(self.adv_img_s,self.orig_img)),self.beta), tf.float32)
# cond3 = tf.cast(tf.less(tf.subtract(self.adv_img_s, self.orig_img),tf.negative(self.beta)), tf.float32)
# upper = tf.minimum(tf.subtract(self.adv_img_s, self.beta), tf.cast(0.5, tf.float32))
# lower = tf.maximum(tf.add(self.adv_img_s, self.beta), tf.cast(-0.5, tf.float32))
# self.assign_adv_img = tf.multiply(cond1,upper)+tf.multiply(cond2,self.orig_img)+tf.multiply(cond3,lower)
self.zt = tf.divide(self.global_step, self.global_step+tf.cast(3, tf.float32))
"""
x = x - beta if x > beta
x = 0 if x < beta
"""
cond1 = tf.cast(tf.greater(self.mask_vec_s, self.beta), tf.float32)
cond2 = tf.cast(tf.less_equal(self.mask_vec_s, self.beta), tf.float32)
upper = tf.minimum(tf.subtract(self.mask_vec_s, self.beta), tf.cast(1, tf.float32))
self.assign_mask_vec = tf.multiply(cond1,upper) + tf.multiply(cond2, tf.constant(0, tf.float32))
self.assign_mask_vec_s = self.assign_mask_vec+tf.multiply(self.zt, self.assign_mask_vec-self.mask_vec)
# cond4=tf.cast(tf.greater(tf.subtract( self.assign_adv_img, self.orig_img),0), tf.float32)
# cond5=tf.cast(tf.less_equal(tf.subtract( self.assign_adv_img,self.orig_img),0), tf.float32)
# if self.mode == "PP":
# self.assign_adv_img = tf.multiply(cond5,self.assign_adv_img)+tf.multiply(cond4,self.orig_img)
# elif self.mode == "PN":
# self.assign_adv_img = tf.multiply(cond4,self.assign_adv_img)+tf.multiply(cond5,self.orig_img)
# self.assign_adv_img_s = self.assign_adv_img+tf.multiply(self.zt, self.assign_adv_img-self.adv_img)
# cond6=tf.cast(tf.greater(tf.subtract( self.assign_adv_img_s, self.orig_img),0), tf.float32)
# cond7=tf.cast(tf.less_equal(tf.subtract( self.assign_adv_img_s,self.orig_img),0), tf.float32)
# if self.mode == "PP":
# self.assign_adv_img_s = tf.multiply(cond7, self.assign_adv_img_s)+tf.multiply(cond6,self.orig_img)
# elif self.mode == "PN":
# self.assign_adv_img_s = tf.multiply(cond6, self.assign_adv_img_s)+tf.multiply(cond7,self.orig_img)
self.mask_updater = tf.assign(self.mask_vec, self.assign_mask_vec)
self.mask_updater_s = tf.assign(self.mask_vec_s, self.assign_mask_vec_s)
""" Thresholding """
# mask_ones = tf.constant(1, tf.float32)
# mask_zeros = tf.constant(0, tf.float32)
# mask_cond1 = tf.cast(tf.greater(self.img_mask, 0.5), tf.float32)
# mask_cond2 = tf.cast(tf.less_equal(self.img_mask, 0.5), tf.float32)
# self.img_mask_threshold = tf.multiply(mask_cond1, mask_ones)+tf.multiply(mask_cond2, mask_zeros)
# cannot find gradient
self.img_mask = tf.reduce_sum(self.mask_vec * self.mask_mat, axis=0)
self.img_mask = tf.expand_dims(self.img_mask, axis=2)
self.img_mask = tf.expand_dims(self.img_mask, axis=0)
self.adv_img = tf.multiply(self.img_mask, self.orig_img)
# mask_s_cond1 = tf.cast(tf.greater(self.img_mask_s, 0.5), tf.float32)
# mask_s_cond2 = tf.cast(tf.less_equal(self.img_mask_s, 0.5), tf.float32)
# self.img_mask_threshold_s = tf.multiply(mask_s_cond1, mask_ones)+tf.multiply(mask_s_cond2, mask_zeros)
# cannot find gradient
self.img_mask_s = tf.reduce_sum(self.mask_vec_s * self.mask_mat, axis=0)
self.img_mask_s = tf.expand_dims(self.img_mask_s, axis=2)
self.img_mask_s = tf.expand_dims(self.img_mask_s, axis=0)
self.adv_img_s = tf.multiply(self.img_mask_s, self.orig_img)
"""--------------------------------"""
# prediction from attribute classifer
self.delta_img = self.orig_img-self.adv_img
self.delta_img_s = self.orig_img-self.adv_img_s
if self.mode == "PP":
self.attr_score = tf.constant(0, dtype="float32")
self.attr_score_s = tf.constant(0, dtype="float32")
#print(attr_model_list[0].predict(self.adv_img))
#print(loaded_model.predict(self.adv_img))
for i in range(len(attr_model_list)):
self.attr_score = self.attr_score + tf.maximum(attr_model_list[i](self.adv_img) - attr_model_list[i](self.orig_img),tf.constant(0, tf.float32))
self.attr_score_s = self.attr_score_s + tf.maximum(attr_model_list[i](self.adv_img_s) - attr_model_list[i](self.orig_img),tf.constant(0, tf.float32))
#print(self.attr_score.shape)
self.attr_score = tf.squeeze(self.attr_score)
self.attr_score_s = tf.squeeze(self.attr_score_s)
#self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
#self.ImgToEnforceLabel_Score_s = model.predict(self.adv_img_s)
# %%change%%
elif self.mode == "PN":
# self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
# self.ImgToEnforceLabel_Score_s = model.predict(self.adv_img_s)
self.ImgToEnforceLabel_Score = model.predictsym(self.adv_img)
self.ImgToEnforceLabel_Score_s = model.predictsym(self.adv_img_s)
# prediction BEFORE-SOFTMAX of the model
self.delta_img = self.orig_img-self.adv_img
self.delta_img_s = self.orig_img-self.adv_img_s
if self.mode == "PP":
# self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
# self.ImgToEnforceLabel_Score_s = model.predict(self.adv_img_s)
self.ImgToEnforceLabel_Score = model.predictsym(self.adv_img)
self.ImgToEnforceLabel_Score_s = model.predictsym(self.adv_img_s)
elif self.mode == "PN":
# self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
# self.ImgToEnforceLabel_Score_s = model.predict(self.adv_img_s)
self.ImgToEnforceLabel_Score = model.predictsym(self.adv_img)
self.ImgToEnforceLabel_Score_s = model.predictsym(self.adv_img_s)
# distance to the input data
self.L2_dist = tf.reduce_sum(tf.square(self.img_mask),[1,2,3])
self.L2_dist_s = tf.reduce_sum(tf.square(self.img_mask_s),[1,2,3])
self.L1_dist = tf.reduce_sum(tf.abs(self.img_mask),[1,2,3])
self.L1_dist_s = tf.reduce_sum(tf.abs(self.img_mask_s),[1,2,3])
self.EN_dist = self.L2_dist + tf.multiply(self.L1_dist, self.beta)
self.EN_dist_s = self.L2_dist_s + tf.multiply(self.L1_dist_s, self.beta)
# compute the probability of the label class versus the maximum other
self.target_lab_score = tf.reduce_sum((self.target_lab)*self.ImgToEnforceLabel_Score,1)
target_lab_score_s = tf.reduce_sum((self.target_lab)*self.ImgToEnforceLabel_Score_s,1)
self.max_nontarget_lab_score = tf.reduce_max((1-self.target_lab)*self.ImgToEnforceLabel_Score - (self.target_lab*10000),1)
max_nontarget_lab_score_s = tf.reduce_max((1-self.target_lab)*self.ImgToEnforceLabel_Score_s - (self.target_lab*10000),1)
if self.mode == "PP":
Loss_Attack = tf.maximum(0.0, self.max_nontarget_lab_score - self.target_lab_score + self.kappa)
Loss_Attack_s = tf.maximum(0.0, max_nontarget_lab_score_s - target_lab_score_s + self.kappa)
elif self.mode == "PN":
Loss_Attack = tf.maximum(0.0, -self.max_nontarget_lab_score + self.target_lab_score + self.kappa)
Loss_Attack_s = tf.maximum(0.0, -max_nontarget_lab_score_s + target_lab_score_s + self.kappa)
# sum up the losses
self.Loss_L1Dist = tf.reduce_sum(self.L1_dist)
self.Loss_L1Dist_s = tf.reduce_sum(self.L1_dist_s)
self.Loss_L2Dist = tf.reduce_sum(self.L2_dist)
self.Loss_L2Dist_s = tf.reduce_sum(self.L2_dist_s)
self.Loss_Attack = tf.reduce_sum(self.const*Loss_Attack)
with tf.name_scope("loss_attack_s"):
self.Loss_Attack_s = tf.reduce_sum(self.const*Loss_Attack_s)
if self.AE:
if self.mode == "PP":
self.Loss_AE_Dist = self.gamma*tf.square(tf.norm(self.AE(self.delta_img)-self.delta_img))
self.Loss_AE_Dist_s = self.gamma*tf.square(tf.norm(self.AE(self.delta_img)-self.delta_img_s))
elif self.mode == "PN":
self.Loss_AE_Dist = self.gamma*tf.square(tf.norm(self.AE(self.adv_img)-self.adv_img))
self.Loss_AE_Dist_s = self.gamma*tf.square(tf.norm(self.AE(self.adv_img_s)-self.adv_img_s))
else:
self.Loss_AE_Dist = tf.constant(0, dtype="float32")
self.Loss_AE_Dist_s = tf.constant(0, dtype="float32")
with tf.name_scope("loss"):
# self.Loss_ToOptimize = self.Loss_Attack_s + self.Loss_L2Dist_s + self.Loss_AE_Dist_s
self.Loss_ToOptimize = self.Loss_Attack_s + tf.multiply(self.gamma,self.attr_score_s)
#self.Loss_Overall = self.Loss_Attack + self.Loss_L2Dist + self.Loss_AE_Dist + tf.multiply(self.beta, self.Loss_L1Dist)
self.Loss_Overall = self.Loss_Attack + tf.multiply(self.gamma,self.attr_score_s) + tf.multiply(self.beta, self.Loss_L1Dist)
#print(self.Loss_Attack.shape, self.Loss_Overall.shape)
self.learning_rate = tf.train.polynomial_decay(self.INIT_LEARNING_RATE, self.global_step, self.MAX_ITERATIONS, 0, power=0.5)
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
start_vars = set(x.name for x in tf.global_variables())
# self.train = optimizer.minimize(self.Loss_ToOptimize, var_list=[self.adv_img_s], global_step=self.global_step)
self.train = optimizer.minimize(self.Loss_ToOptimize, var_list=[self.mask_vec_s], global_step=self.global_step)
end_vars = tf.global_variables()
new_vars = [x for x in end_vars if x.name not in start_vars]
# these are the variables to initialize when we run
self.setup = []
self.setup.append(self.orig_img.assign(self.assign_orig_img))
self.setup.append(self.target_lab.assign(self.assign_target_lab))
self.setup.append(self.const.assign(self.assign_const))
self.setup.append(self.mask_vec.assign(self.assign_mask_vec))
self.setup.append(self.mask_vec_s.assign(self.assign_mask_vec_s))
self.init = tf.variables_initializer(var_list=[self.global_step]+[self.mask_vec_s]+[self.mask_vec]+new_vars)
def check_attributes_celebA(self, attributes, x, y):
"""
Load attribute classifiers and check which attributes in original image x
are modified in adversarial image y
Args:
attributes (str list): list of attributes to load attribute classifiers for
x (numpy.ndarray): original image
y (numpy.ndarray): adversarial image
Returns:
str: string detailing which attributes were added to (or removed from)
x resulting in y
"""
orig_attr_score = np.zeros((len(attributes), 1))
adv_attr_score = np.zeros((len(attributes), 1))
for i in range(len(attributes)):
attr = attributes[i]
# load json and create model
json_file_name = "../../aix360/models/CEM_MAF/simple_{}_model.json".format(
attr)
json_file = open(json_file_name, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
weight_file_name = "../../aix360/models/CEM_MAF/simple_{}_weights.h5".format(
attr)
loaded_model.load_weights(weight_file_name)
orig_attr_score[i] = loaded_model.predict(x)[0]
adv_attr_score[i] = loaded_model.predict(y)[0]
# pre-determined thresholds for changes in prediction values
thresh_pos = np.zeros((len(attributes), 1))
thresh_pos[0] = .15
thresh_pos[1] = .15
thresh_pos[2] = .15
thresh_pos[3] = .15
thresh_pos[4] = .15
thresh_pos[5] = .15
thresh_pos[6] = .1
thresh_pos[7] = .25
thresh_pos[8] = .1
thresh_pos[9] = .15
thresh_pos[10] = .15
thresh_pos[11] = .15
thresh_neg = np.zeros((len(attributes), 1))
thresh_neg[0] = -.25
thresh_neg[1] = -.25
thresh_neg[2] = -.25
thresh_neg[3] = -.25
thresh_neg[4] = -.35
thresh_neg[5] = -.25
thresh_neg[6] = -.12
thresh_neg[7] = -.25
thresh_neg[8] = -.25
thresh_neg[9] = -.25
thresh_neg[10] = -.25
thresh_neg[11] = -.25
changes_abs = adv_attr_score - orig_attr_score
changes = np.zeros((len(attributes), 1))
res = ""
for i in range(len(attributes)):
if changes_abs[i] >= thresh_pos[i]:
changes[i] = 1
elif changes_abs[i] <= thresh_neg[i]:
changes[i] = -1
added = np.where(changes == 1)[0]
for j in range(len(added)):
res += "Added " + attributes[added[j]] + ","
removed = np.where(changes[i] == -1)[0]
for j in range(len(removed)):
res += "Removed " + attributes[removed[j]] + ","
return res[:-1]
def __init__(self, sess, model, attributes, aix360_path, mode, batch_size,
kappa, init_learning_rate, binary_search_steps,
max_iterations, initial_const, gamma, attr_reg,
attr_penalty_reg, latent_square_loss_reg):
"""
Initialize PN explainer object.
Args:
sess (tensorflow.python.client.session.Session): Tensorflow session
model: KerasClassifier that contains a trained model to be explained
attributes (str list): list of attributes to load attribute classifiers for
aix360_path (str): path to aix360 used to determine paths to pretrained attribute classifiers
mode (str): "PN" for pertinent negative or "PP" for pertinent positive
batch_size (int): batch size for how many instances to explain
kappa (float): Confidence parameter that controls difference between prediction of
PN (or PP) and original prediction
init_learning_rate (float): initial learning rate for gradient descent optimizer
binary_search_steps (int): Controls number of random restarts to find best PN
max_iterations (int): Max number iterations to run some version of gradient descent on
PN optimization problem from a single random initialization, i.e., total
number of iterations wll be arg_binary_search_steps * arg_max_iterations
initial_const (int): Constant used for upper/lower bounds in binary search
gamma (float): Penalty parameter encouraging addition of attributes for PN
attr_reg (float): Penalty parameter on regularization of PN to be predicted different from
original image
attr_penalty_reg (float): Penalty regularizing PN from being too different from original image
latent_square_loss_reg (float): Penalty regularizing PN from being too different from original
image in the latent space
"""
# image_size, num_channels, nun_classes = model.image_size, model.num_channels, model.num_labels
# %%change%%
image_size = model._input_shape[0]
num_channels = model._input_shape[2]
nun_classes = model._nb_classes
shape = (batch_size, image_size, image_size, num_channels)
latent_shape = (batch_size, 512)
self.sess = sess
self.INIT_LEARNING_RATE = init_learning_rate
self.MAX_ITERATIONS = max_iterations
self.BINARY_SEARCH_STEPS = binary_search_steps
self.kappa = kappa
self.init_const = initial_const
self.batch_size = batch_size
self.mode = mode
self.gamma = gamma
self.attributes = attributes
self.aix360_path = aix360_path
self.attr_reg = attr_reg
self.attr_penalty_reg = attr_penalty_reg
self.attr_threshold = tf.constant(
0.5, dtype="float32"
) # penalize the attributes of orig_img having scores <= this value
self.latent_square_loss_reg = latent_square_loss_reg
# these are variables to be more efficient in sending data to tf
self.orig_img = tf.Variable(np.zeros(shape), dtype=tf.float32)
self.orig_latent = tf.Variable(np.zeros(latent_shape),
dtype=tf.float32,
name="orig_latent")
self.adv_latent = tf.Variable(np.zeros(latent_shape),
dtype=tf.float32,
name="adv_latent")
self.target_lab = tf.Variable(np.zeros((batch_size, nun_classes)),
dtype=tf.float32)
self.const = tf.Variable(np.zeros(batch_size), dtype=tf.float32)
self.global_step = tf.Variable(0.0, trainable=False)
# and here's what we use to assign them
self.assign_orig_img = tf.placeholder(tf.float32, shape)
self.assign_orig_latent = tf.placeholder(tf.float32,
latent_shape,
name="assign_orig_latent")
self.assign_adv_latent = tf.placeholder(tf.float32,
latent_shape,
name="assign_adv_latent")
self.assign_target_lab = tf.placeholder(tf.float32,
(batch_size, nun_classes),
name="assign_target_label")
self.assign_const = tf.placeholder(tf.float32, [batch_size])
### Load attribute classifier
nn_type = "simple"
#import copy
attr_model_list = []
# attr_threshold_idx=[]
# count=0
for attr in self.attributes:
# load test data into memory using Image Data Generator
# print("Loading data for " + attr + " into memory")
# load json and create model
json_file_name = os.path.join(
aix360_path,
"models/CEM_MAF/{}_{}_model.json".format(nn_type, attr))
json_file = open(json_file_name, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
weight_file_name = os.path.join(
aix360_path,
"models/CEM_MAF/{}_{}_weights.h5".format(nn_type, attr))
loaded_model.load_weights(weight_file_name)
print("Loaded model for " + attr + " from disk")
attr_model_list.append(loaded_model)
# if self.mode == "PP":
# pass
# else:
# attr_threshold_idx=tf.cond(loaded_model(self.orig_img)<= self.attr_threshold, lambda: attr_threshold_idx.append(count), lambda: attr_threshold_idx)
# count=count+1
print("# of attr models is", len(attr_model_list))
# print("# of attr smaller than THR is",len(attr_threshold_idx))
# load gan
# Import official CelebA-HQ networks.
with open(
os.path.join(
aix360_path,
'algorithms/contrastive/progressive_growing_of_gans/karras2018iclr-celebahq-1024x1024.pkl'
), 'rb') as file:
G, D, Gs = pickle.load(file)
# inputs, redundent ...
# in_labels = tf.placeholder(tf.float32, shape=(None, 0))
in_labels = tf.constant(0, shape=(1, 0))
# get the GAN network,
with tf.variable_scope(Gs.scope, reuse=tf.AUTO_REUSE):
out_image = Gs._build_func(self.adv_latent,
in_labels,
**Gs.static_kwargs,
reuse=True)
tanspose_image = tf.transpose(out_image, perm=[0, 2, 3, 1])
resize_image = tf.image.resize_images(tanspose_image, [224, 224])
self.adv_img = tf.clip_by_value(resize_image / 2, -0.5, 0.5)
self.adv_updater = tf.assign(self.adv_latent, self.assign_adv_latent)
"""--------------------------------"""
# prediction BEFORE-SOFTMAX of the model
self.delta_img = self.orig_img - self.adv_img
# %%change%%
if self.mode == "PP":
# self.ImgToEnforceLabel_Score = model.predict(self.delta_img)
self.ImgToEnforceLabel_Score = model.predictsym(self.delta_img)
elif self.mode == "PN":
# self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
self.ImgToEnforceLabel_Score = model.predictsym(self.adv_img)
# Attribute classifier score
self.attr_score = tf.constant(0, dtype="float32")
self.attr_penalty = tf.constant(0, dtype="float32")
if self.mode == "PP":
for i in range(len(attr_model_list)):
self.attr_score = self.attr_score + tf.maximum(
attr_model_list[i](self.adv_img) - attr_model_list[i]
(self.orig_img), tf.constant(0, tf.float32))
self.attr_score = tf.squeeze(self.attr_score)
#self.ImgToEnforceLabel_Score = model.predict(self.adv_img)
#self.ImgToEnforceLabel_Score_s = model.predict(self.adv_img_s)
elif self.mode == "PN":
for i in range(len(attr_model_list)):
self.attr_score = self.attr_score + tf.maximum(
attr_model_list[i](self.orig_img) - attr_model_list[i]
(self.adv_img), tf.constant(0, tf.float32))
self.attr_score = tf.squeeze(self.attr_score)
#self.attr_penalty = tf.squeeze(self.attr_penalty)
self.attr_penalty = self.attr_penalty + tf.multiply(
tf.cond(
tf.squeeze(attr_model_list[i]
(self.orig_img)) <= self.attr_threshold,
lambda: tf.constant(1, tf.float32),
lambda: tf.constant(0, tf.float32)),
tf.squeeze(attr_model_list[i](self.adv_img)))
#tf.maximum(attr_model_list[i](self.orig_img) - attr_model_list[i](self.adv_img),tf.constant(0, tf.float32))
# Sum of attributes penalty in attr_threshold_idx
# self.attr_penalty = tf.constant(0, dtype="float32")
# if len(attr_threshold_idx)==0:
# pass
# else:
# for i in range(len(attr_threshold_idx)):
# self.attr_penalty = self.attr_penalty + attr_model_list[i](self.adv_img)
self.delta_latent = self.orig_latent - self.adv_latent
# distance to the input data
self.L2_img_dist = tf.reduce_sum(tf.square(self.delta_img), [1, 2, 3])
self.L2_latent_dist = tf.reduce_sum(tf.square(self.delta_latent))
# compute the probability of the label class versus the maximum other
self.target_lab_score = tf.reduce_sum(
(self.target_lab) * self.ImgToEnforceLabel_Score, 1)
self.max_nontarget_lab_score = tf.reduce_max(
(1 - self.target_lab) * self.ImgToEnforceLabel_Score -
(self.target_lab * 10000), 1)
if self.mode == "PP":
Loss_Attack = tf.maximum(
0.0, self.max_nontarget_lab_score - self.target_lab_score +
self.kappa)
elif self.mode == "PN":
Loss_Attack = tf.maximum(
0.0, -self.max_nontarget_lab_score + self.target_lab_score +
self.kappa)
# sum up the losses
self.Loss_Latent_L2Dist = tf.reduce_sum(self.latent_square_loss_reg *
self.L2_latent_dist)
self.Loss_Img_L2Dist = tf.reduce_sum(self.gamma * self.L2_img_dist)
self.Loss_Attack = tf.reduce_sum(self.const * Loss_Attack)
self.Loss_attr = tf.reduce_sum(self.attr_reg * self.attr_score)
self.Loss_attr_penalty = tf.reduce_sum(self.attr_penalty_reg *
self.attr_penalty)
self.Loss_Overall = self.Loss_Latent_L2Dist + self.Loss_Img_L2Dist + self.Loss_Attack + self.Loss_attr + self.Loss_attr_penalty
# self.Loss_Overall = self.Loss_Attack
self.learning_rate = tf.train.polynomial_decay(self.INIT_LEARNING_RATE,
self.global_step,
self.MAX_ITERATIONS,
0,
power=0.5)
optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
start_vars = set(x.name for x in tf.global_variables())
self.train = optimizer.minimize(self.Loss_Overall,
var_list=[self.adv_latent],
global_step=self.global_step)
end_vars = tf.global_variables()
new_vars = [x for x in end_vars if x.name not in start_vars]
# these are the variables to initialize when we run
self.setup = []
self.setup.append(self.orig_img.assign(self.assign_orig_img))
self.setup.append(self.orig_latent.assign(self.assign_orig_latent))
self.setup.append(self.adv_latent.assign(self.assign_adv_latent))
self.setup.append(self.target_lab.assign(self.assign_target_lab))
self.setup.append(self.const.assign(self.assign_const))
self.init = tf.variables_initializer(var_list=[self.global_step] +
[self.adv_latent] + new_vars)
def load_codec(codec_prefix, print_summary=False):
# load data
saveFilePrefix = codec_prefix + '_'
# load models
encoder_model_filename = saveFilePrefix + "encoder.json"
decoder_model_filename = saveFilePrefix + "decoder.json"
encoder_weight_filename = saveFilePrefix + "encoder.h5"
decoder_weight_filename = saveFilePrefix + "decoder.h5"
if not os.path.isfile(encoder_model_filename):
raise Exception("The file for encoder model does not exist:{}".format(
encoder_model_filename))
json_file = open(encoder_model_filename, 'r')
encoder = model_from_json(json_file.read(), custom_objects={"tf": tf})
json_file.close()
if not os.path.isfile(encoder_weight_filename):
raise Exception(
"The file for encoder weights does not exist:{}".format(
encoder_weight_filename))
encoder.load_weights(encoder_weight_filename)
if not os.path.isfile(decoder_model_filename):
raise Exception("The file for decoder model does not exist:{}".format(
decoder_model_filename))
json_file = open(decoder_model_filename, 'r')
decoder_temp = model_from_json(json_file.read(), custom_objects={"tf": tf})
json_file.close()
if not os.path.isfile(decoder_weight_filename):
raise Exception(
"The file for decoder weights does not exist:{}".format(
decoder_weight_filename))
decoder_temp.load_weights(decoder_weight_filename)
if print_summary:
print("Encoder summaries")
encoder.summary()
_, encode_H, encode_W, numChannels = encoder.output_shape
# the workaround
# use config to construct the decoder model
# and then load the weights for each layer
# Note that the information in config[0::] is the sequential model for encoder
# thus, we need to exclude the first element
config = decoder_temp.get_config()
config2 = config[1::]
config2[0]['config']['batch_input_shape'] = (None, encode_H, encode_W,
numChannels)
decoder = Sequential.from_config(config2, custom_objects={"tf": tf})
# set weights
cnt = -1
for l in decoder_temp.layers:
cnt += 1
if cnt == 0:
continue
weights = l.get_weights()
decoder.layers[cnt - 1].set_weights(weights)
if print_summary:
print("Decoder summaries")
decoder.summary()
return encoder, decoder
