def SAFA(x_sat, x_grd, keep_prob, dimension, trainable):
vgg_grd = VGG16()
grd_local = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
grd_local = tf.nn.max_pool(grd_local, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
batch, g_height, g_width, channel = grd_local.get_shape().as_list()
grd_w = spatial_aware(grd_local, dimension, trainable, name='spatial_grd')
grd_local = tf.reshape(grd_local, [-1, g_height * g_width, channel])
grd_global = tf.einsum('bic, bid -> bdc', grd_local, grd_w)
grd_global = tf.reshape(grd_global, [-1, dimension*channel])
vgg_sat = VGG16()
sat_local = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
sat_local = tf.nn.max_pool(sat_local, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')
batch, s_height, s_width, channel = sat_local.get_shape().as_list()
sat_w = spatial_aware(sat_local, dimension, trainable, name='spatial_sat')
sat_local = tf.reshape(sat_local, [-1, s_height * s_width, channel])
sat_global = tf.einsum('bic, bid -> bdc', sat_local, sat_w)
sat_global = tf.reshape(sat_global, [-1, dimension*channel])
return tf.nn.l2_normalize(sat_global, dim=1), tf.nn.l2_normalize(grd_global, dim=1)
def two_stream_baseline(x_sat, x_grd, keep_prob, trainable):
with tf.device('/gpu:1'):
vgg_grd = VGG16()
grd_local = vgg_grd.eight_layer_conv_multiscale(
x_grd, keep_prob, trainable, 'VGG_grd')
vgg_sat = VGG16()
sat_local = vgg_sat.eight_layer_conv_multiscale(
x_sat, keep_prob, trainable, 'VGG_sat')
with tf.device('/gpu:0'):
fc = Siamese_FC()
sat_global, grd_global = fc.my_siamese_fc_multiscale(
sat_local, grd_local, trainable, 'dim_reduction')
return sat_global, grd_global
def attack_all():
if args.model == 'resnet':
model = ResNet50(enable_lat =args.enable_lat,
epsilon =args.lat_epsilon,
pro_num =args.lat_pronum,
batch_size =args.model_batchsize,
num_classes = 10,
if_dropout=args.dropout
)
elif args.model == 'vgg':
model = VGG16(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout
)
elif args.model == 'resnet18':
model = ResNet18(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout
)
model.cuda()
model.load_state_dict(torch.load((args.modelpath)))
for eps in range(4,16+1):
test_data_cln, test_data_adv, test_label, test_label_adv = attack_one(model,eps)
if args.generate:
save_data_label(args.savepath, eps, test_data_cln, test_data_adv,test_label, test_label_adv)
def cvm_net_I(x_sat, x_grd, coords_geo, keep_prob, trainable):
with tf.device('/gpu:1'):
# local descriptors / local feature extraction; use only the convolution layers (until conv_5) of vgg for this
vgg_grd = VGG16()
# output of conv_5 layer; the input dimension and output dimension of conv2 layer (512 x 512)
grd_local = vgg_grd.VGG16_conv(x_grd, keep_prob, False, 'VGG_grd')
with tf.variable_scope('netvlad_grd', reuse=tf.AUTO_REUSE):
# embed netvlad on each cnn branch to get global descriptor
# constructor!
# feature size is the depth of the final convolution layer
# grd local
netvlad_grd = lp.NetVLAD(feature_size=512,
max_samples=tf.shape(grd_local)[1] *
tf.shape(grd_local)[2],
cluster_size=64,
output_dim=4096,
gating=True,
add_batch_norm=False,
is_training=trainable)
# forward prop; give input of conv_5 layer
# output is the batch*(vlad vector) i.e 12 * [(K*D) x 1] (64*512) x 1
grd_vlad = netvlad_grd.forward(grd_local)
vgg_sat = VGG16()
# local satellite descriptors ;
sat_local = vgg_sat.VGG16_conv(x_sat, keep_prob, False, 'VGG_sat')
with tf.variable_scope('netvlad_sat', reuse=tf.AUTO_REUSE):
# shape of sat_local is batch_sz x 14 x 14 x 512 (i guess)
# so max samples is 14*14 = 196? is this equal to N?
# is cluster size equal to K or N?
netvlad_sat = lp.NetVLAD(feature_size=512,
max_samples=tf.shape(sat_local)[1] *
tf.shape(sat_local)[2],
cluster_size=64,
output_dim=4096,
gating=True,
add_batch_norm=False,
is_training=trainable)
sat_vlad = netvlad_sat.forward(sat_local)
with tf.device('/gpu:0'):
fc = Siamese_FC()
sat_global, grd_global = fc.siamese_fc(sat_vlad, grd_vlad, trainable,
'dim_reduction')
# grd_global = fc.new_fc_layer(sat_global, coords_geo)
return sat_global, grd_global
def LPN_AWARE_HEATMAP(x_sat, x_grd, keep_prob, dimension, trainable,
multi_loss):
vgg_grd = VGG16()
grd_local = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
grd_local = tf.nn.max_pool(grd_local, [1, 2, 2, 1], [1, 2, 2, 1],
padding='SAME') #b*4*20*512
grd_local_v = grd_local
grd_local = get_block_feature(grd_local, dimension) #batch*4*6*512*8
batch, g_height, g_width, channel, dimension = grd_local.get_shape(
).as_list()
# print(grd_local)
grd_w = block_spatial_aware(grd_local,
dimension,
trainable,
name='b_spatial_grd') #b*24*8
grd_local = tf.reshape(grd_local,
[-1, g_height * g_width, channel, dimension])
grd_global_ = tf.einsum('bicd, bid -> bdc', grd_local, grd_w) #b*8*512
grd_global = tf.reshape(grd_global_, [-1, dimension * channel])
# print('grd_block: ', grd_block.shape)
vgg_sat = VGG16()
sat_local = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
sat_local = tf.nn.max_pool(sat_local, [1, 2, 2, 1], [1, 2, 2, 1],
padding='SAME')
sat_local = get_block_feature(sat_local, dimension) #batch*4*6*512*8
batch, g_height, g_width, channel, dimension = sat_local.get_shape(
).as_list()
sat_w = block_spatial_aware(sat_local,
dimension,
trainable,
name='b_spatial_sat') #b*24*8
sat_local = tf.reshape(sat_local,
[-1, g_height * g_width, channel, dimension])
sat_global_ = tf.einsum('bicd, bid -> bdc', sat_local, sat_w) #b*8*512
sat_global = tf.reshape(sat_global_, [-1, dimension * channel])
# print('sat_block: ', sat_block.shape)
if multi_loss:
return grd_local_v
else:
return tf.nn.l2_normalize(sat_global,
dim=1), tf.nn.l2_normalize(grd_global, dim=1)
def main():
if args.model == 'vgg':
model = VGG16(enable_lat=args.enable_lat,
epsilon=args.anp_epsilon,
pro_num=args.anp_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout)
model.cuda()
model.load_state_dict(torch.load((args.modelpath)))
# if cifar then normalize epsilon from [0,255] to [0,1]
if args.dataset == 'mnist':
eps = args.attack_epsilon
else:
eps = args.attack_epsilon / 255.0
#eps = args.attack_epsilon
attack = Attack(dataroot=args.dataroot,
dataset=args.dataset,
batch_size=args.attack_batchsize,
target_model=model,
criterion=nn.CrossEntropyLoss(),
epsilon=eps,
alpha=args.attack_alpha,
iteration=args.attack_iter)
if args.attack == 'fgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.fgsm(
)
elif args.attack == 'ifgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.i_fgsm(
)
elif args.attack == 'stepll':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.step_ll(
)
elif args.attack == 'pgd':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.PGD()
elif args.attack == 'momentum_ifgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.momentum_ifgsm(
)
print(test_data_adv.size(), test_label.size(), type(test_data_adv))
#test_data, test_label, size = read_data_label('./test_data_cln.p','./test_label.p')
#test_data_adv, test_label_adv, size = read_data_label('./test_data_cln.p','./test_label.p')
'''
test_loader = attack.return_data()
dataiter = iter(test_loader)
images,labels = dataiter.next()
print(images[0])
'''
#test_data_cln, test_data_adv, test_label, test_label_adv = attack.i_fgsm()
#display(test_data_cln, test_data_adv, test_label, test_label_adv)
if args.generate:
save_data_label(args.savepath, test_data_cln, test_data_adv,
test_label, test_label_adv)
def joint_feat_learning(x_sat, x_grd, x_grd_gan, keep_prob, trainable):
with tf.device('/gpu:1'):
vgg_grd = VGG16()
grd_local = vgg_grd.eight_layer_conv_multiscale(
x_grd, keep_prob, trainable, 'VGG_grd')
vgg_sat = VGG16()
sat_local = vgg_sat.eight_layer_conv_multiscale(
x_sat, keep_prob, trainable, 'VGG_sat')
vgg_grd_gan = VGG16()
grd_local_gan = vgg_grd_gan.eight_layer_conv_multiscale(
x_grd_gan, keep_prob, trainable, 'VGG_grd_seg')
with tf.device('/gpu:0'):
fc = Siamese_FC()
sat_global, grd_global, grd_global_gan = fc.three_stream_joint_feat_learning(
sat_local, grd_local, grd_local_gan, trainable, 'dim_reduction')
return sat_global, grd_global, grd_global_gan
def VGG_gp(x_sat, x_grd, keep_prob, trainable):
############## VGG module #################
vgg_grd = VGG16()
grd_vgg = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
vgg_sat = VGG16()
sat_vgg = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
grd_height, grd_width, grd_channel = grd_vgg.get_shape().as_list()[1:]
grd_global = tf.nn.max_pool(grd_vgg, [1, grd_height, grd_width, 1], [1, 1, 1, 1], padding='VALID')
grd_global = tf.reshape(grd_global, [-1, grd_channel])
sat_height, sat_width, sat_channel = sat_vgg.get_shape().as_list()[1:]
sat_global = tf.nn.max_pool(sat_vgg, [1, sat_height, sat_width, 1], [1, 1, 1, 1], padding='VALID')
sat_global = tf.reshape(sat_global, [-1, sat_channel])
return tf.nn.l2_normalize(sat_global, dim=1), tf.nn.l2_normalize(grd_global, dim=1)
def VGG_13_conv_v2_cir(x_sat, x_grd, keep_prob, trainable):
vgg_grd = VGG16(x_grd, keep_prob, trainable, 'VGG_grd')
grd_layer13 = vgg_grd.layer13_output
grd_vgg = vgg_grd.conv2(grd_layer13, 'grd', dimension=16)
grd_vgg = tf.nn.l2_normalize(grd_vgg, axis=[1, 2, 3])
vgg_sat = VGG16_cir(x_sat, keep_prob, trainable, 'VGG_sat')
sat_layer13 = vgg_sat.layer13_output
sat_vgg = vgg_sat.conv2(sat_layer13, 'sat', dimension=16)
sat_matrix, distance, pred_orien = corr_crop_distance(sat_vgg, grd_vgg)
return sat_vgg, grd_vgg, distance, pred_orien
def cvm_net_II(x_sat, x_grd, keep_prob, trainable):
with tf.device('/gpu:1'):
vgg_grd = VGG16()
grd_local = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
vgg_sat = VGG16()
sat_local = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
transnet = TransNet()
trans_sat, trans_grd = transnet.transform(sat_local, grd_local,
keep_prob, trainable,
'transformation')
with tf.variable_scope('netvlad') as scope:
netvlad_sat = lp.NetVLAD(feature_size=512,
max_samples=tf.shape(trans_sat)[1] *
tf.shape(trans_sat)[2],
cluster_size=64,
output_dim=4096,
gating=True,
add_batch_norm=False,
is_training=trainable)
sat_global = netvlad_sat.forward(trans_sat, True)
scope.reuse_variables()
netvlad_grd = lp.NetVLAD(feature_size=512,
max_samples=tf.shape(trans_grd)[1] *
tf.shape(trans_grd)[2],
cluster_size=64,
output_dim=4096,
gating=True,
add_batch_norm=False,
is_training=trainable)
grd_global = netvlad_grd.forward(trans_grd, True)
return sat_global, grd_global
def noise():
distortion_name = [
'gaussian_noise', 'shot_noise', 'impulse_noise', 'speckle_noise',
'gaussian_blur', 'defocus_blur', 'glass_blur', 'snow', 'frost', 'fog',
'brightness', 'contrast', 'elastic_transform', 'motion_blur',
'zoom_blur', 'pixelate', 'jpeg_compression', 'spatter', 'saturate'
]
label_name = 'labels.npy'
#load model
net = VGG16(enable_lat=False,
epsilon=0,
pro_num=1,
batch_size=args.batchsize_noise,
if_dropout=True)
net.cuda()
net.load_state_dict(torch.load(args.model_path_noise))
label_root = args.distortion_root + label_name
y = np.load(label_root)
clean_loader = get_clean_loader(y)
#error_rates = []
for i in range(len(distortion_name)):
data_root = args.distortion_root + distortion_name[i] + '.npy'
label_root = args.distortion_root + label_name
#load data
x = np.load(data_root)
#print(x.shape)
#for j in range(0,10):
# plt.imsave('./cifar-c/{}_{}.png'.format(distortion_name[i],j), x[j])
x = x.transpose((0, 3, 1, 2))
x = x / 255.0
y = np.load(label_root)
#data_loader
test_loader = get_test_loader(x, y)
test_model(net, clean_loader, test_loader, distortion_name[i],
args.model_name)
'''
def CVFT(x_sat, x_grd, keep_prob, trainable):
def conv_layer(x,
kernel_dim,
input_dim,
output_dim,
stride,
trainable,
activated,
name='ot_conv',
activation_function=tf.nn.relu):
with tf.variable_scope(name,
reuse=tf.AUTO_REUSE): # reuse=tf.AUTO_REUSE
weight = tf.get_variable(
name='weights',
shape=[kernel_dim, kernel_dim, input_dim, output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable(
name='biases',
shape=[output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
out = tf.nn.conv2d(
x, weight, strides=[1, stride, stride, 1
], padding='SAME') + bias
if activated:
out = activation_function(out)
return out
def fc_layer(x, trainable, name='ot_fc'):
height, width, channel = x.get_shape().as_list()[1:]
assert channel == 1
in_dimension = height * width
out_dimension = in_dimension**2
input_feature = tf.reshape(x, [-1, height * width])
with tf.variable_scope(name):
weight = tf.get_variable(
name='weights',
shape=[in_dimension, out_dimension],
trainable=trainable,
initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.005),
regularizer=tf.contrib.layers.l2_regularizer(0.01))
bias = tf.get_variable(name='biases',
shape=[out_dimension],
trainable=trainable,
initializer=tf.constant_initializer(
np.eye(in_dimension).reshape(
in_dimension**2)))
out = tf.matmul(input_feature, weight) + bias
out = tf.reshape(out, [-1, in_dimension, in_dimension])
return out
def ot(input_feature, trainable, name='ot'):
height, width, channel = input_feature.get_shape().as_list()[1:]
conv_feature = conv_layer(input_feature,
kernel_dim=1,
input_dim=channel,
output_dim=1,
stride=1,
trainable=trainable,
activated=True,
name=name + 'ot_conv')
fc_feature = fc_layer(conv_feature, trainable, name=name + 'ot_fc')
ot_matrix = sinkhorn(fc_feature * (-100.))
return ot_matrix
def apply_ot(input_feature, ot_matrix):
height, width, channel = input_feature.get_shape().as_list()[1:]
in_dimension = ot_matrix.get_shape().as_list()[1]
reshape_input = tf.transpose(
tf.reshape(input_feature, [-1, in_dimension, channel]), [0, 2, 1])
# shape = [batch, channel, in_dimension]
out = tf.einsum('bci, bio -> bco', reshape_input, ot_matrix)
output_feature = tf.reshape(tf.transpose(out, [0, 2, 1]),
[-1, height, width, channel])
return output_feature
############## VGG module #################
vgg_grd = VGG16()
grd_vgg = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
grd_vgg = conv_layer(grd_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='grd_conv')
vgg_sat = VGG16()
sat_vgg = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
sat_vgg = conv_layer(sat_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='sat_conv')
############## resize #################
height, width, channel = sat_vgg.get_shape().as_list()[1:]
grd_vgg = tf.image.resize_bilinear(grd_vgg, [height, width])
############## OT module ######################
ot_matrix_grd_branch = ot(grd_vgg, trainable, name='ot_grd_branch')
grd_ot = apply_ot(grd_vgg, ot_matrix_grd_branch)
sat_ot = sat_vgg
################# reshape ###################
grd_height, grd_width, grd_channel = grd_ot.get_shape().as_list()[1:]
grd_global = tf.reshape(grd_ot, [-1, grd_height * grd_width * grd_channel])
sat_height, sat_width, sat_channel = sat_ot.get_shape().as_list()[1:]
sat_global = tf.reshape(sat_ot, [-1, sat_height * sat_width * sat_channel])
return tf.nn.l2_normalize(sat_global,
dim=1), tf.nn.l2_normalize(grd_global, dim=1)
def CVFT_LPN(x_sat, x_grd, keep_prob, trainable, block, multi_loss):
def conv_layer(x,
kernel_dim,
input_dim,
output_dim,
stride,
trainable,
activated,
name='ot_conv',
activation_function=tf.nn.relu):
with tf.variable_scope(name,
reuse=tf.AUTO_REUSE): # reuse=tf.AUTO_REUSE
weight = tf.get_variable(
name='weights',
shape=[kernel_dim, kernel_dim, input_dim, output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable(
name='biases',
shape=[output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
out = tf.nn.conv2d(
x, weight, strides=[1, stride, stride, 1
], padding='SAME') + bias
if activated:
out = activation_function(out)
return out
def fc_layer(x, trainable, name='ot_fc'):
height, width, channel = x.get_shape().as_list()[1:]
assert channel == 1
in_dimension = height * width
out_dimension = in_dimension**2
input_feature = tf.reshape(x, [-1, height * width])
with tf.variable_scope(name):
weight = tf.get_variable(
name='weights',
shape=[in_dimension, out_dimension],
trainable=trainable,
initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.005),
regularizer=tf.contrib.layers.l2_regularizer(0.01))
bias = tf.get_variable(name='biases',
shape=[out_dimension],
trainable=trainable,
initializer=tf.constant_initializer(
np.eye(in_dimension).reshape(
in_dimension**2)))
out = tf.matmul(input_feature, weight) + bias
out = tf.reshape(out, [-1, in_dimension, in_dimension])
return out
def ot(input_feature, trainable, name='ot'):
height, width, channel = input_feature.get_shape().as_list()[1:]
conv_feature = conv_layer(input_feature,
kernel_dim=1,
input_dim=channel,
output_dim=1,
stride=1,
trainable=trainable,
activated=True,
name=name + 'ot_conv')
fc_feature = fc_layer(conv_feature, trainable, name=name + 'ot_fc')
ot_matrix = sinkhorn(fc_feature * (-100.))
return ot_matrix
def apply_ot(input_feature, ot_matrix):
height, width, channel = input_feature.get_shape().as_list()[1:]
in_dimension = ot_matrix.get_shape().as_list()[1]
reshape_input = tf.transpose(
tf.reshape(input_feature, [-1, in_dimension, channel]), [0, 2, 1])
# shape = [batch, channel, in_dimension]
out = tf.einsum('bci, bio -> bco', reshape_input, ot_matrix)
output_feature = tf.reshape(tf.transpose(out, [0, 2, 1]),
[-1, height, width, channel])
return output_feature
# column partition
def get_block_feature(input_feature, block):
batch, height, width, channel = input_feature.get_shape().as_list()
dim = height * width * channel
sw = np.floor(width / block).astype(np.int32) #stride
kw = width - (block - 1) * sw #kernel
f_bs = []
for i in range(block):
f_b = input_feature[:, :, i * sw:i * sw + kw, :]
f_bs.append(f_b)
block_feature = tf.stack(f_bs, axis=4) #batch*8*1*64*8
h, w, c, b = block_feature.get_shape().as_list()[1:]
block_feature_ = tf.reshape(block_feature, [-1, h * w * c, block])
# print('block_feature_shape', block_feature_)
return block_feature_
# row partition
# def get_block_feature(input_feature, block):
# batch, height, width, channel = input_feature.get_shape().as_list()
# dim = height * width * channel
# sh = np.floor(height / block).astype(np.int32) #stride
# kh = height - (block - 1) * sh #kernel
# f_bs = []
# for i in range(block):
# f_b = input_feature[:,i*sh:i*sh+kh,:,:]
# f_bs.append(f_b)
# block_feature = tf.stack(f_bs,axis=4) #batch*1*8*64*8
# h, w, c, b = block_feature.get_shape().as_list()[1:]
# block_feature_ = tf.reshape(block_feature, [-1, h*w*c, block])
# # print('block_feature_shape', block_feature_)
# return block_feature_
############## VGG module #################
vgg_grd = VGG16()
grd_vgg = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
grd_vgg = conv_layer(grd_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='grd_conv')
vgg_sat = VGG16()
sat_vgg = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
sat_vgg = conv_layer(sat_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='sat_conv')
############## resize #################
height, width, channel = sat_vgg.get_shape().as_list()[1:]
grd_vgg = tf.image.resize_bilinear(grd_vgg, [height, width])
############## OT module ######################
ot_matrix_grd_branch = ot(grd_vgg, trainable, name='ot_grd_branch')
grd_ot = apply_ot(grd_vgg, ot_matrix_grd_branch)
sat_ot = sat_vgg
################# reshape ###################
if multi_loss:
sat_global_ = get_block_feature(sat_ot, block)
grd_global_ = get_block_feature(grd_ot, block)
else:
grd_height, grd_width, grd_channel = grd_ot.get_shape().as_list()[1:]
grd_global = tf.reshape(grd_ot,
[-1, grd_height * grd_width * grd_channel])
sat_height, sat_width, sat_channel = sat_ot.get_shape().as_list()[1:]
sat_global = tf.reshape(sat_ot,
[-1, sat_height * sat_width * sat_channel])
if multi_loss:
return tf.nn.l2_normalize(sat_global_,
dim=1), tf.nn.l2_normalize(grd_global_,
dim=1)
else:
return tf.nn.l2_normalize(sat_global,
dim=1), tf.nn.l2_normalize(grd_global, dim=1)
def VGG_conv(x_sat, x_grd, keep_prob, trainable):
def conv_layer(x,
kernel_dim,
input_dim,
output_dim,
stride,
trainable,
activated,
name='ot_conv',
activation_function=tf.nn.relu):
with tf.variable_scope(name,
reuse=tf.AUTO_REUSE): # reuse=tf.AUTO_REUSE
weight = tf.get_variable(
name='weights',
shape=[kernel_dim, kernel_dim, input_dim, output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable(
name='biases',
shape=[output_dim],
trainable=trainable,
initializer=tf.contrib.layers.xavier_initializer())
out = tf.nn.conv2d(
x, weight, strides=[1, stride, stride, 1
], padding='SAME') + bias
if activated:
out = activation_function(out)
return out
############## VGG module #################
vgg_grd = VGG16()
grd_vgg = vgg_grd.VGG16_conv(x_grd, keep_prob, trainable, 'VGG_grd')
grd_vgg = conv_layer(grd_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='grd_conv')
vgg_sat = VGG16()
sat_vgg = vgg_sat.VGG16_conv(x_sat, keep_prob, trainable, 'VGG_sat')
sat_vgg = conv_layer(sat_vgg,
kernel_dim=3,
input_dim=512,
output_dim=64,
stride=2,
trainable=trainable,
activated=True,
name='sat_conv')
############## resize #################
height, width, channel = sat_vgg.get_shape().as_list()[1:]
grd_vgg = tf.image.resize_bilinear(grd_vgg, [height, width])
############## reshape #################
grd_height, grd_width, grd_channel = grd_vgg.get_shape().as_list()[1:]
grd_global = tf.reshape(grd_vgg,
[-1, grd_height * grd_width * grd_channel])
sat_height, sat_width, sat_channel = sat_vgg.get_shape().as_list()[1:]
sat_global = tf.reshape(sat_vgg,
[-1, sat_height * sat_width * sat_channel])
return tf.nn.l2_normalize(sat_global,
dim=1), tf.nn.l2_normalize(grd_global, dim=1)
if __name__ == "__main__":
if args.enable_lat:
real_model_path = args.model_path + "lat_param.pkl"
print('loading the ANP model')
else:
real_model_path = args.model_path + "naive_param.pkl"
print('loading the naive model')
if args.test_flag:
args.enable_lat = False
if args.model == 'vgg':
cnn = VGG16(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
cnn.cuda()
if os.path.exists(real_model_path):
cnn.load_state_dict(torch.load(real_model_path))
print('load model.')
else:
print("load failed.")
if args.test_flag:
test_op(cnn)
else:
train_op(cnn)
import tensorflow as tf
from sklearn.metrics import accuracy_score, log_loss
xTest, yTest = Cifar100(train=False)
#xTest, yTest = Cifar10(train=False)
#xTest, yTest = mnist(train=False)
_,w,h,nChannels = xTest.shape
nSamples, nClasses = yTest.shape
sess = tf.Session()
xInput = tf.placeholder(tf.float32, [None, w, h, nChannels])
yInput = tf.placeholder(tf.float32, [None, nClasses])
vgg = VGG16(xInput,0,nClasses, train=False)
logits = vgg.fc8
parameters = np.load("SGD.npz")
vgg.loadW(parameters, sess)
crossEntropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(
labels=yInput, logits=logits))
predictions = tf.nn.softmax(logits)
acc = tf.equal(tf.argmax(predictions,1), tf.argmax(yInput,1))
accuracy = tf.reduce_mean(tf.cast(acc, tf.float32))
y = np.zeros((nSamples, nClasses))
batchSize = 128
step = np.ceil(nSamples/batchSize)
def cal_lip(model_path, data_path):
cleanpath = "/media/dsg3/dsgprivate/lat/test_lip/" + TYPE + "/test_data_cln.p"
labelpath = "/media/dsg3/dsgprivate/lat/test_lip/" + TYPE + "/test_label.p"
model = VGG16(enable_lat=ENABLE_LAT,
epsilon=EPS,
pro_num=PROG,
batch_size=BATCH_SIZE,
if_dropout=IF_DROP)
model.cuda()
if os.path.exists(model_path):
model.load_state_dict(torch.load(model_path))
#print('load model successfully.')
else:
print("load failed.")
# get test_data , test_label from .p file
clean_data, test_label, size = read_data_label(cleanpath, labelpath)
test_data, test_label, size = read_data_label(data_path, labelpath)
if size == 0:
print("reading data failed.")
return
sel_clean = torch.Tensor(LEN, 3, 32, 32)
sel_test = torch.Tensor(LEN, 3, 32, 32)
sel_clean_label = torch.LongTensor(LEN)
sel_test_label = torch.LongTensor(LEN)
j = 0
for i in range(START, test_label.size(0)):
if test_label[i] == 3:
sel_clean[j] = clean_data[i]
sel_test[j] = test_data[i]
sel_clean_label[j] = 3
sel_test_label[j] = 3
j += 1
if j == LEN:
break
'''
sel_clean = clean_data[START:START+LEN]
sel_test = test_data[START:START+LEN]
sel_clean_label = test_label[START:START+LEN]
sel_test_label = test_label[START:START+LEN]
'''
# create dataset
clean_set = Data.TensorDataset(sel_clean, sel_clean_label)
test_set = Data.TensorDataset(sel_test, sel_test_label)
clean_loader = Data.DataLoader(
dataset=clean_set,
batch_size=sel_clean.size(0), # LEN
shuffle=False)
test_loader = Data.DataLoader(
dataset=test_set,
batch_size=sel_test.size(0), # LEN
shuffle=False)
c_lip = 0
criterion = nn.CrossEntropyLoss()
# Test the model
model.eval()
x_cln = 0
loss_cln = 0
for x, y in clean_loader:
x = x.cuda()
x_cln = x.view(sel_clean.size(0), -1)
y = y.cuda()
with torch.no_grad():
h = model(x)
loss = criterion(h, y)
loss_cln = loss.item()
model.train()
model.eval()
x_tst = 0
loss_tst = 0
for x, y in test_loader:
x = x.cuda()
y = y.cuda()
x_tst = x.view(sel_test.size(0), -1)
with torch.no_grad():
h = model(x)
loss = criterion(h, y)
loss_tst = loss.item()
model.train()
dist = 0
for j in range(sel_test.size(0)):
dist += torch.max(abs(x_cln[j] - x_tst[j])) # norm p = inf
dist = dist / LEN
c_lip = abs(loss_cln - loss_tst) / dist
return c_lip
def choose_model():
# switch models
print(args.model)
print(args.layerlist)
if args.model == 'lenet':
cnn = LeNet(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
batch_norm=args.batchnorm,
if_dropout=args.dropout)
elif args.model == 'resnet':
cnn = ResNet50(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'resnet18':
cnn = ResNet18(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg16':
cnn = VGG16(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg11':
cnn = VGG11(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg13':
cnn = VGG13(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg19':
cnn = VGG19(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'densenet':
cnn = DenseNet()
cnn.cuda()
if args.enable_lat:
cnn.choose_layer()
return cnn
def cal_eni(model, data):
cnn = VGG16(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
cnn.cuda()
model_path = model_list[model]
if os.path.exists(model_path):
cnn.load_state_dict(torch.load(model_path))
print('load model successfully.')
else:
print("load failed.")
model = cnn
# get test_data , test_label from .p file
clean_data, test_label, size = read_data_label(args.clean_path,
args.label_path)
test_data, test_label, size = read_data_label(data_list[data],
args.label_path)
if size == 0:
print("reading data failed.")
return
if data == 'clean':
sel_clean = clean_data[args.start_idx:args.start_idx + args.length]
sel_test = test_data[args.start_idx + args.length:args.start_idx +
2 * args.length]
sel_clean_label = test_label[args.start_idx:args.start_idx +
args.length]
sel_test_label = test_label[args.start_idx +
args.length:args.start_idx +
2 * args.length]
else:
sel_clean = clean_data[args.start_idx:args.start_idx + args.length]
sel_test = test_data[args.start_idx:args.start_idx + args.length]
sel_clean_label = test_label[args.start_idx:args.start_idx +
args.length]
sel_test_label = test_label[args.start_idx:args.start_idx +
args.length]
# create dataset
clean_set = Data.TensorDataset(sel_clean, sel_clean_label)
test_set = Data.TensorDataset(sel_test, sel_test_label)
clean_loader = Data.DataLoader(dataset=clean_set,
batch_size=args.length,
shuffle=False)
test_loader = Data.DataLoader(dataset=test_set,
batch_size=args.length,
shuffle=False)
c_eni = 0
criterion = nn.CrossEntropyLoss()
# Test the model
model.eval()
x_cln = 0
loss_cln = 0
for x, y in clean_loader:
x = x.cuda()
x_cln = x.view(sel_clean.size(0), -1)
y = y.cuda()
#print(y)
with torch.no_grad():
h = model(x)
loss = criterion(h, y)
loss_cln = loss.item()
model.train()
model.eval()
x_tst = 0
loss_tst = 0
for x, y in test_loader:
x = x.cuda()
y = y.cuda()
x_tst = x.view(sel_test.size(0), -1)
with torch.no_grad():
h = model(x)
loss = criterion(h, y)
loss_tst = loss.item()
model.train()
dist = 0
for j in range(sel_test.size(0)):
dist += torch.max(abs(x_cln[j] - x_tst[j])) # norm p = inf
dist = dist / args.length
c_eni = abs(loss_cln - loss_tst) / dist
return c_eni
xVal = X
yVal = Y
_, w, h, nChannels = xTrain.shape
nSamples, nClasses = yTrain.shape
print('Number of training samples: ', nSamples, ' Number of classes: ',
nClasses)
print('Number of validation samples: ', xVal.shape[0])
sess = tf.Session(config=config)
xInput = tf.placeholder(tf.float32, [None, w, h, nChannels])
yInput = tf.placeholder(tf.float32, [None, nClasses])
learningRate = tf.placeholder(tf.float32, shape=[])
vgg = VGG16(xInput, 0.5, nClasses)
logits = vgg.fc8
#Metrics
predictions = tf.nn.softmax(logits)
acc = tf.equal(tf.argmax(predictions, 1), tf.argmax(yInput, 1))
accuracy = tf.reduce_mean(tf.cast(acc, tf.float32))
validationL, validationL2, validationA = [], [], []
trainL, trainL2, trainA = [], [], []
#Training parameters
epochs = 200
#lrInit = 0.005 (for Mnist)
lrInit = 0.05
batchSize = 128
data_list = adv_data_resnet
elif target == 'densenet':
data_list = adv_data_densenet
elif target == 'inception':
data_list = adv_data_inception
for data_cat in data_list:
data_path = data_list[data_cat]
test_one(model,data_cat,data_path)
if __name__ == "__main__":
if args.model == 'vgg':
model = VGG16(enable_lat=False,
epsilon=0.6,
pro_num=5,
batch_size=args.batch_size,
if_dropout=True)
elif args.model == 'resnet':
model = ResNet50(enable_lat=False,
epsilon=0.6,
pro_num=5,
batch_size=args.batch_size,
if_dropout=True)
model.cuda()
if os.path.exists(args.model_path):
model.load_state_dict(torch.load(args.model_path))
print('load model.')
else:
print("load failed.")
import numpy as np
from torch.utils.data.sampler import SubsetRandomSampler
import time
# trainloader=Data.DataLoader(md,batch_size=16,shuffle=True, num_workers=12)
torch.cuda.set_device(1)
total_epochs = 640
print_inter = 10
vali_inter = 200
validation_split = 0.2
shuffle_dataset = True
stud_names = [ 'Nut_stud','panel_stud', 'stud', 'T_stud', 'ball_stud']
# stud_names = ['Nut_stud', 'panel_stud', 'stud', 'T_stud', 'ball_stud']
for name in stud_names:
net = VGG()
save_path = './checkpoints/' + name + '/RI1_VGG'
if not os.path.exists(save_path):
os.mkdir(save_path)
md_train = myDataset('./mat/' + name + '/stud_data_RI_train.mat', aug=True, inch=3)
md_test = myDataset('./mat/' + name + '/stud_data_RI_test.mat', aug=False, inch=3)
# dataset_size = len(md)
# indices = list(range(dataset_size))
# split = int(np.floor(validation_split * dataset_size))
# if shuffle_dataset :
# np.random.shuffle(indices)
# train_indices, val_indices = indices[split:], indices[:split]
#
# # Creating PT data samplers and loaders:
# train_sampler = SubsetRandomSampler(train_indices)
# valid_sampler = SubsetRandomSampler(val_indices)
def test_all():
if args.model == 'resnet':
model = ResNet50(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg':
model = VGG16(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
model.cuda()
resnet_model_list = {
'naive-resnet':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/naive/naive_param.pkl",
'new-AT-resnet':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/nat/naive_param.pkl",
'origin-AT-resnet':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/oat/naive_param.pkl",
'ensemble-AT-resnet':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/eat/naive_param.pkl",
'LAT-aaai-resnet':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/aaai/naive_param.pkl",
'DPLAT-resnet50':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/dplat/lat_param.pkl",
'DPLAT-resnet18':
"/media/dsg3/dsgprivate/yuhang/model/resnet50/dplat-18/lat_param.pkl",
}
vgg_model_list = {
'naive-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/naive/naive_param.pkl",
'new-AT-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/nat/naive_param.pkl",
'origin-AT-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/oat/naive_param.pkl",
'ensemble-AT-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/eat/naive_param.pkl",
'LAT-aaai-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/aaai/naive_param.pkl",
'PLAT-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/plat/lat_param.pkl",
'DPLAT-vgg':
"/media/dsg3/dsgprivate/yuhang/model/vgg16/dplat/lat_param.pkl",
}
data_list = {
'k3c0.03-vgg':
"/media/dsg3/dsgprivate/lat/test_cw/test_adv(k3c0.030).p",
'k5c0.05-vgg':
"/media/dsg3/dsgprivate/lat/test_cw/test_adv(k5c0.050).p",
'k3c0.03-resnet':
"/media/dsg3/dsgprivate/lat/test_cw/resnet/test_adv(k3c0.030).p",
'k5c0.05-resnet':
"/media/dsg3/dsgprivate/lat/test_cw/resnet/test_adv(k5c0.050).p",
}
if args.model == 'vgg':
model_list = vgg_model_list
elif args.model == 'resnet':
model_list = resnet_model_list
for target in model_list:
print('------- Now target model is {} ------'.format(target))
model_path = model_list[target]
if target == 'DPLAT-resnet18':
model = ResNet18(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout).cuda()
model.load_state_dict(torch.load(model_path))
for data in data_list:
data_path = data_list[data]
test_one(model, data, data_path)
from ResNet import ResNet50
from VGG import VGG16
from denseNet import DenseNet
from Inception_v2 import Inception_v2
from utils import read_data_label
if args.model == 'resnet':
model = ResNet50(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout)
elif args.model == 'vgg':
model = VGG16(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout)
elif args.model == 'densenet':
model = DenseNet()
elif args.model == 'inception':
model = Inception_v2()
model.cuda()
model.load_state_dict(torch.load((args.modelpath)))
# if cifar then normalize epsilon from [0,255] to [0,1]
if args.dataset == 'cifar10':
eps = args.attack_epsilon / 255.0
else:
eps = args.attack_epsilon
def main():
if args.model == 'resnet':
model = ResNet50(enable_lat =args.enable_lat,
epsilon =args.lat_epsilon,
pro_num =args.lat_pronum,
batch_size =args.model_batchsize,
num_classes = 10,
if_dropout=args.dropout
)
elif args.model == 'vgg':
model = VGG16(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout
)
elif args.model == 'resnet18':
model = ResNet18(enable_lat=args.enable_lat,
epsilon=args.lat_epsilon,
pro_num=args.lat_pronum,
batch_size=args.model_batchsize,
num_classes=10,
if_dropout=args.dropout
)
elif args.model == 'densenet':
model = DenseNet()
elif args.model == 'inception':
model = Inception_v2()
model.cuda()
model.load_state_dict(torch.load((args.modelpath)))
# if cifar then normalize epsilon from [0,255] to [0,1]
'''
if args.dataset == 'cifar10':
eps = args.attack_epsilon / 255.0
else:
eps = args.attack_epsilon
'''
eps = args.attack_epsilon
# the last layer of densenet is F.log_softmax, while CrossEntropyLoss have contained Softmax()
attack = Attack(dataroot = "/media/dsg3/dsgprivate/lat/data/cifar10/",
dataset = args.dataset,
batch_size = args.attack_batchsize,
target_model = model,
criterion = nn.CrossEntropyLoss(),
epsilon = eps,
alpha = args.attack_alpha,
iteration = args.attack_iter)
if args.attack == 'fgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.fgsm()
elif args.attack == 'ifgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.i_fgsm()
elif args.attack == 'stepll':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.step_ll()
elif args.attack == 'pgd':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.PGD()
elif args.attack == 'momentum_ifgsm':
test_data_cln, test_data_adv, test_label, test_label_adv = attack.momentum_ifgsm()
print(test_data_adv.size(),test_label.size(), type(test_data_adv))
#test_data, test_label, size = read_data_label('./test_data_cln.p','./test_label.p')
#test_data_adv, test_label_adv, size = read_data_label('./test_data_cln.p','./test_label.p')
'''
test_loader = attack.return_data()
dataiter = iter(test_loader)
images,labels = dataiter.next()
print(images[0])
'''
#test_data_cln, test_data_adv, test_label, test_label_adv = attack.i_fgsm()
#display(test_data_cln, test_data_adv, test_label, test_label_adv)
if args.generate:
save_data_label(args.savepath, eps, test_data_cln, test_data_adv,test_label, test_label_adv)
