def val_pre_model(source_path, folder, img_dim, architechture):
array_path = os.path.join(source_path, folder)
pre_model_path = os.path.join(source_path, 'pre_model')
# call(['rm', '-rf', pre_model_path])
shutil.rmtree(pre_model_path,ignore_errors=True)
# call(['mkdir', '-p', pre_model_path])
os.makedirs(pre_model_path)
if architechture == 'resnet50':
popped, pre_model = get_resnet_pre_model(img_dim)
elif architechture == 'xception':
popped, pre_model = get_xception_pre_model(img_dim)
else:
popped, pre_model = get_inception_v3_pre_model(img_dim)
for (array, label, array_name, label_name) in tqdm(gen_array_from_dir(array_path)):
if architechture == 'resnet50':
array = resnet_preprocess_input(array[np.newaxis].astype(np.float32))
elif architechture == 'xception':
array = xception_preprocess_input(array[np.newaxis].astype(np.float32))
else:
array = inception_v3_preprocess_input(array[np.newaxis].astype(np.float32))
array_pre_model = np.squeeze(pre_model.predict(array, batch_size=1))
array_name = array_name.split('.')[0]
label_name = label_name.split('.')[0]
img_pre_model_path = os.path.join(pre_model_path, array_name)
label_pre_model_path = os.path.join(pre_model_path, label_name)
np.save(img_pre_model_path, array_pre_model)
np.save(label_pre_model_path, label)
def val_pre_model(source_path, folder, img_dim, architechture):
array_path = os.path.join(source_path, folder)
pre_model_path = os.path.join(source_path, 'pre_model')
shutil.rmtree(pre_model_path,ignore_errors=True)
os.makedirs(pre_model_path)
if architechture == 'resnet50':
popped, pre_model = get_resnet_pre_model(img_dim)
elif architechture == 'xception':
popped, pre_model = get_xception_pre_model(img_dim)
else:
popped, pre_model = get_inception_v3_pre_model(img_dim)
for (array, label, array_name, label_name) in tqdm(gen_array_from_dir(array_path)):
if architechture == 'resnet50':
array = resnet_preprocess_input(array[np.newaxis].astype(np.float32))
elif architechture == 'xception':
array = xception_preprocess_input(array[np.newaxis].astype(np.float32))
else:
array = inception_v3_preprocess_input(array[np.newaxis].astype(np.float32))
array_pre_model = np.squeeze(pre_model.predict(array, batch_size=1))
array_name = array_name.split('.')[0]
label_name = label_name.split('.')[0]
img_pre_model_path = os.path.join(pre_model_path, array_name)
label_pre_model_path = os.path.join(pre_model_path, label_name)
np.save(img_pre_model_path, array_pre_model)
np.save(label_pre_model_path, label)
def gen_minibatches(array_dir, array_names, batch_size, architecture, final = False):
array_names = list(array_names)
while True:
# in place shuffle
np.random.shuffle(array_names)
array_names_mb = array_names[:batch_size]
arrays = []
labels = []
for array_name in array_names_mb:
img_path = os.path.join(array_dir, array_name)
array = np.load(img_path)
if final:
if architecture == 'resnet50':
array = np.squeeze(resnet_preprocess_input(array[np.newaxis].astype(np.float32)))
elif architecture == 'xception':
array = np.squeeze(xception_preprocess_input(array[np.newaxis].astype(np.float32)))
else:
array = np.squeeze(inception_v3_preprocess_input(array[np.newaxis].astype(np.float32)))
arrays.append(array)
labels.append(np.load(img_path.replace('-img-','-label-')))
yield np.array(arrays), np.array(labels)
def get_feats(self, mname, batch_size, filename):
fname, data_directory = self.get_data(filename)
n = len(fname)
print(n)
if mname == 'V':
mod = self.get_model()
features = np.zeros((n, 4096), dtype=np.float32)
else:
mod = self.get_resnet_model()
features = np.zeros((n, 2048), dtype=np.float32)
vgg16_feature = []
imgname = {}
b = int(n / batch_size) + 1
bn = 0
for i in range(0, b):
batch = np.zeros((batch_size, 224, 224, 3))
bn = 0
for k in range(i * batch_size, min((i + 1) * batch_size, n)):
img_path = os.path.join(data_directory, fname[k])
f = fname[k].replace('.jpg', '')
f = int(f[-7:])
imgname[f] = k
img = image.load_img(img_path, target_size=(224, 224))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
if mname == 'V':
img = preprocess_input(img)
else:
img = resnet_preprocess_input(img)
batch[bn, :, :, :] = img
bn += 1
if k % batch_size == 0 and k > 1:
print(str(k) + " Done")
start = time.time()
print("Predicting now at " + str(start))
if n < (i + 1) * batch_size:
batch = batch[0:bn]
features[i * batch_size:min((i + 1) *
batch_size, n), :] = mod.predict(batch)
end = round(time.time() - start, 1)
print("Prediction done in " + str(end))
print(features[i * batch_size:min((i + 1) * batch_size, n), :])
s = 'imgfeature_' + str(filename) + str(batch_size) + '_' + str(
i) + '.pkl'
# pickle.dump([imgname,features],open(s,'wb'))
pickle.dump([imgname, features],
open(os.path.join('D:\\CS\\DLNLP_Project\\data', s),
'wb'))
#print(features)
# s='imgfeature_' + str(data_directory) + '_' + str(batch_size) + '_final.pkl'
# pickle.dump([imgname,features],open(s,'wb'))
s = 'imgfeature_' + str(filename) + str(batch_size) + '_final.pkl'
pickle.dump([imgname, features],
open(os.path.join('D:\\CS\\DLNLP_Project\\data', s), 'wb'))
return imgname, features
def multi_predict(aug_gen, models, architecture):
predicted = []
for img, _ in aug_gen:
if architecture == 'resnet50':
img = resnet_preprocess_input(img[np.newaxis].astype(np.float32))
elif architecture == 'xception':
img = xception_preprocess_input(img[np.newaxis].astype(np.float32))
else:
img = inception_v3_preprocess_input(img[np.newaxis].astype(np.float32))
for model in models:
predicted.append(model.predict(img))
predicted = np.array(predicted).sum(axis=0)
pred_list = list(predicted[0])
return predicted, pred_list
def multi_predict(aug_gen, models, architecture):
predicted = []
for img, _ in aug_gen:
if architecture == 'resnet50':
img = resnet_preprocess_input(img[np.newaxis].astype(np.float32))
elif architecture == 'xception':
img = xception_preprocess_input(img[np.newaxis].astype(np.float32))
else:
img = inception_v3_preprocess_input(img[np.newaxis].astype(np.float32))
for model in models:
predicted.append(model.predict(img))
predicted = np.array(predicted).sum(axis=0)
pred_list = list(predicted[0])
return predicted, pred_list
def run_demo(model_file, img_file, labels_file):
tf.logging.set_verbosity(tf.logging.FATAL)
# read labels
labels = read_labels(labels_file)
# print(labels) #KB
# load model
vqa_model = load_model(model_file)
print("VQA Model loaded..\n")
# load text features and tokenizer
textObj = QuestionFeatures(initialize=False)
textObj.load_tokenizer(
os.path.join(Constants.DIRECTORIES["root"], "tokenizer.pkl"))
# image model and feature pre-processing
img_model = get_resnet_model()
img = image.load_img(img_file, target_size=(224, 224))
img = image.img_to_array(img)
img = np.expand_dims(img, axis=0)
img = resnet_preprocess_input(img)
img_features = np.zeros((1, 2048), dtype=np.float32)
img_features = img_model.predict(img)
print(f'Image @{img_file} loaded...')
while True:
print("\n\nNext Question:")
curr_question = [str(input())]
if curr_question[0].lower() == 'q':
print(f'Thank you for trying the VQA demo, bye!')
break
# convert question to token
question_features = textObj.tokenize(curr_question)
prediction = vqa_model.predict([img_features, question_features])
print(
f'\nAnswer:{np.argmax(prediction)} {labels[np.argmax(prediction)]}'
)
def train_model(project, final=False, last=False):
weight_label = '-' + project['architecture'] + '-weights-'
source_path = project['path']
weights_path = os.path.join(source_path, 'weights')
plot_path = os.path.join(source_path, 'plots')
if last:
weights = 'last_weights'
else:
weights = 'best_weights'
if final:
weight_label += '-final-'
use_path = os.path.join(source_path, 'augmented')
else:
use_path = os.path.join(source_path, 'pre_model')
project['model_round'] += 1
shutil.rmtree(weights_path, ignore_errors=True)
os.makedirs(weights_path)
shutil.rmtree(plot_path, ignore_errors=True)
os.makedirs(plot_path)
img_dim = project['img_dim'] * project['img_size']
conv_dim = project['conv_dim'] * project['img_size']
lr = project['learning_rate']
decay = project['learning_rate_decay']
all_files = os.listdir(use_path)
pre_model_files = list(filter(lambda x: r'-img-' in x, all_files))
label_names = list(filter(lambda x: r'-label-' in x, all_files))
pre_model_files_df = pd.DataFrame({'files': pre_model_files})
pre_model_files_df['suffix'] = pre_model_files_df.apply(
lambda row: row.files.split('.')[-1], axis=1)
pre_model_files_df = pre_model_files_df[pre_model_files_df.suffix == 'npy']
pre_model_files_df['ind'] = pre_model_files_df.apply(
lambda row: row.files.split('-')[0], axis=1).astype(int)
pre_model_files_df['label'] = pre_model_files_df.apply(
lambda row: row.files.split('-')[3], axis=1)
pre_model_files_df_dedup = pre_model_files_df.drop_duplicates(subset='ind')
pre_model_files_df = pre_model_files_df.set_index(['ind'])
pre_model_files.sort()
label_names.sort()
pre_model_files_arr = np.array(pre_model_files)
label_names_arr = np.array(label_names)
labels = [
np.argmax(np.load(os.path.join(use_path, label_name)))
for label_name in label_names
]
best_weights = []
last_weights = []
if project['kfold'] >= 3:
kfold = StratifiedKFold(n_splits=project['kfold'],
shuffle=True,
random_state=project['seed'])
kfold_generator = kfold.split(pre_model_files_df_dedup,
pre_model_files_df_dedup.label)
validate = True
else:
print('Too few k-folds selected, fitting on all data')
kfold_generator = no_folds_generator(pre_model_files_df_dedup)
validate = False
for i, (train, test) in enumerate(kfold_generator):
if project['kfold_every']:
print('----- Fitting Fold', i, '-----')
elif i > 0:
break
weights_name = project['name'] + weight_label + '-kfold-' + str(
i) + '-round-' + str(project['model_round']) + '.hdf5'
plot_name = project['name'] + weight_label + '-kfold-' + str(
i) + '-round-' + str(project['model_round']) + '.png'
if project[weights] is None:
fold_weights = None
else:
fold_weights = project[weights][i]
if final:
if project['architecture'] == 'resnet50':
model = get_resnet_final_model(img_dim, conv_dim,
project['number_categories'],
fold_weights,
project['is_final'])
elif project['architecture'] == 'xception':
model = get_xception_final_model(img_dim, conv_dim,
project['number_categories'],
fold_weights,
project['is_final'])
else:
model = get_inception_v3_final_model(
img_dim, conv_dim, project['number_categories'],
fold_weights, project['is_final'])
for i, layer in enumerate(model.layers[1].layers):
if len(layer.trainable_weights) > 0:
if i < project['final_cutoff']:
mult = 0.01
else:
mult = 0.1
layer.learning_rate_multiplier = [
mult for tw in layer.trainable_weights
]
else:
if project['architecture'] == 'resnet50':
pre_model, model = get_resnet_pre_post_model(
img_dim,
conv_dim,
len(project['categories']),
model_weights=fold_weights)
elif project['architecture'] == 'xception':
pre_model, model = get_xception_pre_post_model(
img_dim,
conv_dim,
len(project['categories']),
model_weights=fold_weights)
else:
pre_model, model = get_inception_v3_pre_post_model(
img_dim,
conv_dim,
len(project['categories']),
model_weights=fold_weights)
pre_model_files_dedup_train = pre_model_files_df_dedup.iloc[train]
train_ind = list(set(pre_model_files_dedup_train.ind))
pre_model_files_train = pre_model_files_df.loc[train_ind]
gen_train = gen_minibatches(use_path,
pre_model_files_train.files,
project['batch_size'],
project['architecture'],
final=final)
number_train_samples = len(pre_model_files_train)
if validate:
pre_model_files_dedup_test = pre_model_files_df_dedup.iloc[test]
test_ind = list(set(pre_model_files_dedup_test.ind))
pre_model_files_test = pre_model_files_df.loc[test_ind]
gen_test = gen_minibatches(use_path,
pre_model_files_test.files,
project['batch_size'],
project['architecture'],
final=final)
number_test_samples = len(pre_model_files_test)
validation_steps = (number_test_samples // project['batch_size'])
weights_checkpoint_file = weights_name.split(
'.'
)[0] + '-kfold-' + str(
i
) + "-improvement-{epoch:02d}-{val_categorical_accuracy:.4f}.hdf5"
checkpoint = ModelCheckpoint(os.path.join(weights_path,
weights_checkpoint_file),
monitor='val_categorical_accuracy',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
else:
gen_test = None
validation_steps = None
callbacks_list = None
steps_per_epoch = (number_train_samples // project['batch_size'])
for j in range(project['rounds']):
optimizer = Adam(lr=lr, decay=decay)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['categorical_accuracy'])
model.fit_generator(gen_train,
steps_per_epoch=steps_per_epoch,
epochs=project['cycle'] * (j + 1),
verbose=1,
validation_data=gen_test,
validation_steps=validation_steps,
initial_epoch=j * project['cycle'],
callbacks=callbacks_list)
model.save_weights(os.path.join(weights_path, weights_name))
last_weights.append(os.path.join(weights_path, weights_name))
weights_names = os.listdir(weights_path)
max_val = -1
max_i = -1
for j, name in enumerate(weights_names):
if name.find(weights_name.split('.')[0]) >= 0:
if (name.find(weight_label) >= 0) and (name.find('improvement')
>= 0):
val = int(name.split('.')[1])
if val > max_val:
max_val = val
max_i = j
if project['plot']:
print('Plotting confusion matrix')
if max_i == -1:
print('Loading last weights:',
os.path.join(weights_path, weights_name))
model.load_weights(os.path.join(weights_path, weights_name))
else:
print('Loading best weights:',
os.path.join(weights_path, weights_names[max_i]))
model.load_weights(
os.path.join(weights_path, weights_names[max_i]))
best_predictions = []
true_labels = []
print('Predicting test files')
if validate:
use_files = pre_model_files_test.files
else:
use_files = pre_model_files_train.files
for array_name in tqdm(use_files):
img_path = os.path.join(use_path, array_name)
img = np.load(img_path)
if final:
if project['architecture'] == 'resnet50':
img = np.squeeze(
resnet_preprocess_input(img[np.newaxis].astype(
np.float32)))
elif project['architecture'] == 'xception':
img = np.squeeze(
xception_preprocess_input(img[np.newaxis].astype(
np.float32)))
else:
img = np.squeeze(
inception_v3_preprocess_input(
img[np.newaxis].astype(np.float32)))
prediction = model.predict(img[np.newaxis])
best_predictions.append(
project['categories'][np.argmax(prediction)])
true_label = np.load(img_path.replace('-img-', '-label-'))
true_labels.append(
project['categories'][np.argmax(true_label)])
cm = confusion_matrix(true_labels, best_predictions,
project['categories'])
plt.clf()
sns.heatmap(pd.DataFrame(cm, project['categories'],
project['categories']),
annot=True,
fmt='g')
plt.xlabel('Actual')
plt.xlabel('Predicted')
plt.xticks(rotation=45, fontsize=8)
plt.yticks(rotation=45, fontsize=8)
plt.title('Confusion matrix for fold: ' + str(i) + '\nweights' +
weights_name)
plt.savefig(os.path.join(plot_path, plot_name))
print('Confusion matrix plot saved:',
colored(os.path.join(plot_path, plot_name), 'magenta'))
if max_i == -1:
best_weights.append(os.path.join(weights_path, weights_name))
else:
best_weights.append(
os.path.join(weights_path, weights_names[max_i]))
project['number_categories'] = len(project['categories'])
project['best_weights'] = best_weights
project['last_weights'] = last_weights
project['is_final'] = final
return project
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('--dataset_dir',
type=str,
default=config.DEFAULT_DATASET_DIR)
parser.add_argument('--batch_size', type=int, default=10)
parser.add_argument('--inv_model_file',
type=str,
help='a saved model (.h5) file')
args = parser.parse_args()
config_keras_backend()
if not args.inv_model_file.endswith('.h5'):
sys.exit('model_file is not a .h5')
inv_model = tf.keras.models.load_model(args.inv_model_file,
compile=False,
custom_objects={'AdamW': AdamW})
inv_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
ds_validation = get_dataset(args.dataset_dir, 'validation',
args.batch_size)
## VGG
vgg_model = VGG19(include_top=True, weights='imagenet', classes=1000)
vgg_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# InceptionV3
inception_model = InceptionV3(include_top=True,
weights='imagenet',
classes=1000)
inception_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
## ResNet
resnet_model = ResNet50(include_top=True, weights='imagenet', classes=1000)
resnet_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# Process batches
iteration = 0
sum1 = 0
sum2 = 0
for images, labels in tfds.as_numpy(ds_validation):
if iteration < 532: #3822:#532:
print('continuing')
iteration += 1
continue
if iteration == 50000:
exit()
labels = np.argmax(labels, axis=1)
adv_imgs = run_attack(False,
'CarliniL2Method',
inception_model,
images,
labels,
batch_size=args.batch_size,
dataset='cifar',
fgsm_epsilon=0.3,
cwl2_confidence=0)
#adv_imgs = run_attack(False, 'DeepFool', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0)
#adv_imgs = run_attack(True, 'FastGradientMethod', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0)
#adv_imgs = run_attack(False, 'ProjectedGradientDescent', inception_model, images, labels, batch_size=10, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0)
## VGG ################################################
#img *= (2.0/255) # normalize to: 0.0~2.0
#img -= 1.0 # subtract mean to make it: -1.0~1.0
#img = np.expand_dims(img, axis=0)
vgg_imgs = []
resnet_imgs = []
inc_imgs = []
flip_imgs = []
inv_imgs = []
adv_vgg_imgs = []
adv_resnet_imgs = []
adv_inc_imgs = []
adv_flip_imgs = []
adv_inv_imgs = []
for ii in range(images.shape[0]):
img = copy.deepcopy(images[ii, :, :, :])
img += 1.0
#img /= (2.0/255)
img *= (255.0 / 2.0)
## VGG
vgg_img = copy.deepcopy(img)
vgg_img = cv2.resize(vgg_img, (224, 224))
vgg_img = vgg_preprocess_input(vgg_img)
vgg_imgs.append(vgg_img)
## Resnet
resnet_img = copy.deepcopy(img)
resnet_img = cv2.resize(resnet_img, (224, 224))
resnet_img = resnet_preprocess_input(resnet_img)
resnet_imgs.append(resnet_img)
## InceptionV3
inc_img = copy.deepcopy(img)
inc_img = cv2.resize(inc_img, (299, 299))
inc_img = inception_preprocess_input(inc_img)
inc_imgs.append(inc_img)
## Flipped
#flip_img = copy.deepcopy(img)
#flip_img = cv2.resize(flip_img, (299, 299))
#flip_img = cv2.flip(flip_img, 1)
#flip_img = inception_preprocess_input(flip_img)
#flip_imgs.append(flip_img)
flip_img = copy.deepcopy(images[ii, :, :, :])
flip_img = cv2.flip(flip_img, 1)
flip_imgs.append(flip_img)
## Inverse
inv_img = copy.deepcopy(images[ii, :, :, :]) #########
inv_img += 1.0
inv_img /= 2.0
inv_img = 1 - inv_img
inv_img *= 255.0
inv_img = cv2.resize(inv_img, (299, 299))
inv_img = inception_preprocess_input(inv_img)
inv_imgs.append(inv_img)
#==========================================
# ADVERSARIAL ---------------
adv_img = copy.deepcopy(adv_imgs[ii, :, :, :])
adv_img += 1.0
#adv_img /= (2.0/255)
adv_img *= (255.0 / 2.0)
# VGG
adv_vgg_img = copy.deepcopy(adv_img)
adv_vgg_img = cv2.resize(adv_vgg_img, (224, 224))
adv_vgg_img = vgg_preprocess_input(adv_vgg_img)
adv_vgg_imgs.append(adv_vgg_img)
# Resnet
adv_resnet_img = copy.deepcopy(adv_img)
adv_resnet_img = cv2.resize(adv_resnet_img, (224, 224))
adv_resnet_img = resnet_preprocess_input(adv_resnet_img)
adv_resnet_imgs.append(adv_resnet_img)
# InceptionV3
adv_inc_img = copy.deepcopy(adv_img)
adv_inc_img = cv2.resize(adv_inc_img, (299, 299))
adv_inc_img = inception_preprocess_input(adv_inc_img)
adv_inc_imgs.append(adv_inc_img)
## Flipped
#adv_flip_img = copy.deepcopy(img)
#adv_flip_img = cv2.resize(adv_flip_img, (299, 299))
#adv_flip_img = cv2.flip(adv_flip_img, 1)
#adv_flip_img = inception_preprocess_input(adv_flip_img)
#adv_flip_imgs.append(adv_flip_img)
adv_flip_img = copy.deepcopy(adv_imgs[ii, :, :, :])
adv_flip_img = cv2.flip(adv_flip_img, 1)
adv_flip_imgs.append(adv_flip_img)
## Inverse
##test on inverse Inceptionv3
adv_inv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) #########
adv_inv_img += 1.0
adv_inv_img /= 2.0
adv_inv_img = 1 - adv_inv_img
adv_inv_img *= 255.0
adv_inv_img = cv2.resize(adv_inv_img, (299, 299))
adv_inv_img = inception_preprocess_input(adv_inv_img)
adv_inv_imgs.append(adv_inv_img)
# Horizontal Flipping
# test on Resnet
vgg_imgs = np.asarray(vgg_imgs)
resnet_imgs = np.asarray(resnet_imgs)
inc_imgs = np.asarray(inc_imgs)
flip_imgs = np.asarray(flip_imgs)
inv_imgs = np.asarray(inv_imgs)
adv_vgg_imgs = np.asarray(adv_vgg_imgs)
adv_resnet_imgs = np.asarray(adv_resnet_imgs)
adv_inc_imgs = np.asarray(adv_inc_imgs)
adv_flip_imgs = np.asarray(adv_flip_imgs)
adv_inv_imgs = np.asarray(adv_inv_imgs)
# Default ResNet accuracy
_, results1 = resnet_model.evaluate(x=resnet_imgs, y=labels, verbose=0)
_, results2 = vgg_model.evaluate(x=vgg_imgs, y=labels, verbose=0)
_, results3 = inception_model.evaluate(x=inc_imgs, y=labels, verbose=0)
_, results4 = inception_model.evaluate(x=flip_imgs,
y=labels,
verbose=0)
_, results5 = inv_model.evaluate(x=inv_imgs, y=labels, verbose=0)
# print('-----------------------------------------------------')
_, results6 = resnet_model.evaluate(x=adv_resnet_imgs,
y=labels,
verbose=0)
_, results7 = vgg_model.evaluate(x=adv_vgg_imgs, y=labels, verbose=0)
_, results8 = inception_model.evaluate(x=adv_inc_imgs,
y=labels,
verbose=0)
_, results9 = inception_model.evaluate(x=adv_flip_imgs,
y=labels,
verbose=0)
_, results10 = inv_model.evaluate(x=adv_inv_imgs, y=labels, verbose=0)
print(iteration)
print(results1, results6)
print(results2, results7)
print(results3, results8)
print(results4, results9)
print(results5, results10)
# Print the figure images
INDEX = 1
# Original image
orig = copy.deepcopy(inc_imgs[INDEX])
orig /= 2.0
orig += 0.5
orig *= 255.0
# Adversarial image
adv = copy.deepcopy(adv_inc_imgs[INDEX])
adv /= 2.0
adv += 0.5
adv *= 255.0
#Perturbation image
diff = adv - orig
print(np.amax(diff))
#exit()
# Flip image
flip = copy.deepcopy(flip_imgs[INDEX])
flip /= 2.0
flip += 0.5
flip *= 255.0
# Save images
imageio.imwrite('pandas/panda_orig.png',
np.reshape(orig, (299, 299, 3)))
imageio.imwrite('pandas/panda_adv.png', np.reshape(adv, (299, 299, 3)))
imageio.imwrite('pandas/panda_diff.png',
np.reshape(diff, (299, 299, 3)))
imageio.imwrite('pandas/panda_flip.png',
np.reshape(flip, (299, 299, 3)))
print(labels)
print('Inception---original-------------------------')
#preds = inception_model.predict(np.reshape(inc_imgs[INDEX],(1,299,299,3)))
#print('confidence:', inc_decode_predictions(preds, top=1)[0])
preds = inception_model.predict(inc_imgs)
print('IncV3 Predicted:', np.argmax(preds, axis=1))
print('IncV3 Predicted:', np.amax(preds, axis=1))
print()
print('VGG---original-------------------------')
#preds = vgg_model.predict(np.reshape(vgg_imgs[INDEX],(1,224,224,3)))
#print('confidence:', vgg_decode_predictions(preds, top=1)[0])
preds = vgg_model.predict(vgg_imgs)
print('VGG Predicted:', np.argmax(preds, axis=1))
print('VGG Predicted:', np.amax(preds, axis=1))
print()
print('ResNet---original-------------------------')
#preds = resnet_model.predict(np.reshape(resnet_imgs[INDEX],(1,224,224,3)))
#print('confidence:', resnet_decode_predictions(preds, top=1)[0])
preds = resnet_model.predict(resnet_imgs)
print('ResNet Predicted:', np.argmax(preds, axis=1))
print('ResNet Predicted:', np.amax(preds, axis=1))
print()
print('Inception---adv-------------------------')
#preds = inception_model.predict(np.reshape(adv_inc_imgs[INDEX],(1,299,299,3)))
#print('confidence:', inc_decode_predictions(preds, top=1)[0])
preds = inception_model.predict(adv_inc_imgs)
print('Adv IncV3 Predicted:', np.argmax(preds, axis=1))
print('Adv IncV3 Predicted:', np.amax(preds, axis=1))
print()
print('VGG---adv-------------------------')
#preds = vgg_model.predict(np.reshape(adv_vgg_imgs[INDEX],(1,224,224,3)))
#print('confidence:', vgg_decode_predictions(preds, top=1)[0])
preds = vgg_model.predict(adv_vgg_imgs)
print('Adv VGG Predicted:', np.argmax(preds, axis=1))
print('Adv VGG Predicted:', np.amax(preds, axis=1))
print()
print('ResNet---adv-------------------------')
#preds = resnet_model.predict(np.reshape(adv_resnet_imgs[INDEX],(1,224,224,3)))
#print('confidence:', resnet_decode_predictions(preds, top=1)[0])
preds = resnet_model.predict(adv_resnet_imgs)
print('Adv ResNet Predicted:', np.argmax(preds, axis=1))
print('Adv ResNet Predicted:', np.amax(preds, axis=1))
print()
print('Inception---flip-------------------------')
#preds = inception_model.predict(np.reshape(adv_flip_imgs[INDEX],(1,299,299,3)))
#print('confidence:', inc_decode_predictions(preds, top=1)[0])
preds = inception_model.predict(adv_flip_imgs)
print('flip Predicted:', np.argmax(preds, axis=1))
print('flip Predicted:', np.amax(preds, axis=1))
print()
#print('Accuracies--------------------------')
#print('flip accuracy:', inc_decode_predictions(preds, top=3)[0])
exit()
with open("output_pgd_untarg_batch-20_norm-2.txt", "a") as myfile:
myfile.write(
str(results1) + ' ' + str(results2) + ' ' + str(results3) +
' ' + str(results4) + ' ' + str(results5) + ' ' +
str(results6) + ' ' + str(results7) + ' ' + str(results8) +
' ' + str(results9) + ' ' + str(results10) + '\n')
# Distances
norm_diffs_1 = [
np.linalg.norm(
np.subtract(adv_inc_imgs[ii].flatten(),
inc_imgs[ii].flatten()), 1)
for ii in range(inc_imgs.shape[0])
]
norm_diffs_2 = [
np.linalg.norm(
np.subtract(adv_inc_imgs[ii].flatten(),
inc_imgs[ii].flatten()), 2)
for ii in range(inc_imgs.shape[0])
]
norm_diffs_inf = [
np.linalg.norm(
np.subtract(adv_inc_imgs[ii].flatten(),
inc_imgs[ii].flatten()), np.inf)
for ii in range(inc_imgs.shape[0])
]
print(np.mean(norm_diffs_1), np.mean(norm_diffs_2),
np.mean(norm_diffs_inf))
with open("distances_pgd_untarg_batch-20_norm-2.txt", "a") as myfile:
myfile.write(
str(np.mean(norm_diffs_1)) + ' ' + str(np.mean(norm_diffs_2)) +
' ' + str(np.mean(norm_diffs_inf)) + '\n')
iteration += 1
#exit()
#results = resnet_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000))
#print('RESNET test loss, test acc:', results)
#results = vgg_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000))
#print('VGG test loss, test acc:', results)
# labels = np.argmax(labels, axis=1)
#
# #results = model.evaluate(
# # x=images, y=to_categorical(labels, 1000))
# #print('test loss, test acc:', results)
# total = total + images.shape[0]
# print(total)
exit()
results = resnet_model.evaluate(x=ds_validation,
steps=50000 // args.batch_size)
print('test loss, test acc:', results)
clear_keras_session()
def train_model(project, final = False, last = False):
weight_label = '-' + project['architecture'] + '-weights-'
source_path = project['path']
weights_path = os.path.join(source_path, 'weights')
plot_path = os.path.join(source_path, 'plots')
if last:
weights = 'last_weights'
else:
weights = 'best_weights'
if final:
weight_label += '-final-'
use_path = os.path.join(source_path, 'augmented')
else:
use_path = os.path.join(source_path, 'pre_model')
project['model_round'] += 1
shutil.rmtree(weights_path,ignore_errors=True)
os.makedirs(plot_path)
img_dim = project['img_dim'] * project['img_size']
conv_dim = project['conv_dim'] * project['img_size']
lr = project['learning_rate']
decay = project['learning_rate_decay']
all_files = os.listdir(use_path)
pre_model_files = list(filter(lambda x: r'-img-' in x, all_files))
label_names = list(filter(lambda x: r'-label-' in x, all_files))
pre_model_files_df = pd.DataFrame({'files': pre_model_files})
pre_model_files_df['suffix'] = pre_model_files_df.apply(lambda row: row.files.split('.')[-1], axis = 1)
pre_model_files_df = pre_model_files_df[pre_model_files_df.suffix == 'npy']
pre_model_files_df['ind'] = pre_model_files_df.apply(lambda row: row.files.split('-')[0], axis = 1).astype(int)
pre_model_files_df['label'] = pre_model_files_df.apply(lambda row: row.files.split('-')[3], axis = 1)
pre_model_files_df_dedup = pre_model_files_df.drop_duplicates(subset='ind')
pre_model_files_df = pre_model_files_df.set_index(['ind'])
pre_model_files.sort()
label_names.sort()
pre_model_files_arr = np.array(pre_model_files)
label_names_arr = np.array(label_names)
labels = [np.argmax(np.load(os.path.join(use_path, label_name))) for label_name in label_names]
best_weights = []
last_weights = []
if project['kfold'] >= 3:
kfold = StratifiedKFold(n_splits=project['kfold'], shuffle=True, random_state = project['seed'])
kfold_generator = kfold.split(pre_model_files_df_dedup, pre_model_files_df_dedup.label)
validate = True
else:
print('Too few k-folds selected, fitting on all data')
kfold_generator = no_folds_generator(pre_model_files_df_dedup)
validate = False
for i, (train, test) in enumerate(kfold_generator):
if project['kfold_every']:
print('----- Fitting Fold', i, '-----')
elif i > 0:
break
weights_name = project['name'] + weight_label + '-kfold-' + str(i) + '-round-' + str(project['model_round']) +'.hdf5'
plot_name = project['name'] + weight_label + '-kfold-' + str(i) + '-round-' + str(project['model_round']) +'.png'
if project[weights] is None:
fold_weights = None
else:
fold_weights = project[weights][i]
if final:
if project['architecture'] == 'resnet50':
model = get_resnet_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final'])
elif project['architecture'] == 'xception':
model = get_xception_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final'])
else:
model = get_inception_v3_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final'])
for i, layer in enumerate(model.layers[1].layers):
if len(layer.trainable_weights) > 0:
if i < project['final_cutoff']:
mult = 0.01
else:
mult = 0.1
layer.learning_rate_multiplier = [mult for tw in layer.trainable_weights]
else:
if project['architecture'] == 'resnet50':
pre_model, model = get_resnet_pre_post_model(img_dim,
conv_dim,
len(project['categories']),
model_weights = fold_weights)
elif project['architecture'] == 'xception':
pre_model, model = get_xception_pre_post_model(img_dim,
conv_dim,
len(project['categories']),
model_weights = fold_weights)
else:
pre_model, model = get_inception_v3_pre_post_model(img_dim,
conv_dim,
len(project['categories']),
model_weights = fold_weights)
pre_model_files_dedup_train = pre_model_files_df_dedup.iloc[train]
train_ind = list(set(pre_model_files_dedup_train.ind))
pre_model_files_train = pre_model_files_df.loc[train_ind]
gen_train = gen_minibatches(use_path, pre_model_files_train.files, project['batch_size'], project['architecture'], final = final)
number_train_samples = len(pre_model_files_train)
if validate:
pre_model_files_dedup_test = pre_model_files_df_dedup.iloc[test]
test_ind = list(set(pre_model_files_dedup_test.ind))
pre_model_files_test = pre_model_files_df.loc[test_ind]
gen_test = gen_minibatches(use_path, pre_model_files_test.files, project['batch_size'], project['architecture'], final = final)
number_test_samples = len(pre_model_files_test)
validation_steps = (number_test_samples // project['batch_size'])
weights_checkpoint_file = weights_name.split('.')[0] + '-kfold-' + str(i) + "-improvement-{epoch:02d}-{val_categorical_accuracy:.4f}.hdf5"
checkpoint = ModelCheckpoint(os.path.join(weights_path, weights_checkpoint_file),
monitor='val_categorical_accuracy',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
else:
gen_test = None
validation_steps = None
callbacks_list = None
steps_per_epoch = (number_train_samples // project['batch_size'])
for j in range(project['rounds']):
optimizer = Adam(lr = lr, decay = decay)
model.compile(optimizer = optimizer,
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy'])
model.fit_generator(gen_train,
steps_per_epoch = steps_per_epoch,
epochs = project['cycle'] * (j + 1),
verbose = 1,
validation_data = gen_test,
validation_steps = validation_steps,
initial_epoch = j * project['cycle'],
callbacks = callbacks_list)
model.save_weights(os.path.join(weights_path, weights_name))
last_weights.append(os.path.join(weights_path, weights_name))
weights_names = os.listdir(weights_path)
max_val = -1
max_i = -1
for j, name in enumerate(weights_names):
if name.find(weights_name.split('.')[0]) >= 0:
if (name.find(weight_label) >= 0) and (name.find('improvement') >= 0):
val = int(name.split('.')[1])
if val > max_val:
max_val = val
max_i = j
if project['plot']:
print('Plotting confusion matrix')
if max_i == -1:
print('Loading last weights:', os.path.join(weights_path, weights_name))
model.load_weights(os.path.join(weights_path, weights_name))
else:
print('Loading best weights:', os.path.join(weights_path, weights_names[max_i]))
model.load_weights(os.path.join(weights_path, weights_names[max_i]))
best_predictions = []
true_labels = []
print('Predicting test files')
if validate:
use_files = pre_model_files_test.files
else:
use_files = pre_model_files_train.files
for array_name in tqdm(use_files):
img_path = os.path.join(use_path, array_name)
img = np.load(img_path)
if final:
if project['architecture'] == 'resnet50':
img = np.squeeze(resnet_preprocess_input(img[np.newaxis].astype(np.float32)))
elif project['architecture'] == 'xception':
img = np.squeeze(xception_preprocess_input(img[np.newaxis].astype(np.float32)))
else:
img = np.squeeze(inception_v3_preprocess_input(img[np.newaxis].astype(np.float32)))
prediction = model.predict(img[np.newaxis])
best_predictions.append(project['categories'][np.argmax(prediction)])
true_label = np.load(img_path.replace('-img-','-label-'))
true_labels.append(project['categories'][np.argmax(true_label)])
cm = confusion_matrix(true_labels, best_predictions, project['categories'])
plt.clf()
sns.heatmap(pd.DataFrame(cm, project['categories'], project['categories']), annot = True, fmt = 'g')
plt.xlabel('Actual')
plt.xlabel('Predicted')
plt.xticks(rotation = 45, fontsize = 8)
plt.yticks(rotation = 45, fontsize = 8)
plt.title('Confusion matrix for fold: ' + str(i) + '\nweights' + weights_name)
plt.savefig(os.path.join(plot_path, plot_name))
print('Confusion matrix plot saved:', colored(os.path.join(plot_path, plot_name), 'magenta'))
if max_i == -1:
best_weights.append(os.path.join(weights_path, weights_name))
else:
best_weights.append(os.path.join(weights_path, weights_names[max_i]))
project['number_categories'] = len(project['categories'])
project['best_weights'] = best_weights
project['last_weights'] = last_weights
project['is_final'] = final
return project
def main():
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument('--dataset_dir',
type=str,
default=config.DEFAULT_DATASET_DIR)
parser.add_argument('--batch_size', type=int, default=20)
parser.add_argument('--inv_model_file',
type=str,
help='a saved model (.h5) file')
args = parser.parse_args()
config_keras_backend()
if not args.inv_model_file.endswith('.h5'):
sys.exit('model_file is not a .h5')
inv_model = tf.keras.models.load_model(args.inv_model_file,
compile=False,
custom_objects={'AdamW': AdamW})
inv_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
ds_validation = get_dataset(args.dataset_dir, 'validation',
args.batch_size)
## VGG
vgg_model = VGG19(include_top=True, weights='imagenet', classes=1000)
vgg_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# InceptionV3
inception_model = InceptionV3(include_top=True,
weights='imagenet',
classes=1000)
inception_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
## ResNet
resnet_model = ResNet50(include_top=True, weights='imagenet', classes=1000)
resnet_model.compile(optimizer='sgd',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# Process batches
iteration = 0
sum1 = 0
sum2 = 0
for images, labels in tfds.as_numpy(ds_validation):
if iteration < 199:
print('continuing')
iteration += 1
continue
if iteration == 500:
exit()
labels = np.argmax(labels, axis=1)
#adv_imgs = run_attack(True, 'CarliniL2Method', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0)
#adv_imgs = run_attack(False, 'DeepFool', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0)
adv_imgs = run_attack(False,
'FastGradientMethod',
inception_model,
images,
labels,
batch_size=args.batch_size,
dataset='cifar',
fgsm_epsilon=0.3,
cwl2_confidence=0)
#adv_imgs = run_attack(False, 'ProjectedGradientDescent', inception_model, images, labels, batch_size=10, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0)
## VGG ################################################
#img *= (2.0/255) # normalize to: 0.0~2.0
#img -= 1.0 # subtract mean to make it: -1.0~1.0
#img = np.expand_dims(img, axis=0)
vgg_imgs = []
resnet_imgs = []
inc_imgs = []
flip_imgs = []
inv_imgs = []
adv_vgg_imgs = []
adv_resnet_imgs = []
adv_inc_imgs = []
adv_flip_imgs = []
adv_inv_imgs = []
for ii in range(images.shape[0]):
img = copy.deepcopy(images[ii, :, :, :])
img += 1.0
#img /= (2.0/255)
img *= (255.0 / 2.0)
## VGG
vgg_img = copy.deepcopy(img)
vgg_img = cv2.resize(vgg_img, (224, 224))
vgg_img = vgg_preprocess_input(vgg_img)
vgg_imgs.append(vgg_img)
## Resnet
resnet_img = copy.deepcopy(img)
resnet_img = cv2.resize(resnet_img, (224, 224))
resnet_img = resnet_preprocess_input(resnet_img)
resnet_imgs.append(resnet_img)
## InceptionV3
inc_img = copy.deepcopy(img)
inc_img = cv2.resize(inc_img, (299, 299))
inc_img = inception_preprocess_input(inc_img)
inc_imgs.append(inc_img)
## Flipped
#flip_img = copy.deepcopy(img)
#flip_img = cv2.resize(flip_img, (299, 299))
#flip_img = cv2.flip(flip_img, 1)
#flip_img = inception_preprocess_input(flip_img)
#flip_imgs.append(flip_img)
flip_img = copy.deepcopy(images[ii, :, :, :])
flip_img = cv2.flip(flip_img, 1)
flip_imgs.append(flip_img)
## Inverse
inv_img = copy.deepcopy(images[ii, :, :, :]) #########
inv_img += 1.0
inv_img /= 2.0
inv_img = 1 - inv_img
inv_img *= 255.0
inv_img = cv2.resize(inv_img, (299, 299))
inv_img = inception_preprocess_input(inv_img)
inv_imgs.append(inv_img)
#==========================================
# ADVERSARIAL ---------------
adv_img = copy.deepcopy(adv_imgs[ii, :, :, :])
adv_img += 1.0
#adv_img /= (2.0/255)
adv_img *= (255.0 / 2.0)
# VGG
adv_vgg_img = copy.deepcopy(adv_img)
adv_vgg_img = cv2.resize(adv_vgg_img, (224, 224))
adv_vgg_img = vgg_preprocess_input(adv_vgg_img)
adv_vgg_imgs.append(adv_vgg_img)
# Resnet
adv_resnet_img = copy.deepcopy(adv_img)
adv_resnet_img = cv2.resize(adv_resnet_img, (224, 224))
adv_resnet_img = resnet_preprocess_input(adv_resnet_img)
adv_resnet_imgs.append(adv_resnet_img)
# InceptionV3
adv_inc_img = copy.deepcopy(adv_img)
adv_inc_img = cv2.resize(adv_inc_img, (299, 299))
adv_inc_img = inception_preprocess_input(adv_inc_img)
adv_inc_imgs.append(adv_inc_img)
## Flipped
#adv_flip_img = copy.deepcopy(img)
#adv_flip_img = cv2.resize(adv_flip_img, (299, 299))
#adv_flip_img = cv2.flip(adv_flip_img, 1)
#adv_flip_img = inception_preprocess_input(adv_flip_img)
#adv_flip_imgs.append(adv_flip_img)
adv_flip_img = copy.deepcopy(adv_imgs[ii, :, :, :])
adv_flip_img = cv2.flip(adv_flip_img, 1)
adv_flip_imgs.append(adv_flip_img)
## Inverse
##test on inverse Inceptionv3
adv_inv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) #########
adv_inv_img += 1.0
adv_inv_img /= 2.0
adv_inv_img = 1 - adv_inv_img
adv_inv_img *= 255.0
adv_inv_img = cv2.resize(adv_inv_img, (299, 299))
adv_inv_img = inception_preprocess_input(adv_inv_img)
adv_inv_imgs.append(adv_inv_img)
# Horizontal Flipping
# test on Resnet
vgg_imgs = np.asarray(vgg_imgs)
resnet_imgs = np.asarray(resnet_imgs)
inc_imgs = np.asarray(inc_imgs)
flip_imgs = np.asarray(flip_imgs)
inv_imgs = np.asarray(inv_imgs)
adv_vgg_imgs = np.asarray(adv_vgg_imgs)
adv_resnet_imgs = np.asarray(adv_resnet_imgs)
adv_inc_imgs = np.asarray(adv_inc_imgs)
adv_flip_imgs = np.asarray(adv_flip_imgs)
adv_inv_imgs = np.asarray(adv_inv_imgs)
# Default ResNet accuracy
_, results1 = resnet_model.evaluate(x=resnet_imgs, y=labels, verbose=0)
_, results2 = vgg_model.evaluate(x=vgg_imgs, y=labels, verbose=0)
_, results3 = inception_model.evaluate(x=inc_imgs, y=labels, verbose=0)
_, results4 = inception_model.evaluate(x=flip_imgs,
y=labels,
verbose=0)
_, results5 = inv_model.evaluate(x=inv_imgs, y=labels, verbose=0)
# print('-----------------------------------------------------')
_, results6 = resnet_model.evaluate(x=adv_resnet_imgs,
y=labels,
verbose=0)
_, results7 = vgg_model.evaluate(x=adv_vgg_imgs, y=labels, verbose=0)
_, results8 = inception_model.evaluate(x=adv_inc_imgs,
y=labels,
verbose=0)
_, results9 = inception_model.evaluate(x=adv_flip_imgs,
y=labels,
verbose=0)
_, results10 = inv_model.evaluate(x=adv_inv_imgs, y=labels, verbose=0)
print(iteration)
print(results1, results6)
print(results2, results7)
print(results3, results8)
print(results4, results9)
print(results5, results10)
with open("kot_fgsm_untarg.txt", "a") as myfile:
myfile.write(
str(results1) + ' ' + str(results2) + ' ' + str(results3) +
' ' + str(results4) + ' ' + str(results5) + ' ' +
str(results6) + ' ' + str(results7) + ' ' + str(results8) +
' ' + str(results9) + ' ' + str(results10) + '\n')
iteration += 1
#exit()
#results = resnet_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000))
#print('RESNET test loss, test acc:', results)
#results = vgg_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000))
#print('VGG test loss, test acc:', results)
# labels = np.argmax(labels, axis=1)
#
# #results = model.evaluate(
# # x=images, y=to_categorical(labels, 1000))
# #print('test loss, test acc:', results)
# total = total + images.shape[0]
# print(total)
exit()
results = resnet_model.evaluate(x=ds_validation,
steps=50000 // args.batch_size)
print('test loss, test acc:', results)
clear_keras_session()
