def val_pre_model(source_path, folder, img_dim, architechture): array_path = os.path.join(source_path, folder) pre_model_path = os.path.join(source_path, 'pre_model') # call(['rm', '-rf', pre_model_path]) shutil.rmtree(pre_model_path,ignore_errors=True) # call(['mkdir', '-p', pre_model_path]) os.makedirs(pre_model_path) if architechture == 'resnet50': popped, pre_model = get_resnet_pre_model(img_dim) elif architechture == 'xception': popped, pre_model = get_xception_pre_model(img_dim) else: popped, pre_model = get_inception_v3_pre_model(img_dim) for (array, label, array_name, label_name) in tqdm(gen_array_from_dir(array_path)): if architechture == 'resnet50': array = resnet_preprocess_input(array[np.newaxis].astype(np.float32)) elif architechture == 'xception': array = xception_preprocess_input(array[np.newaxis].astype(np.float32)) else: array = inception_v3_preprocess_input(array[np.newaxis].astype(np.float32)) array_pre_model = np.squeeze(pre_model.predict(array, batch_size=1)) array_name = array_name.split('.')[0] label_name = label_name.split('.')[0] img_pre_model_path = os.path.join(pre_model_path, array_name) label_pre_model_path = os.path.join(pre_model_path, label_name) np.save(img_pre_model_path, array_pre_model) np.save(label_pre_model_path, label)
def val_pre_model(source_path, folder, img_dim, architechture): array_path = os.path.join(source_path, folder) pre_model_path = os.path.join(source_path, 'pre_model') shutil.rmtree(pre_model_path,ignore_errors=True) os.makedirs(pre_model_path) if architechture == 'resnet50': popped, pre_model = get_resnet_pre_model(img_dim) elif architechture == 'xception': popped, pre_model = get_xception_pre_model(img_dim) else: popped, pre_model = get_inception_v3_pre_model(img_dim) for (array, label, array_name, label_name) in tqdm(gen_array_from_dir(array_path)): if architechture == 'resnet50': array = resnet_preprocess_input(array[np.newaxis].astype(np.float32)) elif architechture == 'xception': array = xception_preprocess_input(array[np.newaxis].astype(np.float32)) else: array = inception_v3_preprocess_input(array[np.newaxis].astype(np.float32)) array_pre_model = np.squeeze(pre_model.predict(array, batch_size=1)) array_name = array_name.split('.')[0] label_name = label_name.split('.')[0] img_pre_model_path = os.path.join(pre_model_path, array_name) label_pre_model_path = os.path.join(pre_model_path, label_name) np.save(img_pre_model_path, array_pre_model) np.save(label_pre_model_path, label)
def gen_minibatches(array_dir, array_names, batch_size, architecture, final = False): array_names = list(array_names) while True: # in place shuffle np.random.shuffle(array_names) array_names_mb = array_names[:batch_size] arrays = [] labels = [] for array_name in array_names_mb: img_path = os.path.join(array_dir, array_name) array = np.load(img_path) if final: if architecture == 'resnet50': array = np.squeeze(resnet_preprocess_input(array[np.newaxis].astype(np.float32))) elif architecture == 'xception': array = np.squeeze(xception_preprocess_input(array[np.newaxis].astype(np.float32))) else: array = np.squeeze(inception_v3_preprocess_input(array[np.newaxis].astype(np.float32))) arrays.append(array) labels.append(np.load(img_path.replace('-img-','-label-'))) yield np.array(arrays), np.array(labels)
def get_feats(self, mname, batch_size, filename): fname, data_directory = self.get_data(filename) n = len(fname) print(n) if mname == 'V': mod = self.get_model() features = np.zeros((n, 4096), dtype=np.float32) else: mod = self.get_resnet_model() features = np.zeros((n, 2048), dtype=np.float32) vgg16_feature = [] imgname = {} b = int(n / batch_size) + 1 bn = 0 for i in range(0, b): batch = np.zeros((batch_size, 224, 224, 3)) bn = 0 for k in range(i * batch_size, min((i + 1) * batch_size, n)): img_path = os.path.join(data_directory, fname[k]) f = fname[k].replace('.jpg', '') f = int(f[-7:]) imgname[f] = k img = image.load_img(img_path, target_size=(224, 224)) img = image.img_to_array(img) img = np.expand_dims(img, axis=0) if mname == 'V': img = preprocess_input(img) else: img = resnet_preprocess_input(img) batch[bn, :, :, :] = img bn += 1 if k % batch_size == 0 and k > 1: print(str(k) + " Done") start = time.time() print("Predicting now at " + str(start)) if n < (i + 1) * batch_size: batch = batch[0:bn] features[i * batch_size:min((i + 1) * batch_size, n), :] = mod.predict(batch) end = round(time.time() - start, 1) print("Prediction done in " + str(end)) print(features[i * batch_size:min((i + 1) * batch_size, n), :]) s = 'imgfeature_' + str(filename) + str(batch_size) + '_' + str( i) + '.pkl' # pickle.dump([imgname,features],open(s,'wb')) pickle.dump([imgname, features], open(os.path.join('D:\\CS\\DLNLP_Project\\data', s), 'wb')) #print(features) # s='imgfeature_' + str(data_directory) + '_' + str(batch_size) + '_final.pkl' # pickle.dump([imgname,features],open(s,'wb')) s = 'imgfeature_' + str(filename) + str(batch_size) + '_final.pkl' pickle.dump([imgname, features], open(os.path.join('D:\\CS\\DLNLP_Project\\data', s), 'wb')) return imgname, features
def multi_predict(aug_gen, models, architecture): predicted = [] for img, _ in aug_gen: if architecture == 'resnet50': img = resnet_preprocess_input(img[np.newaxis].astype(np.float32)) elif architecture == 'xception': img = xception_preprocess_input(img[np.newaxis].astype(np.float32)) else: img = inception_v3_preprocess_input(img[np.newaxis].astype(np.float32)) for model in models: predicted.append(model.predict(img)) predicted = np.array(predicted).sum(axis=0) pred_list = list(predicted[0]) return predicted, pred_list
def multi_predict(aug_gen, models, architecture): predicted = [] for img, _ in aug_gen: if architecture == 'resnet50': img = resnet_preprocess_input(img[np.newaxis].astype(np.float32)) elif architecture == 'xception': img = xception_preprocess_input(img[np.newaxis].astype(np.float32)) else: img = inception_v3_preprocess_input(img[np.newaxis].astype(np.float32)) for model in models: predicted.append(model.predict(img)) predicted = np.array(predicted).sum(axis=0) pred_list = list(predicted[0]) return predicted, pred_list
def run_demo(model_file, img_file, labels_file): tf.logging.set_verbosity(tf.logging.FATAL) # read labels labels = read_labels(labels_file) # print(labels) #KB # load model vqa_model = load_model(model_file) print("VQA Model loaded..\n") # load text features and tokenizer textObj = QuestionFeatures(initialize=False) textObj.load_tokenizer( os.path.join(Constants.DIRECTORIES["root"], "tokenizer.pkl")) # image model and feature pre-processing img_model = get_resnet_model() img = image.load_img(img_file, target_size=(224, 224)) img = image.img_to_array(img) img = np.expand_dims(img, axis=0) img = resnet_preprocess_input(img) img_features = np.zeros((1, 2048), dtype=np.float32) img_features = img_model.predict(img) print(f'Image @{img_file} loaded...') while True: print("\n\nNext Question:") curr_question = [str(input())] if curr_question[0].lower() == 'q': print(f'Thank you for trying the VQA demo, bye!') break # convert question to token question_features = textObj.tokenize(curr_question) prediction = vqa_model.predict([img_features, question_features]) print( f'\nAnswer:{np.argmax(prediction)} {labels[np.argmax(prediction)]}' )
def train_model(project, final=False, last=False): weight_label = '-' + project['architecture'] + '-weights-' source_path = project['path'] weights_path = os.path.join(source_path, 'weights') plot_path = os.path.join(source_path, 'plots') if last: weights = 'last_weights' else: weights = 'best_weights' if final: weight_label += '-final-' use_path = os.path.join(source_path, 'augmented') else: use_path = os.path.join(source_path, 'pre_model') project['model_round'] += 1 shutil.rmtree(weights_path, ignore_errors=True) os.makedirs(weights_path) shutil.rmtree(plot_path, ignore_errors=True) os.makedirs(plot_path) img_dim = project['img_dim'] * project['img_size'] conv_dim = project['conv_dim'] * project['img_size'] lr = project['learning_rate'] decay = project['learning_rate_decay'] all_files = os.listdir(use_path) pre_model_files = list(filter(lambda x: r'-img-' in x, all_files)) label_names = list(filter(lambda x: r'-label-' in x, all_files)) pre_model_files_df = pd.DataFrame({'files': pre_model_files}) pre_model_files_df['suffix'] = pre_model_files_df.apply( lambda row: row.files.split('.')[-1], axis=1) pre_model_files_df = pre_model_files_df[pre_model_files_df.suffix == 'npy'] pre_model_files_df['ind'] = pre_model_files_df.apply( lambda row: row.files.split('-')[0], axis=1).astype(int) pre_model_files_df['label'] = pre_model_files_df.apply( lambda row: row.files.split('-')[3], axis=1) pre_model_files_df_dedup = pre_model_files_df.drop_duplicates(subset='ind') pre_model_files_df = pre_model_files_df.set_index(['ind']) pre_model_files.sort() label_names.sort() pre_model_files_arr = np.array(pre_model_files) label_names_arr = np.array(label_names) labels = [ np.argmax(np.load(os.path.join(use_path, label_name))) for label_name in label_names ] best_weights = [] last_weights = [] if project['kfold'] >= 3: kfold = StratifiedKFold(n_splits=project['kfold'], shuffle=True, random_state=project['seed']) kfold_generator = kfold.split(pre_model_files_df_dedup, pre_model_files_df_dedup.label) validate = True else: print('Too few k-folds selected, fitting on all data') kfold_generator = no_folds_generator(pre_model_files_df_dedup) validate = False for i, (train, test) in enumerate(kfold_generator): if project['kfold_every']: print('----- Fitting Fold', i, '-----') elif i > 0: break weights_name = project['name'] + weight_label + '-kfold-' + str( i) + '-round-' + str(project['model_round']) + '.hdf5' plot_name = project['name'] + weight_label + '-kfold-' + str( i) + '-round-' + str(project['model_round']) + '.png' if project[weights] is None: fold_weights = None else: fold_weights = project[weights][i] if final: if project['architecture'] == 'resnet50': model = get_resnet_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) elif project['architecture'] == 'xception': model = get_xception_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) else: model = get_inception_v3_final_model( img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) for i, layer in enumerate(model.layers[1].layers): if len(layer.trainable_weights) > 0: if i < project['final_cutoff']: mult = 0.01 else: mult = 0.1 layer.learning_rate_multiplier = [ mult for tw in layer.trainable_weights ] else: if project['architecture'] == 'resnet50': pre_model, model = get_resnet_pre_post_model( img_dim, conv_dim, len(project['categories']), model_weights=fold_weights) elif project['architecture'] == 'xception': pre_model, model = get_xception_pre_post_model( img_dim, conv_dim, len(project['categories']), model_weights=fold_weights) else: pre_model, model = get_inception_v3_pre_post_model( img_dim, conv_dim, len(project['categories']), model_weights=fold_weights) pre_model_files_dedup_train = pre_model_files_df_dedup.iloc[train] train_ind = list(set(pre_model_files_dedup_train.ind)) pre_model_files_train = pre_model_files_df.loc[train_ind] gen_train = gen_minibatches(use_path, pre_model_files_train.files, project['batch_size'], project['architecture'], final=final) number_train_samples = len(pre_model_files_train) if validate: pre_model_files_dedup_test = pre_model_files_df_dedup.iloc[test] test_ind = list(set(pre_model_files_dedup_test.ind)) pre_model_files_test = pre_model_files_df.loc[test_ind] gen_test = gen_minibatches(use_path, pre_model_files_test.files, project['batch_size'], project['architecture'], final=final) number_test_samples = len(pre_model_files_test) validation_steps = (number_test_samples // project['batch_size']) weights_checkpoint_file = weights_name.split( '.' )[0] + '-kfold-' + str( i ) + "-improvement-{epoch:02d}-{val_categorical_accuracy:.4f}.hdf5" checkpoint = ModelCheckpoint(os.path.join(weights_path, weights_checkpoint_file), monitor='val_categorical_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] else: gen_test = None validation_steps = None callbacks_list = None steps_per_epoch = (number_train_samples // project['batch_size']) for j in range(project['rounds']): optimizer = Adam(lr=lr, decay=decay) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['categorical_accuracy']) model.fit_generator(gen_train, steps_per_epoch=steps_per_epoch, epochs=project['cycle'] * (j + 1), verbose=1, validation_data=gen_test, validation_steps=validation_steps, initial_epoch=j * project['cycle'], callbacks=callbacks_list) model.save_weights(os.path.join(weights_path, weights_name)) last_weights.append(os.path.join(weights_path, weights_name)) weights_names = os.listdir(weights_path) max_val = -1 max_i = -1 for j, name in enumerate(weights_names): if name.find(weights_name.split('.')[0]) >= 0: if (name.find(weight_label) >= 0) and (name.find('improvement') >= 0): val = int(name.split('.')[1]) if val > max_val: max_val = val max_i = j if project['plot']: print('Plotting confusion matrix') if max_i == -1: print('Loading last weights:', os.path.join(weights_path, weights_name)) model.load_weights(os.path.join(weights_path, weights_name)) else: print('Loading best weights:', os.path.join(weights_path, weights_names[max_i])) model.load_weights( os.path.join(weights_path, weights_names[max_i])) best_predictions = [] true_labels = [] print('Predicting test files') if validate: use_files = pre_model_files_test.files else: use_files = pre_model_files_train.files for array_name in tqdm(use_files): img_path = os.path.join(use_path, array_name) img = np.load(img_path) if final: if project['architecture'] == 'resnet50': img = np.squeeze( resnet_preprocess_input(img[np.newaxis].astype( np.float32))) elif project['architecture'] == 'xception': img = np.squeeze( xception_preprocess_input(img[np.newaxis].astype( np.float32))) else: img = np.squeeze( inception_v3_preprocess_input( img[np.newaxis].astype(np.float32))) prediction = model.predict(img[np.newaxis]) best_predictions.append( project['categories'][np.argmax(prediction)]) true_label = np.load(img_path.replace('-img-', '-label-')) true_labels.append( project['categories'][np.argmax(true_label)]) cm = confusion_matrix(true_labels, best_predictions, project['categories']) plt.clf() sns.heatmap(pd.DataFrame(cm, project['categories'], project['categories']), annot=True, fmt='g') plt.xlabel('Actual') plt.xlabel('Predicted') plt.xticks(rotation=45, fontsize=8) plt.yticks(rotation=45, fontsize=8) plt.title('Confusion matrix for fold: ' + str(i) + '\nweights' + weights_name) plt.savefig(os.path.join(plot_path, plot_name)) print('Confusion matrix plot saved:', colored(os.path.join(plot_path, plot_name), 'magenta')) if max_i == -1: best_weights.append(os.path.join(weights_path, weights_name)) else: best_weights.append( os.path.join(weights_path, weights_names[max_i])) project['number_categories'] = len(project['categories']) project['best_weights'] = best_weights project['last_weights'] = last_weights project['is_final'] = final return project
def main(): parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument('--dataset_dir', type=str, default=config.DEFAULT_DATASET_DIR) parser.add_argument('--batch_size', type=int, default=10) parser.add_argument('--inv_model_file', type=str, help='a saved model (.h5) file') args = parser.parse_args() config_keras_backend() if not args.inv_model_file.endswith('.h5'): sys.exit('model_file is not a .h5') inv_model = tf.keras.models.load_model(args.inv_model_file, compile=False, custom_objects={'AdamW': AdamW}) inv_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ds_validation = get_dataset(args.dataset_dir, 'validation', args.batch_size) ## VGG vgg_model = VGG19(include_top=True, weights='imagenet', classes=1000) vgg_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) # InceptionV3 inception_model = InceptionV3(include_top=True, weights='imagenet', classes=1000) inception_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ## ResNet resnet_model = ResNet50(include_top=True, weights='imagenet', classes=1000) resnet_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) # Process batches iteration = 0 sum1 = 0 sum2 = 0 for images, labels in tfds.as_numpy(ds_validation): if iteration < 532: #3822:#532: print('continuing') iteration += 1 continue if iteration == 50000: exit() labels = np.argmax(labels, axis=1) adv_imgs = run_attack(False, 'CarliniL2Method', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0) #adv_imgs = run_attack(False, 'DeepFool', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0) #adv_imgs = run_attack(True, 'FastGradientMethod', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0) #adv_imgs = run_attack(False, 'ProjectedGradientDescent', inception_model, images, labels, batch_size=10, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0) ## VGG ################################################ #img *= (2.0/255) # normalize to: 0.0~2.0 #img -= 1.0 # subtract mean to make it: -1.0~1.0 #img = np.expand_dims(img, axis=0) vgg_imgs = [] resnet_imgs = [] inc_imgs = [] flip_imgs = [] inv_imgs = [] adv_vgg_imgs = [] adv_resnet_imgs = [] adv_inc_imgs = [] adv_flip_imgs = [] adv_inv_imgs = [] for ii in range(images.shape[0]): img = copy.deepcopy(images[ii, :, :, :]) img += 1.0 #img /= (2.0/255) img *= (255.0 / 2.0) ## VGG vgg_img = copy.deepcopy(img) vgg_img = cv2.resize(vgg_img, (224, 224)) vgg_img = vgg_preprocess_input(vgg_img) vgg_imgs.append(vgg_img) ## Resnet resnet_img = copy.deepcopy(img) resnet_img = cv2.resize(resnet_img, (224, 224)) resnet_img = resnet_preprocess_input(resnet_img) resnet_imgs.append(resnet_img) ## InceptionV3 inc_img = copy.deepcopy(img) inc_img = cv2.resize(inc_img, (299, 299)) inc_img = inception_preprocess_input(inc_img) inc_imgs.append(inc_img) ## Flipped #flip_img = copy.deepcopy(img) #flip_img = cv2.resize(flip_img, (299, 299)) #flip_img = cv2.flip(flip_img, 1) #flip_img = inception_preprocess_input(flip_img) #flip_imgs.append(flip_img) flip_img = copy.deepcopy(images[ii, :, :, :]) flip_img = cv2.flip(flip_img, 1) flip_imgs.append(flip_img) ## Inverse inv_img = copy.deepcopy(images[ii, :, :, :]) ######### inv_img += 1.0 inv_img /= 2.0 inv_img = 1 - inv_img inv_img *= 255.0 inv_img = cv2.resize(inv_img, (299, 299)) inv_img = inception_preprocess_input(inv_img) inv_imgs.append(inv_img) #========================================== # ADVERSARIAL --------------- adv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) adv_img += 1.0 #adv_img /= (2.0/255) adv_img *= (255.0 / 2.0) # VGG adv_vgg_img = copy.deepcopy(adv_img) adv_vgg_img = cv2.resize(adv_vgg_img, (224, 224)) adv_vgg_img = vgg_preprocess_input(adv_vgg_img) adv_vgg_imgs.append(adv_vgg_img) # Resnet adv_resnet_img = copy.deepcopy(adv_img) adv_resnet_img = cv2.resize(adv_resnet_img, (224, 224)) adv_resnet_img = resnet_preprocess_input(adv_resnet_img) adv_resnet_imgs.append(adv_resnet_img) # InceptionV3 adv_inc_img = copy.deepcopy(adv_img) adv_inc_img = cv2.resize(adv_inc_img, (299, 299)) adv_inc_img = inception_preprocess_input(adv_inc_img) adv_inc_imgs.append(adv_inc_img) ## Flipped #adv_flip_img = copy.deepcopy(img) #adv_flip_img = cv2.resize(adv_flip_img, (299, 299)) #adv_flip_img = cv2.flip(adv_flip_img, 1) #adv_flip_img = inception_preprocess_input(adv_flip_img) #adv_flip_imgs.append(adv_flip_img) adv_flip_img = copy.deepcopy(adv_imgs[ii, :, :, :]) adv_flip_img = cv2.flip(adv_flip_img, 1) adv_flip_imgs.append(adv_flip_img) ## Inverse ##test on inverse Inceptionv3 adv_inv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) ######### adv_inv_img += 1.0 adv_inv_img /= 2.0 adv_inv_img = 1 - adv_inv_img adv_inv_img *= 255.0 adv_inv_img = cv2.resize(adv_inv_img, (299, 299)) adv_inv_img = inception_preprocess_input(adv_inv_img) adv_inv_imgs.append(adv_inv_img) # Horizontal Flipping # test on Resnet vgg_imgs = np.asarray(vgg_imgs) resnet_imgs = np.asarray(resnet_imgs) inc_imgs = np.asarray(inc_imgs) flip_imgs = np.asarray(flip_imgs) inv_imgs = np.asarray(inv_imgs) adv_vgg_imgs = np.asarray(adv_vgg_imgs) adv_resnet_imgs = np.asarray(adv_resnet_imgs) adv_inc_imgs = np.asarray(adv_inc_imgs) adv_flip_imgs = np.asarray(adv_flip_imgs) adv_inv_imgs = np.asarray(adv_inv_imgs) # Default ResNet accuracy _, results1 = resnet_model.evaluate(x=resnet_imgs, y=labels, verbose=0) _, results2 = vgg_model.evaluate(x=vgg_imgs, y=labels, verbose=0) _, results3 = inception_model.evaluate(x=inc_imgs, y=labels, verbose=0) _, results4 = inception_model.evaluate(x=flip_imgs, y=labels, verbose=0) _, results5 = inv_model.evaluate(x=inv_imgs, y=labels, verbose=0) # print('-----------------------------------------------------') _, results6 = resnet_model.evaluate(x=adv_resnet_imgs, y=labels, verbose=0) _, results7 = vgg_model.evaluate(x=adv_vgg_imgs, y=labels, verbose=0) _, results8 = inception_model.evaluate(x=adv_inc_imgs, y=labels, verbose=0) _, results9 = inception_model.evaluate(x=adv_flip_imgs, y=labels, verbose=0) _, results10 = inv_model.evaluate(x=adv_inv_imgs, y=labels, verbose=0) print(iteration) print(results1, results6) print(results2, results7) print(results3, results8) print(results4, results9) print(results5, results10) # Print the figure images INDEX = 1 # Original image orig = copy.deepcopy(inc_imgs[INDEX]) orig /= 2.0 orig += 0.5 orig *= 255.0 # Adversarial image adv = copy.deepcopy(adv_inc_imgs[INDEX]) adv /= 2.0 adv += 0.5 adv *= 255.0 #Perturbation image diff = adv - orig print(np.amax(diff)) #exit() # Flip image flip = copy.deepcopy(flip_imgs[INDEX]) flip /= 2.0 flip += 0.5 flip *= 255.0 # Save images imageio.imwrite('pandas/panda_orig.png', np.reshape(orig, (299, 299, 3))) imageio.imwrite('pandas/panda_adv.png', np.reshape(adv, (299, 299, 3))) imageio.imwrite('pandas/panda_diff.png', np.reshape(diff, (299, 299, 3))) imageio.imwrite('pandas/panda_flip.png', np.reshape(flip, (299, 299, 3))) print(labels) print('Inception---original-------------------------') #preds = inception_model.predict(np.reshape(inc_imgs[INDEX],(1,299,299,3))) #print('confidence:', inc_decode_predictions(preds, top=1)[0]) preds = inception_model.predict(inc_imgs) print('IncV3 Predicted:', np.argmax(preds, axis=1)) print('IncV3 Predicted:', np.amax(preds, axis=1)) print() print('VGG---original-------------------------') #preds = vgg_model.predict(np.reshape(vgg_imgs[INDEX],(1,224,224,3))) #print('confidence:', vgg_decode_predictions(preds, top=1)[0]) preds = vgg_model.predict(vgg_imgs) print('VGG Predicted:', np.argmax(preds, axis=1)) print('VGG Predicted:', np.amax(preds, axis=1)) print() print('ResNet---original-------------------------') #preds = resnet_model.predict(np.reshape(resnet_imgs[INDEX],(1,224,224,3))) #print('confidence:', resnet_decode_predictions(preds, top=1)[0]) preds = resnet_model.predict(resnet_imgs) print('ResNet Predicted:', np.argmax(preds, axis=1)) print('ResNet Predicted:', np.amax(preds, axis=1)) print() print('Inception---adv-------------------------') #preds = inception_model.predict(np.reshape(adv_inc_imgs[INDEX],(1,299,299,3))) #print('confidence:', inc_decode_predictions(preds, top=1)[0]) preds = inception_model.predict(adv_inc_imgs) print('Adv IncV3 Predicted:', np.argmax(preds, axis=1)) print('Adv IncV3 Predicted:', np.amax(preds, axis=1)) print() print('VGG---adv-------------------------') #preds = vgg_model.predict(np.reshape(adv_vgg_imgs[INDEX],(1,224,224,3))) #print('confidence:', vgg_decode_predictions(preds, top=1)[0]) preds = vgg_model.predict(adv_vgg_imgs) print('Adv VGG Predicted:', np.argmax(preds, axis=1)) print('Adv VGG Predicted:', np.amax(preds, axis=1)) print() print('ResNet---adv-------------------------') #preds = resnet_model.predict(np.reshape(adv_resnet_imgs[INDEX],(1,224,224,3))) #print('confidence:', resnet_decode_predictions(preds, top=1)[0]) preds = resnet_model.predict(adv_resnet_imgs) print('Adv ResNet Predicted:', np.argmax(preds, axis=1)) print('Adv ResNet Predicted:', np.amax(preds, axis=1)) print() print('Inception---flip-------------------------') #preds = inception_model.predict(np.reshape(adv_flip_imgs[INDEX],(1,299,299,3))) #print('confidence:', inc_decode_predictions(preds, top=1)[0]) preds = inception_model.predict(adv_flip_imgs) print('flip Predicted:', np.argmax(preds, axis=1)) print('flip Predicted:', np.amax(preds, axis=1)) print() #print('Accuracies--------------------------') #print('flip accuracy:', inc_decode_predictions(preds, top=3)[0]) exit() with open("output_pgd_untarg_batch-20_norm-2.txt", "a") as myfile: myfile.write( str(results1) + ' ' + str(results2) + ' ' + str(results3) + ' ' + str(results4) + ' ' + str(results5) + ' ' + str(results6) + ' ' + str(results7) + ' ' + str(results8) + ' ' + str(results9) + ' ' + str(results10) + '\n') # Distances norm_diffs_1 = [ np.linalg.norm( np.subtract(adv_inc_imgs[ii].flatten(), inc_imgs[ii].flatten()), 1) for ii in range(inc_imgs.shape[0]) ] norm_diffs_2 = [ np.linalg.norm( np.subtract(adv_inc_imgs[ii].flatten(), inc_imgs[ii].flatten()), 2) for ii in range(inc_imgs.shape[0]) ] norm_diffs_inf = [ np.linalg.norm( np.subtract(adv_inc_imgs[ii].flatten(), inc_imgs[ii].flatten()), np.inf) for ii in range(inc_imgs.shape[0]) ] print(np.mean(norm_diffs_1), np.mean(norm_diffs_2), np.mean(norm_diffs_inf)) with open("distances_pgd_untarg_batch-20_norm-2.txt", "a") as myfile: myfile.write( str(np.mean(norm_diffs_1)) + ' ' + str(np.mean(norm_diffs_2)) + ' ' + str(np.mean(norm_diffs_inf)) + '\n') iteration += 1 #exit() #results = resnet_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000)) #print('RESNET test loss, test acc:', results) #results = vgg_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000)) #print('VGG test loss, test acc:', results) # labels = np.argmax(labels, axis=1) # # #results = model.evaluate( # # x=images, y=to_categorical(labels, 1000)) # #print('test loss, test acc:', results) # total = total + images.shape[0] # print(total) exit() results = resnet_model.evaluate(x=ds_validation, steps=50000 // args.batch_size) print('test loss, test acc:', results) clear_keras_session()
def train_model(project, final = False, last = False): weight_label = '-' + project['architecture'] + '-weights-' source_path = project['path'] weights_path = os.path.join(source_path, 'weights') plot_path = os.path.join(source_path, 'plots') if last: weights = 'last_weights' else: weights = 'best_weights' if final: weight_label += '-final-' use_path = os.path.join(source_path, 'augmented') else: use_path = os.path.join(source_path, 'pre_model') project['model_round'] += 1 shutil.rmtree(weights_path,ignore_errors=True) os.makedirs(plot_path) img_dim = project['img_dim'] * project['img_size'] conv_dim = project['conv_dim'] * project['img_size'] lr = project['learning_rate'] decay = project['learning_rate_decay'] all_files = os.listdir(use_path) pre_model_files = list(filter(lambda x: r'-img-' in x, all_files)) label_names = list(filter(lambda x: r'-label-' in x, all_files)) pre_model_files_df = pd.DataFrame({'files': pre_model_files}) pre_model_files_df['suffix'] = pre_model_files_df.apply(lambda row: row.files.split('.')[-1], axis = 1) pre_model_files_df = pre_model_files_df[pre_model_files_df.suffix == 'npy'] pre_model_files_df['ind'] = pre_model_files_df.apply(lambda row: row.files.split('-')[0], axis = 1).astype(int) pre_model_files_df['label'] = pre_model_files_df.apply(lambda row: row.files.split('-')[3], axis = 1) pre_model_files_df_dedup = pre_model_files_df.drop_duplicates(subset='ind') pre_model_files_df = pre_model_files_df.set_index(['ind']) pre_model_files.sort() label_names.sort() pre_model_files_arr = np.array(pre_model_files) label_names_arr = np.array(label_names) labels = [np.argmax(np.load(os.path.join(use_path, label_name))) for label_name in label_names] best_weights = [] last_weights = [] if project['kfold'] >= 3: kfold = StratifiedKFold(n_splits=project['kfold'], shuffle=True, random_state = project['seed']) kfold_generator = kfold.split(pre_model_files_df_dedup, pre_model_files_df_dedup.label) validate = True else: print('Too few k-folds selected, fitting on all data') kfold_generator = no_folds_generator(pre_model_files_df_dedup) validate = False for i, (train, test) in enumerate(kfold_generator): if project['kfold_every']: print('----- Fitting Fold', i, '-----') elif i > 0: break weights_name = project['name'] + weight_label + '-kfold-' + str(i) + '-round-' + str(project['model_round']) +'.hdf5' plot_name = project['name'] + weight_label + '-kfold-' + str(i) + '-round-' + str(project['model_round']) +'.png' if project[weights] is None: fold_weights = None else: fold_weights = project[weights][i] if final: if project['architecture'] == 'resnet50': model = get_resnet_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) elif project['architecture'] == 'xception': model = get_xception_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) else: model = get_inception_v3_final_model(img_dim, conv_dim, project['number_categories'], fold_weights, project['is_final']) for i, layer in enumerate(model.layers[1].layers): if len(layer.trainable_weights) > 0: if i < project['final_cutoff']: mult = 0.01 else: mult = 0.1 layer.learning_rate_multiplier = [mult for tw in layer.trainable_weights] else: if project['architecture'] == 'resnet50': pre_model, model = get_resnet_pre_post_model(img_dim, conv_dim, len(project['categories']), model_weights = fold_weights) elif project['architecture'] == 'xception': pre_model, model = get_xception_pre_post_model(img_dim, conv_dim, len(project['categories']), model_weights = fold_weights) else: pre_model, model = get_inception_v3_pre_post_model(img_dim, conv_dim, len(project['categories']), model_weights = fold_weights) pre_model_files_dedup_train = pre_model_files_df_dedup.iloc[train] train_ind = list(set(pre_model_files_dedup_train.ind)) pre_model_files_train = pre_model_files_df.loc[train_ind] gen_train = gen_minibatches(use_path, pre_model_files_train.files, project['batch_size'], project['architecture'], final = final) number_train_samples = len(pre_model_files_train) if validate: pre_model_files_dedup_test = pre_model_files_df_dedup.iloc[test] test_ind = list(set(pre_model_files_dedup_test.ind)) pre_model_files_test = pre_model_files_df.loc[test_ind] gen_test = gen_minibatches(use_path, pre_model_files_test.files, project['batch_size'], project['architecture'], final = final) number_test_samples = len(pre_model_files_test) validation_steps = (number_test_samples // project['batch_size']) weights_checkpoint_file = weights_name.split('.')[0] + '-kfold-' + str(i) + "-improvement-{epoch:02d}-{val_categorical_accuracy:.4f}.hdf5" checkpoint = ModelCheckpoint(os.path.join(weights_path, weights_checkpoint_file), monitor='val_categorical_accuracy', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] else: gen_test = None validation_steps = None callbacks_list = None steps_per_epoch = (number_train_samples // project['batch_size']) for j in range(project['rounds']): optimizer = Adam(lr = lr, decay = decay) model.compile(optimizer = optimizer, loss = 'categorical_crossentropy', metrics = ['categorical_accuracy']) model.fit_generator(gen_train, steps_per_epoch = steps_per_epoch, epochs = project['cycle'] * (j + 1), verbose = 1, validation_data = gen_test, validation_steps = validation_steps, initial_epoch = j * project['cycle'], callbacks = callbacks_list) model.save_weights(os.path.join(weights_path, weights_name)) last_weights.append(os.path.join(weights_path, weights_name)) weights_names = os.listdir(weights_path) max_val = -1 max_i = -1 for j, name in enumerate(weights_names): if name.find(weights_name.split('.')[0]) >= 0: if (name.find(weight_label) >= 0) and (name.find('improvement') >= 0): val = int(name.split('.')[1]) if val > max_val: max_val = val max_i = j if project['plot']: print('Plotting confusion matrix') if max_i == -1: print('Loading last weights:', os.path.join(weights_path, weights_name)) model.load_weights(os.path.join(weights_path, weights_name)) else: print('Loading best weights:', os.path.join(weights_path, weights_names[max_i])) model.load_weights(os.path.join(weights_path, weights_names[max_i])) best_predictions = [] true_labels = [] print('Predicting test files') if validate: use_files = pre_model_files_test.files else: use_files = pre_model_files_train.files for array_name in tqdm(use_files): img_path = os.path.join(use_path, array_name) img = np.load(img_path) if final: if project['architecture'] == 'resnet50': img = np.squeeze(resnet_preprocess_input(img[np.newaxis].astype(np.float32))) elif project['architecture'] == 'xception': img = np.squeeze(xception_preprocess_input(img[np.newaxis].astype(np.float32))) else: img = np.squeeze(inception_v3_preprocess_input(img[np.newaxis].astype(np.float32))) prediction = model.predict(img[np.newaxis]) best_predictions.append(project['categories'][np.argmax(prediction)]) true_label = np.load(img_path.replace('-img-','-label-')) true_labels.append(project['categories'][np.argmax(true_label)]) cm = confusion_matrix(true_labels, best_predictions, project['categories']) plt.clf() sns.heatmap(pd.DataFrame(cm, project['categories'], project['categories']), annot = True, fmt = 'g') plt.xlabel('Actual') plt.xlabel('Predicted') plt.xticks(rotation = 45, fontsize = 8) plt.yticks(rotation = 45, fontsize = 8) plt.title('Confusion matrix for fold: ' + str(i) + '\nweights' + weights_name) plt.savefig(os.path.join(plot_path, plot_name)) print('Confusion matrix plot saved:', colored(os.path.join(plot_path, plot_name), 'magenta')) if max_i == -1: best_weights.append(os.path.join(weights_path, weights_name)) else: best_weights.append(os.path.join(weights_path, weights_names[max_i])) project['number_categories'] = len(project['categories']) project['best_weights'] = best_weights project['last_weights'] = last_weights project['is_final'] = final return project
def main(): parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument('--dataset_dir', type=str, default=config.DEFAULT_DATASET_DIR) parser.add_argument('--batch_size', type=int, default=20) parser.add_argument('--inv_model_file', type=str, help='a saved model (.h5) file') args = parser.parse_args() config_keras_backend() if not args.inv_model_file.endswith('.h5'): sys.exit('model_file is not a .h5') inv_model = tf.keras.models.load_model(args.inv_model_file, compile=False, custom_objects={'AdamW': AdamW}) inv_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ds_validation = get_dataset(args.dataset_dir, 'validation', args.batch_size) ## VGG vgg_model = VGG19(include_top=True, weights='imagenet', classes=1000) vgg_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) # InceptionV3 inception_model = InceptionV3(include_top=True, weights='imagenet', classes=1000) inception_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ## ResNet resnet_model = ResNet50(include_top=True, weights='imagenet', classes=1000) resnet_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) # Process batches iteration = 0 sum1 = 0 sum2 = 0 for images, labels in tfds.as_numpy(ds_validation): if iteration < 199: print('continuing') iteration += 1 continue if iteration == 500: exit() labels = np.argmax(labels, axis=1) #adv_imgs = run_attack(True, 'CarliniL2Method', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0) #adv_imgs = run_attack(False, 'DeepFool', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0) adv_imgs = run_attack(False, 'FastGradientMethod', inception_model, images, labels, batch_size=args.batch_size, dataset='cifar', fgsm_epsilon=0.3, cwl2_confidence=0) #adv_imgs = run_attack(False, 'ProjectedGradientDescent', inception_model, images, labels, batch_size=10, dataset='cifar', fgsm_epsilon=0.1, cwl2_confidence=0) ## VGG ################################################ #img *= (2.0/255) # normalize to: 0.0~2.0 #img -= 1.0 # subtract mean to make it: -1.0~1.0 #img = np.expand_dims(img, axis=0) vgg_imgs = [] resnet_imgs = [] inc_imgs = [] flip_imgs = [] inv_imgs = [] adv_vgg_imgs = [] adv_resnet_imgs = [] adv_inc_imgs = [] adv_flip_imgs = [] adv_inv_imgs = [] for ii in range(images.shape[0]): img = copy.deepcopy(images[ii, :, :, :]) img += 1.0 #img /= (2.0/255) img *= (255.0 / 2.0) ## VGG vgg_img = copy.deepcopy(img) vgg_img = cv2.resize(vgg_img, (224, 224)) vgg_img = vgg_preprocess_input(vgg_img) vgg_imgs.append(vgg_img) ## Resnet resnet_img = copy.deepcopy(img) resnet_img = cv2.resize(resnet_img, (224, 224)) resnet_img = resnet_preprocess_input(resnet_img) resnet_imgs.append(resnet_img) ## InceptionV3 inc_img = copy.deepcopy(img) inc_img = cv2.resize(inc_img, (299, 299)) inc_img = inception_preprocess_input(inc_img) inc_imgs.append(inc_img) ## Flipped #flip_img = copy.deepcopy(img) #flip_img = cv2.resize(flip_img, (299, 299)) #flip_img = cv2.flip(flip_img, 1) #flip_img = inception_preprocess_input(flip_img) #flip_imgs.append(flip_img) flip_img = copy.deepcopy(images[ii, :, :, :]) flip_img = cv2.flip(flip_img, 1) flip_imgs.append(flip_img) ## Inverse inv_img = copy.deepcopy(images[ii, :, :, :]) ######### inv_img += 1.0 inv_img /= 2.0 inv_img = 1 - inv_img inv_img *= 255.0 inv_img = cv2.resize(inv_img, (299, 299)) inv_img = inception_preprocess_input(inv_img) inv_imgs.append(inv_img) #========================================== # ADVERSARIAL --------------- adv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) adv_img += 1.0 #adv_img /= (2.0/255) adv_img *= (255.0 / 2.0) # VGG adv_vgg_img = copy.deepcopy(adv_img) adv_vgg_img = cv2.resize(adv_vgg_img, (224, 224)) adv_vgg_img = vgg_preprocess_input(adv_vgg_img) adv_vgg_imgs.append(adv_vgg_img) # Resnet adv_resnet_img = copy.deepcopy(adv_img) adv_resnet_img = cv2.resize(adv_resnet_img, (224, 224)) adv_resnet_img = resnet_preprocess_input(adv_resnet_img) adv_resnet_imgs.append(adv_resnet_img) # InceptionV3 adv_inc_img = copy.deepcopy(adv_img) adv_inc_img = cv2.resize(adv_inc_img, (299, 299)) adv_inc_img = inception_preprocess_input(adv_inc_img) adv_inc_imgs.append(adv_inc_img) ## Flipped #adv_flip_img = copy.deepcopy(img) #adv_flip_img = cv2.resize(adv_flip_img, (299, 299)) #adv_flip_img = cv2.flip(adv_flip_img, 1) #adv_flip_img = inception_preprocess_input(adv_flip_img) #adv_flip_imgs.append(adv_flip_img) adv_flip_img = copy.deepcopy(adv_imgs[ii, :, :, :]) adv_flip_img = cv2.flip(adv_flip_img, 1) adv_flip_imgs.append(adv_flip_img) ## Inverse ##test on inverse Inceptionv3 adv_inv_img = copy.deepcopy(adv_imgs[ii, :, :, :]) ######### adv_inv_img += 1.0 adv_inv_img /= 2.0 adv_inv_img = 1 - adv_inv_img adv_inv_img *= 255.0 adv_inv_img = cv2.resize(adv_inv_img, (299, 299)) adv_inv_img = inception_preprocess_input(adv_inv_img) adv_inv_imgs.append(adv_inv_img) # Horizontal Flipping # test on Resnet vgg_imgs = np.asarray(vgg_imgs) resnet_imgs = np.asarray(resnet_imgs) inc_imgs = np.asarray(inc_imgs) flip_imgs = np.asarray(flip_imgs) inv_imgs = np.asarray(inv_imgs) adv_vgg_imgs = np.asarray(adv_vgg_imgs) adv_resnet_imgs = np.asarray(adv_resnet_imgs) adv_inc_imgs = np.asarray(adv_inc_imgs) adv_flip_imgs = np.asarray(adv_flip_imgs) adv_inv_imgs = np.asarray(adv_inv_imgs) # Default ResNet accuracy _, results1 = resnet_model.evaluate(x=resnet_imgs, y=labels, verbose=0) _, results2 = vgg_model.evaluate(x=vgg_imgs, y=labels, verbose=0) _, results3 = inception_model.evaluate(x=inc_imgs, y=labels, verbose=0) _, results4 = inception_model.evaluate(x=flip_imgs, y=labels, verbose=0) _, results5 = inv_model.evaluate(x=inv_imgs, y=labels, verbose=0) # print('-----------------------------------------------------') _, results6 = resnet_model.evaluate(x=adv_resnet_imgs, y=labels, verbose=0) _, results7 = vgg_model.evaluate(x=adv_vgg_imgs, y=labels, verbose=0) _, results8 = inception_model.evaluate(x=adv_inc_imgs, y=labels, verbose=0) _, results9 = inception_model.evaluate(x=adv_flip_imgs, y=labels, verbose=0) _, results10 = inv_model.evaluate(x=adv_inv_imgs, y=labels, verbose=0) print(iteration) print(results1, results6) print(results2, results7) print(results3, results8) print(results4, results9) print(results5, results10) with open("kot_fgsm_untarg.txt", "a") as myfile: myfile.write( str(results1) + ' ' + str(results2) + ' ' + str(results3) + ' ' + str(results4) + ' ' + str(results5) + ' ' + str(results6) + ' ' + str(results7) + ' ' + str(results8) + ' ' + str(results9) + ' ' + str(results10) + '\n') iteration += 1 #exit() #results = resnet_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000)) #print('RESNET test loss, test acc:', results) #results = vgg_model.evaluate(x=adv_imgs, y=to_categorical(labels, 1000)) #print('VGG test loss, test acc:', results) # labels = np.argmax(labels, axis=1) # # #results = model.evaluate( # # x=images, y=to_categorical(labels, 1000)) # #print('test loss, test acc:', results) # total = total + images.shape[0] # print(total) exit() results = resnet_model.evaluate(x=ds_validation, steps=50000 // args.batch_size) print('test loss, test acc:', results) clear_keras_session()