def thickness_sim(model, data): labels = utils.create_one_hot(data['labels'].astype('uint')) thicknesses = list(data.keys())[:-1] numdata = len(thicknesses) acc = np.zeros(numdata) for i,t in zip(range(numdata),thicknesses): acc[i] = test_model(model,data[t],labels) return acc
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) digits_data = utils.load_processed_data('digits_og_and_optimal') digits = digits_data['optimal_lw'] labels = utils.create_one_hot(digits_data['labels'].astype('uint')) ensemble_size = 20 epochs = 50 small_digits = reshape_fun(np.array(list(map(scale_down, digits)))) small_digits = utils.normalize_data(small_digits) trials = 5 for t in range(1,trials+1): gc.collect() l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True) train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics = ['acc']) train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval),callbacks=[es]) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) results = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun) #entropy = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'entropy') #c_error = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'c_error') #results['c_error'][0] = c_error #results['entropy'][0] = entropy filename = "saltpepper_norm_trial-%s" % t utils.save_processed_data(results, filename)
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [26,28,30] taus2 = [2,4,5] digits1 = list(map(reshape_fun, [digits_data[t] for t in taus])) digits1 = list(map(utils.normalize_data, digits1)) digits2 = list(map(reshape_fun, [digits_data[t] for t in taus2])) digits2 = list(map(utils.normalize_data, digits2)) digits_og = digits_data2['lecunn_big'] digits_og = salt_and_pepper(digits_og,72000) digits_og = reshape_fun(utils.normalize_data(digits_og)) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 trials = 5 d1_correct = [] d1_wrong = [] d2_wrong = [] d2_correct = [] dog_wrong = [] dog_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['acc']) train_model.fit(l_xtrain,l_ytrain,epochs = epochs, validation_data = (l_xval,l_yval)) dog_preds = merge_model.predict([digits_og]*ensemble_size) correct, wrong = bin_entropies(d2_preds, d2_labels) dog_correct.extend(correct) dog_wrong.extend(wrong) for d in digits1: digits_preds = merge_model.predict([d]*ensemble_size) correct, wrong = bin_entropies(digits_preds,d_labels) d1.extend(wrong) d1.extend(correct) for d in digits2: digits_preds = merge_model.predict([d]*ensemble_size) correct, wrong = bin_entropies(digits_preds,d_labels) d2.extend(wrong) d2.extend(correct) ensemble = { 'd1_correct' : d1_correct, 'd1_wrong' : d1_wrong, 'd2_correct' : d2_correct, 'd2_wrong' : d2_wrong, 'lecunn_correct' : dog_correct, 'lecunn_wrong' : dog_wrong } return ensemble
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) custom_digits_dict = utils.load_processed_data("combined_testing_data") digits_labels = custom_digits_dict['labels'] digits_taus = list(custom_digits_dict.keys())[:-1] digits_data = list(map(reshape_fun, [custom_digits_dict[t] for t in digits_taus])) digits_data = list(map(utils.normalize_data, digits_data)) digits_labels = utils.create_one_hot(digits_labels.astype('uint')) for t in range(trials): print("===== START TRIAL %s =====" % (t + 1)) # Preparing Results # Classification Error ensemble_cerror = [] #t_ensemble_adv_cerror = [] digits_cerror = [] #t_digits_adv_cerror = [] # Prediction Entropy entropy_ensemble = [] #t_entropy_adv_ensemble = [] digits_entropy = [] #t_digits_adv_entropy = [] # Voting entropy #entropy_vote = [] #entropy_adv_vote = [] #digits_vote = [] #digits_adv_entropy = [] epochs = 5 l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) # Without adveserial training inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") losses = list( map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) ) train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval)) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) c_error = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'accuracy' ) entropy = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'entropy' ) ensemble_cerror.append(c_error) entropy_ensemble.append(entropy) #entropy_vote.append(calc_vote_entropy(model_list,m,xtest)) d_cerror,d_entropy = test_digits(merge_model,model_list,ensemble_size,digits_data,digits_labels) digits_cerror.append(d_cerror) digits_entropy.append(d_entropy) #digits_vote.append(d_vote) # Adveserial training #inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average") #losses = list( # map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) #) #train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) #train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval)) #c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' ) #entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' ) # #t_ensemble_adv_cerror.append(c_error) #t_entropy_adv_ensemble.append(entropy) # #d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels) # #t_digits_adv_cerror.append(d_cerror) #t_digits_adv_entropy.append(d_entropy) filename1 = 'adv_mnist_results_5-20-50-trial%s' % t filename2 = 'adv_digits_results_5-20-50-trial%s' % t mnist_results = { 'ensemble_cerror' : ensemble_cerror, 'ensemble_entropy' : entropy_ensemble #'ensemble_adv_cerror' : ensemble_adv_cerror, #'ensemble_adv_entropy' : entropy_adv_ensemble, #'voting_entropy' : entropy_vote #'voting_adv_entropy' : entropy_adv_vote } digits_results = { 'ensemble_cerror' : digits_cerror, 'ensemble_entropy' : digits_entropy #'ensemble_adv_cerror' : digits_adv_cerror, #'ensemble_adv_entropy' : digits_adv_entropy, #'voting_entropy' : digits_vote #'voting_adv_entropy' : digits_adv_vote } utils.save_processed_data(mnist_results,filename1) utils.save_processed_data(digits_results,filename2) return digits_taus, mnist_results, digits_results
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, _ = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type=None) train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, validation_data=(l_xval, l_yval)) mnist_mpreds = np.array(list(map(lambda m: m.predict(xtest), model_list))) mnist_mnwrong = [] mnist_mncorrect = [] for mp in mnist_mpreds: correct, wrong = bin_entropies(mp, ytest) mnist_mnwrong.extend(wrong) mnist_mncorrect.extend(correct) d2_preds = np.array(list(map(lambda m: m.predict(digits_og), model_list))) d2_mnwrong = [] d2_mncorrect = [] for mp in d2_preds: correct, wrong = bin_entropies(mp, d2_labels) d2_mnwrong.extend(wrong) d2_mncorrect.extend(correct) digits_mnwrong = [] digits_mncorrect = [] for d in digits: for m in model_list: preds = m.predict(d) correct, wrong = bin_entropies(preds, d_labels) digits_mnwrong.extend(wrong) digits_mncorrect.extend(correct) individual = { 'mnist_correct': mnist_mncorrect, 'mnist_wrong': mnist_mnwrong, 'digits_correct': digits_mncorrect, 'digits_wrong': digits_mnwrong, 'lecunn_correct': d2_mncorrect, 'lecunn_wrong': d2_mnwrong } return individual
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 trials = 5 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") losses = list( map( lambda m: ann.adveserial_loss( klosses.categorical_crossentropy, m, eps=0.01), model_list)) train_model.compile(optimizer='adam', loss=losses, metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, validation_data=(l_xval, l_yval)) mnist_preds = merge_model.predict([xtest] * ensemble_size) correct, wrong = bin_entropies(mnist_preds, ytest) mnist_correct.extend(correct) mnist_wrong.extend(wrong) d2_preds = merge_model.predict([digits_og] * ensemble_size) correct, wrong = bin_entropies(d2_preds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) correct, wrong = bin_entropies(digits_preds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { 'mnist_correct': mnist_correct, 'mnist_wrong': mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
nchunks = ensemblesize // chunksize xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } reshape_fun = reshape_funs['conv'] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) utils.setup_gpu_session() digits_data = utils.load_processed_data("combined_testing_data") taus = list(digits_data.keys())[:-1] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) labels = utils.create_one_hot(digits_data['labels'].astype('uint')) mnist_mpreds = [] digits_mpreds = {} mnist_mcerr = [] digits_mcerr = {} mnist_mbits = [] digits_mbits = {} for t in taus: digits_mpreds[t] = [] digits_mcerr[t] = [] digits_mbits[t] = [] for _ in range(nchunks): print("====== START TRAINING NEURAL NETWORK MODEL ======")
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('digits_og_and_optimal') digits = digits_data['optimal_lw'] digits = utils.normalize_data(digits) digits_og = digits_data['lecunn_big'] digits_og_small = reshape_fun(digits_data['lecunn']) digits_og_small = utils.normalize_data(digits_og_small) labels = utils.create_one_hot(digits_data['labels'].astype('uint')) ensemble_size = 20 epochs = 3 small_digits = reshape_fun(np.array(list(map(scale_down, digits)))) trials = 5 for t in range(1, trials + 1): gc.collect() l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, batch_size=100, epochs=epochs, validation_data=(l_xval, l_yval)) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) results_linewidth = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun) entropy = ann.test_model(merge_model, [small_digits] * ensemble_size, labels, metric='entropy') c_error = ann.test_model(merge_model, [small_digits] * ensemble_size, labels, metric='c_error') results_linewidth['c_error'][0] = c_error results_linewidth['entropy'][0] = entropy results_lecunn = test_digits(merge_model, digits_og, labels, ensemble_size, reshape_fun) entropy = ann.test_model(merge_model, [digits_og_small] * ensemble_size, labels, metric='entropy') c_error = ann.test_model(merge_model, [digits_og_small] * ensemble_size, labels, metric='c_error') results_lecunn['c_error'][0] = c_error results_lecunn['entropy'][0] = entropy total_results = { 'optimal_lw': results_linewidth, 'lecunn': results_lecunn } filename = "saltpepper_random_trial-%s" % t utils.save_processed_data(total_results, filename)
def experiment(network_conf_json, reshape_mode="conv"): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain) xm_data = utils.load_processed_data("xiaoming_digits") ob_data = utils.load_processed_data("Oleks_digits") xm_taus = np.array(list(xm_data.keys())[:-1]) ob_taus = np.array(list(ob_data.keys())[:-1]) xm_digits = list(map(reshape_fun, [xm_data[t] for t in xm_taus])) xm_labels = utils.create_one_hot(xm_data["labels"]) ob_digits = list(map(reshape_fun, [ob_data[t] for t in ob_taus])) ob_labels = utils.create_one_hot(ob_data["labels"]) # Building the ensemble models and training the networks print("===== Building the ensemble of size %s =====" % ensemble_size) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_conf_json], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['accuracy']) train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size, batch_size=100, verbose=1, validation_data=([xval] * ensemble_size, [yval] * ensemble_size), epochs=num_epochs) mnist_c_errors, mnist_pred_bits, mnist_class_bits = test_digits( [xtest], ytest, merge_model, model_list) xm_c_errors, xm_pred_bits, xm_class_bits = test_digits( xm_digits, xm_labels, merge_model, model_list) ob_c_errors, ob_pred_bits, ob_class_bits = test_digits( ob_digits, ob_labels, merge_model, model_list) mnist_values = { "c_error": mnist_c_errors[0], "pred_bits": mnist_pred_bits[0], "class_bits": mnist_class_bits[0] } xm_results = { "taus": xm_taus, "c_error": xm_c_errors, "pred_bits": xm_pred_bits, "class_bits": xm_class_bits } ob_results = { "taus": ob_taus, "c_error": ob_c_errors, "pred_bits": ob_pred_bits, "class_bits": ob_class_bits } return mnist_values, xm_results, ob_results
validation_data=(xval, yval), epochs=epochs) print("====== CALCULATING MNIST ACCURACY ======") mnist_accuracy = ann.test_model(model, xtest, ytest, "accuracy") print("====== LOAD CUSTOM DIGITS FROM OLEKSANDR AND HENRY ======") xm_digits, xm_labels = utils.load_image_data(xm_digits_path, side=286, padding=40, unpad=False) ob_digits, ob_labels = utils.load_image_data(ob_digits_path, side=286, padding=40, unpad=False) xm_labels = utils.create_one_hot(xm_labels) ob_labels = utils.create_one_hot(ob_labels) combined_data = np.concatenate((xm_digits, ob_digits)) combined_labels = np.concatenate((xm_labels, ob_labels)) print("Analysing Henry's Digits") xm_r, xm_acc, xm_tau, xm_new_dig = thickness_sim(model, xm_digits, xm_labels, mnist_linethickness) print("== FINISH ==\n") print("Analysing Oleksandr's Digits") ob_r, ob_acc, ob_tau, ob_new_dig = thickness_sim(model, ob_digits, ob_labels, mnist_linethickness) print("== FINISH ==\n") print("Analysing COmbined digit set") comb_r, comb_acc, comb_tau, comb_new_data = thickness_sim(
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data_more') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [25, 27, 30] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = digits_data2['optimal_lw'] d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 50 trials = 10 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): inputs = [] outputs = [] model_list = [] for e in range(ensemble_size): model = Sequential() model.add( layers.Dense(200, input_dim=784, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(10, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) model.compile(optimizer=opt.Adam(), loss="categorical_crossentropy", metrics=['acc']) model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, validation_split=(1 / 6), callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs) merge_model = Model(inputs=inputs, outputs=layers.Average()(outputs)) #mnist_preds = merge_model.predict([xtest]*ensemble_size) #mnist_mem_preds = np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,2,0) #correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds ,ytest) #mnist_correct.extend(correct) #mnist_wrong.extend(wrong) sp_digits = apply_salt_pepper(digits_og) for s_d in sp_digits: s_d = utils.normalize_data(reshape_fun(s_d)) d2_preds = merge_model.predict([s_d] * ensemble_size) d2_mempreds = np.array( list(map(lambda m: m.predict(s_d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) mempreds = np.array(list(map(lambda m: m.predict(d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(digits_preds, mempreds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { #'mnist_correct' : mnist_correct, #'mnist_wrong' : mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
def experiment(network_conf_json, reshape_mode="conv"): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain) mnist_c_errors = [] mnist_pred_bits = [] mnist_class_bits = [] xm_c_errors = [] xm_pred_bits = [] xm_class_bits = [] ob_c_errors = [] ob_pred_bits = [] ob_class_bits = [] xm_data = utils.load_processed_data("xiaoming_digits") ob_data = utils.load_processed_data("Oleks_digits") xm_digits = reshape_fun(utils.normalize_data(list(xm_data.values())[0])) xm_labels = utils.create_one_hot(xm_data["labels"]) ob_digits = reshape_fun(utils.normalize_data(list(ob_data.values())[0])) ob_labels = utils.create_one_hot(ob_data["labels"]) for ensemble_size in ensemble_sizes: # Building the ensemble models and training the networks print("===== Building the ensemble of size %s =====" % ensemble_size) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_conf_json], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['accuracy']) train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size, batch_size=100, verbose=1, validation_data=([xval] * ensemble_size, [yval] * ensemble_size), epochs=num_epochs) # Calculating classification errors print("===== Calculating classification errors =====") mnist_c_errors.append( ann.test_model(merge_model, [xtest] * ensemble_size, ytest, metric="c_error")) xm_c_errors.append( ann.test_model(merge_model, [xm_digits] * ensemble_size, xm_labels, metric="c_error")) ob_c_errors.append( ann.test_model(merge_model, [ob_digits] * ensemble_size, ob_labels, metric="c_error")) # Calculating ensemble prediciton entropy print("===== Calculating ensemble prediciton entropy =====") mnist_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([xtest] * ensemble_size)))) xm_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([xm_digits] * ensemble_size)))) ob_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([ob_digits] * ensemble_size)))) # Calculating ensemble members classification entropyc_error print( "===== Calculating ensemble members classification entropy =====") mnist_class_bits.append(np.mean(calc_class_entropy(model_list, xtest))) xm_class_bits.append(np.mean(calc_class_entropy(model_list, xm_digits))) ob_class_bits.append(np.mean(calc_class_entropy(model_list, ob_digits))) mnist_results = { "c_error": mnist_c_errors, "pred_bits": mnist_pred_bits, "class_bits": mnist_class_bits } xm_results = { "c_error": xm_c_errors, "pred_bits": xm_pred_bits, "class_bits": xm_class_bits } ob_results = { "c_error": ob_c_errors, "pred_bits": ob_pred_bits, "class_bits": ob_class_bits } return mnist_results, xm_results, ob_results
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 50 trials = 10 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) es = clb.EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") #print(np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,0,2).shape) train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, batch_size=100, validation_data=(l_xval, l_yval), callbacks=[es]) mnist_preds = merge_model.predict([xtest] * ensemble_size) mnist_mem_preds = np.array(train_model.predict( [xtest] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds, ytest) mnist_correct.extend(correct) mnist_wrong.extend(wrong) d2_preds = merge_model.predict([digits_og] * ensemble_size) d2_mempreds = np.array(train_model.predict( [digits_og] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) mempreds = np.array(train_model.predict( [d] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(digits_preds, mempreds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { 'mnist_correct': mnist_correct, 'mnist_wrong': mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) custom_digits_dict = utils.load_processed_data( "combined_testing_data_more") digits_labels = custom_digits_dict['labels'] digits_taus = [t for t in custom_digits_dict.keys() if t != "labels"] digits_data = list( map(reshape_fun, [custom_digits_dict[t] for t in digits_taus])) digits_data = list(map(utils.normalize_data, digits_data)) digits_labels = utils.create_one_hot(digits_labels.astype('uint')) for tr in range(1, trials + 1): gc.collect() print("==== TRIAL %s ====" % tr) # Preparing Results # Classification Error ensemble_cerror = [] #t_ensemble_adv_cerror = [] digits_cerror = [] #t_digits_adv_cerror = [] # Prediction Entropy entropy_ensemble = [] #t_entropy_adv_ensemble = [] digits_entropy = [] #t_digits_adv_entropy = [] # Voting entropy #entropy_vote = [] #entropy_adv_vote = [] #digits_vote = [] #digits_adv_entropy = [] epochs = 50 for m in ensemble_sizes: print('Working now with ensemble of size m = %s' % m) l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(m): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) # Without adveserial training es = clb.EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=m, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs=epochs, validation_data=(l_xval, l_yval), callbacks=[es]) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) c_error = ann.test_model(merge_model, [xtest] * m, ytest, metric='accuracy') entropy = ann.test_model(merge_model, [xtest] * m, ytest, metric='entropy') ensemble_cerror.append(c_error) entropy_ensemble.append(entropy) #entropy_vote.append(calc_vote_entropy(model_list,m,xtest)) d_cerror, d_entropy = test_digits(merge_model, model_list, m, digits_data, digits_labels) digits_cerror.append(d_cerror) digits_entropy.append(d_entropy) gc.collect() #digits_vote.append(d_vote) # Adveserial training #inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average") #losses = list( # map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) #) #train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) #train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval)) #c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' ) #entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' ) # #t_ensemble_adv_cerror.append(c_error) #t_entropy_adv_ensemble.append(entropy) # #d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels) # #t_digits_adv_cerror.append(d_cerror) #t_digits_adv_entropy.append(d_entropy) filename1 = 'mnist_results_20-trial%s' % tr filename2 = 'digits_results_20-trial%s' % tr mnist_results = { 'ensemble_cerror': ensemble_cerror, 'ensemble_entropy': entropy_ensemble #'ensemble_adv_cerror' : ensemble_adv_cerror, #'ensemble_adv_entropy' : entropy_adv_ensemble, #'voting_entropy' : entropy_vote #'voting_adv_entropy' : entropy_adv_vote } digits_results = { 'ensemble_cerror': digits_cerror, 'ensemble_entropy': digits_entropy #'ensemble_adv_cerror' : digits_adv_cerror, #'ensemble_adv_entropy' : digits_adv_entropy, #'voting_entropy' : digits_vote #'voting_adv_entropy' : digits_adv_vote } utils.save_processed_data(mnist_results, filename1) utils.save_processed_data(digits_results, filename2) return digits_taus, mnist_results, digits_results
utils.setup_gpu_session() pathXM = os.path.join(".", "images", "XiaoMing_Digits") pathOB = os.path.join(".", "images", "60 Images") imgsXM, labelsXM = utils.load_image_data(pathXM) imgsOB, labelsOB = utils.load_image_data(pathOB) xtrain, ytrain, xtest, ytest = utils.load_mnist(normalize=True) xtrain, xtest = xtrain.reshape(60000, 28, 28, 1), xtest.reshape(10000, 28, 28, 1) img_size = imgsXM.shape[0] ximgXM = imgsXM.reshape(img_size, 28, 28, 1) ximgXM = utils.normalize_data(ximgXM) yimgXM = utils.create_one_hot(labelsXM) img_size = imgsOB.shape[0] ximgOB = imgsOB.reshape(img_size, 28, 28, 1) ximgOB = utils.normalize_data(ximgOB) yimgOB = utils.create_one_hot(labelsOB) imgs_combined = np.concatenate((ximgXM, ximgOB)) labels_combined = np.concatenate((yimgXM, yimgOB)) for i in range(trials): t_xtrain, t_ytrain, xval, yval = utils.create_validation(xtrain, ytrain) print("starting trial %s" % (i + 1)) ## Realizing and training the data ## uses TensorBoard to monitor the progress model = ann.parse_model_js(network_model)