def __init__(self, args): super(Trainer, self).__init__() self.args = args self.exp_name = self.experiment_name(args) self.best_roc = 0 self.use_cuda = args.gpu >= 0 and torch.cuda.is_available() if self.use_cuda: torch.cuda.set_device(args.gpu) self.filtered_triplets, self.meta_train_task_triplets, self.meta_valid_task_triplets, self.meta_test_task_triplets, \ self.meta_train_task_entity_to_triplets, self.meta_valid_task_entity_to_triplets, self.meta_test_task_entity_to_triplets \ = utils.load_processed_data('./Dataset/processed_data/{}'.format(args.data)) self.meta_task_entity = np.concatenate( (list(self.meta_train_task_entity_to_triplets.keys()), list(self.meta_valid_task_entity_to_triplets.keys()), list(self.meta_test_task_entity_to_triplets.keys()))) self.load_pretrain_embedding() self.load_model() print(self.model) if self.use_cuda: self.model.cuda() self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.args.lr, weight_decay=self.args.weight_decay)
def __init__(self, args): super(Trainer, self).__init__() self.args = args self.exp_name = self.experiment_name(args) self.best_mrr = 0 self.use_cuda = args.gpu >= 0 and torch.cuda.is_available() if self.use_cuda: torch.cuda.set_device(args.gpu) self.entity2id, self.relation2id, self.train_triplets, self.valid_triplets, self.test_triplets = utils.load_data( './Dataset/raw_data/{}'.format(args.data)) self.filtered_triplets, self.meta_train_task_triplets, self.meta_valid_task_triplets, self.meta_test_task_triplets, \ self.meta_train_task_entity_to_triplets, self.meta_valid_task_entity_to_triplets, self.meta_test_task_entity_to_triplets \ = utils.load_processed_data('./Dataset/processed_data/{}'.format(args.data)) self.all_triplets = torch.LongTensor( np.concatenate((self.train_triplets, self.valid_triplets, self.test_triplets))) self.meta_task_entity = np.concatenate( (list(self.meta_train_task_entity_to_triplets.keys()), list(self.meta_valid_task_entity_to_triplets.keys()), list(self.meta_test_task_entity_to_triplets.keys()))) self.meta_task_triplets = torch.LongTensor( np.concatenate( (self.meta_train_task_triplets, self.meta_valid_task_triplets, self.meta_test_task_triplets))) self.meta_task_test_entity = torch.LongTensor( np.array(list(self.meta_test_task_entity_to_triplets.keys()))) self.load_pretrain_embedding(data=args.data, model=args.pre_train_model) self.load_model(model=args.model) if self.use_cuda: self.model.cuda() self.all_triplets = self.all_triplets.cuda() self.meta_task_triplets = self.meta_task_triplets.cuda() self.meta_task_test_entity = self.meta_task_test_entity.cuda() self.head_relation_triplets = self.all_triplets[:, :2] self.tail_relation_triplets = torch.stack( (self.all_triplets[:, 2], self.all_triplets[:, 1])).transpose(0, 1) self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.args.lr, weight_decay=self.args.weight_decay)
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) digits_data = utils.load_processed_data('digits_og_and_optimal') digits = digits_data['optimal_lw'] labels = utils.create_one_hot(digits_data['labels'].astype('uint')) ensemble_size = 20 epochs = 50 small_digits = reshape_fun(np.array(list(map(scale_down, digits)))) small_digits = utils.normalize_data(small_digits) trials = 5 for t in range(1,trials+1): gc.collect() l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True) train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics = ['acc']) train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval),callbacks=[es]) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) results = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun) #entropy = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'entropy') #c_error = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'c_error') #results['c_error'][0] = c_error #results['entropy'][0] = entropy filename = "saltpepper_norm_trial-%s" % t utils.save_processed_data(results, filename)
def __init__(self, data_type, dataset, batch_size, char_vocab, fw_bigram_vocab, bw_bigram_vocab, tag_vocab, shuffle=True): self.data = load_processed_data(data_type, dataset) self.data_size = len(self.data) self.batch_size = batch_size self.steps = int(np.ceil(self.data_size / self.batch_size)) self.indices = np.arange(self.data_size) self.shuffle = shuffle self.char_vocab = char_vocab self.fw_bigram_vocab = fw_bigram_vocab self.bw_bigram_vocab = bw_bigram_vocab self.tag_vocab = tag_vocab self.n_class = len(self.tag_vocab) self.n_gaze = len(self.data[0].gaze_match)
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, _ = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type=None) train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, validation_data=(l_xval, l_yval)) mnist_mpreds = np.array(list(map(lambda m: m.predict(xtest), model_list))) mnist_mnwrong = [] mnist_mncorrect = [] for mp in mnist_mpreds: correct, wrong = bin_entropies(mp, ytest) mnist_mnwrong.extend(wrong) mnist_mncorrect.extend(correct) d2_preds = np.array(list(map(lambda m: m.predict(digits_og), model_list))) d2_mnwrong = [] d2_mncorrect = [] for mp in d2_preds: correct, wrong = bin_entropies(mp, d2_labels) d2_mnwrong.extend(wrong) d2_mncorrect.extend(correct) digits_mnwrong = [] digits_mncorrect = [] for d in digits: for m in model_list: preds = m.predict(d) correct, wrong = bin_entropies(preds, d_labels) digits_mnwrong.extend(wrong) digits_mncorrect.extend(correct) individual = { 'mnist_correct': mnist_mncorrect, 'mnist_wrong': mnist_mnwrong, 'digits_correct': digits_mncorrect, 'digits_wrong': digits_mnwrong, 'lecunn_correct': d2_mncorrect, 'lecunn_wrong': d2_mnwrong } return individual
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 trials = 5 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") losses = list( map( lambda m: ann.adveserial_loss( klosses.categorical_crossentropy, m, eps=0.01), model_list)) train_model.compile(optimizer='adam', loss=losses, metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, validation_data=(l_xval, l_yval)) mnist_preds = merge_model.predict([xtest] * ensemble_size) correct, wrong = bin_entropies(mnist_preds, ytest) mnist_correct.extend(correct) mnist_wrong.extend(wrong) d2_preds = merge_model.predict([digits_og] * ensemble_size) correct, wrong = bin_entropies(d2_preds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) correct, wrong = bin_entropies(digits_preds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { 'mnist_correct': mnist_correct, 'mnist_wrong': mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
ensemblesize = 100 chunksize = 20 nchunks = ensemblesize // chunksize xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } reshape_fun = reshape_funs['conv'] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) utils.setup_gpu_session() digits_data = utils.load_processed_data("combined_testing_data") taus = list(digits_data.keys())[:-1] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) labels = utils.create_one_hot(digits_data['labels'].astype('uint')) mnist_mpreds = [] digits_mpreds = {} mnist_mcerr = [] digits_mcerr = {} mnist_mbits = [] digits_mbits = {} for t in taus: digits_mpreds[t] = [] digits_mcerr[t] = []
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [13, 14, 15] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = reshape_fun(digits_data2['lecunn']) digits_og = utils.normalize_data(digits_og) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 50 trials = 10 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) es = clb.EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") #print(np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,0,2).shape) train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, batch_size=100, validation_data=(l_xval, l_yval), callbacks=[es]) mnist_preds = merge_model.predict([xtest] * ensemble_size) mnist_mem_preds = np.array(train_model.predict( [xtest] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds, ytest) mnist_correct.extend(correct) mnist_wrong.extend(wrong) d2_preds = merge_model.predict([digits_og] * ensemble_size) d2_mempreds = np.array(train_model.predict( [digits_og] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) mempreds = np.array(train_model.predict( [d] * ensemble_size)).transpose(1, 2, 0) correct, wrong = bin_entropies(digits_preds, mempreds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { 'mnist_correct': mnist_correct, 'mnist_wrong': mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('digits_og_and_optimal') digits = digits_data['optimal_lw'] digits = utils.normalize_data(digits) digits_og = digits_data['lecunn_big'] digits_og_small = reshape_fun(digits_data['lecunn']) digits_og_small = utils.normalize_data(digits_og_small) labels = utils.create_one_hot(digits_data['labels'].astype('uint')) ensemble_size = 20 epochs = 3 small_digits = reshape_fun(np.array(list(map(scale_down, digits)))) trials = 5 for t in range(1, trials + 1): gc.collect() l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, batch_size=100, epochs=epochs, validation_data=(l_xval, l_yval)) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) results_linewidth = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun) entropy = ann.test_model(merge_model, [small_digits] * ensemble_size, labels, metric='entropy') c_error = ann.test_model(merge_model, [small_digits] * ensemble_size, labels, metric='c_error') results_linewidth['c_error'][0] = c_error results_linewidth['entropy'][0] = entropy results_lecunn = test_digits(merge_model, digits_og, labels, ensemble_size, reshape_fun) entropy = ann.test_model(merge_model, [digits_og_small] * ensemble_size, labels, metric='entropy') c_error = ann.test_model(merge_model, [digits_og_small] * ensemble_size, labels, metric='c_error') results_lecunn['c_error'][0] = c_error results_lecunn['entropy'][0] = entropy total_results = { 'optimal_lw': results_linewidth, 'lecunn': results_lecunn } filename = "saltpepper_random_trial-%s" % t utils.save_processed_data(total_results, filename)
def experiment(network_conf_json, reshape_mode="conv"): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain) xm_data = utils.load_processed_data("xiaoming_digits") ob_data = utils.load_processed_data("Oleks_digits") xm_taus = np.array(list(xm_data.keys())[:-1]) ob_taus = np.array(list(ob_data.keys())[:-1]) xm_digits = list(map(reshape_fun, [xm_data[t] for t in xm_taus])) xm_labels = utils.create_one_hot(xm_data["labels"]) ob_digits = list(map(reshape_fun, [ob_data[t] for t in ob_taus])) ob_labels = utils.create_one_hot(ob_data["labels"]) # Building the ensemble models and training the networks print("===== Building the ensemble of size %s =====" % ensemble_size) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_conf_json], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['accuracy']) train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size, batch_size=100, verbose=1, validation_data=([xval] * ensemble_size, [yval] * ensemble_size), epochs=num_epochs) mnist_c_errors, mnist_pred_bits, mnist_class_bits = test_digits( [xtest], ytest, merge_model, model_list) xm_c_errors, xm_pred_bits, xm_class_bits = test_digits( xm_digits, xm_labels, merge_model, model_list) ob_c_errors, ob_pred_bits, ob_class_bits = test_digits( ob_digits, ob_labels, merge_model, model_list) mnist_values = { "c_error": mnist_c_errors[0], "pred_bits": mnist_pred_bits[0], "class_bits": mnist_class_bits[0] } xm_results = { "taus": xm_taus, "c_error": xm_c_errors, "pred_bits": xm_pred_bits, "class_bits": xm_class_bits } ob_results = { "taus": ob_taus, "c_error": ob_c_errors, "pred_bits": ob_pred_bits, "class_bits": ob_class_bits } return mnist_values, xm_results, ob_results
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data_more') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [25, 27, 30] digits = list(map(reshape_fun, [digits_data[t] for t in taus])) digits = list(map(utils.normalize_data, digits)) digits_og = digits_data2['optimal_lw'] d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 50 trials = 10 mnist_correct = [] mnist_wrong = [] digits_wrong = [] digits_correct = [] d2_wrong = [] d2_correct = [] for t in range(trials): inputs = [] outputs = [] model_list = [] for e in range(ensemble_size): model = Sequential() model.add( layers.Dense(200, input_dim=784, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(10, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True) model.compile(optimizer=opt.Adam(), loss="categorical_crossentropy", metrics=['acc']) model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, validation_split=(1 / 6), callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs) merge_model = Model(inputs=inputs, outputs=layers.Average()(outputs)) #mnist_preds = merge_model.predict([xtest]*ensemble_size) #mnist_mem_preds = np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,2,0) #correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds ,ytest) #mnist_correct.extend(correct) #mnist_wrong.extend(wrong) sp_digits = apply_salt_pepper(digits_og) for s_d in sp_digits: s_d = utils.normalize_data(reshape_fun(s_d)) d2_preds = merge_model.predict([s_d] * ensemble_size) d2_mempreds = np.array( list(map(lambda m: m.predict(s_d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels) d2_correct.extend(correct) d2_wrong.extend(wrong) for d in digits: digits_preds = merge_model.predict([d] * ensemble_size) mempreds = np.array(list(map(lambda m: m.predict(d), model_list))).transpose(1, 2, 0) correct, wrong = bin_entropies(digits_preds, mempreds, d_labels) digits_wrong.extend(wrong) digits_correct.extend(correct) ensemble = { #'mnist_correct' : mnist_correct, #'mnist_wrong' : mnist_wrong, 'digits_correct': digits_correct, 'digits_wrong': digits_wrong, 'lecunn_correct': d2_correct, 'lecunn_wrong': d2_wrong } return ensemble
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) test_data = utils.load_processed_data('notmnist') letters = list(test_data.keys()) ensemble_size = 20 epochs = 50 trials = 10 results = { 'A': [], 'B': [], 'C': [], 'D': [], 'E': [], 'F': [], 'G': [], 'H': [], 'I': [], 'J': [] } for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) es = clb.EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=ensemble_size, merge_type="Average") #print(np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,0,2).shape) train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) train_model.fit(l_xtrain, l_ytrain, epochs=epochs, batch_size=100, validation_data=(l_xval, l_yval), callbacks=[es]) for letter in letters: inputs = test_data[letter] inputs = utils.normalize_data(inputs) inputs = reshape_fun(inputs) preds = merge_model.predict([inputs] * ensemble_size) mem_preds = np.array(train_model.predict( [inputs] * ensemble_size)).transpose(1, 2, 0) bits = list(map(stats.entropy, preds)) s_q = list(map(calc_pred_vars, mem_preds)) results[letter].extend(list(zip(bits, s_q))) return results
def experiment(network_conf_json, reshape_mode="conv"): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain) mnist_c_errors = [] mnist_pred_bits = [] mnist_class_bits = [] xm_c_errors = [] xm_pred_bits = [] xm_class_bits = [] ob_c_errors = [] ob_pred_bits = [] ob_class_bits = [] xm_data = utils.load_processed_data("xiaoming_digits") ob_data = utils.load_processed_data("Oleks_digits") xm_digits = reshape_fun(utils.normalize_data(list(xm_data.values())[0])) xm_labels = utils.create_one_hot(xm_data["labels"]) ob_digits = reshape_fun(utils.normalize_data(list(ob_data.values())[0])) ob_labels = utils.create_one_hot(ob_data["labels"]) for ensemble_size in ensemble_sizes: # Building the ensemble models and training the networks print("===== Building the ensemble of size %s =====" % ensemble_size) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_conf_json], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['accuracy']) train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size, batch_size=100, verbose=1, validation_data=([xval] * ensemble_size, [yval] * ensemble_size), epochs=num_epochs) # Calculating classification errors print("===== Calculating classification errors =====") mnist_c_errors.append( ann.test_model(merge_model, [xtest] * ensemble_size, ytest, metric="c_error")) xm_c_errors.append( ann.test_model(merge_model, [xm_digits] * ensemble_size, xm_labels, metric="c_error")) ob_c_errors.append( ann.test_model(merge_model, [ob_digits] * ensemble_size, ob_labels, metric="c_error")) # Calculating ensemble prediciton entropy print("===== Calculating ensemble prediciton entropy =====") mnist_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([xtest] * ensemble_size)))) xm_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([xm_digits] * ensemble_size)))) ob_pred_bits.append( np.mean( calc_shannon_entropy( merge_model.predict([ob_digits] * ensemble_size)))) # Calculating ensemble members classification entropyc_error print( "===== Calculating ensemble members classification entropy =====") mnist_class_bits.append(np.mean(calc_class_entropy(model_list, xtest))) xm_class_bits.append(np.mean(calc_class_entropy(model_list, xm_digits))) ob_class_bits.append(np.mean(calc_class_entropy(model_list, ob_digits))) mnist_results = { "c_error": mnist_c_errors, "pred_bits": mnist_pred_bits, "class_bits": mnist_class_bits } xm_results = { "c_error": xm_c_errors, "pred_bits": xm_pred_bits, "class_bits": xm_class_bits } ob_results = { "c_error": ob_c_errors, "pred_bits": ob_pred_bits, "class_bits": ob_class_bits } return mnist_results, xm_results, ob_results
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) custom_digits_dict = utils.load_processed_data("combined_testing_data") digits_labels = custom_digits_dict['labels'] digits_taus = list(custom_digits_dict.keys())[:-1] digits_data = list(map(reshape_fun, [custom_digits_dict[t] for t in digits_taus])) digits_data = list(map(utils.normalize_data, digits_data)) digits_labels = utils.create_one_hot(digits_labels.astype('uint')) for t in range(trials): print("===== START TRIAL %s =====" % (t + 1)) # Preparing Results # Classification Error ensemble_cerror = [] #t_ensemble_adv_cerror = [] digits_cerror = [] #t_digits_adv_cerror = [] # Prediction Entropy entropy_ensemble = [] #t_entropy_adv_ensemble = [] digits_entropy = [] #t_digits_adv_entropy = [] # Voting entropy #entropy_vote = [] #entropy_adv_vote = [] #digits_vote = [] #digits_adv_entropy = [] epochs = 5 l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) # Without adveserial training inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") losses = list( map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) ) train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval)) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) c_error = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'accuracy' ) entropy = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'entropy' ) ensemble_cerror.append(c_error) entropy_ensemble.append(entropy) #entropy_vote.append(calc_vote_entropy(model_list,m,xtest)) d_cerror,d_entropy = test_digits(merge_model,model_list,ensemble_size,digits_data,digits_labels) digits_cerror.append(d_cerror) digits_entropy.append(d_entropy) #digits_vote.append(d_vote) # Adveserial training #inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average") #losses = list( # map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) #) #train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) #train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval)) #c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' ) #entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' ) # #t_ensemble_adv_cerror.append(c_error) #t_entropy_adv_ensemble.append(entropy) # #d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels) # #t_digits_adv_cerror.append(d_cerror) #t_digits_adv_entropy.append(d_entropy) filename1 = 'adv_mnist_results_5-20-50-trial%s' % t filename2 = 'adv_digits_results_5-20-50-trial%s' % t mnist_results = { 'ensemble_cerror' : ensemble_cerror, 'ensemble_entropy' : entropy_ensemble #'ensemble_adv_cerror' : ensemble_adv_cerror, #'ensemble_adv_entropy' : entropy_adv_ensemble, #'voting_entropy' : entropy_vote #'voting_adv_entropy' : entropy_adv_vote } digits_results = { 'ensemble_cerror' : digits_cerror, 'ensemble_entropy' : digits_entropy #'ensemble_adv_cerror' : digits_adv_cerror, #'ensemble_adv_entropy' : digits_adv_entropy, #'voting_entropy' : digits_vote #'voting_adv_entropy' : digits_adv_vote } utils.save_processed_data(mnist_results,filename1) utils.save_processed_data(digits_results,filename2) return digits_taus, mnist_results, digits_results
"activation" : "relu" }, { "type" : "Dense", "units" : 10, "activation" : "softmax" } ] } ''' epochs = 3 trials = 100 xtrain, ytrain, xtest, ytest = utils.load_mnist() digits_data = utils.load_processed_data('digits_og_and_optimal') digits_big = digits_data['lecunn_big'] digits_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) digits = digits_data['lecunn'] def resize_helper(img): sides = 280 return cv2.resize(img, (sides, sides), interpolation=cv2.INTER_CUBIC) def calc_linewidths(data): taus = np.zeros(data.shape[0]) for i, d in zip(range(data.shape[0]), data): taus[i] = dutils.intern_calc_linewidth(d)
def experiment(network_model, reshape_mode = 'mlp'): reshape_funs = { "conv" : lambda d : d.reshape(-1,28,28,1), "mlp" : lambda d : d.reshape(-1,784) } xtrain,ytrain,xtest,ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest) digits_data = utils.load_processed_data('combined_testing_data') digits_data2 = utils.load_processed_data('digits_og_and_optimal') taus = [26,28,30] taus2 = [2,4,5] digits1 = list(map(reshape_fun, [digits_data[t] for t in taus])) digits1 = list(map(utils.normalize_data, digits1)) digits2 = list(map(reshape_fun, [digits_data[t] for t in taus2])) digits2 = list(map(utils.normalize_data, digits2)) digits_og = digits_data2['lecunn_big'] digits_og = salt_and_pepper(digits_og,72000) digits_og = reshape_fun(utils.normalize_data(digits_og)) d_labels = utils.create_one_hot(digits_data['labels'].astype('uint')) d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint')) ensemble_size = 20 epochs = 3 trials = 5 d1_correct = [] d1_wrong = [] d2_wrong = [] d2_correct = [] dog_wrong = [] dog_correct = [] for t in range(trials): l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(ensemble_size): t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average") train_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['acc']) train_model.fit(l_xtrain,l_ytrain,epochs = epochs, validation_data = (l_xval,l_yval)) dog_preds = merge_model.predict([digits_og]*ensemble_size) correct, wrong = bin_entropies(d2_preds, d2_labels) dog_correct.extend(correct) dog_wrong.extend(wrong) for d in digits1: digits_preds = merge_model.predict([d]*ensemble_size) correct, wrong = bin_entropies(digits_preds,d_labels) d1.extend(wrong) d1.extend(correct) for d in digits2: digits_preds = merge_model.predict([d]*ensemble_size) correct, wrong = bin_entropies(digits_preds,d_labels) d2.extend(wrong) d2.extend(correct) ensemble = { 'd1_correct' : d1_correct, 'd1_wrong' : d1_wrong, 'd2_correct' : d2_correct, 'd2_wrong' : d2_wrong, 'lecunn_correct' : dog_correct, 'lecunn_wrong' : dog_wrong } return ensemble
def experiment(network_model, reshape_mode='mlp'): reshape_funs = { "conv": lambda d: d.reshape(-1, 28, 28, 1), "mlp": lambda d: d.reshape(-1, 784) } xtrain, ytrain, xtest, ytest = utils.load_mnist() reshape_fun = reshape_funs[reshape_mode] xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest) custom_digits_dict = utils.load_processed_data( "combined_testing_data_more") digits_labels = custom_digits_dict['labels'] digits_taus = [t for t in custom_digits_dict.keys() if t != "labels"] digits_data = list( map(reshape_fun, [custom_digits_dict[t] for t in digits_taus])) digits_data = list(map(utils.normalize_data, digits_data)) digits_labels = utils.create_one_hot(digits_labels.astype('uint')) for tr in range(1, trials + 1): gc.collect() print("==== TRIAL %s ====" % tr) # Preparing Results # Classification Error ensemble_cerror = [] #t_ensemble_adv_cerror = [] digits_cerror = [] #t_digits_adv_cerror = [] # Prediction Entropy entropy_ensemble = [] #t_entropy_adv_ensemble = [] digits_entropy = [] #t_digits_adv_entropy = [] # Voting entropy #entropy_vote = [] #entropy_adv_vote = [] #digits_vote = [] #digits_adv_entropy = [] epochs = 50 for m in ensemble_sizes: print('Working now with ensemble of size m = %s' % m) l_xtrain = [] l_xval = [] l_ytrain = [] l_yval = [] for _ in range(m): t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation( xtrain, ytrain, (1 / 6)) l_xtrain.append(t_xtrain) l_xval.append(t_xval) l_ytrain.append(t_ytrain) l_yval.append(t_yval) # Without adveserial training es = clb.EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True) inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble( [network_model], pop_per_type=m, merge_type="Average") train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs=epochs, validation_data=(l_xval, l_yval), callbacks=[es]) merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc']) c_error = ann.test_model(merge_model, [xtest] * m, ytest, metric='accuracy') entropy = ann.test_model(merge_model, [xtest] * m, ytest, metric='entropy') ensemble_cerror.append(c_error) entropy_ensemble.append(entropy) #entropy_vote.append(calc_vote_entropy(model_list,m,xtest)) d_cerror, d_entropy = test_digits(merge_model, model_list, m, digits_data, digits_labels) digits_cerror.append(d_cerror) digits_entropy.append(d_entropy) gc.collect() #digits_vote.append(d_vote) # Adveserial training #inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average") #losses = list( # map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list) #) #train_model.compile(optimizer="adam", loss=losses, metrics = ['acc']) #train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval)) #c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' ) #entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' ) # #t_ensemble_adv_cerror.append(c_error) #t_entropy_adv_ensemble.append(entropy) # #d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels) # #t_digits_adv_cerror.append(d_cerror) #t_digits_adv_entropy.append(d_entropy) filename1 = 'mnist_results_20-trial%s' % tr filename2 = 'digits_results_20-trial%s' % tr mnist_results = { 'ensemble_cerror': ensemble_cerror, 'ensemble_entropy': entropy_ensemble #'ensemble_adv_cerror' : ensemble_adv_cerror, #'ensemble_adv_entropy' : entropy_adv_ensemble, #'voting_entropy' : entropy_vote #'voting_adv_entropy' : entropy_adv_vote } digits_results = { 'ensemble_cerror': digits_cerror, 'ensemble_entropy': digits_entropy #'ensemble_adv_cerror' : digits_adv_cerror, #'ensemble_adv_entropy' : digits_adv_entropy, #'voting_entropy' : digits_vote #'voting_adv_entropy' : digits_adv_vote } utils.save_processed_data(mnist_results, filename1) utils.save_processed_data(digits_results, filename2) return digits_taus, mnist_results, digits_results