def __init__(self, args):
super(Trainer, self).__init__()
self.args = args
self.exp_name = self.experiment_name(args)
self.best_roc = 0
self.use_cuda = args.gpu >= 0 and torch.cuda.is_available()
if self.use_cuda:
torch.cuda.set_device(args.gpu)
self.filtered_triplets, self.meta_train_task_triplets, self.meta_valid_task_triplets, self.meta_test_task_triplets, \
self.meta_train_task_entity_to_triplets, self.meta_valid_task_entity_to_triplets, self.meta_test_task_entity_to_triplets \
= utils.load_processed_data('./Dataset/processed_data/{}'.format(args.data))
self.meta_task_entity = np.concatenate(
(list(self.meta_train_task_entity_to_triplets.keys()),
list(self.meta_valid_task_entity_to_triplets.keys()),
list(self.meta_test_task_entity_to_triplets.keys())))
self.load_pretrain_embedding()
self.load_model()
print(self.model)
if self.use_cuda:
self.model.cuda()
self.optimizer = torch.optim.Adam(self.model.parameters(),
lr=self.args.lr,
weight_decay=self.args.weight_decay)
def __init__(self, args):
super(Trainer, self).__init__()
self.args = args
self.exp_name = self.experiment_name(args)
self.best_mrr = 0
self.use_cuda = args.gpu >= 0 and torch.cuda.is_available()
if self.use_cuda:
torch.cuda.set_device(args.gpu)
self.entity2id, self.relation2id, self.train_triplets, self.valid_triplets, self.test_triplets = utils.load_data(
'./Dataset/raw_data/{}'.format(args.data))
self.filtered_triplets, self.meta_train_task_triplets, self.meta_valid_task_triplets, self.meta_test_task_triplets, \
self.meta_train_task_entity_to_triplets, self.meta_valid_task_entity_to_triplets, self.meta_test_task_entity_to_triplets \
= utils.load_processed_data('./Dataset/processed_data/{}'.format(args.data))
self.all_triplets = torch.LongTensor(
np.concatenate((self.train_triplets, self.valid_triplets,
self.test_triplets)))
self.meta_task_entity = np.concatenate(
(list(self.meta_train_task_entity_to_triplets.keys()),
list(self.meta_valid_task_entity_to_triplets.keys()),
list(self.meta_test_task_entity_to_triplets.keys())))
self.meta_task_triplets = torch.LongTensor(
np.concatenate(
(self.meta_train_task_triplets, self.meta_valid_task_triplets,
self.meta_test_task_triplets)))
self.meta_task_test_entity = torch.LongTensor(
np.array(list(self.meta_test_task_entity_to_triplets.keys())))
self.load_pretrain_embedding(data=args.data,
model=args.pre_train_model)
self.load_model(model=args.model)
if self.use_cuda:
self.model.cuda()
self.all_triplets = self.all_triplets.cuda()
self.meta_task_triplets = self.meta_task_triplets.cuda()
self.meta_task_test_entity = self.meta_task_test_entity.cuda()
self.head_relation_triplets = self.all_triplets[:, :2]
self.tail_relation_triplets = torch.stack(
(self.all_triplets[:, 2], self.all_triplets[:,
1])).transpose(0, 1)
self.optimizer = torch.optim.Adam(self.model.parameters(),
lr=self.args.lr,
weight_decay=self.args.weight_decay)
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
digits_data = utils.load_processed_data('digits_og_and_optimal')
digits = digits_data['optimal_lw']
labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
ensemble_size = 20
epochs = 50
small_digits = reshape_fun(np.array(list(map(scale_down, digits))))
small_digits = utils.normalize_data(small_digits)
trials = 5
for t in range(1,trials+1):
gc.collect()
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)
train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics = ['acc'])
train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval),callbacks=[es])
merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc'])
results = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun)
#entropy = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'entropy')
#c_error = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'c_error')
#results['c_error'][0] = c_error
#results['entropy'][0] = entropy
filename = "saltpepper_norm_trial-%s" % t
utils.save_processed_data(results, filename)
def __init__(self,
data_type,
dataset,
batch_size,
char_vocab,
fw_bigram_vocab,
bw_bigram_vocab,
tag_vocab,
shuffle=True):
self.data = load_processed_data(data_type, dataset)
self.data_size = len(self.data)
self.batch_size = batch_size
self.steps = int(np.ceil(self.data_size / self.batch_size))
self.indices = np.arange(self.data_size)
self.shuffle = shuffle
self.char_vocab = char_vocab
self.fw_bigram_vocab = fw_bigram_vocab
self.bw_bigram_vocab = bw_bigram_vocab
self.tag_vocab = tag_vocab
self.n_class = len(self.tag_vocab)
self.n_gaze = len(self.data[0].gaze_match)
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [13, 14, 15]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = reshape_fun(digits_data2['lecunn'])
digits_og = utils.normalize_data(digits_og)
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, _ = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type=None)
train_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
train_model.fit(l_xtrain,
l_ytrain,
epochs=epochs,
validation_data=(l_xval, l_yval))
mnist_mpreds = np.array(list(map(lambda m: m.predict(xtest), model_list)))
mnist_mnwrong = []
mnist_mncorrect = []
for mp in mnist_mpreds:
correct, wrong = bin_entropies(mp, ytest)
mnist_mnwrong.extend(wrong)
mnist_mncorrect.extend(correct)
d2_preds = np.array(list(map(lambda m: m.predict(digits_og), model_list)))
d2_mnwrong = []
d2_mncorrect = []
for mp in d2_preds:
correct, wrong = bin_entropies(mp, d2_labels)
d2_mnwrong.extend(wrong)
d2_mncorrect.extend(correct)
digits_mnwrong = []
digits_mncorrect = []
for d in digits:
for m in model_list:
preds = m.predict(d)
correct, wrong = bin_entropies(preds, d_labels)
digits_mnwrong.extend(wrong)
digits_mncorrect.extend(correct)
individual = {
'mnist_correct': mnist_mncorrect,
'mnist_wrong': mnist_mnwrong,
'digits_correct': digits_mncorrect,
'digits_wrong': digits_mnwrong,
'lecunn_correct': d2_mncorrect,
'lecunn_wrong': d2_mnwrong
}
return individual
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [13, 14, 15]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = reshape_fun(digits_data2['lecunn'])
digits_og = utils.normalize_data(digits_og)
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
trials = 5
mnist_correct = []
mnist_wrong = []
digits_wrong = []
digits_correct = []
d2_wrong = []
d2_correct = []
for t in range(trials):
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
losses = list(
map(
lambda m: ann.adveserial_loss(
klosses.categorical_crossentropy, m, eps=0.01),
model_list))
train_model.compile(optimizer='adam', loss=losses, metrics=['acc'])
train_model.fit(l_xtrain,
l_ytrain,
epochs=epochs,
validation_data=(l_xval, l_yval))
mnist_preds = merge_model.predict([xtest] * ensemble_size)
correct, wrong = bin_entropies(mnist_preds, ytest)
mnist_correct.extend(correct)
mnist_wrong.extend(wrong)
d2_preds = merge_model.predict([digits_og] * ensemble_size)
correct, wrong = bin_entropies(d2_preds, d2_labels)
d2_correct.extend(correct)
d2_wrong.extend(wrong)
for d in digits:
digits_preds = merge_model.predict([d] * ensemble_size)
correct, wrong = bin_entropies(digits_preds, d_labels)
digits_wrong.extend(wrong)
digits_correct.extend(correct)
ensemble = {
'mnist_correct': mnist_correct,
'mnist_wrong': mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
ensemblesize = 100
chunksize = 20
nchunks = ensemblesize // chunksize
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
reshape_fun = reshape_funs['conv']
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
utils.setup_gpu_session()
digits_data = utils.load_processed_data("combined_testing_data")
taus = list(digits_data.keys())[:-1]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
mnist_mpreds = []
digits_mpreds = {}
mnist_mcerr = []
digits_mcerr = {}
mnist_mbits = []
digits_mbits = {}
for t in taus:
digits_mpreds[t] = []
digits_mcerr[t] = []
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [13, 14, 15]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = reshape_fun(digits_data2['lecunn'])
digits_og = utils.normalize_data(digits_og)
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 50
trials = 10
mnist_correct = []
mnist_wrong = []
digits_wrong = []
digits_correct = []
d2_wrong = []
d2_correct = []
for t in range(trials):
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
es = clb.EarlyStopping(monitor='val_loss',
patience=2,
restore_best_weights=True)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
#print(np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,0,2).shape)
train_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
train_model.fit(l_xtrain,
l_ytrain,
epochs=epochs,
batch_size=100,
validation_data=(l_xval, l_yval),
callbacks=[es])
mnist_preds = merge_model.predict([xtest] * ensemble_size)
mnist_mem_preds = np.array(train_model.predict(
[xtest] * ensemble_size)).transpose(1, 2, 0)
correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds, ytest)
mnist_correct.extend(correct)
mnist_wrong.extend(wrong)
d2_preds = merge_model.predict([digits_og] * ensemble_size)
d2_mempreds = np.array(train_model.predict(
[digits_og] * ensemble_size)).transpose(1, 2, 0)
correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels)
d2_correct.extend(correct)
d2_wrong.extend(wrong)
for d in digits:
digits_preds = merge_model.predict([d] * ensemble_size)
mempreds = np.array(train_model.predict(
[d] * ensemble_size)).transpose(1, 2, 0)
correct, wrong = bin_entropies(digits_preds, mempreds, d_labels)
digits_wrong.extend(wrong)
digits_correct.extend(correct)
ensemble = {
'mnist_correct': mnist_correct,
'mnist_wrong': mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('digits_og_and_optimal')
digits = digits_data['optimal_lw']
digits = utils.normalize_data(digits)
digits_og = digits_data['lecunn_big']
digits_og_small = reshape_fun(digits_data['lecunn'])
digits_og_small = utils.normalize_data(digits_og_small)
labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
small_digits = reshape_fun(np.array(list(map(scale_down, digits))))
trials = 5
for t in range(1, trials + 1):
gc.collect()
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
train_model.fit(x=l_xtrain,
y=l_ytrain,
verbose=1,
batch_size=100,
epochs=epochs,
validation_data=(l_xval, l_yval))
merge_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
results_linewidth = test_digits(merge_model, digits, labels,
ensemble_size, reshape_fun)
entropy = ann.test_model(merge_model, [small_digits] * ensemble_size,
labels,
metric='entropy')
c_error = ann.test_model(merge_model, [small_digits] * ensemble_size,
labels,
metric='c_error')
results_linewidth['c_error'][0] = c_error
results_linewidth['entropy'][0] = entropy
results_lecunn = test_digits(merge_model, digits_og, labels,
ensemble_size, reshape_fun)
entropy = ann.test_model(merge_model,
[digits_og_small] * ensemble_size,
labels,
metric='entropy')
c_error = ann.test_model(merge_model,
[digits_og_small] * ensemble_size,
labels,
metric='c_error')
results_lecunn['c_error'][0] = c_error
results_lecunn['entropy'][0] = entropy
total_results = {
'optimal_lw': results_linewidth,
'lecunn': results_lecunn
}
filename = "saltpepper_random_trial-%s" % t
utils.save_processed_data(total_results, filename)
def experiment(network_conf_json, reshape_mode="conv"):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain)
xm_data = utils.load_processed_data("xiaoming_digits")
ob_data = utils.load_processed_data("Oleks_digits")
xm_taus = np.array(list(xm_data.keys())[:-1])
ob_taus = np.array(list(ob_data.keys())[:-1])
xm_digits = list(map(reshape_fun, [xm_data[t] for t in xm_taus]))
xm_labels = utils.create_one_hot(xm_data["labels"])
ob_digits = list(map(reshape_fun, [ob_data[t] for t in ob_taus]))
ob_labels = utils.create_one_hot(ob_data["labels"])
# Building the ensemble models and training the networks
print("===== Building the ensemble of size %s =====" % ensemble_size)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_conf_json], pop_per_type=ensemble_size, merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['accuracy'])
train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size,
batch_size=100,
verbose=1,
validation_data=([xval] * ensemble_size,
[yval] * ensemble_size),
epochs=num_epochs)
mnist_c_errors, mnist_pred_bits, mnist_class_bits = test_digits(
[xtest], ytest, merge_model, model_list)
xm_c_errors, xm_pred_bits, xm_class_bits = test_digits(
xm_digits, xm_labels, merge_model, model_list)
ob_c_errors, ob_pred_bits, ob_class_bits = test_digits(
ob_digits, ob_labels, merge_model, model_list)
mnist_values = {
"c_error": mnist_c_errors[0],
"pred_bits": mnist_pred_bits[0],
"class_bits": mnist_class_bits[0]
}
xm_results = {
"taus": xm_taus,
"c_error": xm_c_errors,
"pred_bits": xm_pred_bits,
"class_bits": xm_class_bits
}
ob_results = {
"taus": ob_taus,
"c_error": ob_c_errors,
"pred_bits": ob_pred_bits,
"class_bits": ob_class_bits
}
return mnist_values, xm_results, ob_results
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data_more')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [25, 27, 30]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = digits_data2['optimal_lw']
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 50
trials = 10
mnist_correct = []
mnist_wrong = []
digits_wrong = []
digits_correct = []
d2_wrong = []
d2_correct = []
for t in range(trials):
inputs = []
outputs = []
model_list = []
for e in range(ensemble_size):
model = Sequential()
model.add(
layers.Dense(200,
input_dim=784,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(200,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(200,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(10,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',
patience=5,
restore_best_weights=True)
model.compile(optimizer=opt.Adam(),
loss="categorical_crossentropy",
metrics=['acc'])
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
validation_split=(1 / 6),
callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_model = Model(inputs=inputs, outputs=layers.Average()(outputs))
#mnist_preds = merge_model.predict([xtest]*ensemble_size)
#mnist_mem_preds = np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,2,0)
#correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds ,ytest)
#mnist_correct.extend(correct)
#mnist_wrong.extend(wrong)
sp_digits = apply_salt_pepper(digits_og)
for s_d in sp_digits:
s_d = utils.normalize_data(reshape_fun(s_d))
d2_preds = merge_model.predict([s_d] * ensemble_size)
d2_mempreds = np.array(
list(map(lambda m: m.predict(s_d),
model_list))).transpose(1, 2, 0)
correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels)
d2_correct.extend(correct)
d2_wrong.extend(wrong)
for d in digits:
digits_preds = merge_model.predict([d] * ensemble_size)
mempreds = np.array(list(map(lambda m: m.predict(d),
model_list))).transpose(1, 2, 0)
correct, wrong = bin_entropies(digits_preds, mempreds, d_labels)
digits_wrong.extend(wrong)
digits_correct.extend(correct)
ensemble = {
#'mnist_correct' : mnist_correct,
#'mnist_wrong' : mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
test_data = utils.load_processed_data('notmnist')
letters = list(test_data.keys())
ensemble_size = 20
epochs = 50
trials = 10
results = {
'A': [],
'B': [],
'C': [],
'D': [],
'E': [],
'F': [],
'G': [],
'H': [],
'I': [],
'J': []
}
for t in range(trials):
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
es = clb.EarlyStopping(monitor='val_loss',
patience=2,
restore_best_weights=True)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
#print(np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,0,2).shape)
train_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
train_model.fit(l_xtrain,
l_ytrain,
epochs=epochs,
batch_size=100,
validation_data=(l_xval, l_yval),
callbacks=[es])
for letter in letters:
inputs = test_data[letter]
inputs = utils.normalize_data(inputs)
inputs = reshape_fun(inputs)
preds = merge_model.predict([inputs] * ensemble_size)
mem_preds = np.array(train_model.predict(
[inputs] * ensemble_size)).transpose(1, 2, 0)
bits = list(map(stats.entropy, preds))
s_q = list(map(calc_pred_vars, mem_preds))
results[letter].extend(list(zip(bits, s_q)))
return results
def experiment(network_conf_json, reshape_mode="conv"):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
xtrain, ytrain, xval, yval = utils.create_validation(xtrain, ytrain)
mnist_c_errors = []
mnist_pred_bits = []
mnist_class_bits = []
xm_c_errors = []
xm_pred_bits = []
xm_class_bits = []
ob_c_errors = []
ob_pred_bits = []
ob_class_bits = []
xm_data = utils.load_processed_data("xiaoming_digits")
ob_data = utils.load_processed_data("Oleks_digits")
xm_digits = reshape_fun(utils.normalize_data(list(xm_data.values())[0]))
xm_labels = utils.create_one_hot(xm_data["labels"])
ob_digits = reshape_fun(utils.normalize_data(list(ob_data.values())[0]))
ob_labels = utils.create_one_hot(ob_data["labels"])
for ensemble_size in ensemble_sizes:
# Building the ensemble models and training the networks
print("===== Building the ensemble of size %s =====" % ensemble_size)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_conf_json],
pop_per_type=ensemble_size,
merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['accuracy'])
train_model.fit([xtrain] * ensemble_size, [ytrain] * ensemble_size,
batch_size=100,
verbose=1,
validation_data=([xval] * ensemble_size,
[yval] * ensemble_size),
epochs=num_epochs)
# Calculating classification errors
print("===== Calculating classification errors =====")
mnist_c_errors.append(
ann.test_model(merge_model, [xtest] * ensemble_size,
ytest,
metric="c_error"))
xm_c_errors.append(
ann.test_model(merge_model, [xm_digits] * ensemble_size,
xm_labels,
metric="c_error"))
ob_c_errors.append(
ann.test_model(merge_model, [ob_digits] * ensemble_size,
ob_labels,
metric="c_error"))
# Calculating ensemble prediciton entropy
print("===== Calculating ensemble prediciton entropy =====")
mnist_pred_bits.append(
np.mean(
calc_shannon_entropy(
merge_model.predict([xtest] * ensemble_size))))
xm_pred_bits.append(
np.mean(
calc_shannon_entropy(
merge_model.predict([xm_digits] * ensemble_size))))
ob_pred_bits.append(
np.mean(
calc_shannon_entropy(
merge_model.predict([ob_digits] * ensemble_size))))
# Calculating ensemble members classification entropyc_error
print(
"===== Calculating ensemble members classification entropy =====")
mnist_class_bits.append(np.mean(calc_class_entropy(model_list, xtest)))
xm_class_bits.append(np.mean(calc_class_entropy(model_list,
xm_digits)))
ob_class_bits.append(np.mean(calc_class_entropy(model_list,
ob_digits)))
mnist_results = {
"c_error": mnist_c_errors,
"pred_bits": mnist_pred_bits,
"class_bits": mnist_class_bits
}
xm_results = {
"c_error": xm_c_errors,
"pred_bits": xm_pred_bits,
"class_bits": xm_class_bits
}
ob_results = {
"c_error": ob_c_errors,
"pred_bits": ob_pred_bits,
"class_bits": ob_class_bits
}
return mnist_results, xm_results, ob_results
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
custom_digits_dict = utils.load_processed_data("combined_testing_data")
digits_labels = custom_digits_dict['labels']
digits_taus = list(custom_digits_dict.keys())[:-1]
digits_data = list(map(reshape_fun, [custom_digits_dict[t] for t in digits_taus]))
digits_data = list(map(utils.normalize_data, digits_data))
digits_labels = utils.create_one_hot(digits_labels.astype('uint'))
for t in range(trials):
print("===== START TRIAL %s =====" % (t + 1))
# Preparing Results
# Classification Error
ensemble_cerror = []
#t_ensemble_adv_cerror = []
digits_cerror = []
#t_digits_adv_cerror = []
# Prediction Entropy
entropy_ensemble = []
#t_entropy_adv_ensemble = []
digits_entropy = []
#t_digits_adv_entropy = []
# Voting entropy
#entropy_vote = []
#entropy_adv_vote = []
#digits_vote = []
#digits_adv_entropy = []
epochs = 5
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
# Without adveserial training
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
losses = list(
map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
)
train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval))
merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc'])
c_error = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'accuracy' )
entropy = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'entropy' )
ensemble_cerror.append(c_error)
entropy_ensemble.append(entropy)
#entropy_vote.append(calc_vote_entropy(model_list,m,xtest))
d_cerror,d_entropy = test_digits(merge_model,model_list,ensemble_size,digits_data,digits_labels)
digits_cerror.append(d_cerror)
digits_entropy.append(d_entropy)
#digits_vote.append(d_vote)
# Adveserial training
#inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average")
#losses = list(
# map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
#)
#train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
#train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval))
#c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' )
#entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' )
#
#t_ensemble_adv_cerror.append(c_error)
#t_entropy_adv_ensemble.append(entropy)
#
#d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels)
#
#t_digits_adv_cerror.append(d_cerror)
#t_digits_adv_entropy.append(d_entropy)
filename1 = 'adv_mnist_results_5-20-50-trial%s' % t
filename2 = 'adv_digits_results_5-20-50-trial%s' % t
mnist_results = {
'ensemble_cerror' : ensemble_cerror,
'ensemble_entropy' : entropy_ensemble
#'ensemble_adv_cerror' : ensemble_adv_cerror,
#'ensemble_adv_entropy' : entropy_adv_ensemble,
#'voting_entropy' : entropy_vote
#'voting_adv_entropy' : entropy_adv_vote
}
digits_results = {
'ensemble_cerror' : digits_cerror,
'ensemble_entropy' : digits_entropy
#'ensemble_adv_cerror' : digits_adv_cerror,
#'ensemble_adv_entropy' : digits_adv_entropy,
#'voting_entropy' : digits_vote
#'voting_adv_entropy' : digits_adv_vote
}
utils.save_processed_data(mnist_results,filename1)
utils.save_processed_data(digits_results,filename2)
return digits_taus, mnist_results, digits_results
"activation" : "relu"
},
{
"type" : "Dense",
"units" : 10,
"activation" : "softmax"
}
]
}
'''
epochs = 3
trials = 100
xtrain, ytrain, xtest, ytest = utils.load_mnist()
digits_data = utils.load_processed_data('digits_og_and_optimal')
digits_big = digits_data['lecunn_big']
digits_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
digits = digits_data['lecunn']
def resize_helper(img):
sides = 280
return cv2.resize(img, (sides, sides), interpolation=cv2.INTER_CUBIC)
def calc_linewidths(data):
taus = np.zeros(data.shape[0])
for i, d in zip(range(data.shape[0]), data):
taus[i] = dutils.intern_calc_linewidth(d)
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [26,28,30]
taus2 = [2,4,5]
digits1 = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits1 = list(map(utils.normalize_data, digits1))
digits2 = list(map(reshape_fun, [digits_data[t] for t in taus2]))
digits2 = list(map(utils.normalize_data, digits2))
digits_og = digits_data2['lecunn_big']
digits_og = salt_and_pepper(digits_og,72000)
digits_og = reshape_fun(utils.normalize_data(digits_og))
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
trials = 5
d1_correct = []
d1_wrong = []
d2_wrong = []
d2_correct = []
dog_wrong = []
dog_correct = []
for t in range(trials):
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
train_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['acc'])
train_model.fit(l_xtrain,l_ytrain,epochs = epochs, validation_data = (l_xval,l_yval))
dog_preds = merge_model.predict([digits_og]*ensemble_size)
correct, wrong = bin_entropies(d2_preds, d2_labels)
dog_correct.extend(correct)
dog_wrong.extend(wrong)
for d in digits1:
digits_preds = merge_model.predict([d]*ensemble_size)
correct, wrong = bin_entropies(digits_preds,d_labels)
d1.extend(wrong)
d1.extend(correct)
for d in digits2:
digits_preds = merge_model.predict([d]*ensemble_size)
correct, wrong = bin_entropies(digits_preds,d_labels)
d2.extend(wrong)
d2.extend(correct)
ensemble = {
'd1_correct' : d1_correct,
'd1_wrong' : d1_wrong,
'd2_correct' : d2_correct,
'd2_wrong' : d2_wrong,
'lecunn_correct' : dog_correct,
'lecunn_wrong' : dog_wrong
}
return ensemble
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
custom_digits_dict = utils.load_processed_data(
"combined_testing_data_more")
digits_labels = custom_digits_dict['labels']
digits_taus = [t for t in custom_digits_dict.keys() if t != "labels"]
digits_data = list(
map(reshape_fun, [custom_digits_dict[t] for t in digits_taus]))
digits_data = list(map(utils.normalize_data, digits_data))
digits_labels = utils.create_one_hot(digits_labels.astype('uint'))
for tr in range(1, trials + 1):
gc.collect()
print("==== TRIAL %s ====" % tr)
# Preparing Results
# Classification Error
ensemble_cerror = []
#t_ensemble_adv_cerror = []
digits_cerror = []
#t_digits_adv_cerror = []
# Prediction Entropy
entropy_ensemble = []
#t_entropy_adv_ensemble = []
digits_entropy = []
#t_digits_adv_entropy = []
# Voting entropy
#entropy_vote = []
#entropy_adv_vote = []
#digits_vote = []
#digits_adv_entropy = []
epochs = 50
for m in ensemble_sizes:
print('Working now with ensemble of size m = %s' % m)
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(m):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
# Without adveserial training
es = clb.EarlyStopping(monitor='val_loss',
patience=2,
restore_best_weights=True)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=m, merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
train_model.fit(x=l_xtrain,
y=l_ytrain,
verbose=1,
epochs=epochs,
validation_data=(l_xval, l_yval),
callbacks=[es])
merge_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
c_error = ann.test_model(merge_model, [xtest] * m,
ytest,
metric='accuracy')
entropy = ann.test_model(merge_model, [xtest] * m,
ytest,
metric='entropy')
ensemble_cerror.append(c_error)
entropy_ensemble.append(entropy)
#entropy_vote.append(calc_vote_entropy(model_list,m,xtest))
d_cerror, d_entropy = test_digits(merge_model, model_list, m,
digits_data, digits_labels)
digits_cerror.append(d_cerror)
digits_entropy.append(d_entropy)
gc.collect()
#digits_vote.append(d_vote)
# Adveserial training
#inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average")
#losses = list(
# map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
#)
#train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
#train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval))
#c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' )
#entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' )
#
#t_ensemble_adv_cerror.append(c_error)
#t_entropy_adv_ensemble.append(entropy)
#
#d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels)
#
#t_digits_adv_cerror.append(d_cerror)
#t_digits_adv_entropy.append(d_entropy)
filename1 = 'mnist_results_20-trial%s' % tr
filename2 = 'digits_results_20-trial%s' % tr
mnist_results = {
'ensemble_cerror': ensemble_cerror,
'ensemble_entropy': entropy_ensemble
#'ensemble_adv_cerror' : ensemble_adv_cerror,
#'ensemble_adv_entropy' : entropy_adv_ensemble,
#'voting_entropy' : entropy_vote
#'voting_adv_entropy' : entropy_adv_vote
}
digits_results = {
'ensemble_cerror': digits_cerror,
'ensemble_entropy': digits_entropy
#'ensemble_adv_cerror' : digits_adv_cerror,
#'ensemble_adv_entropy' : digits_adv_entropy,
#'voting_entropy' : digits_vote
#'voting_adv_entropy' : digits_adv_vote
}
utils.save_processed_data(mnist_results, filename1)
utils.save_processed_data(digits_results, filename2)
return digits_taus, mnist_results, digits_results
