def main():
config = utils.read_config()
elogger = logger.get_logger()
# initialize arrays for short-term and long-term traffic features
speed_array = 'speeds'
time_array = 'times'
short_ttf = [
[collections.defaultdict(lambda: {speed_array: [], time_array: []}) for _ in range(256)] for _ in range(256)
]
long_ttf = [
[collections.defaultdict(lambda: {speed_array: [], time_array: []}) for _ in range(256)] for _ in range(256)
]
for data_file in config['data']:
elogger.info('Generating G and T paths and extracting traffic features on {} ...'.format(data_file))
data = utils.read_data(data_file)
define_travel_grid_path(data, config['coords'], short_ttf, long_ttf, args.grid_size)
elogger.info('Saving extended with G and T paths data in {}{}.\n'.format(args.data_destination_folder, data_file))
utils.save_processed_data(data, args.data_destination_folder, data_file)
elogger.info('Aggregate historical traffic features ...')
utils.aggregate_historical_data(short_ttf, long_ttf)
elogger.info('Saving extracted traffic features in {}'.format(args.ttf_destination_folder))
utils.save_extracted_traffic_features(short_ttf, long_ttf, args.ttf_destination_folder)
def load_process_digits(pathlist):
labels = np.array([])
combined_data_dict = {}
for t in range(min_thickness, max_thickness + 1):
combined_data_dict[t] = []
dataset_index = 0
t = time.time()
for p in pathlist:
dataset_index += 1
digits_data, labels_data = utils.load_image_data(p,
side=200,
padding=40)
labels = np.concatenate((labels, labels_data))
#digits_data = normalize_digit_thickness(digits_data)
#digits_data = change_thickness(digits_data,min_thickness)
tau_data_dict = build_thickness_data(digits_data, min_thickness,
max_thickness)
processed_digit_copy = copy.deepcopy(tau_data_dict)
processed_digit_copy["labels"] = labels_data
filename = 'Dataset-%i-save' % dataset_index
utils.save_processed_data(processed_digit_copy, filename)
for k in tau_data_dict.keys():
d = tau_data_dict[k]
combined_data_dict[k].append(d)
print(labels.shape)
for t in combined_data_dict.keys():
combined_data_dict[t] = np.concatenate(combined_data_dict[t], axis=0)
elapsed = time.time() - t
print("This ran for : %s" % (datetime.timedelta(seconds=elapsed)))
return combined_data_dict, labels
def replace_null_with_zero(file_name: str, column_name: str,
**context) -> Path:
df = load_data(file_name, raw_data=True)
clean_series = df[column_name].fillna(0)
saved_file_path = get_saved_file_path(context[TASK].task_id)
save_processed_data(clean_series, saved_file_path, use_index=False)
return saved_file_path
def create_xy_column(**context) -> Path:
df = get_upstream_tasks_output_concat_df(context, remove_none_values=True)
xy_series = df[X].astype(str) + "/" + df[Y].astype(str)
xy_series.name = XY
saved_file_path = get_saved_file_path(context[TASK].task_id)
save_processed_data(xy_series, saved_file_path, use_index=False)
return saved_file_path
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
digits_data = utils.load_processed_data('digits_og_and_optimal')
digits = digits_data['optimal_lw']
labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
ensemble_size = 20
epochs = 50
small_digits = reshape_fun(np.array(list(map(scale_down, digits))))
small_digits = utils.normalize_data(small_digits)
trials = 5
for t in range(1,trials+1):
gc.collect()
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)
train_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics = ['acc'])
train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval),callbacks=[es])
merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc'])
results = test_digits(merge_model, digits, labels, ensemble_size, reshape_fun)
#entropy = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'entropy')
#c_error = ann.test_model(merge_model, [small_digits]*ensemble_size, labels, metric = 'c_error')
#results['c_error'][0] = c_error
#results['entropy'][0] = entropy
filename = "saltpepper_norm_trial-%s" % t
utils.save_processed_data(results, filename)
layers.Dense(10,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',
patience=10,
restore_best_weights=True)
model.compile(optimizer=opt.Adam(),
loss="categorical_crossentropy",
metrics=['acc'])
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
validation_split=(1 / 6),
callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_layer = layers.Average()(
outputs) if ensemble_size > 1 else outputs
ensemble = Model(inputs=inputs, outputs=merge_layer)
pred = ensemble.predict([notmnist] * ensemble_size)
h = list(map(stats.entropy, pred))
results[ensemble_size].extend(h)
utils.save_processed_data(results, 'notmnist_sim-trial-%s' % (t + 1))
images_path = os.path.join(".", "images")
xm_digits_path = os.path.join(images_path, "XiaoMing_Digits")
ob_digits_path = os.path.join(images_path, "60 Images")
m_digits_path = os.path.join(images_path, "mnumbers")
hubben1_path = os.path.join(images_path, "Size1")
hubben2_path = os.path.join(images_path, "Size2")
hubben3_path = os.path.join(images_path, "Size3")
hubben4_path = os.path.join(images_path, "Size4")
digit_paths = [
# xm_digits_path
ob_digits_path
#, m_digits_path
#, hubben1_path
#, hubben2_path
#, hubben3_path
#, hubben4_path
]
print("===== LOADING DIGITS =====")
data_dict, labels = load_process_digits(digit_paths)
print(data_dict)
data_dict['labels'] = labels
utils.save_processed_data(data_dict, 'combined_testing_data')
print("Script done running")
digits_mbits[t].extend(d_mbits[t])
mnist_preds = np.mean(np.array(mnist_mpreds), axis=0)
mnist_cerr = calc_cerror(mnist_preds, ytest)
mnist_bits = np.mean(list(map(entropy, mnist_preds)))
digits_cerr = {}
digits_bits = {}
for t in taus:
preds = np.mean(np.array(digits_mpreds[t]), axis=0)
digits_cerr[t] = calc_cerror(preds, labels)
digits_bits[t] = np.mean(list(map(entropy, preds)))
results = {
"ensembles": {
"mnist_bits": mnist_bits,
"mnist_cerr": mnist_cerr,
"digits_cerr": digits_cerr,
"digits_bits": digits_bits
},
"individuals": {
"mnist_bits": mnist_mbits,
"mnist_cerr": mnist_mcerr,
"digits_cerr": digits_mcerr,
"digits_bits": digits_mbits
}
}
utils.save_processed_data(results, "cnn_results_ltsim_100")
def calc_acc():
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
acc_mnist = np.zeros(trials)
bits_mnist = np.zeros(trials)
acc_digits = np.zeros(trials)
bits_digits = np.zeros(trials)
cnn_acc_mnist = np.zeros(trials)
cnn_bits_mnist = np.zeros(trials)
cnn_acc_digits = np.zeros(trials)
cnn_bits_digits = np.zeros(trials)
for i in range(trials):
t_xtrain, t_ytrain, xval, yval = utils.create_validation(
xtrain, ytrain)
reshape_fun = reshape_funs['mlp']
t_xtrain, c_xtest = reshape_fun(t_xtrain), reshape_fun(xtest)
xval = reshape_fun(xval)
c_digits = reshape_fun(digits)
model = ann.parse_model_js(network_model2)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
model.fit(t_xtrain, t_ytrain, epochs=5, validation_data=(xval, yval))
acc_mnist[i] = ann.test_model(model, c_xtest, ytest, metric='accuracy')
bits_mnist[i] = ann.test_model(model, c_xtest, ytest, metric='entropy')
acc_digits[i] = ann.test_model(model,
c_digits,
digits_labels,
metric='accuracy')
bits_digits[i] = ann.test_model(model,
c_digits,
digits_labels,
metric='entropy')
reshape_fun = reshape_funs['conv']
t_xtrain, c_xtest = reshape_fun(t_xtrain), reshape_fun(xtest)
xval = reshape_fun(xval)
c_digits = reshape_fun(digits)
model = ann.parse_model_js(network_model1)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
model.fit(t_xtrain, t_ytrain, epochs=3, validation_data=(xval, yval))
cnn_acc_mnist[i] = ann.test_model(model,
c_xtest,
ytest,
metric='accuracy')
cnn_bits_mnist[i] = ann.test_model(model,
c_xtest,
ytest,
metric='entropy')
cnn_acc_digits[i] = ann.test_model(model,
c_digits,
digits_labels,
metric='accuracy')
cnn_bits_digits[i] = ann.test_model(model,
c_digits,
digits_labels,
metric='entropy')
results_mlp = {
'mnist_acc': acc_mnist,
'mnist_bits': bits_mnist,
'digits_acc': acc_digits,
'digits_bits': bits_digits
}
results_cnn = {
'mnist_acc': cnn_acc_mnist,
'mnist_bits': cnn_bits_mnist,
'digits_acc': cnn_acc_digits,
'digits_bits': cnn_bits_digits
}
utils.save_processed_data(results_mlp, '100-trials-mlp')
utils.save_processed_data(results_cnn, '100-trials-cnn')
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('digits_og_and_optimal')
digits = digits_data['optimal_lw']
digits = utils.normalize_data(digits)
digits_og = digits_data['lecunn_big']
digits_og_small = reshape_fun(digits_data['lecunn'])
digits_og_small = utils.normalize_data(digits_og_small)
labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
ensemble_size = 20
epochs = 3
small_digits = reshape_fun(np.array(list(map(scale_down, digits))))
trials = 5
for t in range(1, trials + 1):
gc.collect()
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=ensemble_size, merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
train_model.fit(x=l_xtrain,
y=l_ytrain,
verbose=1,
batch_size=100,
epochs=epochs,
validation_data=(l_xval, l_yval))
merge_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
results_linewidth = test_digits(merge_model, digits, labels,
ensemble_size, reshape_fun)
entropy = ann.test_model(merge_model, [small_digits] * ensemble_size,
labels,
metric='entropy')
c_error = ann.test_model(merge_model, [small_digits] * ensemble_size,
labels,
metric='c_error')
results_linewidth['c_error'][0] = c_error
results_linewidth['entropy'][0] = entropy
results_lecunn = test_digits(merge_model, digits_og, labels,
ensemble_size, reshape_fun)
entropy = ann.test_model(merge_model,
[digits_og_small] * ensemble_size,
labels,
metric='entropy')
c_error = ann.test_model(merge_model,
[digits_og_small] * ensemble_size,
labels,
metric='c_error')
results_lecunn['c_error'][0] = c_error
results_lecunn['entropy'][0] = entropy
total_results = {
'optimal_lw': results_linewidth,
'lecunn': results_lecunn
}
filename = "saltpepper_random_trial-%s" % t
utils.save_processed_data(total_results, filename)
individual = {
'd1_correct': d1_mncorrect,
'd1_wrong': d1_mnwrong,
'd2_correct': d2_mncorrect,
'd2_wrong': d2_mnwrong,
'lecunn_correct': d3_mncorrect,
'lecunn_wrong': d3_mnwrong
}
return individual
utils.setup_gpu_session()
individual = experiment(network_model1, 'mlp')
utils.save_processed_data(individual, 'individual_bin')
#plt.figure()
#plt.subplot(231)
#plt.hist(individual['d1_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(232)
#plt.hist(individual['d2_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(233)
#plt.hist(individual['lecunn_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(234)
def experiment(network_model, reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
custom_digits_dict = utils.load_processed_data("combined_testing_data")
digits_labels = custom_digits_dict['labels']
digits_taus = list(custom_digits_dict.keys())[:-1]
digits_data = list(map(reshape_fun, [custom_digits_dict[t] for t in digits_taus]))
digits_data = list(map(utils.normalize_data, digits_data))
digits_labels = utils.create_one_hot(digits_labels.astype('uint'))
for t in range(trials):
print("===== START TRIAL %s =====" % (t + 1))
# Preparing Results
# Classification Error
ensemble_cerror = []
#t_ensemble_adv_cerror = []
digits_cerror = []
#t_digits_adv_cerror = []
# Prediction Entropy
entropy_ensemble = []
#t_entropy_adv_ensemble = []
digits_entropy = []
#t_digits_adv_entropy = []
# Voting entropy
#entropy_vote = []
#entropy_adv_vote = []
#digits_vote = []
#digits_adv_entropy = []
epochs = 5
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(ensemble_size):
t_xtrain,t_ytrain,t_xval,t_yval = utils.create_validation(xtrain,ytrain,(1/6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
# Without adveserial training
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=ensemble_size, merge_type="Average")
losses = list(
map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
)
train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
train_model.fit(x=l_xtrain,y=l_ytrain, verbose=1,batch_size=100, epochs = epochs,validation_data=(l_xval,l_yval))
merge_model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=['acc'])
c_error = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'accuracy' )
entropy = ann.test_model(merge_model, [xtest]*ensemble_size, ytest, metric = 'entropy' )
ensemble_cerror.append(c_error)
entropy_ensemble.append(entropy)
#entropy_vote.append(calc_vote_entropy(model_list,m,xtest))
d_cerror,d_entropy = test_digits(merge_model,model_list,ensemble_size,digits_data,digits_labels)
digits_cerror.append(d_cerror)
digits_entropy.append(d_entropy)
#digits_vote.append(d_vote)
# Adveserial training
#inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average")
#losses = list(
# map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
#)
#train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
#train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval))
#c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' )
#entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' )
#
#t_ensemble_adv_cerror.append(c_error)
#t_entropy_adv_ensemble.append(entropy)
#
#d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels)
#
#t_digits_adv_cerror.append(d_cerror)
#t_digits_adv_entropy.append(d_entropy)
filename1 = 'adv_mnist_results_5-20-50-trial%s' % t
filename2 = 'adv_digits_results_5-20-50-trial%s' % t
mnist_results = {
'ensemble_cerror' : ensemble_cerror,
'ensemble_entropy' : entropy_ensemble
#'ensemble_adv_cerror' : ensemble_adv_cerror,
#'ensemble_adv_entropy' : entropy_adv_ensemble,
#'voting_entropy' : entropy_vote
#'voting_adv_entropy' : entropy_adv_vote
}
digits_results = {
'ensemble_cerror' : digits_cerror,
'ensemble_entropy' : digits_entropy
#'ensemble_adv_cerror' : digits_adv_cerror,
#'ensemble_adv_entropy' : digits_adv_entropy,
#'voting_entropy' : digits_vote
#'voting_adv_entropy' : digits_adv_vote
}
utils.save_processed_data(mnist_results,filename1)
utils.save_processed_data(digits_results,filename2)
return digits_taus, mnist_results, digits_results
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_model = Model(inputs = inputs, outputs = layers.Average()(outputs))
preds = merge_model.predict([notmnist]*ensemble_size)
mem_preds = np.array(list(map(lambda m : m.predict(notmnist), model_list))).transpose(1,2,0)
print(mem_preds.shape)
bits = list(map(stats.entropy,preds))
s_q = list(map(calc_pred_vars,mem_preds))
results.extend(list(zip(bits,s_q)))
return results
utils.setup_gpu_session()
ensemble = experiment('mlp')
utils.save_processed_data(ensemble , "distribution_not_mnist")
#plt.figure()
#plt.subplot(221)
#plt.hist(ensemble['mnist_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(222)
#plt.hist(ensemble['mnist_wrong'],color = 'red')
#plt.xlabel('entropy')
#plt.ylabel('nwrong')
#plt.subplot(223)
#plt.hist(ensemble['digits_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(224)
ensemble = {
#'mnist_correct' : mnist_correct,
#'mnist_wrong' : mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
utils.setup_gpu_session()
ensemble = experiment(network_model1, 'mlp')
utils.save_processed_data(ensemble, "error_entropy5sep-bins")
#plt.figure()
#plt.subplot(221)
#plt.hist(ensemble['mnist_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(222)
#plt.hist(ensemble['mnist_wrong'],color = 'red')
#plt.xlabel('entropy')
#plt.ylabel('nwrong')
#plt.subplot(223)
#plt.hist(ensemble['digits_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(224)
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
callbacks=[es],
validation_split=(1 / 6))
members.append(model)
i = 0
for ensemble_size in range(2, ensemble_size_top + 1):
inputs = []
outputs = []
members_to_use = members[i:i + ensemble_size]
for m in members_to_use:
inputs.extend(m.inputs)
outputs.extend(m.outputs)
print((outputs))
ensemble = Model(inputs=inputs, outputs=layers.Average()(outputs))
ensemble.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
accuracy = ann.test_model(ensemble, [xtest] * ensemble_size, ytest,
'accuracy')
i += ensemble_size
t_accuracies.append(accuracy)
t_accuracies = np.array(t_accuracies)
utils.save_processed_data(t_accuracies,
'ensemble_sizesim-trial-%s' % (t + 1))
images_path = os.path.join(".", "images")
xm_digits_path = os.path.join(images_path, "XiaoMing_Digits")
ob_digits_path = os.path.join(images_path, "60 Images")
m_digits_path = os.path.join(images_path, "mnumbers")
hubben1_path = os.path.join(images_path, "Size1")
hubben2_path = os.path.join(images_path, "Size2")
hubben3_path = os.path.join(images_path, "Size3")
hubben4_path = os.path.join(images_path, "Size4")
digit_paths = [
xm_digits_path, ob_digits_path, m_digits_path, hubben1_path, hubben2_path,
hubben3_path, hubben4_path
]
print("===== LOADING DIGITS =====")
bw, bw_small, optimal_lw, labels = load_process_digits(digit_paths)
data_dict = {
'lecunn_big': bw,
'lecunn': bw_small,
'optimal_lw': optimal_lw,
'labels': labels
}
utils.save_processed_data(data_dict, 'digits_og_and_optimal')
print("Script done running")
for img in images:
img_arr = load_image(os.path.join(letter_path, img), True)
if (not (img_arr is None)):
img_arr = dutils.unpad_img(img_arr)
if (has_shape(img_arr)):
img_list.append(img_arr)
img_list = list(map(dutils.unpad_img, img_list))
img_list = list(
map(
lambda img: dutils.center_box_image(
dutils.resize_image(img, 20), 20, 4), img_list))
return np.array(img_list)
path = os.path.join('notMNIST_small')
letters_dict = {}
letters = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
for letter in letters:
l_list = load_letter(os.path.join(path, letter))
count, _, _ = l_list.shape
l_index = np.random.permutation(count)
letters_dict[letter] = l_list[l_index[:100]]
return letters_dict
not_mnist = create_not_mnist()
utils.save_processed_data(not_mnist, 'notMNIST1000')
individual = {
'mnist_correct': mnist_mncorrect,
'mnist_wrong': mnist_mnwrong,
'digits_correct': digits_mncorrect,
'digits_wrong': digits_mnwrong,
'lecunn_correct': d2_mncorrect,
'lecunn_wrong': d2_mnwrong
}
return individual
utils.setup_gpu_session()
individual = experiment(network_model1, 'mlp')
utils.save_processed_data(individual, 'individual_entropy_bins')
#plt.figure()
#plt.subplot(231)
#plt.hist(individual['mnist_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(232)
#plt.hist(individual['lecunn_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(233)
#plt.hist(individual['digits_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('n_correct')
#plt.subplot(234)
#d2_wrong.extend(wrong)
#
#for d in digits:
# digits_preds = merge_model.predict([d]*ensemble_size)
# mempreds = np.array(train_model.predict([d]*ensemble_size)).transpose(1,2,0)
# correct, wrong = bin_entropies(digits_preds,mempreds,d_labels)
# digits_wrong.extend(wrong)
# digits_correct.extend(correct)
ensemble = {'mnist_correct': mnist_correct, 'mnist_wrong': mnist_wrong}
return ensemble
ensemble = experiment(network_model1, 'mlp')
utils.save_processed_data(ensemble, "lookalikes_entropy5sep-bins")
#plt.figure()
#plt.subplot(221)
#plt.hist(ensemble['mnist_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(222)
#plt.hist(ensemble['mnist_wrong'],color = 'red')
#plt.xlabel('entropy')
#plt.ylabel('nwrong')
#plt.subplot(223)
#plt.hist(ensemble['digits_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(224)
d2.extend(wrong)
d2.extend(correct)
ensemble = {
'd1_correct' : d1_correct,
'd1_wrong' : d1_wrong,
'd2_correct' : d2_correct,
'd2_wrong' : d2_wrong,
'lecunn_correct' : dog_correct,
'lecunn_wrong' : dog_wrong
}
return ensemble
ensemble = experiment(network_model2, 'conv')
utils.save_processed_data(ensemble , "cnn_entropy5trial-bins")
#plt.figure()
#plt.subplot(221)
#plt.hist(ensemble['mnist_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(222)
#plt.hist(ensemble['mnist_wrong'],color = 'red')
#plt.xlabel('entropy')
#plt.ylabel('nwrong')
#plt.subplot(223)
#plt.hist(ensemble['digits_correct'],color = 'blue')
#plt.xlabel('entropy')
#plt.ylabel('ncorrect')
#plt.subplot(224)
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
custom_digits_dict = utils.load_processed_data(
"combined_testing_data_more")
digits_labels = custom_digits_dict['labels']
digits_taus = [t for t in custom_digits_dict.keys() if t != "labels"]
digits_data = list(
map(reshape_fun, [custom_digits_dict[t] for t in digits_taus]))
digits_data = list(map(utils.normalize_data, digits_data))
digits_labels = utils.create_one_hot(digits_labels.astype('uint'))
for tr in range(1, trials + 1):
gc.collect()
print("==== TRIAL %s ====" % tr)
# Preparing Results
# Classification Error
ensemble_cerror = []
#t_ensemble_adv_cerror = []
digits_cerror = []
#t_digits_adv_cerror = []
# Prediction Entropy
entropy_ensemble = []
#t_entropy_adv_ensemble = []
digits_entropy = []
#t_digits_adv_entropy = []
# Voting entropy
#entropy_vote = []
#entropy_adv_vote = []
#digits_vote = []
#digits_adv_entropy = []
epochs = 50
for m in ensemble_sizes:
print('Working now with ensemble of size m = %s' % m)
l_xtrain = []
l_xval = []
l_ytrain = []
l_yval = []
for _ in range(m):
t_xtrain, t_ytrain, t_xval, t_yval = utils.create_validation(
xtrain, ytrain, (1 / 6))
l_xtrain.append(t_xtrain)
l_xval.append(t_xval)
l_ytrain.append(t_ytrain)
l_yval.append(t_yval)
# Without adveserial training
es = clb.EarlyStopping(monitor='val_loss',
patience=2,
restore_best_weights=True)
inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble(
[network_model], pop_per_type=m, merge_type="Average")
train_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
train_model.fit(x=l_xtrain,
y=l_ytrain,
verbose=1,
epochs=epochs,
validation_data=(l_xval, l_yval),
callbacks=[es])
merge_model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=['acc'])
c_error = ann.test_model(merge_model, [xtest] * m,
ytest,
metric='accuracy')
entropy = ann.test_model(merge_model, [xtest] * m,
ytest,
metric='entropy')
ensemble_cerror.append(c_error)
entropy_ensemble.append(entropy)
#entropy_vote.append(calc_vote_entropy(model_list,m,xtest))
d_cerror, d_entropy = test_digits(merge_model, model_list, m,
digits_data, digits_labels)
digits_cerror.append(d_cerror)
digits_entropy.append(d_entropy)
gc.collect()
#digits_vote.append(d_vote)
# Adveserial training
#inputs, outputs, train_model, model_list, merge_model = ann.build_ensemble([network_model], pop_per_type=m, merge_type="Average")
#losses = list(
# map( lambda m : ann.adveserial_loss(klosses.categorical_crossentropy,m,eps=0.01), model_list)
#)
#train_model.compile(optimizer="adam", loss=losses, metrics = ['acc'])
#train_model.fit(x=l_xtrain, y=l_ytrain, verbose=1, epochs = epochs ,validation_data=(l_xval,l_yval))
#c_error = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'c_error' )
#entropy = ann.test_model(merge_model, [xtest]*m, ytest, metric = 'entropy' )
#
#t_ensemble_adv_cerror.append(c_error)
#t_entropy_adv_ensemble.append(entropy)
#
#d_cerror,d_entropy,d_vote = test_digits(merge_model,model_list,m,digits_data,digits_labels)
#
#t_digits_adv_cerror.append(d_cerror)
#t_digits_adv_entropy.append(d_entropy)
filename1 = 'mnist_results_20-trial%s' % tr
filename2 = 'digits_results_20-trial%s' % tr
mnist_results = {
'ensemble_cerror': ensemble_cerror,
'ensemble_entropy': entropy_ensemble
#'ensemble_adv_cerror' : ensemble_adv_cerror,
#'ensemble_adv_entropy' : entropy_adv_ensemble,
#'voting_entropy' : entropy_vote
#'voting_adv_entropy' : entropy_adv_vote
}
digits_results = {
'ensemble_cerror': digits_cerror,
'ensemble_entropy': digits_entropy
#'ensemble_adv_cerror' : digits_adv_cerror,
#'ensemble_adv_entropy' : digits_adv_entropy,
#'voting_entropy' : digits_vote
#'voting_adv_entropy' : digits_adv_vote
}
utils.save_processed_data(mnist_results, filename1)
utils.save_processed_data(digits_results, filename2)
return digits_taus, mnist_results, digits_results
