experiment)
meta_data_file = os.path.join(experiment_dir, 'results', 'metadata.json')
ids = [9313361527064236, 7598351433769390]
cmap = cm.get_cmap('jet')
colors = ["#0066ff", "green", "red"]
label = ["NSAS", "EARN+NSAS", "EARN"]
markers = ["*", "D"] * 2
phase = 'test'
for j, id in enumerate(ids):
chain_data_path = os.path.join(
experiment_dir,
results_manager.get_results_by_id(meta_data_file, str(id)),
'results_ensembles.pkl')
dataset = results_manager.get_fieldval_by_id(meta_data_file, str(id),
'dataset')[0][12:-18]
chain = io.read_pickle(chain_data_path)
acc, time, params = get_acc_time_param_from_results(chain, phase)
acc_params_subplot(acc,
params,
colors[j],
"",
label=label[j],
print_front=True,
marker=markers[j])
acc_time_subplot(acc,
time,
return sys
if __name__ == "__main__":
id_fastest = "__trigger_classifier_0.10000000000000003_V001_VGG13_ref_0____trigger_classifier_0" \
".4_V001_ResNeXt29_32x4d_ref_0__V001_VGG11_ref_0"
experiment_dir = os.path.join(os.environ['FCM'], 'Examples', 'compute',
'chain_genetic_algorithm')
metadata_file = os.path.join(experiment_dir, 'results', 'metadata.json')
id = "7062152700584889"
dataset = results_manager.get_fieldval_by_id(metadata_file, id, 'dataset')
results_chain_path = os.path.join(
experiment_dir, results_manager.get_results_by_id(metadata_file, id))
results_chain = io.read_pickle(results_chain_path)
fastest = results_chain[id_fastest]
acc_result = fastest.test['system'].accuracy
time_result = fastest.test['system'].time
# Build same ensemble
classifiers = [
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation',
'V001_VGG13_ref_0'),
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation',
'V001_ResNeXt29_32x4d_ref_0'),
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation', 'V001_VGG11_ref_0')
# 2) Group hypervolume results by EARN parameters
# 3) Print the parameter configuration that:
# 3.1) Obtains higher hypervolume in average
# 3.3) Obtains the higher median hypervolume
# 3.2) Obtains the higher minimum hypervolume
avail_params = [
"population", "offspring", "iterations", "step_th", "pm", "rm", "k"
]
dictval2tuple = lambda d: tuple([d[k] for k in avail_params])
hvolume_by_configuration = {}
# Compute hypervolume per EARN run
for id in ids:
print("Processing %s" % id)
GA_res_loc = results_manager.get_results_by_id(metadataset, id)
GA_params = results_manager.get_fieldval_by_id(metadataset, id,
"params")[0]
individuals_gen = io.read_pickle(
os.path.join(GA_res_loc, 'individuals_fitness_per_generation.pkl'))
R = io.read_pickle(os.path.join(GA_res_loc, 'results_ensembles.pkl'))
a = np.array(GA_params["a"])
hv = hvolume_evolution(R, individuals_gen, a,
phase)[-1] # Only interested in the last run
key = dictval2tuple(GA_params)
hvs_all = hvolume_by_configuration.get(key, [])
hvs_all.append(hv)
hvolume_by_configuration[key] = hvs_all
"pm": 0.2,
"rm": 0.8,
"k": 1,
"a": [
1,
1,
1
]
}
ids = results_manager.get_ids_by_fieldval(metadata_file, "params", params_match)
improvements = np.zeros((3, 3))
ref = None
for id in ids:
R_path = results_manager.get_results_by_id(metadata_file, id)
individuals_fitness_generation = io.read_pickle(os.path.join(R_path, 'individuals_fitness_per_generation.pkl'))
R_dict = io.read_pickle(os.path.join(R_path, 'results_ensembles.pkl'))
# Last generation of ensembles
last_generation = individuals_fitness_generation[-1][0]
R_dict_last = {ensemble_id: R_dict[ensemble_id] for ensemble_id in last_generation}
# Most accurate NN as reference point
if ref is None:
r_ref = get_most_accurate_nn_result(R_dict, phase)
ref = np.array([1-r_ref["system"].accuracy, r_ref["system"].time, r_ref["system"].params])
# Evaluation results to numpy array
obj = results_to_numpy(R_dict_last, phase)
1943013241626030,
9283423602893484,
4453353588233325,
1600106112124184,
6104062115638786]
label = ["Bagging average",
"Bagging voting",
"Bagging max",
"Boosting average",
"Boosting voting",
"Boosting max",
"Chain"]
cmap = cm.get_cmap('jet')
colors = cmap(np.linspace(0, 1.0, 3))
linestyle = ["-", "-", "-", "--", "--", "--", "-."]
for j, id in enumerate(ids):
result_location = os.path.join(result_manager.get_results_by_id(metadata_file, id), "results_ensembles")
exec_params = result_manager.get_fieldval_by_id(metadata_file, id, "params")[0]
R = io.read_pickle(result_location)
if "hain" in label[j]:
R_models = io.read_pickle(os.path.join(result_manager.get_results_by_id(metadata_file, id), "models.pkl"))
plot_optimal_combinations_chain(R, R_models, resolution=50, label=label[j], color=colors[j % 3], linestyle=linestyle[j])
# plot_optimal_combinations_merger(R, resolution=50, label=label[j], color=colors[j%3], linestyle=linestyle[j])
plt.legend()
plt.show()
subplot_most_fit_chain(evaluation_results, ax)
ax[0].legend()
ax[1].legend()
ax[2].legend()
return ax
if __name__ == "__main__":
plt.rcParams.update({'font.size': 6})
experiment = 'bagging_boosting_of_chains_GA'
id = 4289297487379981
experiment_dir = os.path.join(os.environ['FCM'], 'Examples', 'compute', experiment)
meta_data_file = os.path.join(experiment_dir, 'results', 'metadata.json')
earn_evaluations_location = results_manager.get_results_by_id(meta_data_file, id)
sys.modules['Source'] = Source # Due to refactor of the module
earn_evaluations = io.read_pickle(earn_evaluations_location + '/results_ensembles.pkl')
ax = plot_acc_time_param_tradeoff(earn_evaluations)
"""
brute_force_location = os.path.join(os.environ['FCM'],
"Examples/compute/fully_connected_chain/results/sota_models_caltech256-32-dev_validation/3971529594480967/results_ensembles.pkl")
brute_force_results = io.read_pickle(brute_force_location)
subplot_acc_time_paretto_front(brute_force_results, ax[0], color='green')
subplot_acc_params_paretto_front(brute_force_results, ax[1], color='green')
ax[0].legend()
ax[1].legend()
"""
models_front = front.get_front_time_accuracy(models, phase="test")
sorted_models_front = front.sort_results_by_time(models_front,
phase="test")
# 1) G.A. Chain ensembles
GA_results_metadata_file = os.path.join(os.environ['FCM'], 'Examples',
'compute',
'chain_genetic_algorithm',
'results', 'metadata.json')
GA_ids = results_manager.get_ids_by_fieldval(GA_results_metadata_file,
'dataset', dataset)[-3:]
speedup = []
for id in GA_ids:
global_path = results_manager.get_results_by_id(
GA_results_metadata_file, id)
#global_path = os.path.join(os.environ['FCM'], 'examples', 'Compute', 'chain_genetic_algorithm', local_path)
GA_chains = io.read_pickle(global_path)
front_GA_chain = front.get_front_time_accuracy(GA_chains, phase="test")
sorted_front_GA_chain = front.sort_results_by_time(front_GA_chain,
phase="test")
speedup.append(
front.get_speedup_front(sorted_models_front,
sorted_front_GA_chain,
e=1e6,
phase="test"))
if len(speedup) == 1:
speedup.append(speedup[-1])
print("Genetic Algorithm:")