def test_accuracy(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# System Evaluator computes accuracy
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
acc = R.test['system'].accuracy
# Compute accuracy manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
gt = c_dict_0['test']['gt']
logits_0 = c_dict_0['test']['logits']
logits_1 = c_dict_1['test']['logits']
average = (logits_0 + logits_1) / 2
acc_manual = np.sum(np.argmax(average, 1) == gt) / len(gt)
self.assertEqual(acc, acc_manual)
def test_time_threshold_1(self):
self.t1.update_threshold(1.1)
self.t2.update_threshold(1.1)
metadata1 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_DenseNet121_ref_0.pkl"
)
metadata2 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_DenseNet121_ref_0.pkl"
)
metadata3 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_MobileNetV2_ref_0.pkl"
)
ids = metadata1["test"]["id"]
inference_t = metadata1["metrics"]["time"]/128.0 + \
metadata2["metrics"]["time"]/128.0 + \
metadata3["metrics"]["time"]/128.0
true_time = len(ids) * inference_t
input = torch.range(0, len(ids) - 1).long()
self.ensemble(input)
self.assertEqual(self.ensemble.get_processing_time(), true_time)
def check_valid_classifier_metrics(c):
classifier_dict = io.read_pickle(c.classifier_file, verbose=False)
if 'time' not in classifier_dict['metrics']:
warnings.warn("WARNING in Classifier: Time measurement not found, set to 0 instead")
classifier_dict['metrics']['time'] = 0
io.save_pickle(c.classifier_file, classifier_dict)
if 'times' not in classifier_dict['metrics']:
warnings.warn("WARNING in Classifier: Times list measurement not found, set to time instead")
classifier_dict['metrics']['times'] = np.array([classifier_dict['metrics']['time']])
io.save_pickle(c.classifier_file, classifier_dict)
if 'params' not in classifier_dict['metrics']:
warnings.warn("WARNING in Classifier: # Params measurement not found, set to 0 instead")
classifier_dict['metrics']['params'] = 0
io.save_pickle(c.classifier_file, classifier_dict)
if 'ops' not in classifier_dict['metrics']:
warnings.warn("WARNING in Classifier: # Ops measurement not found, set to 0 instead")
classifier_dict['metrics']['ops'] = 0
io.save_pickle(c.classifier_file, classifier_dict)
classifier_dict = io.read_pickle(c.classifier_file, verbose=False)
assert classifier_dict['metrics']['time'] >= 0, "ERROR in Classifier: Time should be positive"
assert np.all(np.array(classifier_dict['metrics']['times']) >= 0) and len(classifier_dict['metrics']['times']) > 0, \
"ERROR in Classifier: Time should be positive"
assert classifier_dict['metrics']['params'] >= 0, "ERROR in Classifier: # Params should be positive"
assert classifier_dict['metrics']['ops'] >= 0, "ERROR in Classifier: # Ops should be positive"
def plot_paretto_fronts(n):
toPlot = io.read_pickle("./results/models")
myplt.plot_accuracy_time(toPlot)
for i in range(n + 1):
print(i)
toPlot = dict(
(k, v)
for k, v in io.read_pickle("./results/front" + str(i)).items()
if k not in toPlot)
myplt.plot_accuracy_time(toPlot, system_color=myplt.color[i])
"""
def sort_models_params(models):
size = [c_dict['metrics']['params'] for c_dict in [io.read_pickle(model) for model in models]]
position = np.argsort(size)
models_sorted = ["" for x in range(len(models))]
for i in range(len(models)):
models_sorted[position[i]] = models[i]
return models_sorted
def __get_trigger_raw_data(data, phase):
if phase == "test":
data_source = data.source.test_path
elif phase == "train":
data_source = data.source.train_path
else: # phase == "val":
data_source = data.source.val_path
dataset = io.read_pickle(data_source)
L = dataset['logits']
P = softmax(L)
ths = dataset['ths']
gt = dataset['gt']
ids = dataset['id']
sort = np.sort(P, axis=1)
diff = sort[:, -1] - sort[:, -2]
sample_size = len(ids)
Y = np.zeros((sample_size, len(ths) + 1)) # Predictions for the trigger
Or = np.zeros(sample_size)
for ith, th in enumerate(ths):
Y[:, ith] = np.array([diff < th])
Or = np.logical_or(Or, Y[:, ith])
Y[:, -1] = np.logical_not(
Or) # Executed by classifier before trigger if none matches
raw_data = make.make_classifier_raw_data(Y, gt == np.argmax(L, axis=1),
ids)
return raw_data
def check_valid_classifier(c):
if c.HasField('classifier_file'):
classifier_dict = io.read_pickle(c.classifier_file, verbose=False)
check_valid_classifier_structure(classifier_dict)
check_valid_classifier_raw_data(classifier_dict)
check_valid_classifier_metrics(c)
else:
raise Exception("ERROR in Classifier: classifier_file field should be specified.")
def plot_paretto_fronts(n, res_dir):
toPlot = io.read_pickle(os.path.join(res_dir, "models"))
plt.figure(0)
plt.xscale("log")
myplt.plot_accuracy_time(toPlot)
toPlot = io.read_pickle(os.path.join(res_dir, "front0"))
for i in range(1, n + 1):
toPlot = dict(
("system_" + k, v)
for k, v in io.read_pickle(os.path.join(res_dir, "front" +
str(i))).items()
if k not in toPlot)
myplt.plot_accuracy_time(toPlot, system_color=myplt.color[i])
plt.legend("Iteration 1", "Iteration 2")
plt.show()
def update_dataset(c_file, train_path, test_path, val_path, th):
# Create dataset
model = io.read_pickle(c_file)
# Test
dataset_test = model['test']
if th >= 0: dataset_test['th'] = th
io.save_pickle(test_path, dataset_test)
"""
def __init__(self, path_to_pickle: str, split=Split.VAL):
# Call component's class constructor
self.path = path_to_pickle
self.split = split
metadata = read_pickle(self.path)
parameters = metadata['metrics']['params']
super().__init__(p=parameters)
# This way predictions can be manipulated on GPU
self.register_buffer("predictions", None)
def evaluate(sys, results, c, check_classifiers, classifier_dict, input_ids,
phase):
contribution = {}
c_dict = io.read_pickle(c.classifier.classifier_file)
L, gt, ids = eval_utils.get_Lgtid(c_dict, phase, input_ids)
n_inputs = len(input_ids) if input_ids is not None else len(
c_dict[phase]['id'])
predictions_trigger = np.argmax(L, axis=1)
results[c.id] = eval.create_metrics_classifier(c_dict, predictions_trigger,
gt, n_inputs)
eval.update_metrics_system(results,
c_dict,
n_inputs,
input_ids=input_ids,
phase=phase)
contribution['gt'] = {}
contribution['predictions'] = {}
contribution['logits'] = {}
contribution['time_instance'] = {}
contribution['model'] = {}
for i, c_id in enumerate(c.component_ids):
c_next = sys.get(c_id)
mask = (predictions_trigger == i)
ids_next = ids[mask]
contribution_component = eval.__evaluate(sys,
results,
c_next,
check_classifiers,
classifier_dict,
input_ids=ids_next,
phase=phase)
contribution['gt'].update(contribution_component['gt'])
contribution['predictions'].update(
contribution_component['predictions'])
contribution['logits'].update(contribution_component['logits'])
if contribution['model']:
contribution['model'].update(
dict([(k, contribution['model'][k] + v)
for k, v in contribution_component['model'].items()]))
else:
contribution['model'].update(contribution_component['model'])
if classifier_dict is not None:
contribution['logits'].update(contribution_component['logits'])
for k, v in contribution_component['time_instance'].items():
contribution['time_instance'][
k] = contribution['time_instance'][
k] + v if k in contribution['time_instance'] else v
return contribution
def test_parameter_count(self):
metadata1 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_DenseNet121_ref_0.pkl"
)
metadata2 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_DenseNet121_ref_0.pkl"
)
metadata3 = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_MobileNetV2_ref_0.pkl"
)
true_count = metadata1["metrics"]["params"] + \
metadata2["metrics"]["params"] + \
metadata3["metrics"]["params"]
self.assertEqual(self.ensemble.get_num_parameters(), true_count)
def update_dataset(model_file, th, train_path, test_path):
model = io.read_pickle(model_file)
# Test
dataset_test = model['test']
dataset_test['th'] = th
io.save_pickle(test_path, dataset_test)
# Train
dataset_train = model['train']
dataset_train['th'] = th
io.save_pickle(train_path, dataset_train)
def test_threshold_1(self):
chain = self.__create_ensemble()
c_id_extend = 'DenseNet-161'
c_file_extend = os.path.join(os.environ['FCM'], 'Definitions',
'Classifiers',
'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
extend_merged_chain(chain, 'ResNet18', c_id_extend, 1.1, c_file_extend)
R = evaluate(chain, chain.get_start())
acc_chain = R.test['system'].accuracy
time_chain = R.test['system'].time
ops_chain = R.test['system'].ops
params_chain = R.test['system'].params
# Accuracy classifier by hand
dict_classifier = io.read_pickle(
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl'))
acc_net0 = np.sum(
np.argmax(dict_classifier['test']['logits'], 1) ==
dict_classifier['test']['gt']) / len(dict_classifier['test']['gt'])
time_net0 = dict_classifier['metrics']['time']
params_net0 = dict_classifier['metrics']['params']
ops_net0 = dict_classifier['metrics']['ops']
# Accuracy classifier by hand
dict_classifier = io.read_pickle(c_file_extend)
acc_net1 = np.sum(
np.argmax(dict_classifier['test']['logits'], 1) ==
dict_classifier['test']['gt']) / len(dict_classifier['test']['gt'])
time_net1 = dict_classifier['metrics']['time']
params_net1 = dict_classifier['metrics']['params']
ops_net1 = dict_classifier['metrics']['ops']
correct = acc_chain == acc_net1 and \
math.isclose(time_net0/128 + time_net1/128, time_chain/5e3) and \
ops_chain/5e3 == ops_net0 + ops_net1 and \
params_chain == params_net0 + params_net1 + 1
self.assertEqual(correct, True)
def test_time(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# Evaluation time of ensemble
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
time = R.test['system'].time
# Compute CIFAR-10 evaluation time manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
time_0 = c_dict_0['metrics']['time']/128 * 5e3
time_1 = c_dict_1['metrics']['time']/128 * 5e3
self.assertAlmostEqual(time_1+time_0, time)
def test_params(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# Evaluation time of ensemble
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
params = R.test['system'].params
# Compute CIFAR-10 evaluation time manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
params_0 = c_dict_0['metrics']['params']
params_1 = c_dict_1['metrics']['params']
self.assertEqual(params_0+params_1, params)
def evaluate_pytorch(ensemble: System, phases: List[Split], dataset=None, device='cpu') -> Results:
import torch
eval = Results()
eval.train = dict()
eval.test = dict()
eval.val = dict()
eval.test['system'] = Metrics()
eval.train['system'] = Metrics()
eval.val['system'] = Metrics()
ensemble.to(device)
metadata = io.read_pickle(ensemble.get_classifiers()[0].get_model())
if Split.TRAIN in phases:
ensemble.set_evaluation_split(Split.TRAIN)
input = torch.arange(len(metadata["train"]["id"])).to(device)
gt_labels = torch.tensor(metadata["train"]["gt"]).to(device)
# Run
predictions = ensemble(input) # Evaluate ensemble
# Performance
eval.train['system'].accuracy = (torch.sum(torch.eq(predictions.argmax(dim=1), gt_labels)).float() / gt_labels.numel()).item()
eval.train['system'].time = ensemble.get_processing_time()
eval.train['system'].params = ensemble.get_num_parameters()
if Split.TEST in phases:
ensemble.set_evaluation_split(Split.TEST)
input = torch.arange(len(metadata["test"]["id"])).to(device)
gt_labels = torch.tensor(metadata["test"]["gt"]).to(device)
# Run
predictions = ensemble(input) # Evaluate ensemble
# Performance
eval.test['system'].accuracy = (torch.sum(torch.eq(predictions.argmax(dim=1), gt_labels)).float() / gt_labels.numel()).item()
eval.test['system'].time = ensemble.get_processing_time()
eval.test['system'].params = ensemble.get_num_parameters()
if Split.VAL in phases:
ensemble.set_evaluation_split(Split.VAL)
input = torch.arange(len(metadata["test"]["id"])).to(device)
gt_labels = torch.tensor(metadata["test"]["gt"]).to(device)
# Run
predictions = ensemble(input) # Evaluate ensemble
# Metrics
eval.val['system'].accuracy = (torch.sum(torch.eq(predictions.argmax(dim=1), gt_labels)).float() / gt_labels.numel()).item()
eval.val['system'].time = ensemble.get_processing_time()
eval.val['system'].params = ensemble.get_num_parameters()
return eval
def evaluate(work, records, tid):
lock = Lock()
while True:
m, th, m_ = work.get()
print(m, th, m_)
small_dict = io.read_pickle(m)
test_images = len(small_dict['test']['gt'])
train_images = len(small_dict['train']['gt'])
data_path = "../../Data/"
trigger_train_dataset = os.path.join(data_path,
"train_trigger_" + str(tid))
test_train_dataset = os.path.join(data_path,
"test_trigger_" + str(tid))
sys = sb.SystemBuilder(verbose=False)
# Complex classifier
bigClassifier = make.make_classifier("big", m_)
sys.add_classifier(bigClassifier)
# Data
source = make.make_source(trigger_train_dataset, test_train_dataset, 2)
data = make.make_data("trigger_data",
train_images,
test_images,
source=source)
sys.add_data(data)
update_dataset(m, th, trigger_train_dataset, test_train_dataset)
# Trigger
trigger = make.make_trigger(
"trigger",
make.make_empty_classifier(data_id="trigger_data"), ["big"],
model="probability")
sys.add_trigger(trigger)
# Simple classifier
smallClassifier = make.make_classifier("small", m, "trigger")
sys.add_classifier(smallClassifier)
results = eval.evaluate(sys, "small", check_classifiers=False)
records["system_" + m + ';' + m_ + ';' + str(th)] = results.test
lock.acquire()
if m_ not in records: # Evaluate individual models in order to plot
records[m_] = eval.evaluate(sys, 'big').test
lock.release()
work.task_done()
def test_accuracy(self):
chain = create_chain()
extend_chain(chain)
replace(chain)
R = evaluate(chain, chain.get_start())
# Accuracy classifier by hand
dict_classifier = io.read_pickle(
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_cifar10-32-dev_validation',
'V001_DenseNet169_ref_0.pkl'))
acc_net0 = np.sum(
np.argmax(dict_classifier['test']['logits'], 1) ==
dict_classifier['test']['gt']) / len(dict_classifier['test']['gt'])
self.assertEqual(acc_net0, R.test['system'].accuracy)
def test_predictions_threshold_1(self):
self.t1.update_threshold(1.1)
self.t2.update_threshold(1.1)
# Dataset info
metadata = read_pickle(
os.environ["FCM"] +
"/Definitions/Classifiers/sota_models_cifar10-32-dev_validation/V001_DenseNet121_ref_0.pkl"
)
# time = metadata["test"]["gt"]
ids = metadata["test"]["id"]
input = torch.range(0, len(ids) - 1).long()
predictions = self.ensemble(input)
predictions_classifier_c3 = self.c3(input)
equal = torch.equal(predictions, predictions_classifier_c3)
self.assertEqual(True, equal)
def update_dataset(c_file, train_path, test_path, val_path, th):
# Create dataset
model = io.read_pickle(c_file)
# Test
dataset_test = model['test']
if th >= 0: dataset_test['th'] = th
io.save_pickle(test_path, dataset_test)
dataset_train = model['train']
if th >= 0: dataset_train['th'] = th
io.save_pickle(train_path, dataset_train)
# Validation
dataset_val = model['val']
if th >= 0: dataset_val['th'] = th
io.save_pickle(val_path, dataset_val)
def forward(self, x=None):
if len(self.get_classifiers()) == 0:
raise ValueError("System empty of classifiers")
if x is None:
metadata = read_pickle(self.get_classifiers()[0].get_model())
if self.split == Split.TRAIN:
num_samples = len(metadata["train"]["id"])
x = arange(num_samples).long()
elif self.split == Split.TEST:
num_samples = len(metadata["test"]["id"])
x = arange(num_samples).long()
elif self.split == Split.VAL:
num_samples = len(metadata["val"]["id"])
x = arange(num_samples).long()
else:
raise ValueError("Dataset split not recognized")
return self.graph(x)
def forward(self, ids: torch.LongTensor) -> torch.Tensor:
"""
Receives the indices of a set of samples, and returns the precomputed predictions of those samples
:param ids: Index of samples in the dataset
:return: Predictions of the classifier for those indices
"""
metadata = read_pickle(self.path)
time_batch_128 = metadata['metrics']['time']
if self.split == Split.TRAIN:
precomputed_pred = metadata['train']['logits']
elif self.split == Split.TEST:
precomputed_pred = metadata['test']['logits']
else:
precomputed_pred = metadata['val']['logits']
precomputed_pred = torch.from_numpy(precomputed_pred)
self.update_processing_time(ids.numel() * time_batch_128 / 128.0)
self.predictions = precomputed_pred[ids].to(ids.device)
return self.predictions
def __get_trigger_raw_data(data, phase):
if phase == "test":
data_source = data.source.test_path
elif phase == "train":
data_source = data.source.train_path
else:
data_source = data.source.val_path
dataset = io.read_pickle(data_source)
P = dataset['predictions'] # Models for voting protocol
n = dataset['n']
gt = dataset['gt']
ids = dataset['ids']
agree = np.apply_along_axis(np.bincount, 1, P, None, P.shape[2])
agree_max = np.max(agree, axis=1)
y = np.transpose(np.append(np.array([agree_max < n]), np.array([agree_max >= n]), axis=0))
raw_data = make.make_classifier_raw_data(y, gt == np.argmax(P, axis=1), ids)
return raw_data
def __get_trigger_raw_data(data, phase):
if phase == "test":
data_source = data.source.test_path
elif phase == "train":
data_source = data.source.train_path
else:
data_source = data.source.val_path
dataset = io.read_pickle(data_source)
L = dataset['logits']
P = softmax(L)
th = dataset['th']
gt = dataset['gt']
ids = dataset['id']
sort = np.sort(P, axis=1)
diff = np.array(sort[:, -1] - sort[:, -2])
y = np.column_stack((diff < th, diff >= th)).astype(np.int)
raw_data = make.make_classifier_raw_data(y, gt == np.argmax(L, axis=1),
ids)
return raw_data
sys = sb.SystemBuilder(verbose=True)
name2 = m_
bigClassifier = make.make_empty_classifier("big")
bigClassifier.classifier_file = name2
sys.add_classifier(bigClassifier)
trigger = make.make_trigger("probability_threshold_trigger",
make.make_empty_classifier(), ["big"])
sys.add_trigger(trigger)
name1 = m
smallClassifier = make.make_empty_classifier("small",
"probability_threshold_trigger")
model1_dict = io.read_pickle(name1, suffix="", verbose=False)
smallClassifier.classifier_file = name1
sys.add_classifier(smallClassifier)
classifier_trigger = build_train_trigger(model1_dict, th)
trigger = make.make_trigger("probability_threshold_trigger",
classifier_trigger, ["big"])
trigger.id = "probability_threshold_trigger"
sys.replace(trigger.id, trigger)
metrics = eval.evaluate(sys, "small", check_classifiers=False)
eval.pretty_print(metrics)
# cl = sys.build_classifier_dict("small")
# io.save_pickle(Classifier_Path+"system_"+net1+"_"+net2+"th=0.7", cl)
X = [record[key].test['system'].params for key in ref_keys]
Y = [record[key].test['system'].accuracy for key in ref_keys]
plt.scatter(X, Y, color='black', s=20)
def show():
plt.legend()
plt.show()
def save(f):
plt.save_fig(f)
if __name__ == "__main__":
method = 0
plt.figure(0)
toPlot_chain = io.read_pickle(
"./probability_threshold/results/imagenet/R_all")
plt.figure(0)
plot_accuracy_parameters(toPlot_chain, system_color='blue')
#plot_accuracy_parameters(toPlot, system_color='green')
plt.figure(1)
plot_accuracy_time(toPlot_chain, system_color='blue')
#plot_accuracy_time(toPlot, system_color='green')
plt.show()
# plot_accuracy_time(result_AVERAGE_4_models)
os.environ['FCM'], 'Examples', 'compute',
'genetic_algorithm_multinode', 'results',
'sota_models_cifar10-40-dev_validation',
'cifar10_8nodes_800population_400offspring_0',
'multinode_metainfo.json')
execution_time = 0
n_nodes = 8
R_all = {}
with open(multinode_meta_file, 'r') as handle:
results = json.load(handle)
for i, iteration in enumerate(results):
max_exec_offspring_time = sum(get_n_maximum(iteration, 1))
selection_time = iteration['selection']['exec_time']
execution_time += max_exec_offspring_time + selection_time
# Gather the results
R_iter = io.read_pickle(
os.path.join(os.environ['FCM'], iteration['selection']['R']))
R_all.update(R_iter)
print("Iteration %d: Offspring+evaluate %f, Selection %f" %
(i, max_exec_offspring_time, selection_time))
print(R_all.keys())
print("Execution time: %f" % execution_time)
# Plot solutions
import Examples.study.plot as myplt
myplt.plot_accuracy_time_old(R_all)
myplt.show()
def get_classifier_name(c_file):
return io.read_pickle(c_file)['name']
if __name__ == "__main__":
id_fastest = "__trigger_classifier_0.10000000000000003_V001_VGG13_ref_0____trigger_classifier_0" \
".4_V001_ResNeXt29_32x4d_ref_0__V001_VGG11_ref_0"
experiment_dir = os.path.join(os.environ['FCM'], 'Examples', 'compute',
'chain_genetic_algorithm')
metadata_file = os.path.join(experiment_dir, 'results', 'metadata.json')
id = "7062152700584889"
dataset = results_manager.get_fieldval_by_id(metadata_file, id, 'dataset')
results_chain_path = os.path.join(
experiment_dir, results_manager.get_results_by_id(metadata_file, id))
results_chain = io.read_pickle(results_chain_path)
fastest = results_chain[id_fastest]
acc_result = fastest.test['system'].accuracy
time_result = fastest.test['system'].time
# Build same ensemble
classifiers = [
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation',
'V001_VGG13_ref_0'),
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation',
'V001_ResNeXt29_32x4d_ref_0'),
os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_stl10-32-dev_validation', 'V001_VGG11_ref_0')
]
