def test_accuracy(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# System Evaluator computes accuracy
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
acc = R.test['system'].accuracy
# Compute accuracy manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
gt = c_dict_0['test']['gt']
logits_0 = c_dict_0['test']['logits']
logits_1 = c_dict_1['test']['logits']
average = (logits_0 + logits_1) / 2
acc_manual = np.sum(np.argmax(average, 1) == gt) / len(gt)
self.assertEqual(acc, acc_manual)
def test_params(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# Evaluation time of ensemble
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
params = R.test['system'].params
# Compute CIFAR-10 evaluation time manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
params_0 = c_dict_0['metrics']['params']
params_1 = c_dict_1['metrics']['params']
self.assertEqual(params_0+params_1, params)
def test_time(self):
ensemble = create_ensemble()
c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_DenseNet161_ref_0.pkl')
c_file2 = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation',
'V001_ResNet18_ref_0.pkl')
# Evaluation time of ensemble
add_classifier_to_merger(ensemble, 'Merger', 'DenseNet161', c_file)
R = evaluate(ensemble, ensemble.get_start())
time = R.test['system'].time
# Compute CIFAR-10 evaluation time manually
c_dict_0 = io.read_pickle(c_file2)
c_dict_1 = io.read_pickle(c_file)
time_0 = c_dict_0['metrics']['time']/128 * 5e3
time_1 = c_dict_1['metrics']['time']/128 * 5e3
self.assertAlmostEqual(time_1+time_0, time)
import merge_two_chains
import os
merger = merge_two_chains.merged
from Source.genetic_algorithm.operations_mutation import add_classifier_to_merger
add_classifier_to_merger(merger, 'Merger', '2_PNASNetA', os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers',
'sota_models_cifar10-32-dev_validation',
'V001_PNASNetA_ref_0'))
from Examples.compute.chain_genetic_algorithm.utils import generate_system_id
merger.set_sysid(generate_system_id(merger))
print(merger.get_sysid())
from Source.system_evaluator import evaluate
from Source.system_evaluator_utils import pretty_print
R = evaluate(merger, merger.get_start())
pretty_print(R)
def mutation_operation(P):
offspring = []
p = P[random.randint(0, len(P) - 1)]
operation = random.randint(0, 3) if len(
p.get_message().merger) == 1 else random.randint(0, 2)
# Extend a chain
if operation == 0:
new_p = p.copy()
# Pick a random classifier from the pool of solutions
c_file_new = utils.pick_random_classifier(args)
c_id_new = get_classifier_name(c_file_new)
# Find the tail of any chain to extend
merger = None if 'Merger' not in new_p.get_start(
) else new_p.get_message().merger[0]
n_chains = len(merger.merged_ids) if merger is not None else 1
tail_chains = [
c.id for c in new_p.get_message().classifier
if c.component_id == ""
]
c_id_extend = tail_chains[random.randint(0, n_chains - 1)]
c_id_new = (c_id_extend[0] + '_' if c_id_extend[0] > '0'
and c_id_extend[0] <= '9' else '') + c_id_new
# Perform the operation
om.extend_merged_chain(new_p,
c_id_extend,
c_id_new,
th=utils.pick_random_threshold(args),
c_file_new=c_file_new)
#new_p.set_sysid(utils.generate_system_id(new_p))
offspring.append(new_p)
# Replace a classifier
if operation == 1:
new_p = p.copy()
c_id_existing = utils.pick_random_classifier(args, new_p)
c_file_new = utils.pick_random_classifier(args)
c_id_new = (c_id_existing[0] +
'_' if c_id_existing[0] > '0' and c_id_existing[0] <= '9'
else '') + get_classifier_name(c_file_new)
om.replace_classifier_merger(new_p,
c_id_existing,
c_id_new,
c_file=c_file_new)
#new_p.set_sysid(utils.generate_system_id(new_p))
offspring.append(new_p)
# Update threshold
if operation == 2:
new_p = p.copy()
sign = 2 * (random.random() > 0.5) - 1
om.update_threshold(new_p, utils.pick_random_classifier(args, new_p),
sign * args.step_th)
#new_p.set_sysid(utils.generate_system_id(new_p))
offspring.append(new_p)
# Add classifier to be merged (Assuming 1 merger)
if operation == 3:
merger = p.get_message().merger[0]
merger_id = merger.id
merger_number_chains = len(merger.merged_ids)
if merger_number_chains < 3:
new_p = p.copy()
c_file_new = utils.pick_random_classifier(args)
c_id_new = get_classifier_name(c_file_new)
c_id_new = str(merger_number_chains) + "_" + c_id_new
om.add_classifier_to_merger(new_p, merger_id, c_id_new, c_file_new)
#new_p.set_sysid(utils.generate_system_id(new_p))
offspring.append(new_p)
return offspring