def evaluate_single_models(models, results): for m in models: sys = sb.SystemBuilder() c = make.make_classifier(m, m) sys.add_classifier(c) sys.set_start(m) results[generate_system_id(sys)] = eval.evaluate(sys, sys.get_start(), phases=["test"])
def __create_ensemble(self): ensemble = sb.SystemBuilder() c_id = 'ResNet18' c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation', 'V001_ResNet18_ref_0.pkl') classifier = mutils.make_classifier(c_id, c_file) ensemble.add_classifier(classifier) ensemble.set_start(c_id) return ensemble
def create_ensemble(): ensemble = sb.SystemBuilder() c_id = 'ResNet18' c_file = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', 'sota_models_cifar10-32-dev_validation', 'V001_ResNet18_ref_0.pkl') classifier = mutils.make_classifier(c_id, c_file) merger_id = 'Merger' merger = mutils.make_merger(merger_id, [c_id], merge_type=fcm.Merger.AVERAGE) ensemble.add_classifier(classifier) ensemble.add_merger(merger) ensemble.set_start('Merger') return ensemble
def build_chain(classifiers, id_classifiers, thresholds, id_triggers, data_id): assert len(classifiers) == len( id_triggers ) + 1, "ERROR: Number of triggers in the chain is not consistent" assert len(id_triggers) == len( thresholds), "ERROR: Each trigger should be assigned a threshold" assert len(classifiers) == len( id_classifiers ), "ERROR: Each classifier file should be assigned a classifier id" data_path = os.path.join(os.environ['FCM'], 'Data', data_id) if not os.path.exists(data_path): os.makedirs(data_path) sys = sb.SystemBuilder(verbose=False) for i in range(len(classifiers) - 1): # Create data for the trigger train_path = os.path.join(data_path, id_triggers[i] + "_test.pkl") test_path = os.path.join(data_path, id_triggers[i] + "_train.pkl") val_path = os.path.join(data_path, id_triggers[i] + "_val.pkl") source = make.make_source(train_path, test_path, 2, val_path=val_path) data = make.make_data("data_" + id_triggers[i], 1, 1, source=source) update_dataset(classifiers[i], train_path, test_path, val_path, thresholds[i]) sys.add_data(data) # Build trigger attached to classifier trigger = make.make_trigger(id_triggers[i], make.make_empty_classifier( id='', data_id="data_" + id_triggers[i]), [id_classifiers[i + 1]], model="probability") sys.add_trigger(trigger) # Build classifier c_file = classifiers[i] classifier = make.make_classifier(id_classifiers[i], c_file, component_id=id_triggers[i]) sys.add_classifier(classifier) if i == 0: sys.set_start(id_classifiers[i]) classifier = make.make_classifier(id_classifiers[-1], classifiers[-1]) sys.add_classifier(classifier) return sys
def build_evaluate_chain(files: [str], ths: [float]): assert len(files) > 0 and len(files) == len(ths)+1 sys = sb.SystemBuilder(verbose=False) classifier = make.make_classifier(os.path.basename(files[0]), files[0]) sys.add_classifier(classifier) sys.set_start(classifier.id) # Automatically build the chain with written mutation operations for i, file in enumerate(files[:-1]): extend_merged_chain(sys, os.path.basename(file), os.path.basename(files[i+1]), ths[i], files[i+1]) result = eval.evaluate(sys) return result
def generate_initial_population(): classifier_path = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', args.dataset) P = [] classifier_files = [ os.path.join(classifier_path, f) for f in os.listdir(classifier_path) if ".pkl" in f ] for c_file in classifier_files: sys = sb.SystemBuilder(verbose=False) c_id = get_classifier_name(c_file) classifier = make.make_classifier(c_id, c_file) sys.add_classifier(classifier) sys.set_start(c_id) # sys.set_sysid(utils.generate_system_id(sys)) P.append(sys) return P
def create_evaluate_system(work, results): # Creating system lock = Lock() while True: protocol, subset = work.get() sys = sb.SystemBuilder(verbose=False) classifiers_ids = [] for m in subset: file = m model = make.make_classifier(m, file) sys.add_classifier(model) classifiers_ids.append(model.id) merger = make.make_merger("MERGER", classifiers_ids, merge_type=protocol) sys.add_merger(merger) r = eval.evaluate(sys, merger.id) # if results: results = io.read_pickle('./results/R_'+str(protocol)+'_'+str(n_models)) results['system_' + '_'.join(classifiers_ids) + '_protocol' + str(protocol)] = r work.task_done()
for dataset in datasets: ######################################################################### Classifier_Path = os.environ['FCM']+"/Definitions/Classifiers/" + dataset + "/" model_paths = [Classifier_Path + f for f in os.listdir(Classifier_Path) if ".pkl" in f] out_dir = os.path.join("./results/", dataset) data_path = os.environ['FCM']+"/Datasets/" if not os.path.exists(out_dir): os.makedirs(out_dir) ######################################################################### import Examples.study.paretto_front as paretto R_models = {} for mpath in model_paths: sys = sb.SystemBuilder(verbose=False) classifier_id = os.path.basename(mpath) c = make.make_classifier(classifier_id, mpath) sys.add_classifier(c) sys.set_start(classifier_id) R_models[classifier_id] = eval.evaluate(sys, phases=["test", "val"]) #front = paretto.get_front_time_accuracy(R_models, phase="test") #front_sorted = paretto.sort_results_by_accuracy(front, phase="test") models_paths = [Classifier_Path + k for k, v in paretto.sort_results_by_accuracy(R_models, phase="val")] records = R_models # Combinations for ic0 in range(len(models_paths)): c0 = models_paths[ic0] for th0 in np.arange(0, 1+step_th, step_th):
from Source.system_evaluator_utils import pretty_print import os classifiers = [ os.path.join( os.environ['FCM'], 'Definitions/Classifiers/sota_models_cifar10-32-dev_validation', 'V001_ResNet18_ref_0.pkl'), os.path.join( os.environ['FCM'], 'Definitions/Classifiers/sota_models_cifar10-32-dev_validation', 'V001_ResNet152_ref_0.pkl') ] # Build the system merged_classifiers = sb.SystemBuilder() merger = make.make_merger('Merger', classifiers, fcm.Merger.AVERAGE) merged_classifiers.add_merger(merger) for classifier in classifiers: c = make.make_classifier(classifier, classifier) merged_classifiers.add_classifier(c) merged_classifiers.set_start('Merger') R = evaluate(merged_classifiers, merged_classifiers.get_start()) pretty_print(R) # Manual check import Source.io_util as io import numpy as np c_dict_0 = io.read_pickle(classifiers[0])