def train_trigger(sys, c): """ Trains a trigger and creates its classifier whose name will be classifier's id and will be saved in Definitions/Classifiers/tmp :param sys: System of components :param c: Trigger protobuf definition :return: - """ assert c.HasField( "classifier" ), "ERROR in TRIGGER.CLASSIFIER: A classifier should be specified for the trigger" train_trigger = not c.classifier.HasField( "classifier_file") or c.classifier.classifier_file == "" if train_trigger: assert c.HasField("model"), \ "ERROR in TRIGGER: A model must be specified when training the trigger" assert c.classifier.HasField("data_id"), \ "ERROR in TRIGGER.CLASSIFIER: Training data should be specified when training the trigger" module = importlib.import_module('Definitions.triggers.' + c.model) c_dict = module.train_fit(sys, c.id) if 'TMP' in os.environ: tmp_location = os.path.join(os.environ['FCM'], os.environ['TMP'] + '/') else: tmp_location = os.path.join(os.environ['FCM'], 'Definitions/Classifiers/tmp/') classifier_file = c.id io.save_pickle(tmp_location + classifier_file, c_dict) c.classifier.classifier_file = tmp_location + classifier_file sys.replace(c.id, c)
def save_results(meta_file, meta_data_results, params, R): """ Saves the results pickle and checks meta-Data information about the results is correct and updated :param meta_file: Storage of all the ensemble results for a particular experiment/ensemble type. :param meta_data_results: Meta Data information of a new result :param params: Params of the experiment :return: """ assert os.path.isabs(meta_file), "ERROR Metadata Results: File meta_file should be an absolute path to the file" assert meta_data_results['results'] != "" and meta_data_results['results'] is not None if not os.path.exists(meta_file): with open(meta_file, 'w') as file: json.dump([], file) save_path = os.path.join(os.environ['FCM'], meta_data_results['results']) if not os.path.exists(save_path): os.makedirs(save_path) # 1- Guardar R io.save_pickle(os.path.join(save_path, "results_ensembles.pkl"), R) # 2- Obre el meta_data file i fer update meta_data_results['params'] = params with open(meta_file, 'r') as file: meta_data = json.load(file) with open(meta_file, 'w') as file: meta_data.append(meta_data_results) json.dump(meta_data, file, indent=4)
def update_dataset(c_file, train_path, test_path, val_path, th): # Create dataset model = io.read_pickle(c_file) # Test dataset_test = model['test'] if th >= 0: dataset_test['th'] = th io.save_pickle(test_path, dataset_test) """
def update_dataset(model_file, th, train_path, test_path): model = io.read_pickle(model_file) # Test dataset_test = model['test'] dataset_test['th'] = th io.save_pickle(test_path, dataset_test) # Train dataset_train = model['train'] dataset_train['th'] = th io.save_pickle(train_path, dataset_train)
def check_valid_classifier_metrics(c): classifier_dict = io.read_pickle(c.classifier_file, verbose=False) if 'time' not in classifier_dict['metrics']: warnings.warn("WARNING in Classifier: Time measurement not found, set to 0 instead") classifier_dict['metrics']['time'] = 0 io.save_pickle(c.classifier_file, classifier_dict) if 'times' not in classifier_dict['metrics']: warnings.warn("WARNING in Classifier: Times list measurement not found, set to time instead") classifier_dict['metrics']['times'] = np.array([classifier_dict['metrics']['time']]) io.save_pickle(c.classifier_file, classifier_dict) if 'params' not in classifier_dict['metrics']: warnings.warn("WARNING in Classifier: # Params measurement not found, set to 0 instead") classifier_dict['metrics']['params'] = 0 io.save_pickle(c.classifier_file, classifier_dict) if 'ops' not in classifier_dict['metrics']: warnings.warn("WARNING in Classifier: # Ops measurement not found, set to 0 instead") classifier_dict['metrics']['ops'] = 0 io.save_pickle(c.classifier_file, classifier_dict) classifier_dict = io.read_pickle(c.classifier_file, verbose=False) assert classifier_dict['metrics']['time'] >= 0, "ERROR in Classifier: Time should be positive" assert np.all(np.array(classifier_dict['metrics']['times']) >= 0) and len(classifier_dict['metrics']['times']) > 0, \ "ERROR in Classifier: Time should be positive" assert classifier_dict['metrics']['params'] >= 0, "ERROR in Classifier: # Params should be positive" assert classifier_dict['metrics']['ops'] >= 0, "ERROR in Classifier: # Ops should be positive"
def build_train_trigger(model1_dict, th): classifier_trigger_dict = {} performance = make.make_performance_metrics(**{}) # Train dict L = model1_dict['train']['logits'] dividend = np.sum(np.exp(L), axis=1) P = np.exp(L) / dividend[:, None] sort = np.sort(P, axis=1) # Sort class probabilities diff = sort[:, -1] - sort[:, -2] # Difference logits_trigger = np.empty((diff.shape[0], 2)) logits_trigger[:, 0] = diff < th logits_trigger[:, 1] = diff >= th pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['train']['gt'] train = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['train']['id'])) # Test dict L = model1_dict['test']['logits'] dividend = np.sum(np.exp(L), axis=1) P = np.exp(L) / dividend[:, None] sort = np.sort(P, axis=1) # Sort class probabilities diff = sort[:, -1] - sort[:, -2] # Difference logits_trigger = np.empty((diff.shape[0], 2)) logits_trigger[:, 0] = diff < th logits_trigger[:, 1] = diff >= th pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['test']['gt'] test = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['test']['id'])) classifier_trigger_dict = make.make_classifier_dict( "trigger_classifier", "cifar10", train, test, performance) io.save_pickle( '../../Definitions/Classifiers/tmp/trigger_random_threshold.pkl', classifier_trigger_dict) classifier = make.make_classifier( "trigger_classifier", "../../Definitions/Classifiers/tmp/trigger_random_threshold.pkl") return classifier
def build_train_trigger2(model1_dict, th): classifier_trigger_dict = {} performance = make.make_performance_metrics(**{}) # Train dict L = model1_dict['train']['logits'] dividend = np.sum(np.exp(L), axis=1) P = np.exp(L) / dividend[:, None] max_P = np.max(P, axis=1) logits_trigger = np.empty((max_P.shape[0], 2)) logits_trigger[:, 0] = max_P < th logits_trigger[:, 1] = max_P >= th pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['train']['gt'] train = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['train']['id'])) # Test dict L = model1_dict['test']['logits'] dividend = np.sum(np.exp(L), axis=1) P = np.exp(L) / dividend[:, None] max_P = np.max(P, axis=1) logits_trigger = np.empty((max_P.shape[0], 2)) logits_trigger[:, 0] = max_P < th logits_trigger[:, 1] = max_P >= th pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['test']['gt'] test = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['test']['id'])) classifier_trigger_dict = make.make_classifier_dict( "trigger_classifier", "cifar10", train, test, performance) io.save_pickle( '../../Definitions/Classifiers/tmp/trigger_random_threshold', classifier_trigger_dict) classifier = make.make_classifier( "trigger_classifier", "../../Definitions/Classifiers/tmp/trigger_random_threshold") return classifier
def build_train_trigger3(model1_dict, p): classifier_trigger_dict = {} performance = make.make_performance_metrics(**{}) # Train dict L = model1_dict['train']['logits'] logits_trigger = np.empty((L.shape[0], 2)) logits_trigger[:, 0] = np.random.binomial(1, p, L.shape[0]) logits_trigger[:, 1] = 1 - logits_trigger[:, 0] pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['train']['gt'] train = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['train']['id'])) # Test dict L = model1_dict['test']['logits'] logits_trigger = np.empty((L.shape[0], 2)) logits_trigger[:, 0] = np.random.binomial(1, p, L.shape[0]) logits_trigger[:, 1] = 1 - logits_trigger[:, 0] pred_model1 = np.argmax(L, axis=1) gt_model1 = model1_dict['test']['gt'] test = make.make_classifier_raw_data(logits_trigger, (pred_model1 == gt_model1), np.copy(model1_dict['test']['id'])) classifier_trigger_dict = make.make_classifier_dict( "trigger_classifier", "cifar10", train, test, performance) io.save_pickle( '../../Definitions/Classifiers/tmp/trigger_random_threshold', classifier_trigger_dict) classifier = make.make_classifier( "trigger_classifier", "../../Definitions/Classifiers/tmp/trigger_random_threshold") return classifier
def save_results(R_dict, individuals_fitness_per_generation): meta_data_file = os.path.join(os.environ['FCM'], 'Examples', 'compute', 'genetic_algorithm_multinode', 'results', 'metadata.json') id = str(random.randint(0, 1e16)) results_loc = os.path.join( 'Examples/compute/genetic_algorithm_multinode/results', main.args.dataset, id) comments = main.args.comment meta_data_result = manager_results.metadata_template( id, main.args.dataset, results_loc, comments) params = main.args.__dict__ # Save dictionary of ensemble results manager_results.save_results(meta_data_file, meta_data_result, params, R_dict) # Additionally save ensemble results per generation (list) io.save_pickle( os.path.join(os.environ['FCM'], results_loc, 'individuals_fitness_per_generation.pkl'), individuals_fitness_per_generation)
def update_dataset(c_file, train_path, test_path, val_path, th): # Create dataset model = io.read_pickle(c_file) # Test dataset_test = model['test'] if th >= 0: dataset_test['th'] = th io.save_pickle(test_path, dataset_test) dataset_train = model['train'] if th >= 0: dataset_train['th'] = th io.save_pickle(train_path, dataset_train) # Validation dataset_val = model['val'] if th >= 0: dataset_val['th'] = th io.save_pickle(val_path, dataset_val)
# Info about current generation print("Iteration %d" % iteration) print("TIME: Seconds per generation: %f " % (time.time() - start)) iteration += 1 # Save the results import Examples.metadata_manager_results as manager_results meta_data_file = os.path.join(os.environ['FCM'], 'Examples', 'compute', 'bagging_boosting_of_chains_GA', 'results', 'metadata.json') id = str(random.randint(0, 1e16)) results_loc = os.path.join( 'Examples/compute/bagging_boosting_of_chains_GA/results', args.dataset, id) comments = args.comment meta_data_result = manager_results.metadata_template( id, args.dataset, results_loc, comments) # Save the ensemble evaluation results R_dict_old.update(R_dict) params = args.__dict__ manager_results.save_results(meta_data_file, meta_data_result, params, R_dict_old) io.save_pickle( os.path.join(os.environ['FCM'], results_loc, 'individuals_fitness_per_generation.pkl'), individuals_fitness_per_generation)
# Creating system sys = sb.SystemBuilder(verbose=False) smallClassifier = make.make_empty_classifier("Classifier") sys.add_classifier(smallClassifier) for m_ in models: # Model 2 name2 = Classifier_Path + m_ model2 = make.make_empty_classifier() model2.id = "Classifier" model2.classifier_file = name2 sys.replace(model2.id, model2) evaluate_time_start = time.time() results = eval.evaluate(sys, model2.id) eval_time = time.time() - evaluate_time_start print("Evaluation time:", eval_time) records[m_] = results import Source.io_util as io if not os.path.exists('./models_evaluation/%s' % d): os.makedirs('./models_evaluation/%s' % d) io.save_pickle('./models_evaluation/%s/models.pkl' % d, records) plot_accuracy_time(records) plt.legend() plt.show()
import Source.protobuf.system_builder_serializable as sb import Source.protobuf.make_util as make import Source.io_util as io import os if __name__ == "__main__": dset = "sota_models_cifar10-32-dev_validation" Classifier_Path = os.path.join(os.environ['FCM'], 'Definitions', 'Classifiers', dset) P = [] models = [f for f in os.listdir(Classifier_Path)] # Creating system records = {} for m_ in models: # Model 2 sys = sb.SystemBuilder(verbose=False) classifier = make.make_classifier(m_, os.path.join(Classifier_Path, m_)) sys.add_classifier(classifier) P.append(sys) io.save_pickle('initial_population', P)
trigger = make.make_trigger("probability_threshold_trigger", classifier_trigger, ["big"]) sys.replace(trigger.id, trigger) results = eval.evaluate(sys, "small", check_classifiers=False).test random_times_chain.append(results) r['random_chain'].append(random_times_chain) # Random trigger tree structure simple = make.make_classifier("small", c_simple_file) sys.replace("small", simple) random_times_tree = [] for i in range(5): classifier_trigger = build_train_trigger3(simple_dict, th) trigger = make.make_trigger("probability_threshold_trigger", classifier_trigger, ["big", "small"]) sys.replace(trigger.id, trigger) results = eval.evaluate(sys, "probability_threshold_trigger", check_classifiers=False).test random_times_tree.append(results) r['random_tree'].append(random_times_tree) R["system_" + c_simple_file + "_" + c_complex_file] = r io.save_pickle("./results/R.pkl", R)
] out_dir = os.path.join("./results/", dataset) data_path = "../../Data/" if not os.path.exists(out_dir): os.mkdir(out_dir) ######################################################################### import Examples.study.paretto_front as paretto R_models = {} for model in models: sys = sb.SystemBuilder(verbose=False) c = make.make_classifier("classifier", model) sys.add_classifier(c) R_models[model] = eval.evaluate(sys, c.id).test io.save_pickle(os.path.join(out_dir, "models"), R_models) front_sorted = paretto.sort_results_by_accuracy(R_models) models = [val[0] for val in front_sorted] print(models) sys = sb.SystemBuilder(verbose=False) trigger_train_dataset = os.path.join(data_path, "train_trigger_0") test_train_dataset = os.path.join(data_path, "test_trigger_0") # Model 1 for im, m in enumerate(models): smallClassifier = make.make_classifier("small", m, "trigger") sys.replace("small", smallClassifier)