def black_box_function(config): # print(os.getcwd()) # print(args.config) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() print('done') return evaluater.f1
def black_box_function(learning_rate): # print(os.getcwd()) # print(args.config) args = get_args() config = process_config_UtsClassification_bayes_optimization(args.config, learning_rate) create_dirs([config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() print('done') return evaluater.f1
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '2' try: RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) args = get_args() config = process_config_UtsClassification_bayes_optimization( args.config, PARAMS) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([ config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir ]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() nni.report_final_result(evaluater.f1) print('done') except Exception as e: LOG.exception(e) raise
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '0' for model_name in ['fcn', 'resnet_v2', 'cnn']: for learning_rate in [0.001, 0.0005, 0.0001]: args = get_args() config = process_config_UtsClassification_grid_search_hyperparamters( args.config, model_name, learning_rate) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([ config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir ]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() print('done') K.clear_session()
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '0' # capture the config path from the run arguments # then process the json configuration file # try: args = get_args() config = process_config_UtsClassification(args.config) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print("size of dataloader:") print(sys.getsizeof(data_loader)) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print("size of model:") print(sys.getsizeof(model)) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() print('done')
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '0' # capture the config path from the run arguments # then process the json configuration file # try: split = 10 training_size = [] accuracy = [] precision = [] recall = [] f1 = [] main_dir = '' args = get_args() for i in range(split): config = process_config_VisOverfit(args.config, i) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([ config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir ]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) total_train_size = data_loader.get_train_size() total_test_size = data_loader.get_test_size() print("total_train_size: " + str(total_train_size)) print("total_test_size: " + str(total_test_size)) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') train_size = int(total_train_size / split) * (i + 1) if i == split - 1: print("train_size: " + str(total_train_size)) trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) main_dir = config.main_dir else: print("train_size: " + str(train_size)) train_data = data_loader.get_train_data() X_train = train_data[0][:train_size, :, :] y_train = train_data[1][:train_size, :] trainer = UtsClassificationTrainer(model.model, [X_train, y_train], config) print('Start training the model.') trainer.train() print('Create the evaluater.') evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), config) print('Start evaluating the model.') evaluater.evluate() training_size.append(train_size) accuracy.append(evaluater.acc) precision.append(evaluater.precision) recall.append(evaluater.recall) f1.append(evaluater.f1) print("ss") metrics = { "accuracy": accuracy, "precision": precision, "recall": recall, "f1": f1, "training_size": training_size } plot_trainingsize_metric(metrics, main_dir + 'vis_overfit_trainingsize.png')
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '0' # capture the config path from the run arguments # then process the json configuration file # try: split = 10 training_size = [] best_model_train_loss = [] best_model_val_loss = [] best_model_train_acc = [] best_model_val_acc = [] best_model_train_precision = [] best_model_val_precision = [] best_model_train_recall = [] best_model_val_recall = [] best_model_train_f1 = [] best_model_val_f1 = [] best_model_learning_rate = [] best_model_nb_epoch = [] best_model_train_err = [] best_model_val_err = [] main_dir = '' args = get_args() for i in range(split): config = process_config_VisTrainingSize(args.config, i) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs create_dirs([ config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.log_dir, config.result_dir ]) print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) total_train_size = data_loader.get_train_size() total_test_size = data_loader.get_test_size() print("total_train_size: " + str(total_train_size)) print("total_test_size: " + str(total_test_size)) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') train_size = int(total_train_size / split) * (i + 1) if i == split - 1: print("train_size: " + str(total_train_size)) trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) main_dir = config.main_dir else: print("train_size: " + str(train_size)) train_data = data_loader.get_train_data() X_train = train_data[0][:train_size, :, :] y_train = train_data[1][:train_size, :] trainer = UtsClassificationTrainer(model.model, [X_train, y_train], config) print('Start training the model.') trainer.train() best_model_train_loss.append(trainer.best_model_train_loss) best_model_val_loss.append(trainer.best_model_val_loss) best_model_train_acc.append(trainer.best_model_train_acc) best_model_val_acc.append(trainer.best_model_val_acc) best_model_train_precision.append(trainer.best_model_train_precision) best_model_val_precision.append(trainer.best_model_val_precision) best_model_train_recall.append(trainer.best_model_train_recall) best_model_val_recall.append(trainer.best_model_val_recall) best_model_train_f1.append(trainer.best_model_train_f1) best_model_val_f1.append(trainer.best_model_val_f1) best_model_learning_rate.append(trainer.best_model_learning_rate) best_model_nb_epoch.append(trainer.best_model_nb_epoch) best_model_train_err.append(1 - trainer.best_model_train_acc) best_model_val_err.append(1 - trainer.best_model_val_acc) training_size.append(train_size) print("ss") metrics = { "training_size": training_size, "train_err": best_model_train_err, "val_err": best_model_val_err } plot_trainingsize_metric(metrics, main_dir + '/vis_overfit_trainingsize.png')
def main(): os.environ['CUDA_VISIBLE_DEVICES'] = '2' # capture the config path from the run arguments # then process the json configuration file # try: args = get_args() config, _ = get_config_from_json(args.config) bayes_config = { "algorithm": "bayes", "parameters": { # "model": {"type": "categorical", "values": ['cnn','mlp']}, "learning_rate": { "type": "float", "min": 0.001, "max": 0.01 }, # "batch_size": {"type": "integer", "min": 16, "max": 32}, # "num_epochs": {"type": "integer", "min": 5, "max": 10}, }, "spec": { "maxCombo": 10, "objective": "minimize", "metric": "test_f1", "minSampleSize": 100, "retryAssignLimit": 0, }, "trials": 1, "name": "Bayes", } opt = Optimizer(bayes_config, api_key=config.comet_api_key, project_name=config.exp_name) for exp in opt.get_experiments(): args = get_args() # config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('model'),exp.get_parameter('learning_rate'), # exp.get_parameter('batch_size'), exp.get_parameter('num_epochs')) config = process_config_UtsClassification_bayes_optimization( args.config, exp.get_parameter('learning_rate')) # except: # print("missing or invalid arguments") # exit(0) # create the experiments dirs print('Create the data generator.') data_loader = UtsClassificationDataLoader(config) print('Create the model.') model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses()) print('Create the trainer') trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config) print('Start training the model.') trainer.train() # print('Create the evaluater.') # evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(), # config) # # print('Start evaluating the model.') # evaluater.evluate() exp.log_metric("test_f1", trainer.best_model_val_loss) print('done')