def black_box_function(config):
# print(os.getcwd())
# print(args.config)
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
print('done')
return evaluater.f1
def black_box_function(learning_rate):
# print(os.getcwd())
# print(args.config)
args = get_args()
config = process_config_UtsClassification_bayes_optimization(args.config, learning_rate)
create_dirs([config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir,
config.log_dir, config.result_dir])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
print('done')
return evaluater.f1
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '2'
try:
RECEIVED_PARAMS = nni.get_next_parameter()
LOG.debug(RECEIVED_PARAMS)
PARAMS = generate_default_params()
PARAMS.update(RECEIVED_PARAMS)
args = get_args()
config = process_config_UtsClassification_bayes_optimization(
args.config, PARAMS)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
nni.report_final_result(evaluater.f1)
print('done')
except Exception as e:
LOG.exception(e)
raise
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
for model_name in ['fcn', 'resnet_v2', 'cnn']:
for learning_rate in [0.001, 0.0005, 0.0001]:
args = get_args()
config = process_config_UtsClassification_grid_search_hyperparamters(
args.config, model_name, learning_rate)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir,
config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config,
data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
print('done')
K.clear_session()
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config = process_config_UtsClassification(args.config)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir,
config.log_dir, config.result_dir])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print("size of dataloader:")
print(sys.getsizeof(data_loader))
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(), data_loader.get_nbclasses())
print("size of model:")
print(sys.getsizeof(model))
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model, data_loader.get_train_data(), config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
print('done')
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# capture the config path from the run arguments
# then process the json configuration file
# try:
split = 10
training_size = []
accuracy = []
precision = []
recall = []
f1 = []
main_dir = ''
args = get_args()
for i in range(split):
config = process_config_VisOverfit(args.config, i)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
total_train_size = data_loader.get_train_size()
total_test_size = data_loader.get_test_size()
print("total_train_size: " + str(total_train_size))
print("total_test_size: " + str(total_test_size))
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
train_size = int(total_train_size / split) * (i + 1)
if i == split - 1:
print("train_size: " + str(total_train_size))
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
main_dir = config.main_dir
else:
print("train_size: " + str(train_size))
train_data = data_loader.get_train_data()
X_train = train_data[0][:train_size, :, :]
y_train = train_data[1][:train_size, :]
trainer = UtsClassificationTrainer(model.model, [X_train, y_train],
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
training_size.append(train_size)
accuracy.append(evaluater.acc)
precision.append(evaluater.precision)
recall.append(evaluater.recall)
f1.append(evaluater.f1)
print("ss")
metrics = {
"accuracy": accuracy,
"precision": precision,
"recall": recall,
"f1": f1,
"training_size": training_size
}
plot_trainingsize_metric(metrics,
main_dir + 'vis_overfit_trainingsize.png')
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# capture the config path from the run arguments
# then process the json configuration file
# try:
split = 10
training_size = []
best_model_train_loss = []
best_model_val_loss = []
best_model_train_acc = []
best_model_val_acc = []
best_model_train_precision = []
best_model_val_precision = []
best_model_train_recall = []
best_model_val_recall = []
best_model_train_f1 = []
best_model_val_f1 = []
best_model_learning_rate = []
best_model_nb_epoch = []
best_model_train_err = []
best_model_val_err = []
main_dir = ''
args = get_args()
for i in range(split):
config = process_config_VisTrainingSize(args.config, i)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
total_train_size = data_loader.get_train_size()
total_test_size = data_loader.get_test_size()
print("total_train_size: " + str(total_train_size))
print("total_test_size: " + str(total_test_size))
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
train_size = int(total_train_size / split) * (i + 1)
if i == split - 1:
print("train_size: " + str(total_train_size))
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
main_dir = config.main_dir
else:
print("train_size: " + str(train_size))
train_data = data_loader.get_train_data()
X_train = train_data[0][:train_size, :, :]
y_train = train_data[1][:train_size, :]
trainer = UtsClassificationTrainer(model.model, [X_train, y_train],
config)
print('Start training the model.')
trainer.train()
best_model_train_loss.append(trainer.best_model_train_loss)
best_model_val_loss.append(trainer.best_model_val_loss)
best_model_train_acc.append(trainer.best_model_train_acc)
best_model_val_acc.append(trainer.best_model_val_acc)
best_model_train_precision.append(trainer.best_model_train_precision)
best_model_val_precision.append(trainer.best_model_val_precision)
best_model_train_recall.append(trainer.best_model_train_recall)
best_model_val_recall.append(trainer.best_model_val_recall)
best_model_train_f1.append(trainer.best_model_train_f1)
best_model_val_f1.append(trainer.best_model_val_f1)
best_model_learning_rate.append(trainer.best_model_learning_rate)
best_model_nb_epoch.append(trainer.best_model_nb_epoch)
best_model_train_err.append(1 - trainer.best_model_train_acc)
best_model_val_err.append(1 - trainer.best_model_val_acc)
training_size.append(train_size)
print("ss")
metrics = {
"training_size": training_size,
"train_err": best_model_train_err,
"val_err": best_model_val_err
}
plot_trainingsize_metric(metrics,
main_dir + '/vis_overfit_trainingsize.png')
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '2'
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
bayes_config = {
"algorithm": "bayes",
"parameters": {
# "model": {"type": "categorical", "values": ['cnn','mlp']},
"learning_rate": {
"type": "float",
"min": 0.001,
"max": 0.01
},
# "batch_size": {"type": "integer", "min": 16, "max": 32},
# "num_epochs": {"type": "integer", "min": 5, "max": 10},
},
"spec": {
"maxCombo": 10,
"objective": "minimize",
"metric": "test_f1",
"minSampleSize": 100,
"retryAssignLimit": 0,
},
"trials": 1,
"name": "Bayes",
}
opt = Optimizer(bayes_config,
api_key=config.comet_api_key,
project_name=config.exp_name)
for exp in opt.get_experiments():
args = get_args()
# config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('model'),exp.get_parameter('learning_rate'),
# exp.get_parameter('batch_size'), exp.get_parameter('num_epochs'))
config = process_config_UtsClassification_bayes_optimization(
args.config, exp.get_parameter('learning_rate'))
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
# print('Create the evaluater.')
# evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
# config)
#
# print('Start evaluating the model.')
# evaluater.evluate()
exp.log_metric("test_f1", trainer.best_model_val_loss)
print('done')