def run_model_multiple_times(dataset_dict, dataset_folds_list, num_of_model_runs, model_params, results_dir_path ='./results'):
global log_num
best_index_l =[]
final_train_acc_l = []
final_valid_acc_l = []
final_test_acc_l = []
# TODO: move get model outside the loop, create a logger function that can change logging file destination which is
# all that need to change between runs and then no need for log_num
# https://stackoverflow.com/questions/13839554/how-to-change-filehandle-with-python-logging-on-the-fly-with-different-classes-a
for j in xrange(len(dataset_folds_list)):
dataset_dict['fold'] = dataset_folds_list[j]
for i in xrange(num_of_model_runs):
logger = my_utilities.set_a_logger(str(log_num), dirpath=results_dir_path, filename='run_{}_fold_{}.log'.format(i,j),
console_level=logging.DEBUG, file_level=logging.DEBUG)
log_num += 1
logger.info('Start logging')
logger.info('########## Number of model run: {0} ##########'.format(i))
# TODO: change Dataset so we could do unshuffle and then move it outside the for loop
dataset = parse_image_seg.Dataset(dataset_dict)
model = get_model(dataset, logger, model_params)
with model:
init_model(model)
index, train_acc_at_ind, valid_acc_ma_at_ind, test_acc_at_ind = run_model(model)
best_index_l.append(index)
final_train_acc_l.append(train_acc_at_ind)
final_valid_acc_l.append(valid_acc_ma_at_ind)
final_test_acc_l.append(test_acc_at_ind)
return best_index_l, final_train_acc_l, final_valid_acc_l, final_test_acc_l
def test_model_vars_after_run(self):
args = parser.parse_args()
json_path = os.path.join(args.params_dir, 'image_segmentation_params.json')
assert os.path.isfile(json_path), "No json configuration file found at {}".format(json_path)
dataset_params = param_manager.DatasetParams()
dataset_params.update(json_path)
dataset_dict = dataset_params.dict
dataset = parse_image_seg.Dataset(dataset_dict)
checkpoint_path = "./results/unitest2.ckpt"
model = NeuralNet(dataset, self.logger, self.params)
with model:
model.build_model()
model.train_model()
self.compare_to_ckpt(model, checkpoint_path)
model_params.update(json_path)
params = model_params.dict
# params = param_manager.ModelParams.create_model_params(batch_norm=1)
# params['number of epochs'] = 5
# params['check point flag'] = 1
# params['check point name'] = './results/unitest2'
# params['batch norm'] = 0
# params['activation'] = 'ELU'
json_path = os.path.join(args.params_dir, 'image_segmentation_params.json')
# json_path = os.path.join(args.params_dir, 'abalone.json')
# json_path = os.path.join(args.params_dir, 'car.json')
assert os.path.isfile(
json_path), "No json configuration file found at {}".format(json_path)
dataset_params = param_manager.DatasetParams()
dataset_params.update(json_path)
dataset_dict = dataset_params.dict
# dataset_dict['fold'] = 2
dataset = parse_image_seg.Dataset(dataset_dict)
model = NeuralNet(dataset, logger, params)
with model:
model.build_model()
train_acc_l, valid_acc_l, test_acc_l = model.train_model()
# train_acc, valid_acc, test_acc = model.eval_model()
index, train_acc_at_ind, valid_acc_ma_at_ind, test_acc_at_ind = model.find_best_accuracy(
train_acc_l, valid_acc_l, test_acc_l)
if model.params['check point flag']:
model.save_variables(model.params['check point name'])