def ab_test_2d(self, args, params_a, params_b): '''A/B Test between two different architectures. Trains two different models and compares their performance. Arguments: args: the arguments namespace params_a: The hyperparameters for architecture A params_b: The hyperparameters for architecture B ''' train_paths, valid_paths, test_paths = td.get_train_valid_test_paths( args) in_channels = defines.total_input_channels_from_args(args) if args.channels_last: tensor_shape = (args.read_limit, args.window_size, in_channels) else: tensor_shape = (in_channels, args.read_limit, args.window_size) generate_train = td.tensor_annotation_generator( args, train_paths, tensor_shape) generate_valid = td.tensor_annotation_generator( args, valid_paths, tensor_shape) test = td.load_tensors_and_annotations_from_class_dirs( args, test_paths, per_class_max=args.samples) model_a = self.model_from_params_2d(args, params_a) weight_path = './weights/hyper_opt_a.hd5' model = models.train_model_from_generators(args, model_a, generate_train, generate_valid, weight_path) plots.print_auc_per_class(model_a, [test[0], test[1]], test[2], args.labels) self.performances[str(params_a)] = plots.get_auc( model_a, [test[0], test[1]], test[2], args.labels) model_b = self.model_from_params_2d(args, params_b) weight_path = './weights/hyper_opt_b.hd5' model = models.train_model_from_generators(args, model_b, generate_train, generate_valid, weight_path) plots.print_auc_per_class(model_b, [test[0], test[1]], test[2], args.labels) self.performances[str(params_b)] = plots.get_auc( model_b, [test[0], test[1]], test[2], args.labels) self.write_results_to_file('./param_ab_test_' + args.id + '.txt') for k, v in sorted(self.performances.items()): print(k, '\nGot AUC:', self.performances[k])
def loss_from_params_1d(x): p = x[0] conv_layers = self.conv_layers_sets[int( p[param_keys['conv_layers']])] max_pool_set = self.max_pool_sets_1d[int( p[param_keys['max_pools_1d']])] fc_layers = self.fc_layer_sets[int(p[param_keys['fc']])] try: model = models.build_reference_1d_model_from_args( args, conv_width=int(p[param_keys['conv_width']]), conv_layers=conv_layers, #conv_dropout = float(p[param_keys['conv_dropout']]), #conv_batch_normalize = bool(p[param_keys['conv_batch_normalize']]), #spatial_dropout = bool(p[param_keys['spatial_dropout']]), max_pools=max_pool_set, padding='valid' if bool(p[param_keys['valid_padding']]) else 'same', fc_layers=fc_layers #fc_dropout = float(p[param_keys['fc_dropout']]) ) if model.count_params() > args.max_parameters: print('Model too big') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same model = models.train_model_from_generators( args, model, generate_train, generate_valid, args.output_dir + args.id + '.hd5') loss_and_metrics = model.evaluate_generator( generate_test, steps=args.validation_steps) stats['count'] += 1 print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'], 'iterations', args.iterations, 'init numdata:', args.patience, 'Model size', model.count_params()) print(self.str_from_params_and_keys(p, param_keys)) if args.inspect_model: image_name = args.id + '_hyper_' + str( stats['count']) + '.png' image_path = image_name if args.image_dir is None else args.image_dir + image_name models.inspect_model(args, model, generate_train, generate_valid, image_path=image_path) limit_mem() return loss_and_metrics[0] except ValueError as e: print( str(e) + '\n Impossible architecture perhaps? return 9e9') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same
def random_search_2d(self, args, iterations): '''Random search in hyperparameter space for good architectures. Create a bunch of random architectures and test their performance. Architectures are created from within the bounds defined at the top of this class. Arguments: iterations: how many architectures to try ''' train_paths, valid_paths, test_paths = td.get_train_valid_test_paths( args) in_channels = defines.total_input_channels_from_args(args) if args.channels_last: tensor_shape = (args.read_limit, args.window_size, in_channels) else: tensor_shape = (in_channels, args.read_limit, args.window_size) generate_train = td.tensor_annotation_generator( args, train_paths, tensor_shape) generate_valid = td.tensor_annotation_generator( args, valid_paths, tensor_shape) test = td.load_tensors_and_annotations_from_class_dirs( args, test_paths, per_class_max=args.samples) for i in range(iterations): try: model, params = self.get_random_architecture(args) except ValueError as e: print( 'value error on architecture, skipping this iteration. Error is:\n', str(e)) continue param_str = 'Iteration: ' + str(i) + '\nParameter set:\n' + str( params) + '\nTotal params:' + str(model.count_params()) print(param_str) weight_path = './weights/hyper_opt_' + str(i) + '.hd5' model = models.train_model_from_generators(args, model, generate_train, generate_valid, weight_path) param_str += plots.string_auc_per_class(model, [test[0], test[1]], test[2], args.labels) plots.print_auc_per_class(model, [test[0], test[1]], test[2], args.labels) self.performances[param_str] = plots.get_auc( model, [test[0], test[1]], test[2], args.labels) self.write_results_to_file('./param_opt_2d_' + args.id + '.txt') for k, v in sorted(self.performances.items(), key=operator.itemgetter(1)): print(k, '\nGot AUC:', self.performances[k])
def loss_from_params_mlp(x): p = x[0] layer_set = self.mlp_layer_sets[int(p[param_keys['mlp_fc']])] try: model = models.annotation_multilayer_perceptron_from_args( args, fc_layers=layer_set, dropout=float(p[param_keys['dropout']]), skip_connection=bool(p[param_keys['annotation_shortcut']]), batch_normalization=bool( p[param_keys['batch_normalization']]), batch_normalize_input=bool( p[param_keys['batch_normalize_input']])) if model.count_params() > args.max_parameters: print('Model too big') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same model = models.train_model_from_generators( args, model, generate_train, generate_valid, args.output_dir + args.id + '.hd5') loss_and_metrics = model.evaluate_generator( generate_test, steps=args.validation_steps) stats['count'] += 1 print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'], 'iterations', args.iterations, 'init numdata:', args.patience, 'Model size', model.count_params()) print(self.str_from_params_and_keys(p, param_keys)) if args.inspect_model: image_name = args.id + '_hyper_' + str( stats['count']) + '.png' image_path = image_name if args.image_dir is None else args.image_dir + image_name models.inspect_model(args, model, generate_train, generate_valid, image_path=image_path) limit_mem() return loss_and_metrics[0] except ValueError as e: print( str(e) + '\n Impossible architecture perhaps? return 9e9') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same
def hp_loss_from_params_2d_anno(x): try: model = models.read_tensor_2d_annotation_model_from_args( args, conv_width=int(x['conv_width']), conv_height=int(x['conv_height']), conv_layers=x['conv_layers'], max_pools=x['max_pools_2d'], padding='valid' if bool(x['valid_padding']) else 'same', kernel_single_channel=bool(x['kernel_single_channel']), annotation_units=int(x['annotation_units']), annotation_shortcut=bool(x['annotation_shortcut']), fc_layers=x['fc']) if model.count_params() > args.max_parameters: print('Model too big') return self.max_loss model = models.train_model_from_generators( args, model, generate_train, generate_valid, args.output_dir + args.id + '.hd5') loss_and_metrics = model.evaluate_generator( generate_test, steps=args.patience) stats['count'] += 1 print('Current architecture: ', self.string_from_arch_dict(x)) print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'], 'iterations', args.iterations, 'Model size', model.count_params()) if args.inspect_model: image_name = args.id + '_hyper_' + str( stats['count']) + '.png' image_path = image_name if args.image_dir is None else args.image_dir + image_name models.inspect_model(args, model, generate_train, generate_valid, image_path=image_path) del model return loss_and_metrics[0] except ValueError as e: print(str(e) + '\n Impossible architecture perhaps?') return self.max_loss
def grid_search_2d(self, args): '''Grid search in hyperparameter space over convolution size and max pooling tuples. Grid search is exponentially slow so this is only practical for small sets of hyperparameters. See random_search_2d() below for a much faster hyperparameter optimizer ''' train_paths, valid_paths, test_paths = td.get_train_valid_test_paths( args) in_channels = defines.total_input_channels_from_args(args) if args.channels_last: tensor_shape = (args.read_limit, args.window_size, in_channels) else: tensor_shape = (in_channels, args.read_limit, args.window_size) generate_train = td.tensor_annotation_generator( args, train_paths, tensor_shape) generate_valid = td.tensor_annotation_generator( args, valid_paths, tensor_shape) test = td.load_tensors_and_annotations_from_class_dirs( args, test_paths, per_class_max=args.samples) for c in self.conv_layers_sets: for m in self.max_pool_sets_2d: args.id = 'epochs_' + str(args.epochs) + '_' + args.node for units in c: args.id += '_' + str(units) args.id += '_pools' for pools in m: args.id += '_' + str(pools[0]) + '-' + str(pools[1]) weight_path = './weights/' + args.id + '.hd5' model = models.read_tensor_2d_annotation_model_from_args( args, conv_width=6, conv_layers=c, conv_dropout=0.1, max_pools=m, padding='valid', annotation_units=16, fc_layers=[64], fc_dropout=0.3) model = models.train_model_from_generators( args, model, generate_train, generate_valid, weight_path) plots.plot_roc_per_class(model, [test[0], test[1]], test[2], args.labels, args.id)
def loss_from_params_mlp(x): try: model = models.annotation_multilayer_perceptron_from_args( args, fc_layers=layer_set, #dropout = float(x['dropout']), skip_connection=bool(x['shortcut']), batch_normalization=bool(x['batch_normalization']), batch_normalize_input=bool(x['batch_normalize_input'])) if model.count_params() > args.max_parameters: print('Model too big') return self.max_loss model = models.train_model_from_generators( args, model, generate_train, generate_valid, args.output_dir + args.id + '.hd5') loss_and_metrics = model.evaluate_generator( generate_test, steps=args.patience) stats['count'] += 1 print('Current architecture: ', self.string_from_arch_dict(x)) print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'], 'iterations', args.iterations, 'Model size', model.count_params()) if args.inspect_model: image_name = args.id + '_hyper_' + str( stats['count']) + '.png' image_path = image_name if args.image_dir is None else args.image_dir + image_name models.inspect_model(args, model, generate_train, generate_valid, image_path=image_path) del model return loss_and_metrics[0] except ValueError as e: print( str(e) + '\n Impossible architecture perhaps? return 9e9') return self.max_loss trials = hyperopt.Trials()
def loss_from_params_2d_anno(x): p = x[0] fc_layers = self.fc_layer_sets[int(p[param_keys['fc']])] conv_layers = self.conv_layers_sets[int( p[param_keys['conv_layers']])] max_pool_set = self.max_pool_sets_2d[int( p[param_keys['max_pools_2d']])] #residual_layers = self.residual_layers_sets[int(p[param_keys['residual_layers']])] try: print(self.str_from_params_and_keys(p, param_keys)) model = models.read_tensor_2d_annotation_model_from_args( args, conv_width=int(p[param_keys['conv_width']]), conv_height=int(p[param_keys['conv_height']]), conv_layers=conv_layers, max_pools=max_pool_set, padding='valid' if bool(p[param_keys['valid_padding']]) else 'same', kernel_single_channel=bool( p[param_keys['kernel_single_channel']]), #annotation_units = int(p[param_keys['annotation_units']]), annotation_shortcut=bool( p[param_keys['annotation_shortcut']]), fc_layers=fc_layers, ) if model.count_params() > args.max_parameters: print('Model too big') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same model = models.train_model_from_generators( args, model, generate_train, generate_valid, args.output_dir + args.id + '.hd5') loss_and_metrics = model.evaluate_generator( generate_test, steps=args.validation_steps) stats['count'] += 1 print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'], 'iterations', args.iterations, 'init numdata:', args.patience, 'Model size', model.count_params()) print(self.str_from_params_and_keys(p, param_keys)) if args.inspect_model: image_name = args.id + '_hyper_' + str( stats['count']) + '.png' image_path = image_name if args.image_dir is None else args.image_dir + image_name models.inspect_model(args, model, generate_train, generate_valid, image_path=image_path) limit_mem() return loss_and_metrics[0] except ValueError as e: print(str(e) + '\n Impossible architecture perhaps?') return np.random.uniform( 100, 10000 ) # this is ugly but optimization quits when loss is the same