def ab_test_2d(self, args, params_a, params_b):
'''A/B Test between two different architectures.
Trains two different models and compares their performance.
Arguments:
args: the arguments namespace
params_a: The hyperparameters for architecture A
params_b: The hyperparameters for architecture B
'''
train_paths, valid_paths, test_paths = td.get_train_valid_test_paths(
args)
in_channels = defines.total_input_channels_from_args(args)
if args.channels_last:
tensor_shape = (args.read_limit, args.window_size, in_channels)
else:
tensor_shape = (in_channels, args.read_limit, args.window_size)
generate_train = td.tensor_annotation_generator(
args, train_paths, tensor_shape)
generate_valid = td.tensor_annotation_generator(
args, valid_paths, tensor_shape)
test = td.load_tensors_and_annotations_from_class_dirs(
args, test_paths, per_class_max=args.samples)
model_a = self.model_from_params_2d(args, params_a)
weight_path = './weights/hyper_opt_a.hd5'
model = models.train_model_from_generators(args, model_a,
generate_train,
generate_valid, weight_path)
plots.print_auc_per_class(model_a, [test[0], test[1]], test[2],
args.labels)
self.performances[str(params_a)] = plots.get_auc(
model_a, [test[0], test[1]], test[2], args.labels)
model_b = self.model_from_params_2d(args, params_b)
weight_path = './weights/hyper_opt_b.hd5'
model = models.train_model_from_generators(args, model_b,
generate_train,
generate_valid, weight_path)
plots.print_auc_per_class(model_b, [test[0], test[1]], test[2],
args.labels)
self.performances[str(params_b)] = plots.get_auc(
model_b, [test[0], test[1]], test[2], args.labels)
self.write_results_to_file('./param_ab_test_' + args.id + '.txt')
for k, v in sorted(self.performances.items()):
print(k, '\nGot AUC:', self.performances[k])
def loss_from_params_1d(x):
p = x[0]
conv_layers = self.conv_layers_sets[int(
p[param_keys['conv_layers']])]
max_pool_set = self.max_pool_sets_1d[int(
p[param_keys['max_pools_1d']])]
fc_layers = self.fc_layer_sets[int(p[param_keys['fc']])]
try:
model = models.build_reference_1d_model_from_args(
args,
conv_width=int(p[param_keys['conv_width']]),
conv_layers=conv_layers,
#conv_dropout = float(p[param_keys['conv_dropout']]),
#conv_batch_normalize = bool(p[param_keys['conv_batch_normalize']]),
#spatial_dropout = bool(p[param_keys['spatial_dropout']]),
max_pools=max_pool_set,
padding='valid'
if bool(p[param_keys['valid_padding']]) else 'same',
fc_layers=fc_layers
#fc_dropout = float(p[param_keys['fc_dropout']])
)
if model.count_params() > args.max_parameters:
print('Model too big')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
model = models.train_model_from_generators(
args, model, generate_train, generate_valid,
args.output_dir + args.id + '.hd5')
loss_and_metrics = model.evaluate_generator(
generate_test, steps=args.validation_steps)
stats['count'] += 1
print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'],
'iterations', args.iterations, 'init numdata:',
args.patience, 'Model size', model.count_params())
print(self.str_from_params_and_keys(p, param_keys))
if args.inspect_model:
image_name = args.id + '_hyper_' + str(
stats['count']) + '.png'
image_path = image_name if args.image_dir is None else args.image_dir + image_name
models.inspect_model(args,
model,
generate_train,
generate_valid,
image_path=image_path)
limit_mem()
return loss_and_metrics[0]
except ValueError as e:
print(
str(e) + '\n Impossible architecture perhaps? return 9e9')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
def random_search_2d(self, args, iterations):
'''Random search in hyperparameter space for good architectures.
Create a bunch of random architectures and test their performance.
Architectures are created from within the bounds defined at the top of this class.
Arguments:
iterations: how many architectures to try
'''
train_paths, valid_paths, test_paths = td.get_train_valid_test_paths(
args)
in_channels = defines.total_input_channels_from_args(args)
if args.channels_last:
tensor_shape = (args.read_limit, args.window_size, in_channels)
else:
tensor_shape = (in_channels, args.read_limit, args.window_size)
generate_train = td.tensor_annotation_generator(
args, train_paths, tensor_shape)
generate_valid = td.tensor_annotation_generator(
args, valid_paths, tensor_shape)
test = td.load_tensors_and_annotations_from_class_dirs(
args, test_paths, per_class_max=args.samples)
for i in range(iterations):
try:
model, params = self.get_random_architecture(args)
except ValueError as e:
print(
'value error on architecture, skipping this iteration. Error is:\n',
str(e))
continue
param_str = 'Iteration: ' + str(i) + '\nParameter set:\n' + str(
params) + '\nTotal params:' + str(model.count_params())
print(param_str)
weight_path = './weights/hyper_opt_' + str(i) + '.hd5'
model = models.train_model_from_generators(args, model,
generate_train,
generate_valid,
weight_path)
param_str += plots.string_auc_per_class(model, [test[0], test[1]],
test[2], args.labels)
plots.print_auc_per_class(model, [test[0], test[1]], test[2],
args.labels)
self.performances[param_str] = plots.get_auc(
model, [test[0], test[1]], test[2], args.labels)
self.write_results_to_file('./param_opt_2d_' + args.id + '.txt')
for k, v in sorted(self.performances.items(),
key=operator.itemgetter(1)):
print(k, '\nGot AUC:', self.performances[k])
def loss_from_params_mlp(x):
p = x[0]
layer_set = self.mlp_layer_sets[int(p[param_keys['mlp_fc']])]
try:
model = models.annotation_multilayer_perceptron_from_args(
args,
fc_layers=layer_set,
dropout=float(p[param_keys['dropout']]),
skip_connection=bool(p[param_keys['annotation_shortcut']]),
batch_normalization=bool(
p[param_keys['batch_normalization']]),
batch_normalize_input=bool(
p[param_keys['batch_normalize_input']]))
if model.count_params() > args.max_parameters:
print('Model too big')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
model = models.train_model_from_generators(
args, model, generate_train, generate_valid,
args.output_dir + args.id + '.hd5')
loss_and_metrics = model.evaluate_generator(
generate_test, steps=args.validation_steps)
stats['count'] += 1
print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'],
'iterations', args.iterations, 'init numdata:',
args.patience, 'Model size', model.count_params())
print(self.str_from_params_and_keys(p, param_keys))
if args.inspect_model:
image_name = args.id + '_hyper_' + str(
stats['count']) + '.png'
image_path = image_name if args.image_dir is None else args.image_dir + image_name
models.inspect_model(args,
model,
generate_train,
generate_valid,
image_path=image_path)
limit_mem()
return loss_and_metrics[0]
except ValueError as e:
print(
str(e) + '\n Impossible architecture perhaps? return 9e9')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
def hp_loss_from_params_2d_anno(x):
try:
model = models.read_tensor_2d_annotation_model_from_args(
args,
conv_width=int(x['conv_width']),
conv_height=int(x['conv_height']),
conv_layers=x['conv_layers'],
max_pools=x['max_pools_2d'],
padding='valid' if bool(x['valid_padding']) else 'same',
kernel_single_channel=bool(x['kernel_single_channel']),
annotation_units=int(x['annotation_units']),
annotation_shortcut=bool(x['annotation_shortcut']),
fc_layers=x['fc'])
if model.count_params() > args.max_parameters:
print('Model too big')
return self.max_loss
model = models.train_model_from_generators(
args, model, generate_train, generate_valid,
args.output_dir + args.id + '.hd5')
loss_and_metrics = model.evaluate_generator(
generate_test, steps=args.patience)
stats['count'] += 1
print('Current architecture: ', self.string_from_arch_dict(x))
print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'],
'iterations', args.iterations, 'Model size',
model.count_params())
if args.inspect_model:
image_name = args.id + '_hyper_' + str(
stats['count']) + '.png'
image_path = image_name if args.image_dir is None else args.image_dir + image_name
models.inspect_model(args,
model,
generate_train,
generate_valid,
image_path=image_path)
del model
return loss_and_metrics[0]
except ValueError as e:
print(str(e) + '\n Impossible architecture perhaps?')
return self.max_loss
def grid_search_2d(self, args):
'''Grid search in hyperparameter space over convolution size and max pooling tuples.
Grid search is exponentially slow so this is only practical for small sets of hyperparameters.
See random_search_2d() below for a much faster hyperparameter optimizer
'''
train_paths, valid_paths, test_paths = td.get_train_valid_test_paths(
args)
in_channels = defines.total_input_channels_from_args(args)
if args.channels_last:
tensor_shape = (args.read_limit, args.window_size, in_channels)
else:
tensor_shape = (in_channels, args.read_limit, args.window_size)
generate_train = td.tensor_annotation_generator(
args, train_paths, tensor_shape)
generate_valid = td.tensor_annotation_generator(
args, valid_paths, tensor_shape)
test = td.load_tensors_and_annotations_from_class_dirs(
args, test_paths, per_class_max=args.samples)
for c in self.conv_layers_sets:
for m in self.max_pool_sets_2d:
args.id = 'epochs_' + str(args.epochs) + '_' + args.node
for units in c:
args.id += '_' + str(units)
args.id += '_pools'
for pools in m:
args.id += '_' + str(pools[0]) + '-' + str(pools[1])
weight_path = './weights/' + args.id + '.hd5'
model = models.read_tensor_2d_annotation_model_from_args(
args,
conv_width=6,
conv_layers=c,
conv_dropout=0.1,
max_pools=m,
padding='valid',
annotation_units=16,
fc_layers=[64],
fc_dropout=0.3)
model = models.train_model_from_generators(
args, model, generate_train, generate_valid, weight_path)
plots.plot_roc_per_class(model, [test[0], test[1]], test[2],
args.labels, args.id)
def loss_from_params_mlp(x):
try:
model = models.annotation_multilayer_perceptron_from_args(
args,
fc_layers=layer_set,
#dropout = float(x['dropout']),
skip_connection=bool(x['shortcut']),
batch_normalization=bool(x['batch_normalization']),
batch_normalize_input=bool(x['batch_normalize_input']))
if model.count_params() > args.max_parameters:
print('Model too big')
return self.max_loss
model = models.train_model_from_generators(
args, model, generate_train, generate_valid,
args.output_dir + args.id + '.hd5')
loss_and_metrics = model.evaluate_generator(
generate_test, steps=args.patience)
stats['count'] += 1
print('Current architecture: ', self.string_from_arch_dict(x))
print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'],
'iterations', args.iterations, 'Model size',
model.count_params())
if args.inspect_model:
image_name = args.id + '_hyper_' + str(
stats['count']) + '.png'
image_path = image_name if args.image_dir is None else args.image_dir + image_name
models.inspect_model(args,
model,
generate_train,
generate_valid,
image_path=image_path)
del model
return loss_and_metrics[0]
except ValueError as e:
print(
str(e) + '\n Impossible architecture perhaps? return 9e9')
return self.max_loss
trials = hyperopt.Trials()
def loss_from_params_2d_anno(x):
p = x[0]
fc_layers = self.fc_layer_sets[int(p[param_keys['fc']])]
conv_layers = self.conv_layers_sets[int(
p[param_keys['conv_layers']])]
max_pool_set = self.max_pool_sets_2d[int(
p[param_keys['max_pools_2d']])]
#residual_layers = self.residual_layers_sets[int(p[param_keys['residual_layers']])]
try:
print(self.str_from_params_and_keys(p, param_keys))
model = models.read_tensor_2d_annotation_model_from_args(
args,
conv_width=int(p[param_keys['conv_width']]),
conv_height=int(p[param_keys['conv_height']]),
conv_layers=conv_layers,
max_pools=max_pool_set,
padding='valid'
if bool(p[param_keys['valid_padding']]) else 'same',
kernel_single_channel=bool(
p[param_keys['kernel_single_channel']]),
#annotation_units = int(p[param_keys['annotation_units']]),
annotation_shortcut=bool(
p[param_keys['annotation_shortcut']]),
fc_layers=fc_layers,
)
if model.count_params() > args.max_parameters:
print('Model too big')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
model = models.train_model_from_generators(
args, model, generate_train, generate_valid,
args.output_dir + args.id + '.hd5')
loss_and_metrics = model.evaluate_generator(
generate_test, steps=args.validation_steps)
stats['count'] += 1
print('Loss:', loss_and_metrics[0], '\nCount:', stats['count'],
'iterations', args.iterations, 'init numdata:',
args.patience, 'Model size', model.count_params())
print(self.str_from_params_and_keys(p, param_keys))
if args.inspect_model:
image_name = args.id + '_hyper_' + str(
stats['count']) + '.png'
image_path = image_name if args.image_dir is None else args.image_dir + image_name
models.inspect_model(args,
model,
generate_train,
generate_valid,
image_path=image_path)
limit_mem()
return loss_and_metrics[0]
except ValueError as e:
print(str(e) + '\n Impossible architecture perhaps?')
return np.random.uniform(
100, 10000
) # this is ugly but optimization quits when loss is the same
