def build_callbacks(config, generator, n_samples, other_generators=None):
# For this model, we want to monitor F1 for early stopping and
# model checkpointing. The way to do that is for the metrics callback
# compute F1, put it in the logs dictionary that's passed to
# on_epoch_end, and to pass that to the early stopping and model
# checkpointing callbacks.
controller_callbacks = []
controller_callbacks.append(
keras.callbacks.EarlyStopping(
monitor=config.callback_monitor,
mode=config.callback_monitor_mode,
patience=config.patience,
verbose=1))
if 'persistent' in config.mode:
controller_callbacks.append(
keras.callbacks.ModelCheckpoint(
monitor=config.callback_monitor,
mode=config.callback_monitor_mode,
filepath=config.model_path + config.model_checkpoint_file,
save_best_only=config.save_best_only,
verbose=1))
controller = MetricsCallback(config, generator, n_samples,
callbacks=controller_callbacks,
other_generators=other_generators)
callbacks = []
callbacks.append(controller)
callbacks.append(modeling.callbacks.DenseWeightNormCallback(config))
return callbacks
def build_callbacks(config, generator, n_samples, dictionary, target_map):
callbacks = []
mc = MetricsCallback(config, generator, n_samples, dictionary, target_map)
wn = DenseWeightNormCallback(config)
es = keras.callbacks.EarlyStopping(patience=config.patience, verbose=1)
callbacks.extend([mc, wn, es])
if 'persistent' in config.mode:
cp = keras.callbacks.ModelCheckpoint(
filepath=config.model_path + 'model.h5',
save_best_only=True)
callbacks.append(cp)
return callbacks
def build_callbacks(model, config, X_validation=None, y_validation=None, marshaller=None):
callbacks = []
callbacks.append(keras.callbacks.EarlyStopping(
patience=config.patience, verbose=1))
callbacks.append(DenseWeightNormCallback(config))
if 'persistent' in config.mode:
callbacks.append(keras.callbacks.ModelCheckpoint(
filepath=os.path.join(config.model_path, config.checkpoint_name),
save_best_only=config.save_best_only,
verbose=1))
pred_cb = PredictionCallback(
X_validation,
logger=config.logger,
marshaller=marshaller,
batch_size=config.batch_size)
pred_callbacks = []
if config.confusion_matrix:
if isinstance(model, keras.models.Graph):
assert marshaller is not None
assert X_validation is not None
assert y_validation is not None
confusion_cb = ConfusionMatrix(
X_validation, y_validation,
logger=config.logger,
marshaller=marshaller)
pred_callbacks.append(confusion_cb)
if config.classification_report:
if isinstance(model, keras.models.Graph):
assert marshaller is not None
assert X_validation is not None
assert y_validation is not None
report_cb = ClassificationReport(
X_validation,
y_validation,
logger=config.logger,
marshaller=marshaller)
pred_callbacks.append(report_cb)
for pcb in pred_callbacks:
pred_cb.add(pcb)
if len(pred_callbacks):
callbacks.append(pred_cb)
return callbacks
def build_callbacks(config, generator, n_samples, dictionary, target_map, pool):
callbacks = []
mc = MetricsCallback(config, generator, n_samples, dictionary, target_map)
wn = DenseWeightNormCallback(config)
es = keras.callbacks.EarlyStopping(patience=config.patience, verbose=1)
callbacks.extend([mc, wn, es])
if 'persistent' in config.mode:
cp = keras.callbacks.ModelCheckpoint(
filepath=config.model_path + 'model.h5',
save_best_only=True)
callbacks.append(cp)
if pool is not None:
cc = CurriculumCallback(config.logger, pool,
threshold=config.length_curriculum_change_threshold,
frequency=config.length_curriculum_change_frequency,
monitor=config.length_curriculum_monitor)
callbacks.append(cc)
return callbacks
def main(args):
model_id = build_model_id(args)
model_path = build_model_path(args, model_id)
setup_model_dir(args, model_path)
sys.stdout, sys.stderr = setup_logging(args, model_path)
x_train, y_train = load_model_data(args.train_file,
args.data_name, args.target_name)
x_validation, y_validation = load_model_data(
args.validation_file,
args.data_name, args.target_name)
rng = np.random.RandomState(args.seed)
if args.n_classes > -1:
n_classes = args.n_classes
else:
n_classes = max(y_train)+1
n_classes, target_names, class_weight = load_target_data(args, n_classes)
if len(class_weight) == 0:
n_samples = len(y_train)
print('n_samples', n_samples)
print('classes', range(n_classes))
print('weights', n_samples / (n_classes * np.bincount(y_train)))
class_weight = dict(zip(range(n_classes),
n_samples / (n_classes * np.bincount(y_train))))
print('class_weight', class_weight)
logging.debug("n_classes {0} min {1} max {2}".format(
n_classes, min(y_train), max(y_train)))
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
min_vocab_index = np.min(x_train)
max_vocab_index = np.max(x_train)
logging.debug("min vocab index {0} max vocab index {1}".format(
min_vocab_index, max_vocab_index))
json_cfg = load_model_json(args, x_train, n_classes)
logging.debug("loading model")
sys.path.append(args.model_dir)
import model
from model import build_model
#######################################################################
# Subsetting
#######################################################################
if args.subsetting_function:
subsetter = getattr(model, args.subsetting_function)
else:
subsetter = None
def take_subset(subsetter, path, x, y, y_one_hot, n):
if subsetter is None:
return x[0:n], y[0:n], y_one_hot[0:n]
else:
mask = subsetter(path)
idx = np.where(mask)[0]
idx = idx[0:n]
return x[idx], y[idx], y_one_hot[idx]
x_train, y_train, y_train_one_hot = take_subset(
subsetter, args.train_file,
x_train, y_train, y_train_one_hot,
n=args.n_train)
x_validation, y_validation, y_validation_one_hot = take_subset(
subsetter, args.validation_file,
x_validation, y_validation, y_validation_one_hot,
n=args.n_validation)
#######################################################################
# Preprocessing
#######################################################################
if args.preprocessing_class:
preprocessor = getattr(model, args.preprocessing_class)(seed=args.seed)
else:
preprocessor = modeling.preprocess.NullPreprocessor()
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
model_cfg = ModelConfig(**json_cfg)
logging.info("model_cfg " + str(model_cfg))
model = build_model(model_cfg)
setattr(model, 'stop_training', False)
logging.info('model has {n_params} parameters'.format(
n_params=count_parameters(model)))
if len(args.extra_train_file) > 1:
callbacks = keras.callbacks.CallbackList()
else:
callbacks = []
save_model_info(args, model_path, model_cfg)
if not args.no_save:
if args.save_all_checkpoints:
filepath = model_path + '/model-{epoch:04d}.h5'
else:
filepath = model_path + '/model.h5'
callbacks.append(ModelCheckpoint(
filepath=filepath,
verbose=1,
save_best_only=not args.save_every_epoch))
callback_logger = logging.info if args.log else callable_print
if args.n_epochs < sys.maxsize:
# Number of epochs overrides patience. If the number of epochs
# is specified on the command line, the model is trained for
# exactly that number; otherwise, the model is trained with
# early stopping using the patience specified in the model
# configuration.
callbacks.append(EarlyStopping(
monitor='val_loss', patience=model_cfg.patience, verbose=1))
if args.classification_report:
cr = ClassificationReport(x_validation, y_validation,
callback_logger,
target_names=target_names)
callbacks.append(cr)
if model_cfg.optimizer == 'SGD':
callbacks.append(SingleStepLearningRateSchedule(patience=10))
if len(args.extra_train_file) > 1:
args.extra_train_file.append(args.train_file)
logging.info("Using the following files for training: " +
','.join(args.extra_train_file))
train_file_iter = itertools.cycle(args.extra_train_file)
current_train = args.train_file
callbacks._set_model(model)
callbacks.on_train_begin(logs={})
epoch = batch = 0
while True:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
iteration = batch % len(args.extra_train_file)
logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
epoch=epoch, iteration=iteration, train_file=current_train))
callbacks.on_epoch_begin(epoch, logs={})
n_train = x_train.shape[0]
callbacks.on_batch_begin(batch, logs={'size': n_train})
index_array = np.arange(n_train)
if args.shuffle:
rng.shuffle(index_array)
batches = keras.models.make_batches(n_train, model_cfg.batch_size)
logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
avg_train_loss = avg_train_accuracy = 0.
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(model, keras.models.Graph):
data = {
'input': x_train[batch_ids],
'output': y_train_one_hot[batch_ids]
}
train_loss = model.train_on_batch(data, class_weight=class_weight)
train_accuracy = 0.
else:
train_loss, train_accuracy = model.train_on_batch(
x_train[batch_ids], y_train_one_hot[batch_ids],
accuracy=True, class_weight=class_weight)
batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}
avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)
callbacks.on_batch_end(batch,
logs={'loss': train_loss, 'accuracy': train_accuracy})
logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))
batch += 1
# Validation frequency (this if-block) doesn't necessarily
# occur in the same iteration as beginning of an epoch
# (next if-block), so model.evaluate appears twice here.
kwargs = { 'verbose': 0 if args.log else 1 }
pargs = []
validation_data = {}
if isinstance(model, keras.models.Graph):
validation_data = {
'input': x_validation,
'output': y_validation_one_hot
}
pargs = [validation_data]
else:
pargs = [x_validation, y_validation_one_hot]
kwargs['show_accuracy'] = True
if (iteration + 1) % args.validation_freq == 0:
if isinstance(model, keras.models.Graph):
val_loss = model.evaluate(*pargs, **kwargs)
y_hat = model.predict(validation_data)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = model.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
callbacks.on_epoch_end(epoch, epoch_end_logs)
if batch % len(args.extra_train_file) == 0:
if isinstance(model, keras.models.Graph):
val_loss = model.evaluate(*pargs, **kwargs)
y_hat = model.predict(validation_data)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = model.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
epoch += 1
callbacks.on_epoch_end(epoch, epoch_end_logs)
if model.stop_training:
logging.info("epoch {epoch} iteration {iteration} - done training".format(
epoch=epoch, iteration=iteration))
break
current_train = next(train_file_iter)
x_train, y_train = load_model_data(current_train,
args.data_name, args.target_name)
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
if epoch > args.n_epochs:
break
callbacks.on_train_end(logs={})
else:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
if isinstance(model, keras.models.Graph):
data = {
'input': x_train,
'output': y_train_one_hot
}
validation_data = {
'input': x_validation,
'output': y_validation_one_hot
}
model.fit(data,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
validation_data=validation_data,
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
y_hat = model.predict(validation_data)
print('val_acc %.04f' %
accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1)))
else:
model.fit(x_train, y_train_one_hot,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
show_accuracy=True,
validation_data=(x_validation, y_validation_one_hot),
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
if missinglink_callback.rm_active:
checkpoints_directory = '/output/checkpoints'
if not os.path.exists(checkpoints_directory):
os.mkdir(checkpoints_directory)
checkpoint_format = checkpoints_directory + '/weights_epoch-{epoch:02d}_loss-{loss:.4f}.h5'
save_models_callback = keras.callbacks.ModelCheckpoint(
checkpoint_format,
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=5,
)
callbacks.append(save_models_callback)
tensor_board_path = '/output/tensorboard'
save_tb_callback = keras.callbacks.TensorBoard(log_dir=tensor_board_path)
callbacks.append(save_tb_callback)
def get_transfer_model():
# import inception with pre-trained weights. do not include fully #connected layers
if MODEL == "resnet50":
base_model = applications.ResNet50(weights='imagenet',
include_top=False)
elif MODEL == "mobilenet":
base_model = applications.MobileNet(weights='imagenet',
include_top=False,
input_shape=input_shape)
def minimize(self, seed_input=None, max_iter=200,
input_modifiers=None, grad_modifier=None,
callbacks=None, verbose=True):
"""Performs gradient descent on the input image with respect to defined losses.
Args:
seed_input: An N-dim numpy array of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
Seeded with random noise if set to None. (Default value = None)
max_iter: The maximum number of gradient descent iterations. (Default value = 200)
input_modifiers: A list of [InputModifier](vis.input_modifiers#inputmodifier) instances specifying
how to make `pre` and `post` changes to the optimized input during the optimization process.
`pre` is applied in list order while `post` is applied in reverse order. For example,
`input_modifiers = [f, g]` means that `pre_input = g(f(inp))` and `post_input = f(g(inp))`
grad_modifier: gradient modifier to use. See [grad_modifiers](vis.grad_modifiers.md). If you don't
specify anything, gradients are unchanged. (Default value = None)
callbacks: A list of [OptimizerCallback](vis.callbacks#optimizercallback) instances to trigger.
verbose: Logs individual losses at the end of every gradient descent iteration.
Very useful to estimate loss weight factor(s). (Default value = True)
Returns:
The tuple of `(optimized input, grads with respect to wrt, wrt_value)` after gradient descent iterations.
"""
seed_input = self._get_seed_input(seed_input)
input_modifiers = input_modifiers or []
grad_modifier = _identity if grad_modifier is None else get(grad_modifier)
callbacks = callbacks or []
if verbose:
callbacks.append(_PRINT_CALLBACK)
cache = None
best_loss = float('inf')
best_input = None
grads = None
wrt_value = None
all_losses = []
for i in range(max_iter):
# Apply modifiers `pre` step
for modifier in input_modifiers:
seed_input = modifier.pre(seed_input)
# 0 learning phase for 'test'
computed_values = self.compute_fn([seed_input, 0])
losses = computed_values[:len(self.loss_names)]
named_losses = list(zip(self.loss_names, losses))
overall_loss, grads, wrt_value = computed_values[len(self.loss_names):]
# TODO: theano grads shape is inconsistent for some reason. Patch for now and investigate later.
if grads.shape != wrt_value.shape:
grads = np.reshape(grads, wrt_value.shape)
# Apply grad modifier.
grads = grad_modifier(grads)
# Trigger callbacks
for c in (callbacks):
c.callback(i, self, named_losses, overall_loss, grads, wrt_value)
# Gradient descent update.
# It only makes sense to do this if wrt_tensor is input_tensor. Otherwise shapes wont match for the update.
if self.wrt_tensor is self.input_tensor:
step, cache = self._rmsprop(grads, cache)
seed_input += step
# Apply modifiers `post` step
for modifier in reversed(input_modifiers):
seed_input = modifier.post(seed_input)
all_losses.append(named_losses)
if overall_loss < best_loss:
best_loss = overall_loss.copy()
best_input = seed_input.copy()
# Trigger on_end
for c in callbacks:
c.on_end()
img = best_input[0]
#img = utils.deprocess_input(best_input[0], self.input_range)
return img, grads, wrt_value,all_losses,named_losses,overall_loss
model.summary()
## Train the model
epochs = 75
# Save the model architecture
basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(
seed) + '_usdot_real_reluDecode'
json_string = model.to_json()
open('model_dotnum_' + basename + '_arch.json', 'w').write(json_string)
# history = History()
callbacks = [History()]
weights_file = 'model_dotnum_' + basename + '_weights.h5' # where weights will be saved
callbacks.append(
ModelCheckpoint(filepath=weights_file,
monitor='val_loss',
save_best_only=True))
print('\nTraining the model...')
start = time.time()
model.fit(dataTrain,
labelsTrain,
batch_size=32,
epochs=epochs,
verbose=1,
callbacks=callbacks,
validation_split=0.0,
validation_data=(dataVal, labelsVal),
shuffle=True,
class_weight=None,
sample_weight=None,
def main(args):
model_id = build_model_id(args)
model_path = build_model_path(args, model_id)
setup_model_dir(args, model_path)
sys.stdout, sys.stderr = setup_logging(args, model_path)
x_train, y_train = load_model_data(args.train_file,
args.data_name, args.target_name)
x_validation, y_validation = load_model_data(
args.validation_file,
args.data_name, args.target_name)
rng = np.random.RandomState(args.seed)
if args.n_classes > -1:
n_classes = args.n_classes
else:
n_classes = max(y_train)+1
n_classes, target_names, class_weight = load_target_data(args, n_classes)
if class_weight is None and args.class_weight_auto:
n_samples = len(y_train)
weights = float(n_samples) / (n_classes * np.bincount(y_train))
if args.class_weight_exponent:
weights = weights**args.class_weight_exponent
class_weight = dict(zip(range(n_classes), weights))
if args.verbose:
logging.debug("n_classes {0} min {1} max {2}".format(
n_classes, min(y_train), max(y_train)))
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)
if args.verbose:
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
min_vocab_index = np.min(x_train)
max_vocab_index = np.max(x_train)
if args.verbose:
logging.debug("min vocab index {0} max vocab index {1}".format(
min_vocab_index, max_vocab_index))
json_cfg = load_model_json(args, x_train, n_classes)
if args.verbose:
logging.debug("loading model")
sys.path.append(args.model_dir)
import model
from model import build_model
#######################################################################
# Subsetting
#######################################################################
if args.subsetting_function:
subsetter = getattr(M, args.subsetting_function)
else:
subsetter = None
def take_subset(subsetter, path, x, y, y_one_hot, n):
if subsetter is None:
return x[0:n], y[0:n], y_one_hot[0:n]
else:
mask = subsetter(path)
idx = np.where(mask)[0]
idx = idx[0:n]
return x[idx], y[idx], y_one_hot[idx]
x_train, y_train, y_train_one_hot = take_subset(
subsetter, args.train_file,
x_train, y_train, y_train_one_hot,
n=args.n_train)
x_validation, y_validation, y_validation_one_hot = take_subset(
subsetter, args.validation_file,
x_validation, y_validation, y_validation_one_hot,
n=args.n_validation)
#######################################################################
# Preprocessing
#######################################################################
if args.preprocessing_class:
preprocessor = getattr(M, args.preprocessing_class)(seed=args.seed)
else:
preprocessor = modeling.preprocess.NullPreprocessor()
if args.verbose:
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
model_cfg = ModelConfig(**json_cfg)
if args.verbose:
logging.info("model_cfg " + str(model_cfg))
net = build_model(model_cfg)
setattr(net, 'stop_training', False)
marshaller = None
if isinstance(net, keras.models.Graph):
marshaller = getattr(model, args.graph_marshalling_class)()
logging.info('model has {n_params} parameters'.format(
n_params=count_parameters(net)))
if len(args.extra_train_file) > 1:
callbacks = keras.callbacks.CallbackList()
else:
callbacks = []
save_model_info(args, model_path, model_cfg)
callback_logger = logging.info if args.log else callable_print
#######################################################################
# Callbacks that need validation set predictions.
#######################################################################
pc = PredictionCallback(x_validation, callback_logger,
marshaller=marshaller, batch_size=model_cfg.batch_size)
callbacks.append(pc)
if args.classification_report:
cr = ClassificationReport(x_validation, y_validation,
callback_logger,
target_names=target_names)
pc.add(cr)
if args.confusion_matrix:
cm = ConfusionMatrix(x_validation, y_validation,
callback_logger)
pc.add(cm)
def get_mode(metric_name):
return {
'val_loss': 'min',
'val_acc': 'max',
'val_f1': 'max',
'val_f2': 'max',
'val_f0.5': 'max'
}[metric_name]
if args.early_stopping or args.early_stopping_metric is not None:
es = EarlyStopping(monitor=args.early_stopping_metric,
mode=get_mode(args.early_stopping_metric),
patience=model_cfg.patience,
verbose=1)
cb = DelegatingMetricCallback(
x_validation, y_validation, callback_logger,
delegate=es,
metric_name=args.early_stopping_metric,
marshaller=marshaller)
pc.add(cb)
if not args.no_save:
if args.save_all_checkpoints:
filepath = model_path + '/model-{epoch:04d}.h5'
else:
filepath = model_path + '/model.h5'
mc = ModelCheckpoint(
filepath=filepath,
mode=get_mode(args.checkpoint_metric),
verbose=1,
monitor=args.checkpoint_metric,
save_best_only=not args.save_every_epoch)
cb = DelegatingMetricCallback(
x_validation, y_validation, callback_logger,
delegate=mc,
metric_name=args.checkpoint_metric,
marshaller=marshaller)
pc.add(cb)
if model_cfg.optimizer == 'SGD':
callbacks.append(SingleStepLearningRateSchedule(patience=10))
if len(args.extra_train_file) > 1:
args.extra_train_file.append(args.train_file)
logging.info("Using the following files for training: " +
','.join(args.extra_train_file))
train_file_iter = itertools.cycle(args.extra_train_file)
current_train = args.train_file
callbacks._set_model(net)
callbacks.on_train_begin(logs={})
epoch = batch = 0
while True:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
iteration = batch % len(args.extra_train_file)
logging.info("epoch {epoch} iteration {iteration} - training with {train_file}".format(
epoch=epoch, iteration=iteration, train_file=current_train))
callbacks.on_epoch_begin(epoch, logs={})
n_train = x_train.shape[0]
callbacks.on_batch_begin(batch, logs={'size': n_train})
index_array = np.arange(n_train)
if args.shuffle:
rng.shuffle(index_array)
batches = keras.models.make_batches(n_train, model_cfg.batch_size)
logging.info("epoch {epoch} iteration {iteration} - starting {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
avg_train_loss = avg_train_accuracy = 0.
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(net, keras.models.Graph):
train_data = marshaller.marshal(
x_train[batch_ids], y_train_one_hot[batch_ids])
train_loss = net.train_on_batch(
train_data, class_weight=class_weight)
# It looks like train_on_batch returns a different
# type for graph than sequential models.
train_loss = train_loss[0]
train_accuracy = 0.
else:
train_loss, train_accuracy = net.train_on_batch(
x_train[batch_ids], y_train_one_hot[batch_ids],
accuracy=True, class_weight=class_weight)
batch_end_logs = {'loss': train_loss, 'accuracy': train_accuracy}
avg_train_loss = (avg_train_loss * batch_index + train_loss)/(batch_index + 1)
avg_train_accuracy = (avg_train_accuracy * batch_index + train_accuracy)/(batch_index + 1)
callbacks.on_batch_end(batch,
logs={'loss': train_loss, 'accuracy': train_accuracy})
logging.info("epoch {epoch} iteration {iteration} - finished {n_batches} batches".format(
epoch=epoch, iteration=iteration, n_batches=len(batches)))
logging.info("epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}".format(
epoch=epoch, iteration=iteration, loss=avg_train_loss, acc=avg_train_accuracy))
batch += 1
# Validation frequency (this if-block) doesn't necessarily
# occur in the same iteration as beginning of an epoch
# (next if-block), so net.evaluate appears twice here.
kwargs = {
'batch_size': model_cfg.batch_size,
'verbose': 0 if args.log else 1
}
pargs = []
validation_data = {}
if isinstance(net, keras.models.Graph):
validation_data = marshaller.marshal(
x_validation, y_validation_one_hot)
pargs = [validation_data]
else:
pargs = [x_validation, y_validation_one_hot]
kwargs['show_accuracy'] = True
if (iteration + 1) % args.validation_freq == 0:
if isinstance(net, keras.models.Graph):
val_loss = net.evaluate(*pargs, **kwargs)
y_hat = net.predict(validation_data, batch_size=model_cfg.batch_size)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = net.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
callbacks.on_epoch_end(epoch, epoch_end_logs)
if batch % len(args.extra_train_file) == 0:
if isinstance(net, keras.models.Graph):
val_loss = net.evaluate(*pargs, **kwargs)
y_hat = net.predict(validation_data, batch_size=model_cfg.batch_size)
val_acc = accuracy_score(y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = net.evaluate(
*pargs, **kwargs)
logging.info("epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}".format(
epoch=epoch, iteration=iteration, val_loss=val_loss, val_acc=val_acc))
epoch_end_logs = {'iteration': iteration, 'val_loss': val_loss, 'val_acc': val_acc}
epoch += 1
callbacks.on_epoch_end(epoch, epoch_end_logs)
if net.stop_training:
logging.info("epoch {epoch} iteration {iteration} - done training".format(
epoch=epoch, iteration=iteration))
break
current_train = next(train_file_iter)
x_train, y_train = load_model_data(current_train,
args.data_name, args.target_name)
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
if epoch > args.n_epochs:
break
callbacks.on_train_end(logs={})
else:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
if isinstance(net, keras.models.Graph):
train_data = marshaller.marshal(
x_train, y_train_one_hot)
validation_data = marshaller.marshal(
x_validation, y_validation_one_hot)
net.fit(train_data,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
validation_data=validation_data,
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
else:
net.fit(x_train, y_train_one_hot,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
show_accuracy=True,
validation_data=(x_validation, y_validation_one_hot),
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
def fit(model, *fit_args, plots=['loss'], **fit_kwargs):
"""
Interactive model training in jupyter notebook
Just replace
model.fit(model, x, y, nb_epoch=42, ...)
with
fit(model, x, y, nb_epoch=42, ...)
to get interactive training
"""
nbagg_backend = matplotlib.backends.backend.lower() == 'nbagg'
if not nbagg_backend:
warnings.warn(
'''\nUse nbAgg backend if possible to prevent stacking display of training plots.
Add import matplotlib; matplotlib.use(\'nbagg\') and restart kernel''')
nb_epoch = fit_kwargs.get('nb_epoch', 10)
progress_bar = tqdm.tqdm_notebook(total=nb_epoch)
class LossHistory(keras.callbacks.Callback):
def __init__(self):
super().__init__()
self.fig = plt.figure()
self.vals = dict()
def on_epoch_end(self, epoch, logs={}):
progress_bar.update()
for k in logs.keys():
val_list = self.vals.get(k, [])
val_list.append(logs[k])
self.vals[k] = val_list
def on_train_end(self, logs={}):
progress_bar.close()
cancel_button = ipywidgets.Button(description='Cancel')
def cancel_fitting():
model.stop_training = True
learning_thread.join(5)
print('Cancelled')
cancel_button.on_click(lambda sender: cancel_fitting())
loss_history = LossHistory()
display(cancel_button)
if nbagg_backend:
axis = None
def plot_loss(sender):
nonlocal axis
if axis is None:
axis = plt.gca()
axis.clear()
for n, l in loss_history.vals.items():
if n in plots:
axis.semilogy(l, label=n)
axis.legend(loc='best', frameon=False)
plt.show()
else:
def plot_loss(sender):
for n, l in loss_history.vals.items():
if n in plots:
plt.semilogy(l, label=n)
plt.legend(loc='best', frameon=False)
plt.show()
plot_button = ipywidgets.Button(description='Plot')
plot_button.on_click(plot_loss)
display(plot_button)
callbacks = fit_kwargs.get('callbacks', [])
callbacks = callbacks.copy()
callbacks.append(loss_history)
fit_kwargs['callbacks'] = callbacks
fit_kwargs['verbose'] = False
learning_thread = threading.Thread(
target=lambda: model.fit(*fit_args, **fit_kwargs))
learning_thread.start()
def train(training_h5, model_json, weights_path, epochs, batch_size, s3=False,
s3_bucket=None, using_dataset=False, test_set_index=0):
"""Trains a CNN.
training_h5: Training HDF5 file.
model_json: JSON model file.
weights_path: Output weights HDF5 file.
epochs: Number of training epochs.
batch_size: Batch size.
s3: Whether to periodically dump to Amazon S3. Default False.
s3_bucket: Name of the bucket to dump to. Must be specified iff s3 is True.
using_dataset: Whether the given training file is the Zenodo crowdastro
dataset. Default False (i.e. we are using the generated training file).
test_set_index: Index of the test set to not use for training. Default 0.
"""
if s3 and not s3_bucket:
raise ValueError('Must specify s3_bucket to dump to S3.')
if not s3 and s3_bucket:
raise ValueError('s3_bucket was specified but s3 is False.')
import keras.callbacks
import keras.models
from keras.preprocessing.image import ImageDataGenerator
model = keras.models.model_from_json(model_json.read())
model.compile(loss='binary_crossentropy', optimizer='adadelta')
if not using_dataset:
ir_survey = training_h5.attrs['ir_survey']
n_nonimage_features = config['surveys'][ir_survey]['n_features']
features_name = 'raw_features'
else:
ir_survey = 'wise'
n_nonimage_features = 5
features_name = 'features'
test_set = set(training_h5['test_sets'][test_set_index, :])
n_examples = training_h5[features_name].shape[0]
train_set = [i for i in range(n_examples) if i not in test_set]
del test_set # Just to make sure we don't use it.
training_inputs = training_h5[features_name].value[train_set, :]
training_outputs = training_h5['labels'].value[train_set]
assert training_inputs.shape[0] == training_outputs.shape[0]
# Downsample for class balance.
zero_indices = (training_outputs == 0).nonzero()[0]
one_indices = (training_outputs == 1).nonzero()[0]
subset_zero_indices = numpy.random.choice(
zero_indices, size=(len(one_indices,)), replace=False)
all_indices = numpy.hstack([subset_zero_indices, one_indices])
all_indices.sort()
training_inputs = training_inputs[all_indices]
training_outputs = training_outputs[all_indices]
assert (training_outputs == 1).sum() == (training_outputs == 0).sum()
class DumpToS3(keras.callbacks.Callback):
def __init__(self, weights_path, bucket, period=50):
super().__init__()
self.weights_path = weights_path
self.bucket = bucket
self.period = period
def on_train_begin(self, logs={}):
self.epochs = 0
def on_epoch_end(self, epoch, logs={}):
self.epochs += 1
if self.epochs % self.period == 0:
# Every 50 epochs...
logging.debug('Dumping to S3...')
res = subprocess.check_call(
['aws', 's3', 'cp', self.weights_path,
's3://' + self.bucket + '/', '--region', 'us-east-1'])
logging.info('Dumped to S3: {}'.format(res))
try:
model.load_weights(weights_path)
logging.info('Loaded weights.')
except OSError:
logging.warning('Couldn\'t load weights file. Creating new file...')
pass
callbacks = [
keras.callbacks.ModelCheckpoint(
weights_path,
monitor='val_loss',
verbose=1,
save_best_only=False,
save_weights_only=True,
mode='auto'),
]
if s3:
callbacks.append(DumpToS3(weights_path, s3_bucket))
def create_generator(X, Y):
"""Yields generated images and auxiliary inputs.
https://github.com/fchollet/keras/issues/3386
"""
X_im = X[:, n_nonimage_features:].reshape(
(-1, 1, PATCH_DIAMETER, PATCH_DIAMETER))
X_au = X[:, :n_nonimage_features]
while True:
# Shuffle indices.
idx = numpy.random.permutation(X.shape[0])
# Standard image generator.
datagen = ImageDataGenerator(
data_format='channels_first',
horizontal_flip=True,
vertical_flip=True)
datagen.fit(X_im)
# Shuffle the data before batching using known indices.
batches = datagen.flow(X_im[idx], Y[idx], batch_size=batch_size,
shuffle=False)
idx0 = 0
for batch in batches:
idx1 = idx0 + batch[0].shape[0]
# Yield ((image, aux), label) tuples.
to_yield = ([X_au[idx[idx0:idx1]], batch[0]], batch[1])
yield to_yield
idx0 = idx1
if idx1 >= X.shape[0]:
break
model.fit_generator(create_generator(training_inputs, training_outputs),
steps_per_epoch=training_inputs.shape[0] // batch_size,
epochs=epochs,
callbacks=callbacks,
workers=1)
model.save_weights(weights_path, overwrite=True)
def main():
# Parse arguments.
parser = argparse.ArgumentParser()
kwargs = {
'type': int,
'default': 100,
'help': 'The number of times of learning. default: 100'
}
parser.add_argument('-e', '--epochs', **kwargs)
kwargs = {
'type': int,
'default': 10,
'help': 'The frequency of saving model. default: 10'
}
parser.add_argument('-c', '--checkpoint_interval', **kwargs)
kwargs = {
'type': int,
'default': 1,
'help': 'The number of samples contained per mini batch. default: 1'
}
parser.add_argument('-b', '--batch_size', **kwargs)
kwargs = {
'default':
False,
'action':
'store_true',
'help':
'Whether store all data to GPU. If not specified this option, use both CPU memory and GPU memory.'
}
parser.add_argument('--onmemory', **kwargs)
args = parser.parse_args()
# Prepare training data.
dataset = np.load('./temp/dataset.npz')
train_x = dataset['train_x']
train_y = dataset['train_y']
test_x = dataset['test_x']
test_y = dataset['test_y']
# Prepare tensorflow.
config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
session = tf.Session(config=config)
keras.backend.tensorflow_backend.set_session(session)
# Prepare model.
model = SegNet(shape=(360, 480, 3))
model.compile(loss='binary_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
# Training.
callbacks = []
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
directory = f'./logs/{timestamp}/'
os.makedirs(directory, exist_ok=True)
callbacks.append(keras.callbacks.TensorBoard(log_dir=directory))
filename = 'model-{epoch:04d}.h5'
directory = f'./temp/{timestamp}/'
os.makedirs(directory, exist_ok=True)
callbacks.append(
keras.callbacks.ModelCheckpoint(filepath=f'{directory}{filename}',
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=args.checkpoint_interval))
model.save_weights(f'{directory}{filename}'.format(epoch=0))
if args.onmemory:
model.fit(x=train_x,
y=train_y,
validation_data=(test_x, test_y),
epochs=args.epochs,
batch_size=args.batch_size,
class_weight='balanced',
shuffle=True,
verbose=1,
callbacks=callbacks)
else:
class Generator(keras.utils.Sequence):
def __init__(self, x, y, batch_size, shuffle):
self.x = x
self.y = y
self.batch_size = batch_size
self.indices = np.arange(len(self.x))
self.shuffle = shuffle
assert len(self.x) == len(self.y)
assert len(self.x) % self.batch_size == 0
def __getitem__(self, index):
i = index * self.batch_size
indices = self.indices[i:i + self.batch_size]
x = self.x[indices]
y = self.y[indices]
return x, y
def __len__(self):
return len(self.x) // self.batch_size
def on_epoch_end(self):
if self.shuffle:
self.indices = np.random.permutation(self.indices)
model.fit_generator(generator=Generator(train_x, train_y,
args.batch_size, True),
validation_data=Generator(test_x, test_y,
args.batch_size, False),
epochs=args.epochs,
class_weight='balanced',
shuffle=True,
verbose=1,
callbacks=callbacks)
def main(args):
model_id = build_model_id(args)
model_path = build_model_path(args, model_id)
setup_model_dir(args, model_path)
sys.stdout, sys.stderr = setup_logging(args, model_path)
x_train, y_train = load_model_data(args.train_file, args.data_name,
args.target_name)
x_validation, y_validation = load_model_data(args.validation_file,
args.data_name,
args.target_name)
rng = np.random.RandomState(args.seed)
if args.n_classes > -1:
n_classes = args.n_classes
else:
n_classes = max(y_train) + 1
n_classes, target_names, class_weight = load_target_data(args, n_classes)
logging.debug("n_classes {0} min {1} max {2}".format(
n_classes, min(y_train), max(y_train)))
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
min_vocab_index = np.min(x_train)
max_vocab_index = np.max(x_train)
logging.debug("min vocab index {0} max vocab index {1}".format(
min_vocab_index, max_vocab_index))
json_cfg = load_model_json(args, x_train, n_classes)
logging.debug("loading model")
sys.path.append(args.model_dir)
import model
from model import build_model
if args.subsetting_function:
subsetter = getattr(model, args.subsetting_function)
else:
subsetter = None
def take_subset(subsetter, path, x, y, y_one_hot, n):
if subsetter is None:
return x[0:n], y[0:n], y_one_hot[0:n]
else:
mask = subsetter(path)
idx = np.where(mask)[0]
idx = idx[0:n]
return x[idx], y[idx], y_one_hot[idx]
x_train, y_train, y_train_one_hot = take_subset(subsetter,
args.train_file,
x_train,
y_train,
y_train_one_hot,
n=args.n_train)
x_validation, y_validation, y_validation_one_hot = take_subset(
subsetter,
args.validation_file,
x_validation,
y_validation,
y_validation_one_hot,
n=args.n_validation)
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
model_cfg = ModelConfig(**json_cfg)
logging.info("model_cfg " + str(model_cfg))
model = build_model(model_cfg)
setattr(model, 'stop_training', False)
logging.info('model has {n_params} parameters'.format(
n_params=count_parameters(model)))
if len(args.extra_train_file) > 1:
callbacks = keras.callbacks.CallbackList()
else:
callbacks = []
save_model_info(args, model_path, model_cfg)
if not args.no_save:
callbacks.append(
ModelCheckpoint(filepath=model_path + '/model-{epoch:04d}.h5',
verbose=1,
save_best_only=True))
callback_logger = logging.info if args.log else callable_print
if args.n_epochs < sys.maxsize:
# Number of epochs overrides patience. If the number of epochs
# is specified on the command line, the model is trained for
# exactly that number; otherwise, the model is trained with
# early stopping using the patience specified in the model
# configuration.
callbacks.append(
EarlyStopping(monitor='val_loss',
patience=model_cfg.patience,
verbose=1))
if args.classification_report:
cr = ClassificationReport(x_validation,
y_validation,
callback_logger,
target_names=target_names)
callbacks.append(cr)
if model_cfg.optimizer == 'SGD':
callbacks.append(SingleStepLearningRateSchedule(patience=10))
if len(args.extra_train_file) > 1:
args.extra_train_file.append(args.train_file)
logging.info("Using the following files for training: " +
','.join(args.extra_train_file))
train_file_iter = itertools.cycle(args.extra_train_file)
current_train = args.train_file
callbacks._set_model(model)
callbacks.on_train_begin(logs={})
epoch = batch = 0
while True:
iteration = batch % len(args.extra_train_file)
logging.info(
"epoch {epoch} iteration {iteration} - training with {train_file}"
.format(epoch=epoch,
iteration=iteration,
train_file=current_train))
callbacks.on_epoch_begin(epoch, logs={})
n_train = x_train.shape[0]
callbacks.on_batch_begin(batch, logs={'size': n_train})
index_array = np.arange(n_train)
if args.shuffle:
rng.shuffle(index_array)
batches = keras.models.make_batches(n_train, model_cfg.batch_size)
logging.info(
"epoch {epoch} iteration {iteration} - starting {n_batches} batches"
.format(epoch=epoch,
iteration=iteration,
n_batches=len(batches)))
avg_train_loss = avg_train_accuracy = 0.
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(model, keras.models.Graph):
data = {
'input': x_train[batch_ids],
'output': y_train_one_hot[batch_ids]
}
train_loss = model.train_on_batch(
data, class_weight=class_weight)
train_accuracy = 0.
else:
train_loss, train_accuracy = model.train_on_batch(
x_train[batch_ids],
y_train_one_hot[batch_ids],
accuracy=True,
class_weight=class_weight)
batch_end_logs = {
'loss': train_loss,
'accuracy': train_accuracy
}
avg_train_loss = (avg_train_loss * batch_index +
train_loss) / (batch_index + 1)
avg_train_accuracy = (avg_train_accuracy * batch_index +
train_accuracy) / (batch_index + 1)
callbacks.on_batch_end(batch,
logs={
'loss': train_loss,
'accuracy': train_accuracy
})
logging.info(
"epoch {epoch} iteration {iteration} - finished {n_batches} batches"
.format(epoch=epoch,
iteration=iteration,
n_batches=len(batches)))
logging.info(
"epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}"
.format(epoch=epoch,
iteration=iteration,
loss=avg_train_loss,
acc=avg_train_accuracy))
batch += 1
# Validation frequency (this if-block) doesn't necessarily
# occur in the same iteration as beginning of an epoch
# (next if-block), so model.evaluate appears twice here.
if (iteration + 1) % args.validation_freq == 0:
val_loss, val_acc = model.evaluate(
x_validation,
y_validation_one_hot,
show_accuracy=True,
verbose=0 if args.log else 1)
logging.info(
"epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
.format(epoch=epoch,
iteration=iteration,
val_loss=val_loss,
val_acc=val_acc))
epoch_end_logs = {
'iteration': iteration,
'val_loss': val_loss,
'val_acc': val_acc
}
callbacks.on_epoch_end(epoch, epoch_end_logs)
if batch % len(args.extra_train_file) == 0:
val_loss, val_acc = model.evaluate(
x_validation,
y_validation_one_hot,
show_accuracy=True,
verbose=0 if args.log else 1)
logging.info(
"epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
.format(epoch=epoch,
iteration=iteration,
val_loss=val_loss,
val_acc=val_acc))
epoch_end_logs = {
'iteration': iteration,
'val_loss': val_loss,
'val_acc': val_acc
}
epoch += 1
callbacks.on_epoch_end(epoch, epoch_end_logs)
if model.stop_training:
logging.info(
"epoch {epoch} iteration {iteration} - done training".
format(epoch=epoch, iteration=iteration))
break
current_train = next(train_file_iter)
x_train, y_train = load_model_data(current_train, args.data_name,
args.target_name)
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
if epoch > args.n_epochs:
break
callbacks.on_train_end(logs={})
else:
print('args.n_epochs', args.n_epochs)
if isinstance(model, keras.models.Graph):
data = {'input': x_train, 'output': y_train_one_hot}
validation_data = {
'input': x_validation,
'output': y_validation_one_hot
}
model.fit(
data,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
#show_accuracy=True,
validation_data=validation_data,
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
y_hat = model.predict_classes(data)
print('val_acc %.04f' % accuracy_score(y_validate, y_hat))
else:
model.fit(x_train,
y_train_one_hot,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
show_accuracy=True,
validation_data=(x_validation, y_validation_one_hot),
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
def train(callback=None, out_weights='weights.h5'):
reload(audiotransform)
reload(speechmodel)
model = speechmodel.makeModel()
model.compile(loss='mean_squared_error',
optimizer=keras.optimizers.Nadam(lr=0.00002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004),
metrics=['accuracy'])
paths = []
words = []
for p in sampleSet2(): # or findSounds(words)
try:
raw = load(p, hz=speechmodel.rate)
except:
print "load failed", p
continue
try:
crop = audiotransform.autoCrop(raw, rate=speechmodel.rate)
print 'using %s autocropped to %s samples' % (p, len(crop))
except audiotransform.TooQuiet:
print '%s too quiet' % p
continue
paths.append(p)
word = soundFields(p)['word']
if word not in words:
words.append(word)
repeat = 2
x = numpy.zeros((len(paths) * repeat, speechmodel.xWidth),
dtype=numpy.float)
y = numpy.zeros((len(paths) * repeat, speechmodel.embedSize),
dtype=numpy.float)
for row, p in enumerate(paths * repeat):
audio = load(p, hz=speechmodel.rate)
audio = audiotransform.autoCrop(audio, rate=speechmodel.rate)
#audio = audiotransform.rightPad(audio, speechmodel.goalSize)
audio = audiotransform.randomPad(audio, speechmodel.goalSize, path=p)
audio = audiotransform.randomScale(audio)
m = mfcc(audio, samplerate=speechmodel.rate)
x[row, :] = m.reshape((1, speechmodel.xWidth))
y[row, :] = np_utils.to_categorical(
words.index(soundFields(p)['word']), speechmodel.embedSize)
if callback:
callback.loaded_sound(row, len(paths) * repeat)
callbacks = []
#callbacks.append(keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=1, write_graph=True))
if callback:
callbacks.append(callback)
model.fit(x,
y,
batch_size=500,
epochs=500,
validation_split=.2,
shuffle=True,
callbacks=callbacks)
model.save_weights(out_weights)
with open(out_weights + '.words', 'w') as f:
f.write(json.dumps(words) + '\n')
if callback:
callback.on_save(out_weights, fileSize=os.path.getsize(out_weights))
def train(epochs, img_w, output, gs_output, steps_per_epoch, validation_steps,
minibatch_size, **kwargs):
# Input Parameters
img_h = 64
output_dir = output
c.reset_dir(output_dir)
# Network parameters
pool_size = my_model.pool_size
img_gen = TextImageGenerator(minibatch_size=minibatch_size,
img_w=img_w,
img_h=img_h,
downsample_factor=(pool_size**2))
input_data, y_pred = my_model.create_tensor_io(img_w, img_h,
img_gen.get_output_size())
labels = Input(name='the_labels',
shape=[img_gen.absolute_max_string_len],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])
callbacks = []
callbacks.append(
ModelCheckpoint(
filepath=os.path.join(output_dir, 'weight.{epoch:06d}.hdf5')))
callbacks.append(
ModelCheckpoint(filepath=os.path.join(output_dir, 'weight.best.hdf5'),
save_best_only=True))
callbacks.append(CSVLogger(filename=os.path.join(output_dir, 'log.csv')))
if gs_output != None:
callbacks.append(
g.Copy(os.path.join(output_dir, 'weight.{epoch:06d}.hdf5'),
os.path.join(gs_output, 'weight.{epoch:06d}.hdf5')))
callbacks.append(
g.Copy(os.path.join(output_dir, 'weight.best.hdf5'),
os.path.join(gs_output, 'weight.best.hdf5')))
callbacks.append(
g.Copy(os.path.join(output_dir, 'log.csv'),
os.path.join(gs_output, 'log.csv')))
model.fit_generator(generator=img_gen.next_batch(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=img_gen.next_batch(),
validation_steps=validation_steps,
callbacks=callbacks,
verbose=2)
def train(
training_h5,
model_json,
weights_path,
epochs,
batch_size,
s3=False,
s3_bucket=None,
using_dataset=False,
cnn_train_set_h5=None,
):
"""Trains a CNN.
training_h5: Training HDF5 file.
model_json: JSON model file.
weights_path: Output weights HDF5 file.
epochs: Number of training epochs.
batch_size: Batch size.
s3: Whether to periodically dump to Amazon S3. Default False.
s3_bucket: Name of the bucket to dump to. Must be specified iff s3 is True.
using_dataset: Whether the given training file is the Zenodo crowdastro
dataset. Default False (i.e. we are using the generated training file).
cnn_train_set_h5: HDF5 file specifying the CNN training set. Overrides the
training file if specified. Default None (i.e. not specified).
"""
if s3 and not s3_bucket:
raise ValueError("Must specify s3_bucket to dump to S3.")
if not s3 and s3_bucket:
raise ValueError("s3_bucket was specified but s3 is False.")
import keras.callbacks
import keras.models
model = keras.models.model_from_json(model_json.read())
model.compile(loss="binary_crossentropy", optimizer="adadelta")
if not cnn_train_set_h5:
train_set = training_h5["cnn_train_set"].value
else:
train_set = cnn_train_set_h5["cnn_train_set"].value
if not using_dataset:
ir_survey = training_h5.attrs["ir_survey"]
n_nonimage_features = config["surveys"][ir_survey]["n_features"]
features_name = "raw_features"
else:
ir_survey = "wise"
n_nonimage_features = 5
features_name = "features"
training_inputs = training_h5[features_name].value[train_set, n_nonimage_features:]
training_inputs = training_inputs.reshape((-1, 1, PATCH_DIAMETER, PATCH_DIAMETER))
training_outputs = training_h5["labels"].value[train_set]
assert training_inputs.shape[0] == training_outputs.shape[0]
# Downsample for class balance.
zero_indices = (training_outputs == 0).nonzero()[0]
one_indices = (training_outputs == 1).nonzero()[0]
subset_zero_indices = numpy.random.choice(zero_indices, size=(len(one_indices)), replace=False)
all_indices = numpy.hstack([subset_zero_indices, one_indices])
all_indices.sort()
training_inputs = training_inputs[all_indices]
training_outputs = training_outputs[all_indices]
assert (training_outputs == 1).sum() == (training_outputs == 0).sum()
class DumpToS3(keras.callbacks.Callback):
def __init__(self, weights_path, bucket, period=50):
super().__init__()
self.weights_path = weights_path
self.bucket = bucket
self.period = period
def on_train_begin(self, logs={}):
self.epochs = 0
def on_epoch_end(self, epoch, logs={}):
self.epochs += 1
if self.epochs % self.period == 0:
# Every 50 epochs...
logging.debug("Dumping to S3...")
res = subprocess.check_call(
["aws", "s3", "cp", self.weights_path, "s3://" + self.bucket + "/", "--region", "us-east-1"]
)
logging.info("Dumped to S3: {}".format(res))
try:
model.load_weights(weights_path)
logging.info("Loaded weights.")
except OSError:
logging.warning("Couldn't load weights file. Creating new file...")
pass
callbacks = [
keras.callbacks.ModelCheckpoint(
weights_path, monitor="val_loss", verbose=1, save_best_only=False, save_weights_only=True, mode="auto"
)
]
if s3:
callbacks.append(DumpToS3(weights_path, s3_bucket))
model.fit(training_inputs, training_outputs, batch_size=batch_size, nb_epoch=epochs, callbacks=callbacks)
model.save_weights(weights_path, overwrite=True)
def main(args):
model_id = build_model_id(args)
model_path = build_model_path(args, model_id)
setup_model_dir(args, model_path)
sys.stdout, sys.stderr = setup_logging(args, model_path)
x_train, y_train = load_model_data(args.train_file, args.data_name,
args.target_name)
x_validation, y_validation = load_model_data(args.validation_file,
args.data_name,
args.target_name)
rng = np.random.RandomState(args.seed)
if args.n_classes > -1:
n_classes = args.n_classes
else:
n_classes = max(y_train) + 1
n_classes, target_names, class_weight = load_target_data(args, n_classes)
if class_weight is None and args.class_weight_auto:
n_samples = len(y_train)
weights = float(n_samples) / (n_classes * np.bincount(y_train))
if args.class_weight_exponent:
weights = weights**args.class_weight_exponent
class_weight = dict(zip(range(n_classes), weights))
if args.verbose:
logging.debug("n_classes {0} min {1} max {2}".format(
n_classes, min(y_train), max(y_train)))
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
y_validation_one_hot = np_utils.to_categorical(y_validation, n_classes)
if args.verbose:
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
min_vocab_index = np.min(x_train)
max_vocab_index = np.max(x_train)
if args.verbose:
logging.debug("min vocab index {0} max vocab index {1}".format(
min_vocab_index, max_vocab_index))
json_cfg = load_model_json(args, x_train, n_classes)
if args.verbose:
logging.debug("loading model")
sys.path.append(args.model_dir)
import model
from model import build_model
#######################################################################
# Subsetting
#######################################################################
if args.subsetting_function:
subsetter = getattr(M, args.subsetting_function)
else:
subsetter = None
def take_subset(subsetter, path, x, y, y_one_hot, n):
if subsetter is None:
return x[0:n], y[0:n], y_one_hot[0:n]
else:
mask = subsetter(path)
idx = np.where(mask)[0]
idx = idx[0:n]
return x[idx], y[idx], y_one_hot[idx]
x_train, y_train, y_train_one_hot = take_subset(subsetter,
args.train_file,
x_train,
y_train,
y_train_one_hot,
n=args.n_train)
x_validation, y_validation, y_validation_one_hot = take_subset(
subsetter,
args.validation_file,
x_validation,
y_validation,
y_validation_one_hot,
n=args.n_validation)
#######################################################################
# Preprocessing
#######################################################################
if args.preprocessing_class:
preprocessor = getattr(M, args.preprocessing_class)(seed=args.seed)
else:
preprocessor = modeling.preprocess.NullPreprocessor()
if args.verbose:
logging.debug("y_train_one_hot " + str(y_train_one_hot.shape))
logging.debug("x_train " + str(x_train.shape))
model_cfg = ModelConfig(**json_cfg)
if args.verbose:
logging.info("model_cfg " + str(model_cfg))
net = build_model(model_cfg)
setattr(net, 'stop_training', False)
marshaller = None
if isinstance(net, keras.models.Graph):
marshaller = getattr(model, args.graph_marshalling_class)()
logging.info('model has {n_params} parameters'.format(
n_params=count_parameters(net)))
if len(args.extra_train_file) > 1:
callbacks = keras.callbacks.CallbackList()
else:
callbacks = []
save_model_info(args, model_path, model_cfg)
callback_logger = logging.info if args.log else callable_print
#######################################################################
# Callbacks that need validation set predictions.
#######################################################################
pc = PredictionCallback(x_validation,
callback_logger,
marshaller=marshaller,
batch_size=model_cfg.batch_size)
callbacks.append(pc)
if args.classification_report:
cr = ClassificationReport(x_validation,
y_validation,
callback_logger,
target_names=target_names)
pc.add(cr)
if args.confusion_matrix:
cm = ConfusionMatrix(x_validation, y_validation, callback_logger)
pc.add(cm)
def get_mode(metric_name):
return {
'val_loss': 'min',
'val_acc': 'max',
'val_f1': 'max',
'val_f2': 'max',
'val_f0.5': 'max'
}[metric_name]
if args.early_stopping or args.early_stopping_metric is not None:
es = EarlyStopping(monitor=args.early_stopping_metric,
mode=get_mode(args.early_stopping_metric),
patience=model_cfg.patience,
verbose=1)
cb = DelegatingMetricCallback(x_validation,
y_validation,
callback_logger,
delegate=es,
metric_name=args.early_stopping_metric,
marshaller=marshaller)
pc.add(cb)
if not args.no_save:
if args.save_all_checkpoints:
filepath = model_path + '/model-{epoch:04d}.h5'
else:
filepath = model_path + '/model.h5'
mc = ModelCheckpoint(filepath=filepath,
mode=get_mode(args.checkpoint_metric),
verbose=1,
monitor=args.checkpoint_metric,
save_best_only=not args.save_every_epoch)
cb = DelegatingMetricCallback(x_validation,
y_validation,
callback_logger,
delegate=mc,
metric_name=args.checkpoint_metric,
marshaller=marshaller)
pc.add(cb)
if model_cfg.optimizer == 'SGD':
callbacks.append(SingleStepLearningRateSchedule(patience=10))
if len(args.extra_train_file) > 1:
args.extra_train_file.append(args.train_file)
logging.info("Using the following files for training: " +
','.join(args.extra_train_file))
train_file_iter = itertools.cycle(args.extra_train_file)
current_train = args.train_file
callbacks._set_model(net)
callbacks.on_train_begin(logs={})
epoch = batch = 0
while True:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
iteration = batch % len(args.extra_train_file)
logging.info(
"epoch {epoch} iteration {iteration} - training with {train_file}"
.format(epoch=epoch,
iteration=iteration,
train_file=current_train))
callbacks.on_epoch_begin(epoch, logs={})
n_train = x_train.shape[0]
callbacks.on_batch_begin(batch, logs={'size': n_train})
index_array = np.arange(n_train)
if args.shuffle:
rng.shuffle(index_array)
batches = keras.models.make_batches(n_train, model_cfg.batch_size)
logging.info(
"epoch {epoch} iteration {iteration} - starting {n_batches} batches"
.format(epoch=epoch,
iteration=iteration,
n_batches=len(batches)))
avg_train_loss = avg_train_accuracy = 0.
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(net, keras.models.Graph):
train_data = marshaller.marshal(x_train[batch_ids],
y_train_one_hot[batch_ids])
train_loss = net.train_on_batch(train_data,
class_weight=class_weight)
# It looks like train_on_batch returns a different
# type for graph than sequential models.
train_loss = train_loss[0]
train_accuracy = 0.
else:
train_loss, train_accuracy = net.train_on_batch(
x_train[batch_ids],
y_train_one_hot[batch_ids],
accuracy=True,
class_weight=class_weight)
batch_end_logs = {
'loss': train_loss,
'accuracy': train_accuracy
}
avg_train_loss = (avg_train_loss * batch_index +
train_loss) / (batch_index + 1)
avg_train_accuracy = (avg_train_accuracy * batch_index +
train_accuracy) / (batch_index + 1)
callbacks.on_batch_end(batch,
logs={
'loss': train_loss,
'accuracy': train_accuracy
})
logging.info(
"epoch {epoch} iteration {iteration} - finished {n_batches} batches"
.format(epoch=epoch,
iteration=iteration,
n_batches=len(batches)))
logging.info(
"epoch {epoch} iteration {iteration} - loss: {loss} - acc: {acc}"
.format(epoch=epoch,
iteration=iteration,
loss=avg_train_loss,
acc=avg_train_accuracy))
batch += 1
# Validation frequency (this if-block) doesn't necessarily
# occur in the same iteration as beginning of an epoch
# (next if-block), so net.evaluate appears twice here.
kwargs = {
'batch_size': model_cfg.batch_size,
'verbose': 0 if args.log else 1
}
pargs = []
validation_data = {}
if isinstance(net, keras.models.Graph):
validation_data = marshaller.marshal(x_validation,
y_validation_one_hot)
pargs = [validation_data]
else:
pargs = [x_validation, y_validation_one_hot]
kwargs['show_accuracy'] = True
if (iteration + 1) % args.validation_freq == 0:
if isinstance(net, keras.models.Graph):
val_loss = net.evaluate(*pargs, **kwargs)
y_hat = net.predict(validation_data,
batch_size=model_cfg.batch_size)
val_acc = accuracy_score(
y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = net.evaluate(*pargs, **kwargs)
logging.info(
"epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
.format(epoch=epoch,
iteration=iteration,
val_loss=val_loss,
val_acc=val_acc))
epoch_end_logs = {
'iteration': iteration,
'val_loss': val_loss,
'val_acc': val_acc
}
callbacks.on_epoch_end(epoch, epoch_end_logs)
if batch % len(args.extra_train_file) == 0:
if isinstance(net, keras.models.Graph):
val_loss = net.evaluate(*pargs, **kwargs)
y_hat = net.predict(validation_data,
batch_size=model_cfg.batch_size)
val_acc = accuracy_score(
y_validation, np.argmax(y_hat['output'], axis=1))
else:
val_loss, val_acc = net.evaluate(*pargs, **kwargs)
logging.info(
"epoch {epoch} iteration {iteration} - val_loss: {val_loss} - val_acc: {val_acc}"
.format(epoch=epoch,
iteration=iteration,
val_loss=val_loss,
val_acc=val_acc))
epoch_end_logs = {
'iteration': iteration,
'val_loss': val_loss,
'val_acc': val_acc
}
epoch += 1
callbacks.on_epoch_end(epoch, epoch_end_logs)
if net.stop_training:
logging.info(
"epoch {epoch} iteration {iteration} - done training".
format(epoch=epoch, iteration=iteration))
break
current_train = next(train_file_iter)
x_train, y_train = load_model_data(current_train, args.data_name,
args.target_name)
y_train_one_hot = np_utils.to_categorical(y_train, n_classes)
if epoch > args.n_epochs:
break
callbacks.on_train_end(logs={})
else:
x_train, y_train_one_hot = preprocessor.fit_transform(
x_train, y_train_one_hot)
x_validation, y_validation_one_hot = preprocessor.transform(
x_validation, y_validation_one_hot)
if isinstance(net, keras.models.Graph):
train_data = marshaller.marshal(x_train, y_train_one_hot)
validation_data = marshaller.marshal(x_validation,
y_validation_one_hot)
net.fit(train_data,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
validation_data=validation_data,
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
else:
net.fit(x_train,
y_train_one_hot,
shuffle=args.shuffle,
nb_epoch=args.n_epochs,
batch_size=model_cfg.batch_size,
show_accuracy=True,
validation_data=(x_validation, y_validation_one_hot),
callbacks=callbacks,
class_weight=class_weight,
verbose=2 if args.log else 1)
test_samples = [samples[i] for i in test_index]
X_train, y_train = vectorize_samples(
train_samples, computed_params)
y_train = np.expand_dims(y_train, -1)
model = create_model(params, computed_params)
callbacks = []
model_checkpoint = ModelCheckpoint(
weights_path,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
callbacks.append(model_checkpoint)
early_stopping = EarlyStopping(monitor='val_loss',
patience=5,
verbose=1,
mode='auto')
callbacks.append(early_stopping)
model.fit(X_train,
y_train,
validation_split=0.1,
epochs=100,
verbose=2,
batch_size=batch_size,
callbacks=callbacks)
model.load_weights(weights_path)
print("\tDone: " + str(end - start) + "seconds")
## Train the model
# Save the model architecture
basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(
seed) + '_usdot_synth_ctc'
# basename = 'epoch{:03d}_seed'.format(epochs) + '{:04d}'.format(seed) + '_usdot_real_ctc'
json_string = model.to_json()
filename_model_arch = 'model_dotnum_' + basename + '_arch.json'
open(filename_model_arch, 'w').write(json_string)
# history = History()
callbacks = [History()]
filename_model_weights = 'model_dotnum_' + basename + '_weights.h5' # where weights will be saved
callbacks.append(
ModelCheckpoint(filepath=filename_model_weights,
monitor='val_loss',
save_best_only=True))
inTrain = {
'the_input': dataTrain,
'the_labels': labelsTrain,
'input_length': inputLengthTrain,
'label_length': labelLengthTrain,
'source_str': lineStringsTrain # used for visualization only
}
outTrain = {
'ctc': np.zeros([len(dataTrain)])
} # dummy data for dummy loss function
inVal = {
'the_input': dataVal,
'the_labels': labelsVal,
logger.info("load weight:%s", args.weights_path)
model.load_weights(args.weights_path, by_name=True)
gen = dataset.DataGenerator(config)
train_data_generator = gen.generate(os.path.join(args.data_dir, "train"), train_generator, train_discriminator)
val_data_generator = gen.generate(os.path.join(args.data_dir, "validate"), train_generator, train_discriminator)
# Call back preparation.
model_file_path = './nnmodel/glcic-stage{}-{}'.format(
args.stage, '{epoch:02d}-{val_loss:.2f}.h5')
callbacks = [keras.callbacks.TerminateOnNaN(),
keras.callbacks.TensorBoard(log_dir='./tb_log',histogram_freq=0,write_graph=True,write_images=False),
keras.callbacks.ModelCheckpoint(filepath=model_file_path,verbose=1,save_weights_only=True,save_best_only=False,period=20)]
if args.testimage_path and not args.stage == 2:
# epoch毎にgeneratorの出力を保存
test_data_generator = gen.generate(args.testimage_path,
train_generator, train_discriminator)
inputs, _ = next(test_data_generator)
callbacks.append(SaveGeneratorOutput(gen, config.batch_size, inputs))
model.fit_generator(train_data_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=1,max_queue_size=10,
callbacks=callbacks,
validation_data=val_data_generator,
validation_steps=5)
model_file_path = './nnmodel/glcic-latest-stage{}.h5'.format(args.stage)
model.save_weights(model_file_path)
def train_or_load(model, input_shape, class_ids, model_dir, args):
hp = Hyperparameters(args.model_prototype)
num_classes = len(class_ids)
weights_path = os.path.join(model_dir, 'weights.keras')
if os.path.exists(weights_path) and not args.reset:
# Load weights from file
model.load_weights(weights_path)
print()
print('Weights loaded from', weights_path)
if args.dummy:
# Generate random dummy data for verification of model definition
data = np.random.random((1000,)+input_shape)
labels = np.random.randint(num_classes, size=(1000, 1))
one_hot_labels = keras.utils.to_categorical(labels, num_classes=num_classes) # Convert labels to categorical one-hot encoding
model.fit(data, one_hot_labels, batch_size=32, epochs=epochs, verbose=1)
else:
# Get list of proto-samples and shuffle them
print()
print('Getting sample file list...')
sample_count = 0
now = int(time.time())
proto_samples = list() # List of all proto-samples (in the original dataset without mutation) sorted by classes
for i in range(num_classes):
proto_samples.append(list())
srcdir = os.path.normpath(args.src)
pathlist = sorted(Path(srcdir).glob('**/*.ppm'))
for path in pathlist:
path_in_str = str(path)
sample_dir, sample_filename = os.path.split(path_in_str)
class_id = int(os.path.split(sample_dir)[1])
proto_samples[class_id].append(path_in_str)
sample_count += 1
for i in range(num_classes):
random.shuffle(proto_samples[i])
print('Total of', sample_count, 'samples found.')
# These variables/objects outside of iteration loop maintain their status through the whole training process
sample_offset = 0
history = TrainHistory(args, model_dir)
logdir = os.path.normpath(os.path.join(model_dir, 'logdir'))
if not os.path.exists(logdir):
os.makedirs(logdir)
# The graph is inconsistent with keras
#tensorboard = keras.callbacks.TensorBoard(log_dir=logdir, histogram_freq=epochs/4,
# write_graph=True, write_grads=True, write_images=True)
# Iterations of training
samples_per_class = int(hp.batch_size / num_classes)
hp.batch_size = int(num_classes * samples_per_class) # Enforce strict classes balancing (same amount of samples for each class)
sample_offset = 0
fine_tune = False
for iteration in range(hp.iterations_total):
# Load training data from filesystem
samples = list()
phase = 0
for i in range(len(hp.iterations)-1):
if iteration > hp.iterations[i]:
phase = i
print()
print('Initial iteration:', iteration, 'in', hp.iterations_total, ', phase', phase)
print('Time elapsed:', int(time.time())-now, 'sec')
print('Loading', samples_per_class, 'samples per class with offset', sample_offset)
for i in range(samples_per_class):
batch_shuffled = list()
for class_id in range(num_classes):
sample_pointer = (sample_offset + i) % len(proto_samples[class_id])
#print(class_id, len(samples[class_id]), sample_pointer)
#print(samples[class_id])
sample_path = proto_samples[class_id][sample_pointer]
sample = get_mutations(sample_path, 1, intensity=hp.mintensity[phase])[0]
sample = np.array(cv2.normalize(sample.astype('float'), None, 0.0, 1.0, cv2.NORM_MINMAX)).reshape(input_shape)
batch_shuffled.append([class_id, sample]) # Pair class_id and sample data so it's easily shuffled
random.shuffle(batch_shuffled) # Shuffle in small chunk with balanced data (one sample for each class)
samples += batch_shuffled
labels = np.zeros((len(samples), 1))
data = np.zeros((len(samples),)+input_shape)
for i in range(len(samples)):
labels[i] = samples[i][0]
data[i] = samples[i][1]
one_hot_labels = keras.utils.to_categorical(labels, num_classes=num_classes) # Convert labels to categorical one-hot encoding
# Train the model
callbacks = [
history,
#tensorboard, # The graph is messed and inconsistent with keras
keras.callbacks.ModelCheckpoint(weights_path, monitor=hp.checkpoint_metric, save_best_only=True, verbose=1),
]
if hp.patience:
callbacks.append(keras.callbacks.EarlyStopping(monitor=hp.earlystop_metric, min_delta=hp.min_delta, patience=hp.patience, verbose=1))
lr = hp.learn_rate[phase]
print('Training with learn rate=', lr)
K.update(model.optimizer.lr, lr)
model.fit(data, one_hot_labels,
batch_size=num_classes, epochs=hp.epochs,
verbose=1, callbacks=callbacks,
validation_split=hp.vs, shuffle=False) # Do not use keras internal shuffling so the logic can be controlled
#model.train_on_batch(data, one_hot_labels)
sample_offset += samples_per_class
if sample_offset>=65535: sample_offset = 0
#model.compile(optimizer='adam', loss='mse')
model.compile(loss='mean_squared_error',
optimizer=Adam(lr=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0,
amsgrad=False),
metrics=['mae', 'accuracy'])
#visualize the model
plot_model(model, to_file=path + '/model.png', show_shapes=True)
callbacks = []
# CSVLogger
callbacks.append(CSVLogger(path + "/log.csv"))
from keras.callbacks import ModelCheckpoint
checkpointer = ModelCheckpoint(filepath=path + "/best_weight.h5",
monitor="val_loss",
save_best_only=True,
save_weights_only=True)
callbacks = [checkpointer]
# fitting
history = model.fit(x_train,
y_train,
batch_size=243,
epochs=10000,
verbose=1,
validation_data=(x_test, y_test),
callbacks=callbacks)
def train():
logging.basicConfig(level=logging.INFO)
args = cmd.opts.parse_arguments()
logging.info("Loading dataset...")
augmentation_args = {
'rotation_range': args.rotation_range,
'width_shift_range': args.width_shift_range,
'height_shift_range': args.height_shift_range,
'shear_range': args.shear_range,
'zoom_range': args.zoom_range,
'fill_mode': args.fill_mode,
}
train_generator, train_steps_per_epoch, val_generator, val_steps_per_epoch = datafeed.create_generators(
args.batch_size,
args.train_num,
args.test_num,
shuffle=args.shuffle,
normalize_images=args.normalize,
augment_training=args.augment_training,
augment_validation=args.augment_validation,
augmentation_args=augmentation_args)
# ==========================================================
# ======================Build model ========================
print('=' * 40)
print('Creating and compiling model...')
print('=' * 40)
imgs_train, mask_train = next(train_generator)
_, height, width, channels = imgs_train.shape
_, _, _, classes = mask_train.shape
logging.info("Building model...")
string_to_model = {
"unet": models.unet,
"dilated-unet": models.dilated_unet,
}
model = string_to_model[args.model]
m = model(height=height,
width=width,
channels=channels,
classes=classes,
features=args.features,
depth=args.depth,
padding=args.padding,
temperature=args.temperature,
batchnorm=args.batchnorm,
dropout=args.dropout)
m.summary()
#===============================================================
#======================= Build metrics, lossfunc ===============
if args.loss == 'pixel':
def lossfunc(y_true, y_pred):
return loss.weighted_categorical_crossentropy(
y_true, y_pred, args.loss_weights)
elif args.loss == 'dice':
def lossfunc(y_true, y_pred):
return loss.sorensen_dice_loss(y_true, y_pred, args.loss_weights)
elif args.loss == 'jaccard':
def lossfunc(y_true, y_pred):
return loss.jaccard_loss(y_true, y_pred, args.loss_weights)
else:
raise Exception("Unknown loss ({})".format(args.loss))
#===================== metrics =================================
metrics = []
if args.metrics_F1:
metrics.append(custom_metrics.F1)
if args.metrics_precision:
metrics.append(custom_metrics.precision)
if args.metrics_recall:
metrics.append(custom_metrics.recall)
if args.metrics_jaccard:
metrics.append(custom_metrics.jaccard)
#===================== parse for optimizer ====================
if args.optimizer == 'sgd':
opt = optimizers.SGD(lr=args.learning_rate,
decay=args.decay,
momentum=args.momentum)
elif args.optimizer == 'adam':
opt = optimizers.adam(lr=args.learning_rate, beta_1=0.9, beta_2=0.999)
else:
raise Exception("Unknown optimizer ({})".format(args.loss))
model.compile(loss=lossfunc, optimizer=opt, metrics=metrics)
# ========================================================
# ================== CallBacks ======================
print('=' * 40)
print('Checkpoint config')
print('=' * 40)
checkpoint_folder = args.checkpoint_dir
if not os.path.exists(checkpoint_folder):
os.makedirs(checkpoint_folder)
if args.checkpoint:
if args.loss == 'dice':
filepath = os.path.join(
args.outdir,
"weights-{epoch:02d}-{dice:.4f}--{val_dice:.4f}.hdf5")
monitor = 'val_dice'
mode = 'max'
elif args.loss == 'jaccard':
filepath = os.path.join(
args.outdir,
"weights-{epoch:02d}-{jaccard:.4f}--{val_jaccard:.4f}.hdf5")
monitor = 'val_jaccard'
mode = 'max'
checkpoint = ModelCheckpoint(filepath,
monitor=monitor,
verbose=1,
save_best_only=True,
mode=mode)
callbacks = [checkpoint]
else:
callbacks = []
history = metricsHistory()
callbacks.append(history)
# ========================================================
# ========================================================
print('=' * 40)
print('Fitting model...')
print('=' * 40)
logging.info("Begin training.")
model.fit_generator(train_generator,
epochs=args.epochs,
steps_per_epoch=train_steps_per_epoch,
validation_data=val_generator,
validation_steps=val_steps_per_epoch,
callbacks=callbacks)
m.save(os.path.join(args.outdir, args.outfile))
np.save(checkpoint_folder + "/losses", history.losses)
