def _check_args(self):
""" Perfunctory argument checks and modification.
"""
args = self.args
# Check scp and labels
common.CHK_GE(len(args.valid_scp), 1)
common.CHK_EQ(len(args.valid_labels) % len(args.valid_scp), 0)
labels_per_valid_scp = len(args.valid_labels) / len(args.valid_scp)
common.CHK_EQ(len(args.train_labels), labels_per_valid_scp)
# Check task
if len(args.task) == 1:
args.task = args.task * len(args.train_labels)
common.CHK_EQ(len(args.task), len(args.train_labels))
for task in args.task:
common.CHK_VALS(task, TASKS)
if args.num_classes is not None:
common.CHK_EQ(len(args.task), len(args.num_classes))
if args.task_weights is None:
args.task_weights = [1.0] * len(args.task)
common.CHK_EQ(len(args.task), len(args.task_weights))
# Check others
for layer_type in args.layer_type:
common.CHK_VALS(layer_type, LAYERS.keys())
common.CHK_VALS(args.optimizer, OPTIMIZERS.keys())
common.CHK_VALS(args.lrate, LRATES.keys())
def copy_dnn(from_dnn, to_dnn):
""" Copy weights from from_dnn to to_dnn. The number of layers must match.
"""
common.CHK_EQ(len(from_dnn.layers), len(to_dnn.layers))
for i in range(len(from_dnn.layers)):
to_dnn.layers[i].W.set_value(from_dnn.layers[i].W.get_value())
to_dnn.layers[i].b.set_value(from_dnn.layers[i].b.get_value())
return to_dnn
def accum(self, y_true, y_pred):
classes = preds_to_classes(y_pred)
common.CHK_EQ(len(y_true), len(classes))
for corr, pred in zip(y_true, classes):
if corr not in self.total:
self.total[corr] = 0.0
self.correct[corr] = 0.0
self.total[corr] += 1
self.correct[corr] += int(corr == pred)
def _check_args(self):
""" Perfunctory argument checks.
"""
args = self.args
common.CHK_GE(len(args.valid_scp), 1)
common.CHK_EQ(len(args.valid_scp), len(args.valid_labels))
common.CHK_VALS(args.task, TASKS)
for layer_type in args.layer_type:
common.CHK_VALS(layer_type, LAYERS.keys())
common.CHK_VALS(args.optimizer, OPTIMIZERS.keys())
common.CHK_VALS(args.lrate, LRATES.keys())
def __extract(extract_fn, ark_out, dataset, x, utt_names, utt_frames):
# Set data
feats = dataset.get_data_by_utt_names(utt_names)
common.CHK_EQ(len(feats), np.sum(utt_frames))
x[:len(feats)] = feats
# Get big feature matrix
ext_feats = extract_fn(x[:len(feats)])
# Write to ark for each utterance
start = 0
for utt, frames in zip(utt_names, utt_frames):
end = start + frames
ark_out.write_kaldi_mat(utt, ext_feats[start:end])
start = end
def __nnet_fwd(output_fn, dataset, x, shared_x, utt_names, utt_frames,
log_priors):
# Set data
feats = dataset.get_data_by_utt_names(utt_names)
common.CHK_EQ(len(feats), np.sum(utt_frames))
x[:len(feats)] = feats
shared_x.set_value(x[:len(feats)], borrow=True)
# Process each utterance
start = 0
for utt, frames in zip(utt_names, utt_frames):
end = start + frames
log_probs = np.log(output_fn(start, end))
if log_priors is not None:
log_probs -= log_priors
print_matrix(utt, log_probs)
start = end
def comp_UARs(corr_lbls, pred_lbls):
""" Compute recall rate for each class. Return (recall_rates, classes).
"""
common.CHK_EQ(len(pred_lbls), len(corr_lbls))
correct = {}
count = {}
for pred, corr in zip(pred_lbls, corr_lbls):
if corr not in count:
count[corr] = 0.0
correct[corr] = 0.0
count[corr] += 1
correct[corr] += 1 if pred == corr else 0
recall_rates = []
classes = []
for c in count.keys():
recall_rates.append(correct[c] / count[c])
classes.append(c)
return (recall_rates, classes)
def ali_with_length_read(ali_fname, ordered=True, expand=False):
"""
Read a Kaldi alignment with length file. Returns a dictionary.
If `expand` is False, each value is a list of 2-element tuples
containing the token ID and length in frames.
If `expand` is True, each value is an expanded list of labels.
"""
ret = OrderedDict() if ordered else {}
with open(ali_fname, 'r') as f:
for line in f:
ary = line.strip().replace(';', '').split()
common.CHK_EQ(len(ary) % 2, 1)
if ary[0] in ret:
log('WARNING: duplicate key {}'.format(ary[0]))
ret[ary[0]] = []
for i in range(1, len(ary), 2):
if expand:
ret[ary[0]].extend([ary[i]] * int(ary[i + 1]))
else:
ret[ary[0]].append((ary[i], int(ary[i + 1])))
return ret
def __init__(self, data_gens, buf_cls, shuffle_gens=True, **kwargs):
"""
:type buf_cls: class
:param buf_cls: Class to use for buffered dataset
:type data_gens: list of chaipy.data.incremental.DataGenerator
:param data_gens: For generating individual datasets
:type shuffle_gens: bool
:param shuffle_gens: Shuffle generator list after each iteration
:type kwargs: dict
:param kwargs: Keyword arguments used to initialize `buf_cls`
"""
common.CHK_EQ(type(data_gens), list)
common.CHK_GE(len(data_gens), 1)
self.__buf_dataset = None
self._data_gens = data_gens
self._buf_cls = buf_cls
self._shuffle_gens = shuffle_gens
self._kwargs = kwargs
self.reset()
def AUC(corr_lbls, pred_scores, positive_lbl):
""" Compute AUC (Area Under the Curve) """
common.CHK_EQ(len(pred_scores), len(corr_lbls))
fpr, tpr, _ = metrics.roc_curve(corr_lbls, pred_scores,
pos_label=positive_lbl)
return metrics.auc(fpr, tpr)
def train_mtrnn_v2(model,
buf_train,
buf_valids,
lrate,
validate_on=0,
ntasks=None,
optimizer=None,
task_weights=None,
primary_task=0,
save_dir=None,
pre_validate=False,
restore=False,
loss='sparse_categorical_crossentropy',
metrics=['acc'],
validate_metrics=None,
stateful=False):
"""
Train multi-task RNN given a training and multiple validation sets.
:type model: Keras model
:param model: The model to train
:type buf_train: chaipy.data.temporal.BufferedUttData
:param buf_train: The dataset to train on (multi-label)
:type buf_valids: list of chaipy.data.temporal.BufferedUttData
:param buf_valids: The datasets to validate on (multi-label)
:type lrate: utils.learn_rates.LearningRate
:param lrate: Object to control learning rate decay
:type validate_on: int
:param validate_on: (optional) Which validation set to use for learning
rate schedule. By default, use the first validation set.
:type ntasks: int
:param ntasks: (optional) Number of tasks. If not set, determine automatically.
:type optimizer: keras.optimizers.Optimizer
:param optimizer: (optional) Optimizer to use. If not set, use Adam.
:type task_weights: list
:param task_weights: (optional) Task weights. If not set, use 1.0 everywhere.
:type primary_task: int
:param primary_task: (optional) Validate on this task. If set to -1, use
the average metric across all tasks.
:type save_dir: str
:param save_dir: (optional) If not None, save the most recent intermediate
model to this directory. We only keep the most recent model in this
directory, except for the final model since we expect the caller of
this function to save it manually. The caller is responsible for the
creation and deletion of this directory. The code does not create and
delete the directory automatically.
:type pre_validate: bool
:param pre_validate: (optional) If True, do one validation iteration
before training the model and use this value to bootstrap lrate.
:type restore: bool
:param restore: (optional) If True, restore parameters of the previous model
if new validation error is higher than the lowest error thus far. This
strategy is suitable for less stable learning.
:type loss: str or keras loss function or list
:param loss: (optional) Loss function(s) for training. Use list to apply
different loss functions to different tasks. Accepted types are:
* str: must be a key in `chaipy.common.metrics.OBJECTIVE_FUNCTIONS`
or one recognized by keras.
* function: a symbolic function with signature `fn(y_true, y_pred)`
:type metrics: list of str
:param metrics: (optional) Metrics to report when showing training progress.
Must use the exact Keras name defined by the metric function! For
example, use 'mean_squared_error' instead of 'mse'; use 'acc' instead
of 'accuracy'. If multiple metrics are specified, the first metric is
used to log training errors.
:type validate_metrics: list of str or function
:param validate_metrics: (optional) Which metrics to use for validation.
If unspecified, use training metrics. If multiple metrics are specified,
the first metric is used for actual validation. Valid string metrics are
'acc' or those defined in `chaipy.common.metrics.VALIDATION_METRICS`.
Functions must have the signature `fn(y_true, y_pred)`. If different
tasks need different validation metrics, use a dict that maps from
task ID to a list of validation metrics.
:type validate_metrics: list
:param validate_metrics: (optional) Which metrics to use for validation.
If unspecified, use training metrics. If multiple metrics are specified,
the first metric is used for actual validation. By default, the same
metrics are applied to all tasks. If different tasks need different
validation metrics, use a dict that maps from task ID to a list of
validation metrics. Accepted types are:
* str: must be a key in `chaipy.common.metrics.VALIDATION_METRICS`
* function: must have the signature `fn(y_true, y_pred)`
* class: must be a subclass of `chaipy.common.metrics.AbstractMetric`
:type stateful: bool
:param stateful: (optional) Network is stateful.
:rtype: tuple
:return: (training errors, [validation errors])
"""
common.CHK_EQ(type(buf_valids), list)
if optimizer is None:
optimizer = adam(lr=lrate.get_rate())
if type(loss) == list:
loss = [init_loss(l) for l in loss]
else:
loss = init_loss(loss)
if validate_metrics is None:
validate_metrics = metrics
io.log('... finetuning the model')
train_errs, valid_errs = [], []
for _ in range(len(buf_valids)):
valid_errs.append([])
prev_weights = None
# Do one premptive validation iteration if necessary
if pre_validate:
train_errs.append(-1)
for i in range(len(buf_valids)):
all_valid_errs = validate_mtrnn(model,
buf_valids[i],
ntasks=ntasks,
metrics=validate_metrics,
stateful=stateful)
io.log('** Pre-validate: all validation errors ({}) {}'.format(
i, all_valid_errs))
valid_errs[i].append(__get_metric(all_valid_errs, primary_task))
io.log('** Pre-validate: validation error ({}) {}'.format(
i, valid_errs[i][-1]))
lrate.lowest_error = valid_errs[validate_on][-1]
if restore:
prev_weights = [l.get_weights() for l in model.layers]
# Start training
model.compile(optimizer=optimizer,
loss=loss,
loss_weights=task_weights,
metrics=metrics)
while lrate.get_rate() != 0:
model.optimizer.lr.set_value(lrate.get_rate())
io.log('*** Optimizer state: {}'.format(model.optimizer.get_config()))
# One epoch of training
all_train_errs = _train_mt(model,
buf_train,
ntasks=ntasks,
metrics=metrics,
stateful=stateful,
loss=loss)
io.log('** Epoch {}, lrate {}, all training errors {}'.format(
lrate.epoch, lrate.get_rate(), all_train_errs))
train_errs.append(__get_metric(all_train_errs, primary_task))
io.log('** Epoch {}, lrate {}, training error {}'.format(
lrate.epoch, lrate.get_rate(), train_errs[-1]))
for i in range(len(buf_valids)):
all_valid_errs = validate_mtrnn(model,
buf_valids[i],
ntasks=ntasks,
metrics=validate_metrics,
stateful=stateful)
io.log(
'** Epoch {}, lrate {}, all validation errors ({}) {}'.format(
lrate.epoch, lrate.get_rate(), i, all_valid_errs))
valid_errs[i].append(__get_metric(all_valid_errs, primary_task))
io.log('** Epoch {}, lrate {}, validation error ({}) {}'.format(
lrate.epoch, lrate.get_rate(), i, valid_errs[i][-1]))
prev_error = lrate.lowest_error
lrate.get_next_rate(current_error=valid_errs[validate_on][-1])
io.log('**** Updated lrate: {}'.format(lrate.get_rate()))
# Restore model weights if necessary
if restore:
if valid_errs[validate_on][-1] < prev_error:
prev_weights = [l.get_weights() for l in model.layers]
elif prev_weights is None:
io.log(
'**WARN** error increased but no prev_weights to restore!')
elif lrate.epoch <= lrate.min_epoch_decay_start:
io.log(
'** Only {} training epoch, need at least {} to restore **'
.format(lrate.epoch - 1, lrate.min_epoch_decay_start))
else:
io.log('** Restoring params of previous best model **')
for l, lw in zip(model.layers, prev_weights):
l.set_weights(lw)
idx = numpy.argmin(valid_errs[validate_on])
io.log(
'** Restored: train err = {}, valid errs = {} **'.format(
train_errs[idx],
[valid_errs[i][idx] for i in range(len(buf_valids))]))
# Save intermediate model
if save_dir is not None:
curr_epoch = lrate.epoch - 1
prev_epoch = curr_epoch - 1
curr_fname = os.path.join(save_dir, '{}'.format(curr_epoch))
prev_fname = os.path.join(save_dir, '{}'.format(prev_epoch))
if lrate.get_rate() != 0:
io.json_save('{}.json'.format(curr_fname), model.to_json())
model.save_weights('{}.weights'.format(curr_fname))
for suffix in ['json', 'weights']:
if os.path.exists('{}.{}'.format(prev_fname, suffix)):
os.remove('{}.{}'.format(prev_fname, suffix))
# If restoring, make sure the final err is also the best err
if restore and valid_errs[validate_on][-1] != numpy.min(
valid_errs[validate_on]):
idx = numpy.argmin(valid_errs[validate_on])
train_errs.append(train_errs[idx])
for i in range(len(buf_valids)):
valid_errs[i].append(valid_errs[i][idx])
return (train_errs, valid_errs)