def update_core(self):
# When we pass one iterator and optimizer to StandardUpdater.__init__,
# they are automatically named 'main'.
train_iter = self.get_iterator('main')
optimizer = self.get_optimizer('main')
# Get the next batch ( a list of json files)
batch = train_iter.__next__()
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
# Compute the loss at this time step and accumulate it
loss = optimizer.target(x)
optimizer.target.cleargrads() # Clear the parameter gradients
loss.backward() # Backprop
loss.unchain_backward() # Truncate the graph
# compute the gradient norm to check if it is normal or not
grad_norm = np.sqrt(
self._sum_sqnorm([p.grad for p in optimizer.target.params(False)]))
logging.info('grad norm={}'.format(grad_norm))
if math.isnan(grad_norm):
logging.warning('grad norm is nan. Do not update model.')
else:
optimizer.update()
delete_feat(x)
def evaluate(self):
'''evaluate over iterator'''
iterator = self._iterators['main']
eval_func = self.eval_func or self._targets['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
# for multi gpu calculation
chainer.cuda.get_device_from_id(self.device).use()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
with function.no_backprop_mode():
eval_func(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def update_core(self):
# When we pass one iterator and optimizer to StandardUpdater.__init__,
# they are automatically named 'main'.
train_iter = self.get_iterator('main')
optimizer = self.get_optimizer('main')
# Get the next batch ( a list of json files)
batch = train_iter.__next__()
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
if len(batch[0]) < self.num_gpu:
logging.warning('batch size is less than number of gpus. Ignored')
return
x = converter_kaldi(batch[0], self.reader)
# Compute the loss at this time step and accumulate it
loss = 1. / self.num_gpu * self.model(x)
optimizer.zero_grad() # Clear the parameter gradients
if self.num_gpu > 1:
loss.backward(torch.ones(self.num_gpu)) # Backprop
else:
loss.backward() # Backprop
loss.detach() # Truncate the graph
# compute the gradient norm to check if it is normal or not
grad_norm = torch.nn.utils.clip_grad_norm(
self.model.parameters(), self.grad_clip_threshold)
logging.info('grad norm={}'.format(grad_norm))
if math.isnan(grad_norm):
logging.warning('grad norm is nan. Do not update model.')
else:
optimizer.step()
delete_feat(x)
def evaluate(self):
iterator = self._iterators['main']
if self.eval_hook:
self.eval_hook(self)
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
summary = reporter_module.DictSummary()
for batch in it:
observation = {}
with reporter_module.report_scope(observation):
# read scp files
# x: original json with loaded features
# will be converted to chainer variable later
# batch only has one minibatch utterance, which is specified by batch[0]
x = converter_kaldi(batch[0], self.reader)
self.model.eval()
self.model(x)
delete_feat(x)
summary.add(observation)
return summary.compute_mean()
def update_core(self):
self.setup_workers()
self._send_message(('update', None))
with cuda.Device(self._devices[0]):
from cupy.cuda import nccl
# For reducing memory
self._master.cleargrads()
optimizer = self.get_optimizer('main')
batch = self.get_iterator('main').next()
x = converter_kaldi(batch[0], self.reader)
loss = self._master(x)
self._master.cleargrads()
loss.backward()
loss.unchain_backward()
# NCCL: reduce grads
null_stream = cuda.Stream.null
if self.comm is not None:
gg = gather_grads(self._master)
self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size,
nccl.NCCL_FLOAT,
nccl.NCCL_SUM,
0, null_stream.ptr)
scatter_grads(self._master, gg)
del gg
# check gradient value
grad_norm = np.sqrt(self._sum_sqnorm(
[p.grad for p in optimizer.target.params(False)]))
logging.info('grad norm={}'.format(grad_norm))
# update
if math.isnan(grad_norm):
logging.warning('grad norm is nan. Do not update model.')
else:
optimizer.update()
if self.comm is not None:
gp = gather_params(self._master)
self.comm.bcast(gp.data.ptr, gp.size, nccl.NCCL_FLOAT,
0, null_stream.ptr)
delete_feat(x)
def run(self):
from cupy.cuda import nccl
dev = cuda.Device(self.device)
dev.use()
self.setup()
gp = None
while True:
job, data = self.pipe.recv()
if job == 'finalize':
dev.synchronize()
break
if job == 'update':
# For reducing memory
self.model.cleargrads()
batch = self.iterator.next()
x = converter_kaldi(batch[0], self.reader)
observation = {}
with self.reporter.scope(observation):
loss = self.model(x)
self.model.cleargrads()
loss.backward()
loss.unchain_backward()
del loss
gg = gather_grads(self.model)
null_stream = cuda.Stream.null
self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size,
nccl.NCCL_FLOAT,
nccl.NCCL_SUM, 0,
null_stream.ptr)
del gg
self.model.cleargrads()
gp = gather_params(self.model)
self.comm.bcast(gp.data.ptr, gp.size,
nccl.NCCL_FLOAT, 0,
null_stream.ptr)
scatter_params(self.model, gp)
gp = None
delete_feat(x)
def train(args):
'''Run training'''
# display chainer version
logging.info('chainer version = ' + chainer.__version__)
# seed setting (chainer seed may not need it)
os.environ['CHAINER_SEED'] = str(args.seed)
logging.info('chainer seed = ' + os.environ['CHAINER_SEED'])
# debug mode setting
# 0 would be fastest, but 1 seems to be reasonable
# by considering reproducability
# revmoe type check
if args.debugmode < 2:
chainer.config.type_check = False
logging.info('chainer type check is disabled')
# use determinisitic computation or not
if args.debugmode < 1:
chainer.config.cudnn_deterministic = False
logging.info('chainer cudnn deterministic is disabled')
else:
chainer.config.cudnn_deterministic = True
# check cuda and cudnn availability
if not chainer.cuda.available:
logging.warning('cuda is not available')
if not chainer.cuda.cudnn_enabled:
logging.warning('cudnn is not available')
# get input and output dimension info
with open(args.valid_json, 'rb') as f:
valid_json = json.load(f)['utts']
utts = list(valid_json.keys())
# TODO(nelson) remove in future
if 'input' not in valid_json[utts[0]]:
logging.error("input file format (json) is modified, please redo"
"stage 2: Dictionary and Json Data Preparation")
sys.exit(1)
idim = int(valid_json[utts[0]]['input'][0]['shape'][1])
odim = int(valid_json[utts[0]]['output'][0]['shape'][1])
logging.info('#input dims : ' + str(idim))
logging.info('#output dims: ' + str(odim))
# check attention type
if args.atype not in ['noatt', 'dot', 'location']:
raise NotImplementedError(
'chainer supports only noatt, dot, and location attention.')
# specify attention, CTC, hybrid mode
if args.mtlalpha == 1.0:
mtl_mode = 'ctc'
logging.info('Pure CTC mode')
elif args.mtlalpha == 0.0:
mtl_mode = 'att'
logging.info('Pure attention mode')
else:
mtl_mode = 'mtl'
logging.info('Multitask learning mode')
# specify model architecture
e2e = E2E(idim, odim, args)
model = Loss(e2e, args.mtlalpha)
# write model config
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
model_conf = args.outdir + '/model.conf'
with open(model_conf, 'wb') as f:
logging.info('writing a model config file to' + model_conf)
# TODO(watanabe) use others than pickle, possibly json, and save as a text
pickle.dump((idim, odim, args), f)
for key in sorted(vars(args).keys()):
logging.info('ARGS: ' + key + ': ' + str(vars(args)[key]))
# Set gpu
ngpu = args.ngpu
if ngpu == 1:
gpu_id = 0
# Make a specified GPU current
chainer.cuda.get_device_from_id(gpu_id).use()
model.to_gpu() # Copy the model to the GPU
logging.info('single gpu calculation.')
elif ngpu > 1:
gpu_id = 0
devices = {'main': gpu_id}
for gid in six.moves.xrange(1, ngpu):
devices['sub_%d' % gid] = gid
logging.info('multi gpu calculation (#gpus = %d).' % ngpu)
logging.info('batch size is automatically increased (%d -> %d)' %
(args.batch_size, args.batch_size * args.ngpu))
else:
gpu_id = -1
logging.info('cpu calculation')
# Setup an optimizer
if args.opt == 'adadelta':
optimizer = chainer.optimizers.AdaDelta(eps=args.eps)
elif args.opt == 'adam':
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.GradientClipping(args.grad_clip))
# read json data
with open(args.train_json, 'rb') as f:
train_json = json.load(f)['utts']
with open(args.valid_json, 'rb') as f:
valid_json = json.load(f)['utts']
# set up training iterator and updater
if ngpu <= 1:
# make minibatch list (variable length)
train = make_batchset(train_json, args.batch_size, args.maxlen_in,
args.maxlen_out, args.minibatches)
# hack to make batchsize argument as 1
# actual batchsize is included in a list
train_iter = chainer.iterators.SerialIterator(train, 1)
# set up updater
updater = ChainerSeqUpdaterKaldi(train_iter,
optimizer,
converter=converter_kaldi,
device=gpu_id)
else:
# set up minibatches
train_subsets = []
for gid in six.moves.xrange(ngpu):
# make subset
train_json_subset = {
k: v
for i, (k, v) in enumerate(train_json.viewitems())
if i % ngpu == gid
}
# make minibatch list (variable length)
train_subsets += [
make_batchset(train_json_subset, args.batch_size,
args.maxlen_in, args.maxlen_out,
args.minibatches)
]
# each subset must have same length for MultiprocessParallelUpdater
maxlen = max([len(train_subset) for train_subset in train_subsets])
for train_subset in train_subsets:
if maxlen != len(train_subset):
for i in six.moves.xrange(maxlen - len(train_subset)):
train_subset += [train_subset[i]]
# hack to make batchsize argument as 1
# actual batchsize is included in a list
train_iters = [
chainer.iterators.MultiprocessIterator(train_subsets[gid],
1,
n_processes=1)
for gid in six.moves.xrange(ngpu)
]
# set up updater
updater = ChainerMultiProcessParallelUpdaterKaldi(
train_iters, optimizer, converter=converter_kaldi, devices=devices)
# Set up a trainer
trainer = training.Trainer(updater, (args.epochs, 'epoch'),
out=args.outdir)
# Resume from a snapshot
if args.resume:
chainer.serializers.load_npz(args.resume, trainer)
# set up validation iterator
valid = make_batchset(valid_json, args.batch_size, args.maxlen_in,
args.maxlen_out, args.minibatches)
valid_iter = chainer.iterators.SerialIterator(valid,
1,
repeat=False,
shuffle=False)
# Evaluate the model with the test dataset for each epoch
trainer.extend(
extensions.Evaluator(valid_iter,
model,
converter=converter_kaldi,
device=gpu_id))
# Save attention weight each epoch
if args.num_save_attention > 0 and args.mtlalpha != 1.0:
data = sorted(list(valid_json.items())[:args.num_save_attention],
key=lambda x: int(x[1]['input'][0]['shape'][1]),
reverse=True)
data = converter_kaldi([data], device=gpu_id)
trainer.extend(PlotAttentionReport(model, data,
args.outdir + "/att_ws"),
trigger=(1, 'epoch'))
# Take a snapshot for each specified epoch
trainer.extend(extensions.snapshot(), trigger=(1, 'epoch'))
# Make a plot for training and validation values
trainer.extend(
extensions.PlotReport([
'main/loss', 'validation/main/loss', 'main/loss_ctc',
'validation/main/loss_ctc', 'main/loss_att',
'validation/main/loss_att'
],
'epoch',
file_name='loss.png'))
trainer.extend(
extensions.PlotReport(['main/acc', 'validation/main/acc'],
'epoch',
file_name='acc.png'))
# Save best models
trainer.extend(
extensions.snapshot_object(model, 'model.loss.best'),
trigger=training.triggers.MinValueTrigger('validation/main/loss'))
if mtl_mode is not 'ctc':
trainer.extend(
extensions.snapshot_object(model, 'model.acc.best'),
trigger=training.triggers.MaxValueTrigger('validation/main/acc'))
# epsilon decay in the optimizer
if args.opt == 'adadelta':
if args.criterion == 'acc' and mtl_mode is not 'ctc':
trainer.extend(restore_snapshot(model,
args.outdir + '/model.acc.best'),
trigger=CompareValueTrigger(
'validation/main/acc', lambda best_value,
current_value: best_value > current_value))
trainer.extend(adadelta_eps_decay(args.eps_decay),
trigger=CompareValueTrigger(
'validation/main/acc', lambda best_value,
current_value: best_value > current_value))
elif args.criterion == 'loss':
trainer.extend(restore_snapshot(model,
args.outdir + '/model.loss.best'),
trigger=CompareValueTrigger(
'validation/main/loss', lambda best_value,
current_value: best_value < current_value))
trainer.extend(adadelta_eps_decay(args.eps_decay),
trigger=CompareValueTrigger(
'validation/main/loss', lambda best_value,
current_value: best_value < current_value))
# Write a log of evaluation statistics for each epoch
trainer.extend(extensions.LogReport(trigger=(100, 'iteration')))
report_keys = [
'epoch', 'iteration', 'main/loss', 'main/loss_ctc', 'main/loss_att',
'validation/main/loss', 'validation/main/loss_ctc',
'validation/main/loss_att', 'main/acc', 'validation/main/acc',
'elapsed_time'
]
if args.opt == 'adadelta':
trainer.extend(extensions.observe_value(
'eps', lambda trainer: trainer.updater.get_optimizer('main').eps),
trigger=(100, 'iteration'))
report_keys.append('eps')
trainer.extend(extensions.PrintReport(report_keys),
trigger=(100, 'iteration'))
trainer.extend(extensions.ProgressBar())
# Run the training
trainer.run()
def train(args):
'''Run training'''
# seed setting
torch.manual_seed(args.seed)
# debug mode setting
# 0 would be fastest, but 1 seems to be reasonable
# by considering reproducability
# revmoe type check
if args.debugmode < 2:
chainer.config.type_check = False
logging.info('torch type check is disabled')
# use determinisitic computation or not
if args.debugmode < 1:
torch.backends.cudnn.deterministic = False
logging.info('torch cudnn deterministic is disabled')
else:
torch.backends.cudnn.deterministic = True
# check cuda availability
if not torch.cuda.is_available():
logging.warning('cuda is not available')
# get input and output dimension info
with open(args.valid_label, 'rb') as f:
valid_json = json.load(f)['utts']
utts = list(valid_json.keys())
idim = int(valid_json[utts[0]]['idim'])
odim = int(valid_json[utts[0]]['odim'])
logging.info('#input dims : ' + str(idim))
logging.info('#output dims: ' + str(odim))
# specify attention, CTC, hybrid mode
if args.mtlalpha == 1.0:
mtl_mode = 'ctc'
logging.info('Pure CTC mode')
elif args.mtlalpha == 0.0:
mtl_mode = 'att'
logging.info('Pure attention mode')
else:
mtl_mode = 'mtl'
logging.info('Multitask learning mode')
# specify model architecture
e2e = E2E(idim, odim, args)
model = Loss(e2e, args.mtlalpha)
# write model config
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
model_conf = args.outdir + '/model.conf'
with open(model_conf, 'wb') as f:
logging.info('writing a model config file to' + model_conf)
# TODO(watanabe) use others than pickle, possibly json, and save as a text
pickle.dump((idim, odim, args), f)
for key in sorted(vars(args).keys()):
logging.info('ARGS: ' + key + ': ' + str(vars(args)[key]))
# Set gpu
reporter = model.reporter
ngpu = args.ngpu
if ngpu == 1:
gpu_id = range(ngpu)
logging.info('gpu id: ' + str(gpu_id))
model.cuda()
elif ngpu > 1:
gpu_id = range(ngpu)
logging.info('gpu id: ' + str(gpu_id))
model = DataParallel(model, device_ids=gpu_id)
model.cuda()
logging.info('batch size is automatically increased (%d -> %d)' %
(args.batch_size, args.batch_size * args.ngpu))
args.batch_size *= args.ngpu
else:
gpu_id = [-1]
# Setup an optimizer
if args.opt == 'adadelta':
optimizer = torch.optim.Adadelta(model.parameters(),
rho=0.95,
eps=args.eps)
elif args.opt == 'adam':
optimizer = torch.optim.Adam(model.parameters())
# FIXME: TOO DIRTY HACK
setattr(optimizer, "target", reporter)
setattr(optimizer, "serialize", lambda s: reporter.serialize(s))
# read json data
with open(args.train_label, 'rb') as f:
train_json = json.load(f)['utts']
with open(args.valid_label, 'rb') as f:
valid_json = json.load(f)['utts']
# make minibatch list (variable length)
train = make_batchset(train_json, args.batch_size, args.maxlen_in,
args.maxlen_out, args.minibatches)
valid = make_batchset(valid_json, args.batch_size, args.maxlen_in,
args.maxlen_out, args.minibatches)
# hack to make batchsze argument as 1
# actual bathsize is included in a list
train_iter = chainer.iterators.SerialIterator(train, 1)
valid_iter = chainer.iterators.SerialIterator(valid,
1,
repeat=False,
shuffle=False)
# prepare Kaldi reader
train_reader = lazy_io.read_dict_scp(args.train_feat)
valid_reader = lazy_io.read_dict_scp(args.valid_feat)
# Set up a trainer
updater = PytorchSeqUpdaterKaldi(model, args.grad_clip, train_iter,
optimizer, train_reader, gpu_id)
trainer = training.Trainer(updater, (args.epochs, 'epoch'),
out=args.outdir)
# Resume from a snapshot
if args.resume:
chainer.serializers.load_npz(args.resume, trainer)
if ngpu > 1:
model.module.load_state_dict(
torch.load(args.outdir + '/model.acc.best'))
else:
model.load_state_dict(torch.load(args.outdir + '/model.acc.best'))
model = trainer.updater.model
# Evaluate the model with the test dataset for each epoch
trainer.extend(
PytorchSeqEvaluaterKaldi(model,
valid_iter,
reporter,
valid_reader,
device=gpu_id))
# Save attention weight each epoch
if args.num_save_attention > 0 and args.mtlalpha != 1.0:
data = sorted(valid_json.items()[:args.num_save_attention],
key=lambda x: int(x[1]['ilen']),
reverse=True)
data = converter_kaldi(data, valid_reader)
trainer.extend(PlotAttentionReport(model, data,
args.outdir + "/att_ws"),
trigger=(1, 'epoch'))
# Take a snapshot for each specified epoch
trainer.extend(extensions.snapshot(), trigger=(1, 'epoch'))
# Make a plot for training and validation values
trainer.extend(
extensions.PlotReport([
'main/loss', 'validation/main/loss', 'main/loss_ctc',
'validation/main/loss_ctc', 'main/loss_att',
'validation/main/loss_att'
],
'epoch',
file_name='loss.png'))
trainer.extend(
extensions.PlotReport(['main/acc', 'validation/main/acc'],
'epoch',
file_name='acc.png'))
# Save best models
def torch_save(path, _):
if ngpu > 1:
torch.save(model.module.state_dict(), path)
torch.save(model.module, path + ".pkl")
else:
torch.save(model.state_dict(), path)
torch.save(model, path + ".pkl")
trainer.extend(
extensions.snapshot_object(model,
'model.loss.best',
savefun=torch_save),
trigger=training.triggers.MinValueTrigger('validation/main/loss'))
if mtl_mode is not 'ctc':
trainer.extend(
extensions.snapshot_object(model,
'model.acc.best',
savefun=torch_save),
trigger=training.triggers.MaxValueTrigger('validation/main/acc'))
# epsilon decay in the optimizer
def torch_load(path, obj):
if ngpu > 1:
model.module.load_state_dict(torch.load(path))
else:
model.load_state_dict(torch.load(path))
return obj
if args.opt == 'adadelta':
if args.criterion == 'acc' and mtl_mode is not 'ctc':
trainer.extend(restore_snapshot(model,
args.outdir + '/model.acc.best',
load_fn=torch_load),
trigger=CompareValueTrigger(
'validation/main/acc', lambda best_value,
current_value: best_value > current_value))
trainer.extend(adadelta_eps_decay(args.eps_decay),
trigger=CompareValueTrigger(
'validation/main/acc', lambda best_value,
current_value: best_value > current_value))
elif args.criterion == 'loss':
trainer.extend(restore_snapshot(model,
args.outdir + '/model.loss.best',
load_fn=torch_load),
trigger=CompareValueTrigger(
'validation/main/loss', lambda best_value,
current_value: best_value < current_value))
trainer.extend(adadelta_eps_decay(args.eps_decay),
trigger=CompareValueTrigger(
'validation/main/loss', lambda best_value,
current_value: best_value < current_value))
# Write a log of evaluation statistics for each epoch
trainer.extend(extensions.LogReport(trigger=(100, 'iteration')))
report_keys = [
'epoch', 'iteration', 'main/loss', 'main/loss_ctc', 'main/loss_att',
'validation/main/loss', 'validation/main/loss_ctc',
'validation/main/loss_att', 'main/acc', 'validation/main/acc',
'elapsed_time'
]
if args.opt == 'adadelta':
trainer.extend(extensions.observe_value(
'eps', lambda trainer: trainer.updater.get_optimizer('main').
param_groups[0]["eps"]),
trigger=(100, 'iteration'))
report_keys.append('eps')
trainer.extend(extensions.PrintReport(report_keys),
trigger=(100, 'iteration'))
trainer.extend(extensions.ProgressBar())
# Run the training
trainer.run()
