labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=args.bidirectional)
decoder = GreedyDecoder(labels)
train_dataset = SpectrogramDataset(audio_conf=audio_conf, manifest_filepath=args.train_manifest, labels=labels,
normalize=True, augment=args.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf, manifest_filepath=args.val_manifest, labels=labels,
normalize=True, augment=False)
if not args.distributed:
train_sampler = BucketingSampler(train_dataset, batch_size=args.batch_size)
else:
train_sampler = DistributedBucketingSampler(train_dataset, batch_size=args.batch_size,
num_replicas=args.world_size, rank=args.rank)
train_loader = AudioDataLoader(train_dataset,
num_workers=args.num_workers, batch_sampler=train_sampler)
test_loader = AudioDataLoader(test_dataset, batch_size=args.batch_size,
num_workers=args.num_workers)
print("AudioDataLoader init done !")
if (not args.no_shuffle and start_epoch != 0) or args.no_sorta_grad:
print("Shuffling batches for the following epochs")
train_sampler.shuffle(start_epoch)
if not args.distributed:
model = model.to(device)
else :
model.to(device)
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[device], output_device=[device])
print(model)
print("Number of parameters: %d" % DeepSpeech.get_param_size(model))
# optimizer 实例化 需要 在 DistributedDataParallel 操作之后
parameters = model.parameters()
def main():
args = parser.parse_args()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
if params.rnn_type == 'gru' and params.rnn_act_type != 'tanh':
print "ERROR: GRU does not currently support activations other than tanh"
sys.exit()
if params.rnn_type == 'rnn' and params.rnn_act_type != 'relu':
print "ERROR: We should be using ReLU RNNs"
sys.exit()
print("=======================================================")
for arg in vars(args):
print "***%s = %s " % (arg.ljust(25), getattr(args, arg))
print("=======================================================")
save_folder = args.save_folder
loss_results, cer_results, wer_results = torch.Tensor(
params.epochs), torch.Tensor(params.epochs), torch.Tensor(
params.epochs)
best_wer = None
try:
os.makedirs(save_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
criterion = CTCLoss()
with open(params.labels_path) as label_file:
labels = str(''.join(json.load(label_file)))
audio_conf = dict(sample_rate=params.sample_rate,
window_size=params.window_size,
window_stride=params.window_stride,
window=params.window,
noise_dir=params.noise_dir,
noise_prob=params.noise_prob,
noise_levels=(params.noise_min, params.noise_max))
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=params.train_manifest,
labels=labels,
normalize=True,
augment=params.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=params.val_manifest,
labels=labels,
normalize=True,
augment=False)
train_loader = AudioDataLoader(train_dataset,
batch_size=params.batch_size,
num_workers=1)
test_loader = AudioDataLoader(test_dataset,
batch_size=params.batch_size,
num_workers=1)
rnn_type = params.rnn_type.lower()
assert rnn_type in supported_rnns, "rnn_type should be either lstm, rnn or gru"
model = DeepSpeech(rnn_hidden_size=params.hidden_size,
nb_layers=params.hidden_layers,
labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=True,
rnn_activation=params.rnn_act_type,
bias=params.bias)
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters,
lr=params.lr,
momentum=params.momentum,
nesterov=True,
weight_decay=params.l2)
decoder = GreedyDecoder(labels)
if args.continue_from:
print("Loading checkpoint model %s" % args.continue_from)
package = torch.load(args.continue_from)
model.load_state_dict(package['state_dict'])
optimizer.load_state_dict(package['optim_dict'])
start_epoch = int(package.get(
'epoch', 1)) - 1 # Python index start at 0 for training
start_iter = package.get('iteration', None)
if start_iter is None:
start_epoch += 1 # Assume that we saved a model after an epoch finished, so start at the next epoch.
start_iter = 0
else:
start_iter += 1
avg_loss = int(package.get('avg_loss', 0))
if args.start_epoch != -1:
start_epoch = args.start_epoch
loss_results[:
start_epoch], cer_results[:start_epoch], wer_results[:start_epoch] = package[
'loss_results'][:start_epoch], package[
'cer_results'][:start_epoch], package[
'wer_results'][:start_epoch]
print loss_results
epoch = start_epoch
else:
avg_loss = 0
start_epoch = 0
start_iter = 0
avg_training_loss = 0
if params.cuda:
model = torch.nn.DataParallel(model).cuda()
print(model)
print("Number of parameters: %d" % DeepSpeech.get_param_size(model))
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ctc_time = AverageMeter()
for epoch in range(start_epoch, params.epochs):
model.train()
end = time.time()
for i, (data) in enumerate(train_loader, start=start_iter):
if i == len(train_loader):
break
inputs, targets, input_percentages, target_sizes = data
# measure data loading time
data_time.update(time.time() - end)
inputs = Variable(inputs, requires_grad=False)
target_sizes = Variable(target_sizes, requires_grad=False)
targets = Variable(targets, requires_grad=False)
if params.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
seq_length = out.size(0)
sizes = Variable(input_percentages.mul_(int(seq_length)).int(),
requires_grad=False)
ctc_start_time = time.time()
loss = criterion(out, targets, sizes, target_sizes)
ctc_time.update(time.time() - ctc_start_time)
loss = loss / inputs.size(0) # average the loss by minibatch
loss_sum = loss.data.sum()
inf = float("inf")
if loss_sum == inf or loss_sum == -inf:
print("WARNING: received an inf loss, setting loss value to 0")
loss_value = 0
else:
loss_value = loss.data[0]
avg_loss += loss_value
losses.update(loss_value, inputs.size(0))
# compute gradient
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), params.max_norm)
# SGD step
optimizer.step()
if params.cuda:
torch.cuda.synchronize()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
print(
'Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'CTC Time {ctc_time.val:.3f} ({ctc_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
(epoch + 1), (i + 1),
len(train_loader),
batch_time=batch_time,
data_time=data_time,
ctc_time=ctc_time,
loss=losses))
del loss
del out
avg_loss /= len(train_loader)
print('Training Summary Epoch: [{0}]\t'
'Average Loss {loss:.3f}\t'.format(
epoch + 1,
loss=avg_loss,
))
start_iter = 0 # Reset start iteration for next epoch
total_cer, total_wer = 0, 0
model.eval()
wer, cer = eval_model(model, test_loader, decoder)
loss_results[epoch] = avg_loss
wer_results[epoch] = wer
cer_results[epoch] = cer
print(
'Validation Summary Epoch: [{0}]\t'
'Average WER {wer:.3f}\t'
'Average CER {cer:.3f}\t'.format(epoch + 1, wer=wer, cer=cer))
if args.checkpoint:
file_path = '%s/deepspeech_%d.pth.tar' % (save_folder, epoch + 1)
torch.save(
DeepSpeech.serialize(model,
optimizer=optimizer,
epoch=epoch,
loss_results=loss_results,
wer_results=wer_results,
cer_results=cer_results), file_path)
# anneal lr
optim_state = optimizer.state_dict()
optim_state['param_groups'][0]['lr'] = optim_state['param_groups'][0][
'lr'] / params.learning_anneal
optimizer.load_state_dict(optim_state)
print('Learning rate annealed to: {lr:.6f}'.format(
lr=optim_state['param_groups'][0]['lr']))
if best_wer is None or best_wer > wer:
print("Found better validated model, saving to %s" %
args.model_path)
torch.save(
DeepSpeech.serialize(model,
optimizer=optimizer,
epoch=epoch,
loss_results=loss_results,
wer_results=wer_results,
cer_results=cer_results), args.model_path)
best_wer = wer
avg_loss = 0
#If set to exit at a given accuracy, exit
if params.exit_at_acc and (best_wer <= args.acc):
break
print("=======================================================")
print "***Best WER = ", best_wer
for arg in vars(args):
print "***%s = %s " % (arg.ljust(25), getattr(args, arg))
print("=======================================================")
cutoff_prob=args.cutoff_prob,
beam_width=args.beam_width,
num_processes=args.lm_workers)
elif args.decoder == "greedy":
decoder = GreedyDecoder(model.labels,
blank_index=model.labels.index('_'))
else:
decoder = None
target_decoder = GreedyDecoder(model.labels,
blank_index=model.labels.index('_'))
test_dataset = SpectrogramDataset(audio_conf=model.audio_conf,
manifest_filepath=args.test_manifest,
labels=model.labels,
normalize=True)
test_loader = AudioDataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
wer, cer, output_data = evaluate(test_loader=test_loader,
device=device,
model=model,
decoder=decoder,
target_decoder=target_decoder,
save_output=args.save_output,
verbose=args.verbose,
half=args.half)
print('Test Summary \t'
'Average WER {wer:.3f}\t'
'Average CER {cer:.3f}\t'.format(wer=wer, cer=cer))
if args.save_output is not None:
np.save(args.save_output, output_data)
def decode_dataset(logits, test_dataset, batch_size, lm_alpha, lm_beta, mesh_x,
mesh_y, index, labels, eval):
print("Beginning decode for {}, {}".format(lm_alpha, lm_beta))
test_loader = AudioDataLoader(test_dataset,
batch_size=batch_size,
num_workers=0)
target_decoder = GreedyDecoder(labels, blank_index=labels.index('_'))
decoder = BeamCTCDecoder(labels,
beam_width=args.beam_width,
cutoff_top_n=args.cutoff_top_n,
blank_index=labels.index('_'),
lm_path=args.lm_path,
alpha=lm_alpha,
beta=lm_beta,
num_processes=1)
model_name = re.sub('.json.pth.tar', '', os.path.basename(args.model_path))
ref_file = None
if eval == 'concept':
eval_dir = "%s/%s/%s" % (os.path.dirname(
args.output_path), model_name, index)
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
ref_file = open(
"%s/%s_reference.txt" %
(eval_dir, re.sub('.csv', '', os.path.basename(
args.test_manifest))), 'w')
trans_file = open(
"%s/%s_transcription.txt" %
(eval_dir, re.sub('.csv', '', os.path.basename(
args.test_manifest))), 'w')
total_cer, total_wer = 0, 0
for i, (data) in enumerate(test_loader):
inputs, targets, input_percentages, target_sizes, audio_ids = data
# unflatten targets
split_targets = []
offset = 0
for size in target_sizes:
split_targets.append(targets[offset:offset + size])
offset += size
out = torch.from_numpy(logits[i][0])
sizes = torch.from_numpy(logits[i][1])
decoded_output, _, _, _, _ = decoder.decode(out, sizes)
target_strings = target_decoder.convert_to_strings(split_targets)
wer, cer = 0, 0
for x in range(len(target_strings)):
transcript, reference = decoded_output[x][0], target_strings[x][0]
if eval == 'concept':
ref_file.write(
reference.encode('utf-8') + "(" + audio_ids[x] + ")\n")
trans_file.write(
transcript.encode('utf-8') + "(" + audio_ids[x] + ")\n")
wer_inst = decoder.wer(transcript, reference) / float(
len(reference.split()))
cer_inst = decoder.cer(transcript, reference) / float(
len(reference))
wer += wer_inst
cer += cer_inst
total_cer += cer
total_wer += wer
ref_file.close()
trans_file.close()
wer = total_wer / len(test_loader.dataset)
cer = total_cer / len(test_loader.dataset)
if eval == 'concept': # Concept error rate evaluation
cmd = "perl /lium/buster1/ghannay/deepSpeech2/deepspeech.pytorch/data/eval.sclit_cer.pl %s" % (
eval_dir)
print("cmd ", cmd)
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
coner, error = p.communicate()
print(" coner ", coner)
return [mesh_x, mesh_y, lm_alpha, lm_beta, float(coner) / 100, cer]
else:
return [mesh_x, mesh_y, lm_alpha, lm_beta, wer, cer]
def main():
args = parser.parse_args()
save_folder = args.save_folder
########
"""
loss_results, cer_results, wer_results = torch.Tensor(args.epochs), torch.Tensor(args.epochs), torch.Tensor(
args.epochs)
best_wer = None
if args.visdom:
from visdom import Visdom
viz = Visdom()
opts = [dict(title=args.visdom_id + ' Loss', ylabel='Loss', xlabel='Epoch'),
dict(title=args.visdom_id + ' WER', ylabel='WER', xlabel='Epoch'),
dict(title=args.visdom_id + ' CER', ylabel='CER', xlabel='Epoch')]
viz_windows = [None, None, None]
epochs = torch.arange(1, args.epochs + 1)
if args.tensorboard:
from logger import TensorBoardLogger
try:
os.makedirs(args.log_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
for file in os.listdir(args.log_dir):
file_path = os.path.join(args.log_dir, file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
raise
else:
raise
logger = TensorBoardLogger(args.log_dir)
try:
os.makedirs(save_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
"""
########
########
"""
criterion = CTCLoss()
"""
criterion = nn.CrossEntropyLoss()
class_accu = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum_120 = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum_240 = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum_360 = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum_480 = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
########
with open(args.labels_path) as label_file:
labels = str(''.join(json.load(label_file)))
audio_conf = dict(sample_rate=args.sample_rate,
window_size=args.window_size,
window_stride=args.window_stride,
window=args.window,
noise_dir=args.noise_dir,
noise_prob=args.noise_prob,
noise_levels=(args.noise_min, args.noise_max))
train_dataset = SpectrogramDataset(audio_conf=audio_conf, manifest_filepath=args.train_manifest, labels=labels,
normalize=True, augment=args.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf, manifest_filepath=args.val_manifest, labels=labels,
normalize=True, augment=False)
train_loader = AudioDataLoader(train_dataset, batch_size=args.batch_size,
num_workers=args.num_workers)
test_loader = AudioDataLoader(test_dataset, batch_size=args.batch_size,
num_workers=args.num_workers)
rnn_type = args.rnn_type.lower()
assert rnn_type in supported_rnns, "rnn_type should be either lstm, rnn or gru"
########
"""
model = DeepSpeech(rnn_hidden_size=args.hidden_size,
nb_layers=args.hidden_layers,
labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=True,
cnn_features=args.cnn_features)
"""
model = DeepSpeech(rnn_hidden_size=args.hidden_size,
nb_layers=args.hidden_layers,
labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=True,
cnn_features=args.cnn_features,
kernel=args.kernel,
stride=args.stride)
########
########
#print(list(model.rnns.modules()))
#for rnn in model.rnns.modules():
# print(rnn)#.flatten_parameters()
#def flat_model(model):
# for m in model.modules():
# if isinstance(m, nn.LSTM):
# m.flatten_parameters()
########
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters, lr=args.lr,
momentum=args.momentum, nesterov=True)
########
#scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.learning_rate_decay_epochs, gamma=args.learning_rate_decay_rate)
#scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99)
########
########
"""
decoder = GreedyDecoder(labels)
"""
########
########
"""
if args.continue_from:
print("Loading checkpoint model %s" % args.continue_from)
package = torch.load(args.continue_from)
model.load_state_dict(package['state_dict'])
optimizer.load_state_dict(package['optim_dict'])
start_epoch = int(package.get('epoch', 1)) - 1 # Python index start at 0 for training
start_iter = package.get('iteration', None)
if start_iter is None:
start_epoch += 1 # Assume that we saved a model after an epoch finished, so start at the next epoch.
start_iter = 0
else:
start_iter += 1
avg_loss = int(package.get('avg_loss', 0))
loss_results, cer_results, wer_results = package['loss_results'], package[
'cer_results'], package['wer_results']
if args.visdom and \
package['loss_results'] is not None and start_epoch > 0: # Add previous scores to visdom graph
x_axis = epochs[0:start_epoch]
y_axis = [loss_results[0:start_epoch], wer_results[0:start_epoch], cer_results[0:start_epoch]]
for x in range(len(viz_windows)):
viz_windows[x] = viz.line(
X=x_axis,
Y=y_axis[x],
opts=opts[x],
)
if args.tensorboard and \
package['loss_results'] is not None and start_epoch > 0: # Previous scores to tensorboard logs
for i in range(start_epoch):
info = {
'Avg Train Loss': loss_results[i],
'Avg WER': wer_results[i],
'Avg CER': cer_results[i]
}
for tag, val in info.items():
logger.scalar_summary(tag, val, i + 1)
if not args.no_bucketing and epoch != 0:
print("Using bucketing sampler for the following epochs")
train_dataset = SpectrogramDatasetWithLength(audio_conf=audio_conf, manifest_filepath=args.train_manifest,
labels=labels,
normalize=True, augment=args.augment)
sampler = BucketingSampler(train_dataset)
train_loader.sampler = sampler
else:
avg_loss = 0
start_epoch = 0
start_iter = 0
"""
avg_loss = 0
start_epoch = 0
start_iter = 0
best_train_accu = 0
best_train_accu_sum = 0
best_train_accu_sum_120 = 0
best_train_accu_sum_240 = 0
best_train_accu_sum_360 = 0
best_train_accu_sum_480 = 0
best_test_accu = 0
best_test_accu_sum = 0
best_test_accu_sum_120 = 0
best_test_accu_sum_240 = 0
best_test_accu_sum_360 = 0
best_test_accu_sum_480 = 0
best_avg_loss = float("inf") # sys.float_info.max # 1000000
epoch_70 = None
epoch_90 = None
epoch_95 = None
epoch_99 = None
if args.stride == 1: multiplier = 6
if args.stride == 2: multiplier = 3
if args.stride == 3: multiplier = 2
if args.stride == 4: multiplier = 1 # (Should be 1.5...)
#sample_time_steps = int(args.sample_miliseconds / 10)
loss_begin = round(args.crop_begin / (10 * args.stride))
loss_end = -round(args.crop_end / (10 * args.stride)) or None
print("LOSS BEGIN:", loss_begin)
print("LOSS END:", loss_end)
########
if args.cuda:
model = torch.nn.DataParallel(model).cuda()
print(model)
print("Number of parameters: %d" % DeepSpeech.get_param_size(model))
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
for epoch in range(start_epoch, args.epochs):
########
#scheduler.step()
optim_state_now = optimizer.state_dict()
print('\nLEARNING RATE: {lr:.6f}'.format(lr=optim_state_now['param_groups'][0]['lr']))
class_accu.reset()
class_accu_sum.reset()
class_accu_sum_120.reset()
class_accu_sum_240.reset()
class_accu_sum_360.reset()
class_accu_sum_480.reset()
########
model.train()
end = time.time()
for i, (data) in enumerate(train_loader, start=start_iter):
if i == len(train_loader):
break
########
"""
inputs, targets, input_percentages, target_sizes = data
"""
inputs, targets, input_percentages, target_sizes, speaker_labels = data
########
# measure data loading time
data_time.update(time.time() - end)
inputs = Variable(inputs, requires_grad=False)
########
"""
target_sizes = Variable(target_sizes, requires_grad=False)
targets = Variable(targets, requires_grad=False)
"""
speaker_labels = Variable(speaker_labels, requires_grad=False)
########
if args.cuda:
inputs = inputs.cuda()
########
"""
out = model(inputs)
"""
#temp_random = random.randint(0, (inputs.size(3)-1)-sample_time_steps)
#print("INPUT", inputs[...,temp_random:temp_random+sample_time_steps].size(),temp_random, temp_random+sample_time_steps)
#out = model(inputs[...,temp_random:temp_random+sample_time_steps])
#print("OUTPUT", out.size())
start = random.randint(0, int((inputs.size(3)-1)*(1-args.sample_proportion)))
print("INPUT", inputs.size(3), inputs[...,start:start+int((inputs.size(3))*(args.sample_proportion))].size(),start, start+int((inputs.size(3))*(args.sample_proportion)))
out = model(inputs[...,start:start+int((inputs.size(3))*(args.sample_proportion))])
print("OUTPUT", out.size())
########
out = out.transpose(0, 1) # TxNxH
########
speaker_labels = speaker_labels.cuda(async=True).long()
# Prints the output of the model in a sequence of probabilities of char for each audio...
torch.set_printoptions(profile="full")
####print("OUT: " + str(out.size()), "SPEAKER LABELS:" + str(speaker_labels.size()), "INPUT PERCENTAGES MEAN: " + str(input_percentages.mean()))
"""
seq_length = out.size(0)
sizes = Variable(input_percentages.mul_(int(seq_length)).int(), requires_grad=False)
loss = criterion(out, targets, sizes, target_sizes)
"""
#print(out[:,:,0])
#print("SPEAKER LABELS: " + str(speaker_labels))
#print(out[0][0])
#softmax_output = F.softmax(out).data # This DOES NOT what I want...
#softmax_output_alt = flex_softmax(out, axis=2).data # This is FINE!!! <<<===
#print(softmax_output[0][0])
#print(softmax_output_alt[0][0])
####new_out = torch.sum(out, 0)
####new_out = torch.sum(out[20:], 0)
#print(out.size())
#print(new_out.size())
#print(out[-1].size())
########
########
if args.loss_type == "reg":
#loss_out = out[-1]; loss_speaker_labels = speaker_labels
loss_out = out[round(out.size(0)/2)]; loss_speaker_labels = speaker_labels
#print("LOSS TYPE = REGULAR")
elif args.loss_type == "sum":
loss_out = torch.sum(out[loss_begin:loss_end], 0); loss_speaker_labels = speaker_labels
#print("LOSS TYPE = SUM")
elif args.loss_type == "full":
# Don't know if is ok!!! Don't use!!! => loss_out = out.contiguous().view(-1,48); loss_speaker_labels = speaker_labels.repeat(out.size(0)) #speaker_labels = speaker_labels.expand(20, out.size(0))
# Don't know if is ok!!! Don't use!!! => loss_out = out.contiguous().view(-1,48); loss_speaker_labels = speaker_labels.repeat(1, out.size(0)).squeeze() #speaker_labels = speaker_labels.expand(20, out.size(0))
loss_out = out.contiguous()[loss_begin:loss_end].view(-1,48); loss_speaker_labels = speaker_labels.repeat(out.size(0),1)[loss_begin:loss_end].view(-1) #speaker_labels = speaker_labels.expand(20, out.size(0))
#print("LOSS TYPE = FULL")
print("LOSS_OUT: " + str(loss_out.size()), "SPEAKER LABELS:" + str(loss_speaker_labels.size()))
loss = criterion(loss_out, loss_speaker_labels)
########
loss = loss / inputs.size(0) # average the loss by minibatch
loss_sum = loss.data.sum()
inf = float("inf")
if loss_sum == inf or loss_sum == -inf:
print("WARNING: received an inf loss, setting loss value to 0")
loss_value = 0
else:
loss_value = loss.data[0]
avg_loss += loss_value
losses.update(loss_value, inputs.size(0))
########
#if args.stride == 1: multiplier = 6
#if args.stride == 2: multiplier = 3
#if args.stride == 3: multiplier = 2
#if args.stride == 4: multiplier = 1 #(Should be 1.5...)
#if args.stride == 5: multiplier = 1 #(Should be 1.25...)
class_accu.add(out[round(out.size(0)/2)].data, speaker_labels.data)
class_accu_sum.add(torch.sum(out, 0).data, speaker_labels.data)
#class_accu_sum_120.add(torch.sum(out[1*multiplier:-1*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_240.add(torch.sum(out[2*multiplier:-2*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_360.add(torch.sum(out[3*multiplier:-3*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_480.add(torch.sum(out[4*multiplier:-4*multiplier], 0).data, speaker_labels.data)
####class_accu_sum_120.add(torch.sum(out[round(out.size(0)/2)-1*multiplier:round(out.size(0)/2)+1*multiplier], 0).data, speaker_labels.data)
####class_accu_sum_240.add(torch.sum(out[round(out.size(0)/2)-2*multiplier:round(out.size(0)/2)+2*multiplier], 0).data, speaker_labels.data)
####class_accu_sum_360.add(torch.sum(out[round(out.size(0)/2)-3*multiplier:round(out.size(0)/2)+3*multiplier], 0).data, speaker_labels.data)
####class_accu_sum_480.add(torch.sum(out[round(out.size(0)/2)-4*multiplier:round(out.size(0)/2)+4*multiplier], 0).data, speaker_labels.data)
#accu_out3 = torch.sum(flex_softmax(out[20:], axis=2), 0)
#print(classaccu.value()[0], classaccu.value()[1])
# Cross Entropy Loss for a Sequence (Time Series) of Output?
#output = output.view(-1,29)
#target = target.view(-1)
#criterion = nn.CrossEntropyLoss()
#loss = criterion(output,target)
########
# compute gradient
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), args.max_norm)
# SGD step
optimizer.step()
if args.cuda:
torch.cuda.synchronize()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if not args.silent:
########
"""
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
(epoch + 1), (i + 1), len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses))
"""
print('Epoch: [{0}][{1}/{2}]\t'
# 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'CAR {car:.3f}\t'
'CAR_SUM {car_sum:.3f}\t'
#'CAR_SUM_120 {car_sum_120:.3f}\t'
#'CAR_SUM_240 {car_sum_240:.3f}\t'
#'CAR_SUM_360 {car_sum_360:.3f}\t'
#'CAR_SUM_480 {car_sum_480:.3f}\t'
.format((epoch + 1), (i + 1), len(train_loader), batch_time=batch_time, data_time=data_time,
loss=losses, car=class_accu.value()[0], car_sum=class_accu_sum.value()[0],
# car_sum_240=class_accu_sum_240.value()[0], car_sum_120=class_accu_sum_120.value()[0],
# car_sum_360=class_accu_sum_360.value()[0], car_sum_480=class_accu_sum_480.value()[0]
)
)
########
########
"""
if args.checkpoint_per_batch > 0 and i > 0 and (i + 1) % args.checkpoint_per_batch == 0:
file_path = '%s/deepspeech_checkpoint_epoch_%d_iter_%d.pth.tar' % (save_folder, epoch + 1, i + 1)
print("Saving checkpoint model to %s" % file_path)
torch.save(DeepSpeech.serialize(model, optimizer=optimizer, epoch=epoch, iteration=i,
loss_results=loss_results,
wer_results=wer_results, cer_results=cer_results, avg_loss=avg_loss),
file_path)
"""
########
del loss
del out
########
del loss_out
del speaker_labels
del loss_speaker_labels
########
avg_loss /= len(train_loader)
########
"""
print('Training Summary Epoch: [{0}]\t'
'Average Loss {loss:.3f}\t'.format(
epoch + 1, loss=avg_loss))
"""
if (best_avg_loss > avg_loss): best_avg_loss = avg_loss
print("\nCURRENT EPOCH TRAINING RESULTS:\t", class_accu.value()[0], "\t", class_accu_sum.value()[0],"\t",
#class_accu_sum_120.value()[0], class_accu_sum_240.value()[0], class_accu_sum_360.value()[0], "\t", class_accu_sum_480.value()[0], "\n"
)
if (best_train_accu < class_accu.value()[0]): best_train_accu = class_accu.value()[0]
if (best_train_accu_sum < class_accu_sum.value()[0]): best_train_accu_sum = class_accu_sum.value()[0]
#if (best_train_accu_sum_120 < class_accu_sum_120.value()[0]): best_train_accu_sum_120 = class_accu_sum_120.value()[0]
#if (best_train_accu_sum_240 < class_accu_sum_240.value()[0]): best_train_accu_sum_240 = class_accu_sum_240.value()[0]
#if (best_train_accu_sum_360 < class_accu_sum_360.value()[0]): best_train_accu_sum_360 = class_accu_sum_360.value()[0]
#if (best_train_accu_sum_480 < class_accu_sum_480.value()[0]): best_train_accu_sum_480 = class_accu_sum_480.value()[0]
get_70 = ((class_accu.value()[0] > 70) or (class_accu_sum.value()[0] > 70)
#or (class_accu_sum_120.value()[0] > 70) or (class_accu_sum_240.value()[0] > 70)
#or (class_accu_sum_360.value()[0] > 70) or (class_accu_sum_480.value()[0] > 70)
)
if ((epoch_70 is None) and (get_70 == True)): epoch_70 = epoch + 1
get_90 = ((class_accu.value()[0] > 90) or (class_accu_sum.value()[0] > 90)
#or (class_accu_sum_120.value()[0] > 90) or (class_accu_sum_240.value()[0] > 90)
#or (class_accu_sum_360.value()[0] > 90) or (class_accu_sum_480.value()[0] > 90)
)
if ((epoch_90 is None) and (get_90 == True)): epoch_90 = epoch + 1
get_95 = ((class_accu.value()[0] > 95) or (class_accu_sum.value()[0] > 95)
#or (class_accu_sum_120.value()[0] > 95) or (class_accu_sum_240.value()[0] > 95)
#or (class_accu_sum_360.value()[0] > 95) or (class_accu_sum_480.value()[0] > 95)
)
if ((epoch_95 is None) and (get_95 == True)): epoch_95 = epoch + 1
get_99 = ((class_accu.value()[0] > 99) or (class_accu_sum.value()[0] > 99)
#or (class_accu_sum_120.value()[0] > 99) or (class_accu_sum_240.value()[0] > 99)
#or (class_accu_sum_360.value()[0] > 99) or (class_accu_sum_480.value()[0] > 99)
)
if ((epoch_99 is None) and (get_99 == True)): epoch_99 = epoch + 1
########
start_iter = 0 # Reset start iteration for next epoch
total_cer, total_wer = 0, 0
model.eval()
########
class_accu.reset()
class_accu_sum.reset()
class_accu_sum_120.reset()
class_accu_sum_240.reset()
class_accu_sum_360.reset()
class_accu_sum_480.reset()
########
for i, (data) in enumerate(test_loader): # test
########
"""
inputs, targets, input_percentages, target_sizes = data
"""
inputs, targets, input_percentages, target_sizes, speaker_labels = data
########
inputs = Variable(inputs, volatile=True)
########
speaker_labels = Variable(speaker_labels, requires_grad=False)
speaker_labels = speaker_labels.cuda(async=True).long()
"""
# unflatten targets
split_targets = []
offset = 0
for size in target_sizes:
split_targets.append(targets[offset:offset + size])
offset += size
"""
########
if args.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
########
speaker_labels = speaker_labels.cuda(async=True).long()
# Prints the output of the model in a sequence of probabilities of char for each audio...
torch.set_printoptions(profile="full")
########print("OUT: " + str(out.size()), "NEW OUT:" + str(new_out.size()), "SPEAKER LABELS:" + str(speaker_labels.size()), "INPUT PERCENTAGES MEAN: " + str(input_percentages.mean()))
#print(out[:,:,0])
#print("SPEAKER LABELS: " + str(speaker_labels))
#print(out[0][0])
#softmax_output = F.softmax(out).data # This DOES NOT what I want...
#softmax_output_alt = flex_softmax(out, axis=2).data # This is FINE!!! <<<===
#print(softmax_output[0][0])
#print(softmax_output_alt[0][0])
########
########
#if args.stride == 1: multiplier = 6
#if args.stride == 2: multiplier = 3
#if args.stride == 3: multiplier = 2
#if args.stride == 4: multiplier = 1 #(Should be 1.5...)
#if args.stride == 5: multiplier = 1 #(Should be 1.25...)
class_accu.add(out[round(out.size(0)/2)].data, speaker_labels.data)
class_accu_sum.add(torch.sum(out, 0).data, speaker_labels.data)
class_accu_sum_120.add(torch.sum(out[1*multiplier:-1*multiplier], 0).data, speaker_labels.data)
class_accu_sum_240.add(torch.sum(out[2*multiplier:-2*multiplier], 0).data, speaker_labels.data)
class_accu_sum_360.add(torch.sum(out[3*multiplier:-3*multiplier], 0).data, speaker_labels.data)
class_accu_sum_480.add(torch.sum(out[4*multiplier:-4*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_120.add(torch.sum(out[round(out.size(0)/2)-1*multiplier:round(out.size(0)/2)+1*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_240.add(torch.sum(out[round(out.size(0)/2)-2*multiplier:round(out.size(0)/2)+2*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_360.add(torch.sum(out[round(out.size(0)/2)-3*multiplier:round(out.size(0)/2)+3*multiplier], 0).data, speaker_labels.data)
#class_accu_sum_480.add(torch.sum(out[round(out.size(0)/2)-4*multiplier:round(out.size(0)/2)+4*multiplier], 0).data, speaker_labels.data)
#accu_out3 = torch.sum(flex_softmax(out[20:], axis=2), 0)
#print(classaccu.value()[0], classaccu.value()[1])
# Cross Entropy Loss for a Sequence (Time Series) of Output?
#output = output.view(-1,29)
#target = target.view(-1)
#criterion = nn.CrossEntropyLoss()
#loss = criterion(output,target)
print('Validation Summary Epoch: [{0}]\t'
'CAR {car:.3f}\t'
'CAR_SUM {car_sum:.3f}\t'
'CAR_SUM_120 {car_sum_120:.3f}\t'
'CAR_SUM_240 {car_sum_240:.3f}\t'
'CAR_SUM_360 {car_sum_360:.3f}\t'
'CAR_SUM_480 {car_sum_480:.3f}\t'
.format(epoch + 1, car=class_accu.value()[0], car_sum=class_accu_sum.value()[0],
car_sum_240=class_accu_sum_240.value()[0], car_sum_120=class_accu_sum_120.value()[0],
car_sum_360=class_accu_sum_360.value()[0], car_sum_480=class_accu_sum_480.value()[0]
)
)
"""
seq_length = out.size(0)
sizes = input_percentages.mul_(int(seq_length)).int()
decoded_output = decoder.decode(out.data, sizes)
target_strings = decoder.process_strings(decoder.convert_to_strings(split_targets))
wer, cer = 0, 0
for x in range(len(target_strings)):
wer += decoder.wer(decoded_output[x], target_strings[x]) / float(len(target_strings[x].split()))
cer += decoder.cer(decoded_output[x], target_strings[x]) / float(len(target_strings[x]))
total_cer += cer
total_wer += wer
"""
########
if args.cuda:
torch.cuda.synchronize()
del out
########
"""
wer = total_wer / len(test_loader.dataset)
cer = total_cer / len(test_loader.dataset)
wer *= 100
cer *= 100
loss_results[epoch] = avg_loss
wer_results[epoch] = wer
cer_results[epoch] = cer
print('Validation Summary Epoch: [{0}]\t'
'Average WER {wer:.3f}\t'
'Average CER {cer:.3f}\t'.format(
epoch + 1, wer=wer, cer=cer))
"""
########
########
print("\nCURRENT EPOCH TEST RESULTS:\t", class_accu.value()[0], "\t", class_accu_sum.value()[0],
"\t", class_accu_sum_120.value()[0], "\t", class_accu_sum_240.value()[0],
"\t", class_accu_sum_360.value()[0], "\t", class_accu_sum_480.value()[0], "\n")
if (best_test_accu < class_accu.value()[0]): best_test_accu = class_accu.value()[0]
if (best_test_accu_sum < class_accu_sum.value()[0]): best_test_accu_sum = class_accu_sum.value()[0]
if (best_test_accu_sum_120 < class_accu_sum_120.value()[0]): best_test_accu_sum_120 = class_accu_sum_120.value()[0]
if (best_test_accu_sum_240 < class_accu_sum_240.value()[0]): best_test_accu_sum_240 = class_accu_sum_240.value()[0]
if (best_test_accu_sum_360 < class_accu_sum_360.value()[0]): best_test_accu_sum_360 = class_accu_sum_360.value()[0]
if (best_test_accu_sum_480 < class_accu_sum_480.value()[0]): best_test_accu_sum_480 = class_accu_sum_480.value()[0]
print("\nBEST EPOCH TRAINING RESULTS:\t", best_train_accu, "\t", best_train_accu_sum,
"\t", best_train_accu_sum_120, "\t", best_train_accu_sum_240,
"\t", best_train_accu_sum_360, "\t", best_train_accu_sum_480)
print("\nBEST EPOCH TEST RESULTS:\t", best_test_accu, "\t", best_test_accu_sum,
"\t", best_test_accu_sum_120, "\t", best_test_accu_sum_240,
"\t", best_test_accu_sum_360, "\t", best_test_accu_sum_480)
print("\nEPOCHS 70%, 90%, 95%, 99%:\t", epoch_70, "\t", epoch_90, "\t", epoch_95, "\t", epoch_99)
print("\nBEST AVERAGE LOSS:\t", best_avg_loss, "\n")
########
########
"""
if args.visdom:
# epoch += 1
x_axis = epochs[0:epoch + 1]
y_axis = [loss_results[0:epoch + 1], wer_results[0:epoch + 1], cer_results[0:epoch + 1]]
for x in range(len(viz_windows)):
if viz_windows[x] is None:
viz_windows[x] = viz.line(
X=x_axis,
Y=y_axis[x],
opts=opts[x],
)
else:
viz.line(
X=x_axis,
Y=y_axis[x],
win=viz_windows[x],
update='replace',
)
if args.tensorboard:
info = {
'Avg Train Loss': avg_loss,
'Avg WER': wer,
'Avg CER': cer
}
for tag, val in info.items():
logger.scalar_summary(tag, val, epoch + 1)
if args.log_params:
for tag, value in model.named_parameters():
tag = tag.replace('.', '/')
logger.histo_summary(tag, to_np(value), epoch + 1)
logger.histo_summary(tag + '/grad', to_np(value.grad), epoch + 1)
if args.checkpoint:
file_path = '%s/deepspeech_%d.pth.tar' % (save_folder, epoch + 1)
torch.save(DeepSpeech.serialize(model, optimizer=optimizer, epoch=epoch, loss_results=loss_results,
wer_results=wer_results, cer_results=cer_results),
file_path)
# anneal lr
optim_state = optimizer.state_dict()
optim_state['param_groups'][0]['lr'] = optim_state['param_groups'][0]['lr'] / args.learning_anneal
optimizer.load_state_dict(optim_state)
print('Learning rate annealed to: {lr:.6f}'.format(lr=optim_state['param_groups'][0]['lr']))
if best_wer is None or best_wer > wer:
print("Found better validated model, saving to %s" % args.model_path)
torch.save(DeepSpeech.serialize(model, optimizer=optimizer, epoch=epoch, loss_results=loss_results,
wer_results=wer_results, cer_results=cer_results)
, args.model_path)
best_wer = wer
"""
########
avg_loss = 0
if not args.no_bucketing and epoch == 0:
print("Switching to bucketing sampler for following epochs")
train_dataset = SpectrogramDatasetWithLength(audio_conf=audio_conf, manifest_filepath=args.train_manifest,
labels=labels,
normalize=True, augment=args.augment)
sampler = BucketingSampler(train_dataset)
train_loader.sampler = sampler
window_stride=args.window_stride,
window=args.window,
noise_dir=args.noise_dir,
noise_prob=args.noise_prob,
noise_levels=(args.noise_min, args.noise_max))
else:
print("Must load model!")
exit()
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
labels=labels,
normalize=True,
augment=False)
train_sampler = BucketingSampler(train_dataset, batch_size=1)
train_loader = AudioDataLoader(train_dataset, batch_sampler=train_sampler)
# get the previous output f* & have modified the forward function
with torch.no_grad():
for i, (data) in enumerate(train_loader): # just once
# inputs: Nx1xKxT
inputs, targets, input_percentages, target_sizes = data
input_sizes = input_percentages.mul_(int(inputs.size(3))).int()
inputs = inputs.to(device)
print('input size is:', inputs.size())
# initial M_Noise model
M_model = M_Noise_Deepspeech(package, inputs.size())
M_model.to(device)
# no update to these parameters
for para in M_model.deepspeech_net.parameters():
para.requires_grad = False
if not args.distributed:
train_sampler = BucketingSampler(train_dataset, batch_size=batch_size)
val_sampler = BucketingSampler(val_dataset, batch_size=batch_size)
else:
train_sampler = DistributedBucketingSampler(
train_dataset,
batch_size=batch_size,
num_replicas=args.world_size,
rank=args.rank)
val_sampler = DistributedBucketingSampler(val_dataset,
batch_size=batch_size,
num_replicas=args.world_size,
rank=args.rank)
train_loader = AudioDataLoader(train_dataset,
num_workers=args.num_workers,
batch_sampler=train_sampler)
val_loader = AudioDataLoader(val_dataset,
num_workers=args.num_workers,
batch_sampler=val_sampler)
if (shuffle and start_epoch != 0) or not sorta_grad:
logger.info("Shuffling batches for the following epochs")
train_sampler.shuffle(start_epoch)
val_sampler.shuffle(1)
# Optimizer.
optim_name = train_conf['optimizer']
optim_conf = config[optim_name]
parameters = model.parameters()
learning_rate = train_conf.getfloat('learning_rate')
val_manifest = "data/dev-clean_manifest.csv"
device = "cuda"
model_path = args.model_path
package = torch.load(model_path, map_location=lambda storage, loc: storage)
model = DeepSpeech.load_model_package(package)
labels = model.labels
audio_conf = model.audio_conf
model.to(device)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=val_manifest,
labels=labels,
normalize=True,
speed_volume_perturb=False,
spec_augment=False)
test_loader = AudioDataLoader(test_dataset, batch_size=20, num_workers=4)
decoder = GreedyDecoder(model.labels, blank_index=model.labels.index('_'))
target_decoder = GreedyDecoder(model.labels,
blank_index=model.labels.index('_'))
with torch.no_grad():
evaluate(test_loader,
device,
model,
decoder,
target_decoder,
verbose=True)
print("made it this far")
def main():
args = parser.parse_args()
save_folder = args.save_folder
loss_results, cer_results, wer_results = torch.Tensor(
args.epochs), torch.Tensor(args.epochs), torch.Tensor(args.epochs)
best_wer = None
if args.visdom:
from visdom import Visdom
viz = Visdom()
opts = [
dict(title=args.visdom_id + ' Loss', ylabel='Loss',
xlabel='Epoch'),
dict(title=args.visdom_id + ' WER', ylabel='WER', xlabel='Epoch'),
dict(title=args.visdom_id + ' CER', ylabel='CER', xlabel='Epoch')
]
viz_windows = [None, None, None]
epochs = torch.arange(1, args.epochs + 1)
if args.tensorboard:
from logger import TensorBoardLogger
try:
os.makedirs(args.log_dir)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
for file in os.listdir(args.log_dir):
file_path = os.path.join(args.log_dir, file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
raise
else:
raise
logger = TensorBoardLogger(args.log_dir)
try:
os.makedirs(save_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
criterion = CTCLoss()
with open(args.labels_path) as label_file:
labels = str(''.join(json.load(label_file)))
audio_conf = dict(sample_rate=args.sample_rate,
window_size=args.window_size,
window_stride=args.window_stride,
window=args.window,
noise_dir=args.noise_dir,
noise_prob=args.noise_prob,
noise_levels=(args.noise_min, args.noise_max))
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
labels=labels,
normalize=True,
augment=args.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.val_manifest,
labels=labels,
normalize=True,
augment=False)
train_loader = AudioDataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
test_loader = AudioDataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
rnn_type = args.rnn_type.lower()
assert rnn_type in supported_rnns, "rnn_type should be either lstm, rnn or gru"
model = DeepSpeech(rnn_hidden_size=args.hidden_size,
nb_layers=args.hidden_layers,
labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=True)
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
nesterov=True)
decoder = GreedyDecoder(labels)
if args.continue_from:
print("Loading checkpoint model %s" % args.continue_from)
package = torch.load(args.continue_from)
model.load_state_dict(package['state_dict'])
optimizer.load_state_dict(package['optim_dict'])
start_epoch = int(package.get(
'epoch', 1)) - 1 # Python index start at 0 for training
start_iter = package.get('iteration', None)
if start_iter is None:
start_epoch += 1 # Assume that we saved a model after an epoch finished, so start at the next epoch.
start_iter = 0
else:
start_iter += 1
avg_loss = int(package.get('avg_loss', 0))
loss_results, cer_results, wer_results = package[
'loss_results'], package['cer_results'], package['wer_results']
if args.visdom and \
package['loss_results'] is not None and start_epoch > 0: # Add previous scores to visdom graph
x_axis = epochs[0:start_epoch]
y_axis = [
loss_results[0:start_epoch], wer_results[0:start_epoch],
cer_results[0:start_epoch]
]
for x in range(len(viz_windows)):
viz_windows[x] = viz.line(
X=x_axis,
Y=y_axis[x],
opts=opts[x],
)
if args.tensorboard and \
package['loss_results'] is not None and start_epoch > 0: # Previous scores to tensorboard logs
for i in range(start_epoch):
info = {
'Avg Train Loss': loss_results[i],
'Avg WER': wer_results[i],
'Avg CER': cer_results[i]
}
for tag, val in info.items():
logger.scalar_summary(tag, val, i + 1)
if not args.no_bucketing and epoch != 0:
print("Using bucketing sampler for the following epochs")
train_dataset = SpectrogramDatasetWithLength(
audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
labels=labels,
normalize=True,
augment=args.augment)
sampler = BucketingSampler(train_dataset)
train_loader.sampler = sampler
else:
avg_loss = 0
start_epoch = 0
start_iter = 0
if args.cuda:
model = torch.nn.DataParallel(model).cuda()
print(model)
print("Number of parameters: %d" % DeepSpeech.get_param_size(model))
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
for epoch in range(start_epoch, args.epochs):
model.train()
end = time.time()
for i, (data) in enumerate(train_loader, start=start_iter):
if i == len(train_loader):
break
inputs, targets, input_percentages, target_sizes = data
# measure data loading time
data_time.update(time.time() - end)
inputs = Variable(inputs, requires_grad=False)
target_sizes = Variable(target_sizes, requires_grad=False)
targets = Variable(targets, requires_grad=False)
if args.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
seq_length = out.size(0)
sizes = Variable(input_percentages.mul_(int(seq_length)).int(),
requires_grad=False)
loss = criterion(out, targets, sizes, target_sizes)
loss = loss / inputs.size(0) # average the loss by minibatch
loss_sum = loss.data.sum()
inf = float("inf")
if loss_sum == inf or loss_sum == -inf:
print("WARNING: received an inf loss, setting loss value to 0")
loss_value = 0
else:
loss_value = loss.data[0]
avg_loss += loss_value
losses.update(loss_value, inputs.size(0))
# compute gradient
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), args.max_norm)
# SGD step
optimizer.step()
if args.cuda:
torch.cuda.synchronize()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if not args.silent:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
(epoch + 1), (i + 1),
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
if args.checkpoint_per_batch > 0 and i > 0 and (
i + 1) % args.checkpoint_per_batch == 0:
file_path = '%s/deepspeech_checkpoint_epoch_%d_iter_%d.pth.tar' % (
save_folder, epoch + 1, i + 1)
print("Saving checkpoint model to %s" % file_path)
torch.save(
DeepSpeech.serialize(model,
optimizer=optimizer,
epoch=epoch,
iteration=i,
loss_results=loss_results,
wer_results=wer_results,
cer_results=cer_results,
avg_loss=avg_loss), file_path)
del loss
del out
avg_loss /= len(train_loader)
print('Training Summary Epoch: [{0}]\t'
'Average Loss {loss:.3f}\t'.format(epoch + 1, loss=avg_loss))
start_iter = 0 # Reset start iteration for next epoch
total_cer, total_wer = 0, 0
model.eval()
for i, (data) in enumerate(test_loader): # test
inputs, targets, input_percentages, target_sizes = data
inputs = Variable(inputs, volatile=True)
# unflatten targets
split_targets = []
offset = 0
for size in target_sizes:
split_targets.append(targets[offset:offset + size])
offset += size
if args.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
seq_length = out.size(0)
sizes = input_percentages.mul_(int(seq_length)).int()
decoded_output = decoder.decode(out.data, sizes)
target_strings = decoder.process_strings(
decoder.convert_to_strings(split_targets))
wer, cer = 0, 0
for x in range(len(target_strings)):
wer += decoder.wer(decoded_output[x],
target_strings[x]) / float(
len(target_strings[x].split()))
cer += decoder.cer(decoded_output[x],
target_strings[x]) / float(
len(target_strings[x]))
total_cer += cer
total_wer += wer
if args.cuda:
torch.cuda.synchronize()
del out
wer = total_wer / len(test_loader.dataset)
cer = total_cer / len(test_loader.dataset)
wer *= 100
cer *= 100
loss_results[epoch] = avg_loss
wer_results[epoch] = wer
cer_results[epoch] = cer
print('Validation Summary Epoch: [{0}]\t'
'Average WER {wer:.3f}\t'
'Average CER {cer:.3f}\t'.format(epoch + 1, wer=wer, cer=cer))
if args.visdom:
# epoch += 1
x_axis = epochs[0:epoch + 1]
y_axis = [
loss_results[0:epoch + 1], wer_results[0:epoch + 1],
cer_results[0:epoch + 1]
]
for x in range(len(viz_windows)):
if viz_windows[x] is None:
viz_windows[x] = viz.line(
X=x_axis,
Y=y_axis[x],
opts=opts[x],
)
else:
viz.line(
X=x_axis,
Y=y_axis[x],
win=viz_windows[x],
update='replace',
)
if args.tensorboard:
info = {'Avg Train Loss': avg_loss, 'Avg WER': wer, 'Avg CER': cer}
for tag, val in info.items():
logger.scalar_summary(tag, val, epoch + 1)
if args.log_params:
for tag, value in model.named_parameters():
tag = tag.replace('.', '/')
logger.histo_summary(tag, to_np(value), epoch + 1)
logger.histo_summary(tag + '/grad', to_np(value.grad),
epoch + 1)
if args.checkpoint:
file_path = '%s/deepspeech_%d.pth.tar' % (save_folder, epoch + 1)
torch.save(
DeepSpeech.serialize(model,
optimizer=optimizer,
epoch=epoch,
loss_results=loss_results,
wer_results=wer_results,
cer_results=cer_results), file_path)
# anneal lr
optim_state = optimizer.state_dict()
optim_state['param_groups'][0][
'lr'] = optim_state['param_groups'][0]['lr'] / args.learning_anneal
optimizer.load_state_dict(optim_state)
print('Learning rate annealed to: {lr:.6f}'.format(
lr=optim_state['param_groups'][0]['lr']))
if best_wer is None or best_wer > wer:
print("Found better validated model, saving to %s" %
args.model_path)
torch.save(
DeepSpeech.serialize(model,
optimizer=optimizer,
epoch=epoch,
loss_results=loss_results,
wer_results=wer_results,
cer_results=cer_results), args.model_path)
best_wer = wer
avg_loss = 0
if not args.no_bucketing and epoch == 0:
print("Switching to bucketing sampler for following epochs")
train_dataset = SpectrogramDatasetWithLength(
audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
labels=labels,
normalize=True,
augment=args.augment)
sampler = BucketingSampler(train_dataset)
train_loader.sampler = sampler
batch_size=args.batch_size)
train_sampler_adv = BucketingSampler(train_dataset_adv,
batch_size=args.batch_size)
else:
train_sampler_clean = DistributedBucketingSampler(
train_dataset_clean,
batch_size=args.batch_size,
num_replicas=args.world_size,
rank=args.rank)
train_sampler_adv = DistributedBucketingSampler(
train_dataset_adv,
batch_size=args.batch_size,
num_replicas=args.world_size,
rank=args.rank)
train_loader_clean = AudioDataLoader(train_dataset_clean,
num_workers=args.num_workers,
batch_sampler=train_sampler_clean)
train_loader_adv = AudioDataLoader(train_dataset_adv,
num_workers=args.num_workers,
batch_sampler=train_sampler_adv)
test_loader = AudioDataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
'''
if (not args.no_shuffle and start_epoch != 0) or args.no_sorta_grad:
print("Shuffling batches for the following epochs")
train_sampler.shuffle(start_epoch)
'''
model = model.to(device)
denoiser = denoiser.to(device)
if args.mixed_precision:
log_file = f'{save_folder}/{datetime.now().strftime("%Y%m%d-%H%M%S")}_{test_job}'
logger = config_logger('test', log_file=log_file, console_level='ERROR')
torch.set_grad_enabled(False)
model, _ = load_model(args.model_path)
device = torch.device("cuda" if args.cuda else "cpu")
label_decoder = LabelDecoder(model.labels)
model.eval()
model = model.to(device)
test_dataset = SpectrogramDataset(audio_conf=model.audio_conf,
manifest_filepath=args.test_manifest,
labels=model.labels)
test_sampler = BucketingSampler(test_dataset, batch_size=args.batch_size)
test_loader = AudioDataLoader(test_dataset,
batch_sampler=test_sampler,
num_workers=args.num_workers)
test_sampler.shuffle(1)
total_wer, total_cer, total_ler, num_words, num_chars, num_labels = 0, 0, 0, 0, 0, 0
output_data = []
for i, (data) in tqdm(enumerate(test_loader),
total=len(test_loader),
ascii=True):
inputs, targets, input_sizes, target_sizes, filenames = data
inputs = inputs.to(device)
input_sizes = input_sizes.to(device)
outputs = model.transcribe(inputs, input_sizes)
for i, target in enumerate(targets):
def train(self, **kwargs):
"""
Run optimization to train the model.
Parameters
----------
"""
world_size = kwargs.pop('world_size', 1)
gpu_rank = kwargs.pop('gpu_rank', 0)
rank = kwargs.pop('rank', 0)
dist_backend = kwargs.pop('dist_backend', 'nccl')
dist_url = kwargs.pop('dist_url', None)
os.environ['MASTER_ADDR'] = '127.0.0.1'
os.environ['MASTER_PORT'] = '1234'
main_proc = True
self.distributed = world_size > 1
if self.distributed:
if self.gpu_rank:
torch.cuda.set_device(int(gpu_rank))
dist.init_process_group(backend=dist_backend,
init_method=dist_url,
world_size=world_size,
rank=rank)
print('Initiated process group')
main_proc = rank == 0 # Only the first proc should save models
if main_proc and self.tensorboard:
tensorboard_logger = TensorBoardLogger(self.id,
self.log_dir,
self.log_params,
comment=self.sufix)
if self.distributed:
train_sampler = DistributedBucketingSampler(
self.data_train,
batch_size=self.batch_size,
num_replicas=world_size,
rank=rank)
else:
if self.sampler_type == 'bucketing':
train_sampler = BucketingSampler(self.data_train,
batch_size=self.batch_size,
shuffle=True)
if self.sampler_type == 'random':
train_sampler = RandomBucketingSampler(
self.data_train, batch_size=self.batch_size)
print("Shuffling batches for the following epochs..")
train_sampler.shuffle(self.start_epoch)
train_loader = AudioDataLoader(self.data_train,
num_workers=self.num_workers,
batch_sampler=train_sampler)
val_loader = AudioDataLoader(self.data_val,
batch_size=self.batch_size_val,
num_workers=self.num_workers,
shuffle=True)
if self.tensorboard and self.generate_graph: # TO DO get some audios also
with torch.no_grad():
inputs, targets, input_percentages, target_sizes = next(
iter(train_loader))
input_sizes = input_percentages.mul_(int(inputs.size(3))).int()
tensorboard_logger.add_image(inputs,
input_sizes,
targets,
network=self.model)
self.model = self.model.to(self.device)
parameters = self.model.parameters()
if self.update_rule == 'adam':
optimizer = torch.optim.Adam(parameters,
lr=self.lr,
weight_decay=self.reg)
if self.update_rule == 'sgd':
optimizer = torch.optim.SGD(parameters,
lr=self.lr,
weight_decay=self.reg)
self.model, self.optimizer = amp.initialize(
self.model,
optimizer,
opt_level=self.opt_level,
keep_batchnorm_fp32=self.keep_batchnorm_fp32,
loss_scale=self.loss_scale)
if self.optim_state is not None:
self.optimizer.load_state_dict(self.optim_state)
if self.amp_state is not None:
amp.load_state_dict(self.amp_state)
if self.distributed:
self.model = DistributedDataParallel(self.model)
print(self.model)
if self.criterion_type == 'cross_entropy_loss':
self.criterion = torch.nn.CrossEntropyLoss()
# Useless for now because I don't save.
accuracies_train_iters = []
losses_iters = []
avg_loss = 0
batch_time = AverageMeter()
epoch_time = AverageMeter()
losses = AverageMeter()
start_training = time.time()
for epoch in range(self.start_epoch, self.num_epochs):
print("Start epoch..")
# Put model in train mode
self.model.train()
y_true_train_epoch = np.array([])
y_pred_train_epoch = np.array([])
start_epoch = time.time()
for i, (data) in enumerate(train_loader, start=0):
start_batch = time.time()
print('Start batch..')
if i == len(train_sampler): # QUE pq isso deus
break
inputs, targets, input_percentages, _ = data
input_sizes = input_percentages.mul_(int(inputs.size(3))).int()
inputs = inputs.to(self.device)
targets = targets.to(self.device)
output, loss_value = self._step(inputs, input_sizes, targets)
print('Step finished.')
avg_loss += loss_value
with torch.no_grad():
y_pred = self.decoder.decode(output.detach()).cpu().numpy()
# import pdb; pdb.set_trace()
y_true_train_epoch = np.concatenate(
(y_true_train_epoch, targets.cpu().numpy()
)) # maybe I should do it with tensors?
y_pred_train_epoch = np.concatenate(
(y_pred_train_epoch, y_pred))
inputs_size = inputs.size(0)
del output, inputs, input_percentages
if self.intra_epoch_sanity_check:
with torch.no_grad():
acc, _ = self.check_accuracy(targets.cpu().numpy(),
y_pred=y_pred)
accuracies_train_iters.append(acc)
losses_iters.append(loss_value)
cm = confusion_matrix(targets.cpu().numpy(),
y_pred,
labels=self.labels)
print('[it %i/%i] Confusion matrix train step:' %
((i + 1, len(train_sampler))))
print(pd.DataFrame(cm))
if self.tensorboard:
tensorboard_logger.update(
len(train_loader) * epoch + i + 1, {
'Loss/through_iterations': loss_value,
'Accuracy/train_through_iterations': acc
})
del targets
batch_time.update(time.time() - start_batch)
epoch_time.update(time.time() - start_epoch)
losses.update(loss_value, inputs_size)
# Write elapsed time (and loss) to terminal
print('Epoch: [{0}][{1}/{2}]\t'
'Batch {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Epoch {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
(epoch + 1), (i + 1),
len(train_sampler),
batch_time=batch_time,
data_time=epoch_time,
loss=losses))
# Loss log
avg_loss /= len(train_sampler)
self.loss_epochs.append(avg_loss)
# Accuracy train log
acc_train, _ = self.check_accuracy(y_true_train_epoch,
y_pred=y_pred_train_epoch)
self.accuracy_train_epochs.append(acc_train)
# Accuracy val log
with torch.no_grad():
y_pred_val = np.array([])
targets_val = np.array([])
for data in val_loader:
inputs, targets, input_percentages, _ = data
input_sizes = input_percentages.mul_(int(
inputs.size(3))).int()
_, y_pred_val_batch = self.check_accuracy(
targets.cpu().numpy(),
inputs=inputs,
input_sizes=input_sizes)
y_pred_val = np.concatenate((y_pred_val, y_pred_val_batch))
targets_val = np.concatenate(
(targets_val, targets.cpu().numpy()
)) # TO DO: think of a smarter way to do this later
del inputs, targets, input_percentages
# import pdb; pdb.set_trace()
acc_val, y_pred_val = self.check_accuracy(targets_val,
y_pred=y_pred_val)
self.accuracy_val_epochs.append(acc_val)
cm = confusion_matrix(targets_val, y_pred_val, labels=self.labels)
print('Confusion matrix validation:')
print(pd.DataFrame(cm))
# Write epoch stuff to tensorboard
if self.tensorboard:
tensorboard_logger.update(
epoch + 1, {'Loss/through_epochs': avg_loss},
parameters=self.model.named_parameters)
tensorboard_logger.update(epoch + 1, {
'train': acc_train,
'validation': acc_val
},
together=True,
name='Accuracy/through_epochs')
# Keep track of the best model
if acc_val > self.best_acc_val:
self.best_acc_val = acc_val
self.best_params = {}
for k, v in self.model.named_parameters(
): # TO DO: actually copy model and save later? idk..
self.best_params[k] = v.clone()
# Anneal learning rate. TO DO: find better way to this this specific to every parameter as cs231n does.
for g in self.optimizer.param_groups:
g['lr'] = g['lr'] / self.learning_anneal
print('Learning rate annealed to: {lr:.6f}'.format(lr=g['lr']))
# Shuffle batches order
print("Shuffling batches...")
train_sampler.shuffle(epoch)
# Rechoose batches elements
if self.sampler_type == 'random':
train_sampler.recompute_bins()
end_training = time.time()
if self.tensorboard:
tensorboard_logger.close()
print('Elapsed time in training: %.02f ' %
((end_training - start_training) / 60.0))
def main():
args = parser.parse_args()
torch.set_printoptions(profile="full")
criterion = nn.CrossEntropyLoss()
class_accu_reg = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
class_accu_sum = tnt.meter.ClassErrorMeter(topk=[1], accuracy=True)
audio_conf = dict(sample_rate=args.sample_rate,
window_size=args.window_size,
window_stride=args.window_stride,
window=args.window,
noise_dir=args.noise_dir,
noise_prob=args.noise_prob,
noise_levels=(args.noise_min, args.noise_max))
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
normalize=True,
augment=args.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.val_manifest,
normalize=True,
augment=False)
train_loader = AudioDataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
test_loader = AudioDataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
rnn_type = args.rnn_type.lower()
assert rnn_type in supported_rnns, "rnn_type should be either lstm, rnn or gru"
#print("FIRST LAYER TYPE:\t", args.first_layer_type)
#print("MFCC TRANSFORM:\t\t", args.mfcc)
model = DeepSpeech(rnn_hidden_size=args.hidden_size,
nb_layers=args.hidden_layers,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=True,
cnn_features=args.cnn_features,
kernel=args.kernel,
first_layer_type=args.first_layer_type,
stride=args.stride,
mfcc=args.mfcc)
########
#print(list(model.rnns.modules()))
#for rnn in model.rnns.modules():
# print(rnn)#.flatten_parameters()
#def flat_model(model):
# for m in model.modules():
# if isinstance(m, nn.LSTM):
# m.flatten_parameters()
########
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
nesterov=True)
#scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.learning_rate_decay_epochs, gamma=args.learning_rate_decay_rate)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99)
avg_loss = 0
start_epoch = 0
start_iter = 0
best_train_accu_reg = 0
best_train_accu_sum = 0
best_test_accu_reg = 0
best_test_accu_sum = 0
best_avg_loss = float("inf") # sys.float_info.max # 1000000
epoch_70 = None
epoch_90 = None
epoch_95 = None
epoch_99 = None
utterance_sequence_length = int(args.utterance_miliseconds / 10)
loss_begin = round(args.crop_begin / (10 * args.stride))
loss_end = -round(args.crop_end / (10 * args.stride)) or None
gap = loss_begin
print("LOSS BEGIN:", loss_begin)
print("LOSS END:", loss_end)
if args.cuda:
model = torch.nn.DataParallel(model).cuda()
print(model)
print("Number of parameters: ", DeepSpeech.get_param_size(model))
batch_time = AverageMeter()
data_time = AverageMeter()
#losses = AverageMeter()
print(args, "\n")
for epoch in range(start_epoch, args.epochs):
losses = AverageMeter()
scheduler.step()
optim_state_now = optimizer.state_dict()
print('\nLEARNING RATE: {lr:.6f}'.format(
lr=optim_state_now['param_groups'][0]['lr']))
class_accu_reg.reset()
class_accu_sum.reset()
model.train()
end = time.time()
for i, (data) in enumerate(train_loader, start=start_iter):
if i == len(train_loader):
break
inputs, input_percentages, speaker_labels, mfccs = data
# measure data loading time
data_time.update(time.time() - end)
inputs = Variable(inputs, requires_grad=False)
########
mfccs = Variable(mfccs, requires_grad=False)
if args.mfcc == "true":
inputs = mfccs # <<-- This line makes us to use mfccs...
#print("INPUTS SIZE:", inputs.size())
#print("MFCCS SIZE:", mfccs.size())
########
speaker_labels = Variable(speaker_labels, requires_grad=False)
speaker_labels = speaker_labels.cuda(async=True).long()
if args.cuda:
inputs = inputs.cuda()
########
########
sizes = inputs.size()
inputs = inputs.view(sizes[0], sizes[1] * sizes[2],
sizes[3]) # Collapse feature dimension
#print("INPUTS SIZE: ====>>>>>\t", inputs.size())
#start = 0
#duration = 100
start = random.randint(
0, int((inputs.size(2) - 1) * (1 - args.sample_proportion)))
duration = int((inputs.size(2)) * (args.sample_proportion))
#start = random.randint(0, (inputs.size(3)-1)-utterance_sequence_length)
#duration = utterance_sequence_length
utterances = inputs[
..., start:start +
duration] # <<<<<<====== THIS IS THE MOST IMPORTANT CODE OF THE PROJECT
#print("UTTERS SIZE: ====>>>>>\t", utterances.size(), start, start+duration)
out = model(utterances)
#print("OUTPUT SIZE: ====>>>>>\t", out.size())
out = out.transpose(0, 1) # TxNxH
########
########
# Prints the output of the model in a sequence of probabilities of char for each audio...
#torch.set_printoptions(profile="full")
####print("OUT: " + str(out.size()), "SPEAKER LABELS:" + str(speaker_labels.size()), "INPUT PERCENTAGES MEAN: " + str(input_percentages.mean()))
#print(out[:,:,0])
#print("SPEAKER LABELS: " + str(speaker_labels))
#print(out[0][0])
#softmax_output = F.softmax(out).data # This DOES NOT what I want...
#softmax_output_alt = flex_softmax(out, axis=2).data # This is FINE!!! <<<===
#print(softmax_output[0][0])
#print(softmax_output_alt[0][0])
####new_out = torch.sum(out, 0)
####new_out = torch.sum(out[20:], 0)
#print(out.size())
#print(new_out.size())
#print(out[-1].size())
class_accu_reg.add(out[round(out.size(0) / 2)].data,
speaker_labels.data)
class_accu_sum.add(
torch.sum(out[loss_begin:loss_end], 0).data,
speaker_labels.data)
#class_accu_reg.add(processed_out.data, processed_speaker_labels.data)
if args.loss_type == "reg":
processed_out = out[round(out.size(0) / 2)]
processed_speaker_labels = speaker_labels
if args.loss_type == "mult":
#indices = torch.LongTensor([0,2])
mult = (round(out.size(0) / 4), round(out.size(0) / 2),
round(3 * out.size(0) / 4))
processed_out = out.contiguous()[mult, ...].view(-1, 48)
processed_speaker_labels = speaker_labels.repeat(
out.size(0), 1)[mult, ...].view(-1)
#processed_out = out.contiguous()[(round(out.size(0)/4),round(out.size(0)/2),round(3*out.size(0)/4)),...].view(-1,48)
#processed_speaker_labels = speaker_labels.repeat(out.size(0),1)[(round(out.size(0)/4),round(out.size(0)/2),round(3*out.size(0)/4)),...].view(-1)
#processed_out = out.contiguous()[(loss_begin,round(out.size(0)/2),loss_end),...].view(-1,48)
#processed_speaker_labels = speaker_labels.repeat(out.size(0),1)[(loss_begin,round(out.size(0)/2),loss_end),...].view(-1)
##speaker_labels = speaker_labels.expand(20, out.size(0))
elif args.loss_type == "sum":
sum_begin = round(out.size(0) / 2) - round(out.size(0) / 4)
sum_end = round(out.size(0) / 2) + round(out.size(0) / 4)
processed_out = torch.sum(out[sum_begin:sum_end], 0)
processed_speaker_labels = speaker_labels
#processed_out = torch.sum(out[loss_begin:loss_end], 0)
#processed_speaker_labels = speaker_labels
#processed_out = torch.sum(out, 0)
#processed_speaker_labels = speaker_labels
elif args.loss_type == "full":
full_begin = round(out.size(0) / 2) - round(out.size(0) / 4)
full_end = round(out.size(0) / 2) + round(out.size(0) / 4)
processed_out = out.contiguous()[full_begin:full_end].view(
-1, 48)
processed_speaker_labels = speaker_labels.repeat(
out.size(0), 1)[full_begin:full_end].view(-1)
##speaker_labels = speaker_labels.expand(20, out.size(0))
#processed_out = out.contiguous()[loss_begin:loss_end].view(-1,48)
#processed_speaker_labels = speaker_labels.repeat(out.size(0),1)[loss_begin:loss_end].view(-1)
##speaker_labels = speaker_labels.expand(20, out.size(0))
#processed_out = out.contiguous().view(-1, 48)
#processed_speaker_labels = speaker_labels.repeat(out.size(0),1).view(-1)
##speaker_labels = speaker_labels.expand(20, out.size(0))
#print("PROC OUTPUT: ====>>>>>\t" + str(processed_out.size()))
#print("PROC LABELS: ====>>>>>\t" + str(processed_speaker_labels.size()))
loss = criterion(processed_out, processed_speaker_labels)
loss = loss / inputs.size(0) # average the loss by minibatch
loss_sum = loss.data.sum()
inf = float("inf")
if loss_sum == inf or loss_sum == -inf:
print("WARNING: received an inf loss, setting loss value to 0")
loss_value = 0
else:
loss_value = loss.data[0]
avg_loss += loss_value
losses.update(loss_value, inputs.size(0))
#accu_out3 = torch.sum(flex_softmax(out[20:], axis=2), 0)
#print(classaccu.value()[0], classaccu.value()[1])
# Cross Entropy Loss for a Sequence (Time Series) of Output?
#output = output.view(-1,29)
#target = target.view(-1)
#criterion = nn.CrossEntropyLoss()
#loss = criterion(output,target)
# compute gradient
optimizer.zero_grad()
loss.backward()
#torch.nn.utils.clip_grad_norm(model.parameters(), args.max_norm)
# SGD step
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if not args.silent:
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.8f} ({loss.avg:.8f})\t'
'CARR {carr:.2f}\t'
'CARS {cars:.2f}\t'.format(
(epoch + 1), (i + 1),
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
carr=class_accu_reg.value()[0],
cars=class_accu_sum.value()[0]))
if args.cuda:
torch.cuda.synchronize()
del loss
del out
del processed_out
del speaker_labels
del processed_speaker_labels
avg_loss /= len(train_loader)
if (best_avg_loss > avg_loss): best_avg_loss = avg_loss
print("\nCURRENT EPOCH AVERAGE LOSS:\t", avg_loss)
print("\nCURRENT EPOCH TRAINING RESULTS:\t",
class_accu_reg.value()[0], "\t",
class_accu_sum.value()[0], "\n")
if (best_train_accu_reg < class_accu_reg.value()[0]):
best_train_accu_reg = class_accu_reg.value()[0]
if (best_train_accu_sum < class_accu_sum.value()[0]):
best_train_accu_sum = class_accu_sum.value()[0]
get_70 = (class_accu_reg.value()[0] > 70)
if ((epoch_70 is None) and (get_70 == True)): epoch_70 = epoch + 1
get_90 = (class_accu_reg.value()[0] > 90)
if ((epoch_90 is None) and (get_90 == True)): epoch_90 = epoch + 1
get_95 = (class_accu_reg.value()[0] > 95)
if ((epoch_95 is None) and (get_95 == True)): epoch_95 = epoch + 1
get_99 = (class_accu_reg.value()[0] > 99)
if ((epoch_99 is None) and (get_99 == True)): epoch_99 = epoch + 1
start_iter = 0 # Reset start iteration for next epoch
model.eval()
class_accu_reg.reset()
class_accu_sum.reset()
for i, (data) in enumerate(test_loader): # test
inputs, input_percentages, speaker_labels, mfccs = data
inputs = Variable(inputs, volatile=True)
########
mfccs = Variable(mfccs, requires_grad=False)
if args.mfcc == "true":
inputs = mfccs # <<-- This line makes us to use mfccs...
#print("INPUTS SIZE:", inputs.size())
#print("MFCCS SIZE:", mfccs.size())
########
speaker_labels = Variable(speaker_labels, requires_grad=False)
speaker_labels = speaker_labels.cuda(async=True).long()
if args.cuda:
inputs = inputs.cuda()
########
########
sizes = inputs.size()
inputs = inputs.view(sizes[0], sizes[1] * sizes[2],
sizes[3]) # Collapse feature dimension
#print("INPUTS SIZE: ====>>>>>\t", inputs.size())
#start = round(inputs.size(2)/2)-40
#duration = 80
#start = random.randint(0, int((inputs.size(3)-1)*(1-args.sample_proportion)))
#duration = int((inputs.size(3))*(args.sample_proportion))
#start = random.randint(0, (inputs.size(3)-1)-utterance_sequence_length)
#duration = utterance_sequence_length
utterances = inputs #[...,start:start+duration] # <<<<<<====== THIS IS THE MOST IMPORTANT CODE OF THE PROJECT
#print("UTTERS SIZE: ====>>>>>\t", utterances.size(), start, start+duration)
out = model(utterances)
#print("OUTPUT SIZE: ====>>>>>\t", out.size())
out = out.transpose(0, 1) # TxNxH
########
########
# Prints the output of the model in a sequence of probabilities of char for each audio...
#torch.set_printoptions(profile="full")
########print("OUT: " + str(out.size()), "NEW OUT:" + str(new_out.size()), "SPEAKER LABELS:" + str(speaker_labels.size()), "INPUT PERCENTAGES MEAN: " + str(input_percentages.mean()))
#print(out[:,:,0])
#print("SPEAKER LABELS: " + str(speaker_labels))
#print(out[0][0])
#softmax_output = F.softmax(out).data # This DOES NOT what I want...
#softmax_output_alt = flex_softmax(out, axis=2).data # This is FINE!!! <<<===
#print(softmax_output[0][0])
#print(softmax_output_alt[0][0])
########
#if args.loss_type == "reg":
# processed_out = out[round(out.size(0)/2)]; processed_speaker_labels = speaker_labels
#elif args.loss_type == "sum" or "full":
# #processed_out = torch.sum(out[loss_begin:loss_end], 0); processed_speaker_labels = speaker_labels
# processed_out = torch.sum(out, 0); processed_speaker_labels = speaker_labels
#elif args.loss_type == "full":
# #processed_out = out.contiguous()[loss_begin:loss_end].view(-1,48); processed_speaker_labels = speaker_labels.repeat(out.size(0),1)[loss_begin:loss_end].view(-1) #speaker_labels = speaker_labels.expand(20, out.size(0))
# processed_out = out.contiguous().view(-1, 48); processed_speaker_labels = speaker_labels.repeat(out.size(0),1).view(-1) # speaker_labels = speaker_labels.expand(20, out.size(0))
#print("OUT: " + str(out.size()), "SPEAKER LABELS:" + str(speaker_labels.size()))
#print("PROC OUTPUT: ====>>>>>\t" + str(processed_out.size()))
#print("PROC LABELS: ====>>>>>\t" + str(processed_speaker_labels.size()))
class_accu_reg.add(out[round(out.size(0) / 2)].data,
speaker_labels.data)
class_accu_sum.add(
torch.sum(out[loss_begin:loss_end], 0).data,
speaker_labels.data)
#class_accu_reg.add(processed_out.data, processed_speaker_labels.data)
print('Validation Summary Epoch: [{0}]\t'
'CARR {carr:.2f}\t'
'CARS {cars:.2f}\t'.format(epoch + 1,
carr=class_accu_reg.value()[0],
cars=class_accu_sum.value()[0]))
if args.cuda:
torch.cuda.synchronize()
del out
print("\nCURRENT EPOCH TEST RESULTS:\t",
class_accu_reg.value()[0], "\t",
class_accu_sum.value()[0], "\n")
if (best_test_accu_reg < class_accu_reg.value()[0]):
best_test_accu_reg = class_accu_reg.value()[0]
if (best_test_accu_sum < class_accu_sum.value()[0]):
best_test_accu_sum = class_accu_sum.value()[0]
print("\nBEST AVERAGE LOSS:\t\t", best_avg_loss)
print("\nBEST EPOCH TRAINING RESULTS:\t", best_train_accu_reg, "\t",
best_train_accu_sum)
print("\nBEST EPOCH TEST RESULTS:\t", best_test_accu_reg, "\t",
best_test_accu_sum)
print("\nEPOCHS 70%, 90%, 95%, 99%:\t", epoch_70, "\t", epoch_90, "\t",
epoch_95, "\t", epoch_99, "\n")
torch.save(
DeepSpeech.serialize(model, optimizer=optimizer, epoch=epoch),
args.model_path)
avg_loss = 0
if not args.no_bucketing and epoch == 0:
print("Switching to bucketing sampler for following epochs")
train_dataset = SpectrogramDatasetWithLength(
audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
normalize=True,
augment=args.augment)
sampler = BucketingSampler(train_dataset)
train_loader.sampler = sampler
def main():
args = parser.parse_args()
params.cuda = not bool(args.cpu)
print("Use cuda: {}".format(params.cuda))
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
if params.rnn_type == 'gru' and params.rnn_act_type != 'tanh':
print(
"ERROR: GRU does not currently support activations other than tanh"
)
sys.exit()
if params.rnn_type == 'rnn' and params.rnn_act_type != 'relu':
print("ERROR: We should be using ReLU RNNs")
sys.exit()
print("=======================================================")
for arg in vars(args):
print("***%s = %s " % (arg.ljust(25), getattr(args, arg)))
print("=======================================================")
save_folder = args.save_folder
loss_results, cer_results, wer_results = torch.Tensor(
params.epochs), torch.Tensor(params.epochs), torch.Tensor(
params.epochs)
best_wer = None
try:
os.makedirs(save_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
criterion = CTCLoss()
with open(params.labels_path) as label_file:
labels = str(''.join(json.load(label_file)))
audio_conf = dict(sample_rate=params.sample_rate,
window_size=params.window_size,
window_stride=params.window_stride,
window=params.window,
noise_dir=params.noise_dir,
noise_prob=params.noise_prob,
noise_levels=(params.noise_min, params.noise_max))
if args.use_set == 'libri':
testing_manifest = params.val_manifest + ("_held" if args.hold_idx >= 0
else "")
else:
testing_manifest = params.test_manifest
if args.batch_size_val > 0:
params.batch_size_val = args.batch_size_val
print("Testing on: {}".format(testing_manifest))
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=params.val_manifest,
labels=labels,
normalize=True,
augment=params.augment)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=testing_manifest,
labels=labels,
normalize=True,
augment=False)
train_loader = AudioDataLoader(train_dataset,
batch_size=params.batch_size,
num_workers=1)
test_loader = AudioDataLoader(test_dataset,
batch_size=params.batch_size_val,
num_workers=1)
rnn_type = params.rnn_type.lower()
assert rnn_type in supported_rnns, "rnn_type should be either lstm, rnn or gru"
model = DeepSpeech(rnn_hidden_size=params.hidden_size,
nb_layers=params.hidden_layers,
labels=labels,
rnn_type=supported_rnns[rnn_type],
audio_conf=audio_conf,
bidirectional=False,
rnn_activation=params.rnn_act_type,
bias=params.bias)
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters,
lr=params.lr,
momentum=params.momentum,
nesterov=True,
weight_decay=params.l2)
decoder = GreedyDecoder(labels)
if args.continue_from:
print("Loading checkpoint model %s" % args.continue_from)
package = torch.load(args.continue_from)
model.load_state_dict(package['state_dict'])
optimizer.load_state_dict(package['optim_dict'])
start_epoch = int(package.get(
'epoch', 1)) - 1 # Python index start at 0 for training
start_iter = package.get('iteration', None)
if start_iter is None:
start_epoch += 1 # Assume that we saved a model after an epoch finished, so start at the next epoch.
start_iter = 0
else:
start_iter += 1
avg_loss = int(package.get('avg_loss', 0))
if args.start_epoch != -1:
start_epoch = args.start_epoch
avg_loss = 0
start_epoch = 0
start_iter = 0
avg_training_loss = 0
epoch = 1
else:
avg_loss = 0
start_epoch = 0
start_iter = 0
avg_training_loss = 0
if params.cuda:
model = torch.nn.DataParallel(model).cuda()
# model = torch.nn.parallel.DistributedDataParallel(model).cuda()
print(model)
print("Number of parameters: %d" % DeepSpeech.get_param_size(model))
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
ctc_time = AverageMeter()
for epoch in range(start_epoch, params.epochs):
#################################################################################################################
# The test script only really cares about this section.
#################################################################################################################
model.eval()
wer, cer, trials = eval_model_verbose(model, test_loader, decoder,
params.cuda, args.n_trials)
root = os.getcwd()
outfile = osp.join(
root,
"inference_bs{}_i{}_gpu{}.csv".format(params.batch_size_val,
args.hold_idx, params.cuda))
print("Exporting inference to: {}".format(outfile))
make_file(outfile)
write_line(
outfile, "batch times pre normalized by hold_sec =,{}\n".format(
args.hold_sec))
write_line(outfile, "wer, {}\n".format(wer))
write_line(outfile, "cer, {}\n".format(cer))
write_line(outfile, "bs, {}\n".format(params.batch_size_val))
write_line(outfile, "hold_idx, {}\n".format(args.hold_idx))
write_line(outfile, "cuda, {}\n".format(params.cuda))
write_line(outfile,
"avg batch time, {}\n".format(trials.avg / args.hold_sec))
percentile_50 = np.percentile(
trials.array, 50) / params.batch_size_val / args.hold_sec
write_line(outfile, "50%-tile latency, {}\n".format(percentile_50))
percentile_99 = np.percentile(
trials.array, 99) / params.batch_size_val / args.hold_sec
write_line(outfile, "99%-tile latency, {}\n".format(percentile_99))
write_line(outfile, "through put, {}\n".format(1 / percentile_50))
write_line(outfile, "data\n")
for trial in trials.array:
write_line(outfile, "{}\n".format(trial / args.hold_sec))
loss_results[epoch] = avg_loss
wer_results[epoch] = wer
cer_results[epoch] = cer
print('Validation Summary Epoch: [{0}]\t'
'Average WER {wer:.3f}\t'
'Average CER {cer:.3f}\t'.format(epoch + 1, wer=wer, cer=cer))
# anneal lr
optim_state = optimizer.state_dict()
optim_state['param_groups'][0]['lr'] = optim_state['param_groups'][0][
'lr'] / params.learning_anneal
optimizer.load_state_dict(optim_state)
print('Learning rate annealed to: {lr:.6f}'.format(
lr=optim_state['param_groups'][0]['lr']))
break
print("=======================================================")
print("***Best WER = ", best_wer)
for arg in vars(args):
print("***%s = %s " % (arg.ljust(25), getattr(args, arg)))
print("=======================================================")
# elif args.decoder == "greedy":
# decoder = GreedyDecoder(model.labels, blank_index=model.labels.index('_'))
# else:
# decoder = None
# target_decoder = GreedyDecoder(model.labels, blank_index=model.labels.index('_'))
decoder = MyDecoder(model.labels)
target_decoder = MyDecoder(model.labels)
test_dataset = SpectrogramDataset(audio_conf=model.audio_conf,
manifest_filepath=args.test_manifest,
metadata_file_path=metadata_path,
labels=model.labels,
normalize=True)
test_loader = AudioDataLoader(
test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=True
) # in train, the manifest will be already stratified with only train data so it's ok.
accuracy_mean, acccuracy_std, output_data = evaluate(
test_loader=test_loader,
device=device,
model=model,
decoder=decoder,
target_decoder=target_decoder,
save_output=args.save_output,
verbose=args.verbose,
half=args.half)
print('Test Summary \t'
'Average accuracy {acc_mean:.3f}\t'
'Standard deviation accuracy {acc_std:.3f}\t'.format(
def main():
args = parser.parse_args()
save_folder = args.save_folder
try:
os.makedirs(save_folder)
except OSError as e:
if e.errno == errno.EEXIST:
print('Directory already exists.')
else:
raise
criterion = CTCLoss()
with open(args.labels_path) as label_file:
labels = str(''.join(json.load(label_file)))
audio_conf = dict(sample_rate=args.sample_rate,
window_size=args.window_size,
window_stride=args.window_stride,
window=args.window)
train_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.train_manifest,
labels=labels,
normalize=True)
test_dataset = SpectrogramDataset(audio_conf=audio_conf,
manifest_filepath=args.val_manifest,
labels=labels,
normalize=True)
train_loader = AudioDataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
test_loader = AudioDataLoader(test_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
model = DeepSpeech(rnn_hidden_size=args.hidden_size,
nb_layers=args.hidden_layers,
num_classes=len(labels))
decoder = ArgMaxDecoder(labels)
if args.cuda:
model = torch.nn.DataParallel(model).cuda()
print(model)
parameters = model.parameters()
optimizer = torch.optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
nesterov=True)
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
for epoch in range(args.epochs):
model.train()
end = time.time()
avg_loss = 0
for i, (data) in enumerate(train_loader):
inputs, targets, input_percentages, target_sizes = data
# measure data loading time
data_time.update(time.time() - end)
inputs = Variable(inputs)
target_sizes = Variable(target_sizes)
targets = Variable(targets)
if args.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
seq_length = out.size(0)
sizes = Variable(input_percentages.mul_(int(seq_length)).int())
loss = criterion(out, targets, sizes, target_sizes)
loss = loss / inputs.size(0) # average the loss by minibatch
loss_sum = loss.data.sum()
inf = float("inf")
if loss_sum == inf or loss_sum == -inf:
print("WARNING: received an inf loss, setting loss value to 0")
loss_value = 0
else:
loss_value = loss.data[0]
avg_loss += loss_value
losses.update(loss_value, inputs.size(0))
# compute gradient
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), args.max_norm)
# SGD step
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if not args.silent:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
(epoch + 1), (i + 1),
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
avg_loss /= len(train_loader)
print('Training Summary Epoch: [{0}]\t'
'Average Loss {loss:.3f}\t'.format((epoch + 1), loss=avg_loss))
total_cer, total_wer = 0, 0
for i, (data) in enumerate(test_loader): # test
inputs, targets, input_percentages, target_sizes = data
inputs = Variable(inputs)
# unflatten targets
split_targets = []
offset = 0
for size in target_sizes:
split_targets.append(targets[offset:offset + size])
offset += size
if args.cuda:
inputs = inputs.cuda()
out = model(inputs)
out = out.transpose(0, 1) # TxNxH
seq_length = out.size(0)
sizes = Variable(input_percentages.mul_(int(seq_length)).int())
decoded_output = decoder.decode(out.data, sizes)
target_strings = decoder.process_strings(
decoder.convert_to_strings(split_targets))
wer, cer = 0, 0
for x in range(len(target_strings)):
wer += decoder.wer(decoded_output[x],
target_strings[x]) / float(
len(target_strings[x].split()))
cer += decoder.cer(decoded_output[x],
target_strings[x]) / float(
len(target_strings[x]))
total_cer += cer
total_wer += wer
wer = total_wer / len(test_loader.dataset)
cer = total_cer / len(test_loader.dataset)
print('Validation Summary Epoch: [{0}]\t'
'Average WER {wer:.0f}\t'
'Average CER {cer:.0f}\t'.format((epoch + 1),
wer=wer * 100,
cer=cer * 100))
if args.epoch_save:
file_path = '%s/deepspeech_%d.pth.tar' % (save_folder, epoch)
torch.save(checkpoint(model, args, len(labels), epoch), file_path)
torch.save(checkpoint(model, args, len(labels)), args.final_model_path)
