def get_data(prefix, sibling=True):
r = {}
ref_prefix = "{prefix}/single/".format(prefix=prefix)
if sibling: sample_prefix = "{prefix}/double/".format(prefix=prefix)
else: sample_prefix = "{prefix}/double_far/".format(prefix=prefix)
for bg in benches:
for fg in benches:
reference = csv2dict(ref_prefix+"{fg}".format(fg=fg))
sample = csv2dict(sample_prefix+"/{bg}/{fg}".format(bg=bg,fg=fg))
r[bg,fg] = get_degr(reference, sample)
return r
def main(argv):
data = []
figpath = argv[1]
files = argv[2:]
if not files:
die("please specify datasets to compare")
for f in files:
data += [csv2dict(f)]
reference = data.pop(0)
# calculations
ticks = []
values = []
ratio = []
for k,vref in reference.items():
if k not in data[0]:
print("not in second dataset:", k)
continue
v = data[0][k]
if v == 0 or vref == 0: continue
r = v/vref-1
if abs(r) <0.2: continue # skip parameters that didn't change much
if k == 'cpu-migrations': continue # skip irrelevant counters
ratio += [r]
ticks += [k]
values += [v]
# plotting
barh(list(myrange(-0.4, 1, len(ratio))), ratio)
yticks(range(len(ticks)), ticks, rotation=0)
axvline(linewidth=4, color='g')
xlim([-1.2,5])
subplots_adjust(left=0.34)
xticks(arange(-1,5), ('-100%','0', '100%', '200%', '300%', '400%'))
for i, v in enumerate(values):
color = 'red'
r = ratio[i]
v = values[i]
x = r + 0.07 if r > 0 else r - 0.2
if x > 4: x = 4; color = 'white'
label = "{}".format(int(v))
text(x, i-0.16, label, fontsize=18, color=color)
# bar(list(myrange(-0.4, 1, len(ratio))), ratio)
# xticks(range(len(ticks)), ticks, rotation=60)
# axhline(linewidth=4, color='g')
# subplots_adjust(bottom=0.2)
#savefig("/home/exe/github/perf2013paper/pic/blosc_bars.eps")
savefig(figpath)
show()
def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus,
rank, group_name, hparams):
"""Training and validation logging results to tensorboard and stdout
Params
------
output_directory (string): directory to save checkpoints
log_directory (string) directory to save tensorboard logs
checkpoint_path(string): checkpoint path
n_gpus (int): number of gpus
rank (int): rank of current gpu
hparams (object): comma separated list of "name=value" pairs.
"""
if hparams.distributed_run:
init_distributed(hparams, n_gpus, rank, group_name)
torch.manual_seed(hparams.seed)
torch.cuda.manual_seed(hparams.seed)
model = load_model(hparams)
learning_rate = hparams.learning_rate
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate,
weight_decay=hparams.weight_decay)
if hparams.fp16_run:
from apex import amp
model, optimizer = amp.initialize(
model, optimizer, opt_level='O2')
if hparams.distributed_run:
model = apply_gradient_allreduce(model)
if hparams.use_vae:
criterion = Tacotron2Loss_VAE(hparams)
else:
criterion = Tacotron2Loss()
logger = prepare_directories_and_logger(
output_directory, log_directory, rank, hparams.use_vae)
train_loader, valset, collate_fn = prepare_dataloaders(hparams)
valset_csv = os.path.join(output_directory, log_directory, 'valset.csv')
# list2csv(flatten_list(valset.audiopaths_and_text), valset_csv, delimiter='|')
list2csv(valset.audiopaths_and_text, valset_csv, delimiter='|')
# Load checkpoint if one exists
iteration = 0
epoch_offset = 0
if checkpoint_path is not None:
if warm_start:
model = warm_start_model(
checkpoint_path, model, hparams.ignore_layers)
else:
model, optimizer, _learning_rate, iteration, epoch, step = \
load_checkpoint(checkpoint_path, model, optimizer)
if hparams.use_saved_learning_rate:
learning_rate = _learning_rate
if epoch == 0:
iteration += 1 # next iteration is iteration + 1
epoch_offset = max(0, int(iteration / len(train_loader)))
else:
epoch_offset = epoch
print('epoch offset: {}'.format(epoch_offset))
train_loader = prepare_dataloaders(hparams, epoch_offset, valset,
collate_fn['train'])[0]
print('completing loading model ...')
model.train()
is_overflow = False
# ================ MAIN TRAINNIG LOOP! ===================
track_csv = os.path.join(output_directory, log_directory, 'track.csv')
track_header = ['padding-rate-txt', 'max-len-txt', 'top-len-txt',
'padding-rate-mel', 'max-len-mel', 'top-len-mel', 'batch-size',
'batch-length', 'batch-area', 'mem-use', 'mem-all', 'mem-cached',
'duration', 'iteration', 'epoch', 'step']
if os.path.isfile(track_csv) and checkpoint_path is not None:
print('loading existing {} ...'.format(track_csv))
track = csv2dict(track_csv, header=track_header)
else:
track = {k:[] for k in track_header}
print('start training in epoch {} ~ {} ...'.format(epoch_offset, hparams.epochs))
nbatches = len(train_loader)
for epoch in range(epoch_offset, hparams.epochs):
#if epoch >= 10: break
print("Epoch: {}, #batches: {}".format(epoch, nbatches))
batch_sizes, batch_lengths = [0] * nbatches, [0] * nbatches
for i, batch in enumerate(train_loader):
start = time.perf_counter()
for param_group in optimizer.param_groups:
param_group['lr'] = learning_rate
model.zero_grad()
x, y = model.parse_batch(batch)
y_pred = model(x)
if hparams.use_vae:
loss, recon_loss, kl, kl_weight = criterion(y_pred, y, iteration)
else:
loss = criterion(y_pred, y)
if hparams.distributed_run:
reduced_loss = reduce_tensor(loss.data, n_gpus).item()
else:
reduced_loss = loss.item()
if hparams.fp16_run:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if hparams.fp16_run:
grad_norm = torch.nn.utils.clip_grad_norm_(
amp.master_params(optimizer), hparams.grad_clip_thresh)
is_overflow = math.isnan(grad_norm)
else:
grad_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), hparams.grad_clip_thresh)
optimizer.step()
if not is_overflow and rank == 0:
duration = time.perf_counter() - start
batch_sizes[i], batch_lengths[i] = batch[0].size(0), batch[2].size(2)
batch_capacity = batch_sizes[i] * batch_lengths[i]
mem_all = torch.cuda.memory_allocated() / (1024**2)
mem_cached = torch.cuda.memory_cached() / (1024**2)
mem_use = mem_all + mem_cached
print("{} ({}:{}/{}): ".format(iteration, epoch, i, nbatches), end='')
print("Batch {} ({}X{}) ".format(batch_capacity, batch_sizes[i],
batch_lengths[i]), end='')
print("Mem {:.1f} ({:.1f}+{:.1f}) ".format(mem_use, mem_all,
mem_cached), end='')
print("Train loss {:.3f} Grad Norm {:.3f} {:.2f}s/it".format(
reduced_loss, grad_norm, duration))
input_lengths, gate_padded = batch[1], batch[4]
metadata = (duration, iteration, epoch, i)
track_seq(track, input_lengths, gate_padded, metadata)
padding_rate_txt = track['padding-rate-txt'][-1]
max_len_txt = track['max-len-txt'][-1]
padding_rate_mel = track['padding-rate-mel'][-1]
max_len_mel = track['max-len-mel'][-1]
if hparams.use_vae:
logger.log_training(
reduced_loss, grad_norm, learning_rate, duration,
padding_rate_txt, max_len_txt, padding_rate_mel,
max_len_mel, iteration, recon_loss, kl, kl_weight)
else:
logger.log_training(
reduced_loss, grad_norm, learning_rate, duration,
padding_rate_txt, max_len_txt, padding_rate_mel,
max_len_mel, iteration)
check_by_iter = (hparams.check_by == 'iter') and \
(iteration % hparams.iters_per_checkpoint == 0)
check_by_epoch = (hparams.check_by == 'epoch') and i == 0 and \
(epoch % hparams.epochs_per_checkpoint == 0)
if not is_overflow and (check_by_iter or check_by_epoch):
dict2col(track, track_csv, verbose=True)
val_loss, (mus, emotions) = validate(model, criterion, valset,
iteration, hparams.batch_size, n_gpus, collate_fn['val'], logger,
hparams.distributed_run, rank, hparams.use_vae, pre_batching=False)
if rank == 0:
checkpoint_path = os.path.join(output_directory,
"checkpoint_{}-{}-{}_{:.3f}".format(iteration, epoch, i, val_loss))
save_checkpoint(model, optimizer, learning_rate, iteration,
epoch, i, checkpoint_path)
if hparams.use_vae:
image_scatter_path = os.path.join(output_directory,
"checkpoint_{0}_scatter_val.png".format(iteration))
image_tsne_path = os.path.join(output_directory,
"checkpoint_{0}_tsne_val.png".format(iteration))
imageio.imwrite(image_scatter_path, plot_scatter(mus, emotions))
imageio.imwrite(image_tsne_path, plot_tsne(mus, emotions))
iteration += 1
if hparams.prep_trainset_per_epoch:
train_loader = prepare_dataloaders(hparams, epoch+1, valset,
collate_fn['train'])[0]
nbatches = len(train_loader)
