tfboard_writer = SummaryWriter(tfboard_path)
# exclude cutoff #
print("=====START=====")
prev_dev_loss = 2**64
best_dev_loss = 2**64
niter = 0
for ee in range(EPOCHS) :
print('epoch {}'.format(ee))
start_time = time.time()
# choose standard training or bucket training #
if opts['bucket'] :
if opts['block'] == -1 :
train_rr_a = iter_minibucket(sorted_train_idx_a, BATCHSIZE,
shuffle=False if ee < opts['sortagrad'] else True,
excludes=[])
train_rr_b = iter_minibucket(sorted_train_idx_b, BATCHSIZE,
shuffle=False if ee < opts['sortagrad'] else True,
excludes=[])
else :
train_rr_a = iter_minibucket_block(sorted_train_idx_a,
opts['block'], sorted_train_len,
shuffle=False if ee < opts['sortagrad'] else True,
pad=True, excludes=[])
train_rr_b = iter_minibucket_block(sorted_train_idx_b,
opts['block'], sorted_train_len,
shuffle=False if ee < opts['sortagrad'] else True,
pad=True, excludes=[])
else :
best_dev_loss_ep = 0
ee = 1
while ee <= EPOCHS:
start_time = time.time()
mloss = dict(train=0, dev=0, test=0)
mcount = dict(train=0, dev=0, test=0)
mloss_feat = dict(train=0, dev=0, test=0)
mloss_bernend = dict(train=0, dev=0, test=0)
macc_bernend = dict(train=0, dev=0, test=0)
# choose standard training or bucket training #
if opts['bucket']:
if opts['block'] == -1:
train_rr = iter_minibucket(
sorted_train_idx,
BATCHSIZE,
shuffle=False if ee <= opts['sortagrad'] else True,
excludes=exc_idx['train'])
dev_rr = iter_minibucket(sorted_dev_idx,
BATCHSIZE,
shuffle=False,
excludes=exc_idx['dev'])
test_rr = iter_minibucket(sorted_test_idx,
BATCHSIZE,
shuffle=False,
excludes=exc_idx['test'])
else:
train_rr = iter_minibucket_block(
sorted_train_idx,
opts['block'],
sorted_train_len,
else:
# read from stdin
print('Feat input redirected from previous script ...',
file=sys.stderr)
list_info = yaml.load(sys.stdin.read())
list_kv = [(_info['key'], _info['feat']) for _info in list_info]
list_klen = [(_info['key'], _info['feat_len']) for _info in list_info]
data_iterator_infer = DataIteratorNP(feat_kv=list_kv,
feat_len_kv=list_klen)
group = model.in_size // data_iterator_source.get_feat_dim()
sorted_data_idx = np.argsort(
data_iterator_infer.get_feat_length())[::-1].tolist()
data_rr = iter_minibucket(sorted_data_idx,
opts['chunk'],
shuffle=False,
excludes=[])
list_saved = []
for rr in tqdm(list(data_rr), ascii=True, ncols=50):
curr_feat_list = data_iterator_infer.get_feat_by_index(rr)
feat_mat, feat_len = batch_speech(opts['gpu'],
curr_feat_list,
feat_sil=feat_sil,
group=group,
start_sil=0,
end_sil=0)
pred_out = invert_mel_to_linear(model,
Variable(feat_mat),
feat_len,
group=group)
# convert to cpu & numpy #
plot_attention.plot_softmax_attention(info['att'], None, _path_att_img)
_path_att_npz = os.path.join(tmpdir, info['key'] + '_att.npz')
np.savez(_path_att_npz, key=info['key'], feat=info['att'])
return _path_att_img, _path_att_npz
start = time.time()
# cache list #
list_saved = []
ori_key_pos = dict((x, y) for y, x in enumerate(list_key))
sorted_feat_idx = np.argsort(list_feat_len)[::-1].tolist()
sorted_feat_len = operator.itemgetter(*sorted_feat_idx)(list_feat_len)
if args.chunk is not None:
data_rr = iter_minibucket(sorted_feat_idx,
args.chunk,
shuffle=False,
excludes=[])
elif args.chunkblk is not None:
data_rr = iter_minibucket_block(sorted_feat_idx,
args.chunkblk,
sorted_feat_len,
pad_block=True,
shuffle=False,
excludes=[])
data_rr = [list(reversed(x)) for x in data_rr]
for rr in tqdm(list(data_rr), ascii=True, ncols=50):
curr_key_list = [list_key[rrii] for rrii in rr]
curr_feat_list = [list_feat[rrii]() for rrii in rr] # lazy load call
if args.mode == 'pred':
pass
ee = 1
# logging #
HEADER_ASR = ['SET', '[ASR] LOSS CE', '[ASR] ACC']
HEADER_TTS = [
'SET', '[TTS] LOSS', '[TTS] LOSS FEAT', '[TTS] LOSS BCE',
'[TTS] LOSS SPK EMB', '[TTS] ACC'
]
set_name = ['train', 'dev', 'test', 'unpair']
feat_rr = dict((x, None) for x in set_name)
text_rr = dict((x, None) for x in set_name)
for set_name in ['dev', 'test']:
feat_rr[set_name] = list(
iter_minibucket(sorted_feat_idx[set_name],
FEAT_BATCHSIZE,
shuffle=False,
excludes=exc_idx[set_name + '_feat']))
text_rr[set_name] = list(
iter_minibucket(sorted_text_idx[set_name],
TEXT_BATCHSIZE,
shuffle=False,
excludes=exc_idx[set_name + '_text']))
best_asr_dev = {'loss': 2**64, 'ep': 0} # loss & epoch #
best_tts_dev = {'loss': 2**64, 'ep': 0} # loss & epoch #
logger.info("=====START=====")
while ee <= EPOCHS:
start_time = time.time()
m_asr_loss = dict((x, 0) for x in SET_NAMES)
m_asr_acc = dict((x, 0) for x in SET_NAMES)
map_spk2key = dict((y, x) for x, y in map_key2spk.items())
feat_iterator = DataIteratorNP(feat_kv=_feat_kv, feat_len_kv=_feat_len_kv)
feat_len = feat_iterator.get_feat_length()
sorted_feat_idx = np.argsort(feat_len).tolist()
sorted_feat_len = [feat_len[x] for x in sorted_feat_idx]
feat_stat = pickle.load(open(data_cfg['feat']['stat'], 'rb'))
feat_sil = feat_sil_from_stat(feat_stat)
all_embed = []
all_speaker_id = []
all_key = []
for rr in tqdm(list(
iter_minibucket(sorted_feat_idx, args.batchsize, shuffle=False)),
ncols=60):
curr_feat_list = feat_iterator.get_feat_by_index(rr)
curr_key_list = feat_iterator.get_key_by_index(rr)
curr_speaker_list = [map_key2spk[x] for x in curr_key_list]
curr_speaker_list_id = [map_spk2id[x] for x in curr_speaker_list]
all_speaker_id.extend(curr_speaker_list_id)
all_key.extend(curr_key_list)
feat_mat, feat_len = batch_speech(args.gpu,
curr_feat_list,
feat_sil=feat_sil,
group=1,
start_sil=1,
end_sil=1)
res_embed = model(Variable(feat_mat), feat_len)