def prep_batches(batch):
mode_ = FLAGS["model.emb_mode"]
'''
x_char: char encoding grouped by word
x_word: word encoding
'''
if mode_ == CHAR:
x_char_id, x_expert_features, y_batch = zip(*batch)
elif mode_ == WORD:
x_word_id, x_expert_features, y_batch = zip(*batch)
elif mode_ == CHAR_AND_WORD:
x_char_id, x_word_id, x_expert_features, y_batch = zip(*batch)
elif mode_ == CHARWORD_AND_WORD:
x_charword_id, x_word_id, x_expert_features, y_batch = zip(*batch)
elif mode_ == CHARWORD_AND_WORD_AND_CHAR:
x_charword_id, x_word_id, x_char_id, x_expert_features, y_batch = zip(
*batch)
x_batch = [np.asarray(x_expert_features)]
if mode_ in [CHAR, CHAR_AND_WORD, CHARWORD_AND_WORD_AND_CHAR]:
x_char_id = pad_seq_in_word(x_char_id, FLAGS["data.max_len_chars"])
x_batch.append(x_char_id)
if mode_ in [
WORD, CHAR_AND_WORD, CHARWORD_AND_WORD, CHARWORD_AND_WORD_AND_CHAR
]:
x_word_id = pad_seq_in_word(x_word_id, FLAGS["data.max_len_words"])
x_batch.append(x_word_id)
if mode_ in [CHARWORD_AND_WORD, CHARWORD_AND_WORD_AND_CHAR]:
x_charword_id, x_charword_id_embedding = pad_seq(
x_charword_id, FLAGS["data.max_len_words"],
FLAGS["data.max_len_subwords"], FLAGS["model.emb_dim"])
x_batch.extend([x_charword_id, x_charword_id_embedding])
return x_batch, y_batch
def get_batch_iter(examples, batch_size, augment):
# the transpose ops requires deterministic
# batch size, thus comes the padding
padded = pad_seq(examples, batch_size)
def process(img):
img = bytes_to_file(img)
try:
img_A, img_B = read_split_image(img)
if augment:
# augment the image by:
# 1) enlarge the image
# 2) random crop the image back to its original size
# NOTE: image A and B needs to be in sync as how much
# to be shifted
w, h = img_A.shape
multiplier = random.uniform(1.00, 1.20)
# add an eps to prevent cropping issue
nw = int(multiplier * w) + 1
nh = int(multiplier * h) + 1
shift_x = int(np.ceil(np.random.uniform(0.01, nw - w)))
shift_y = int(np.ceil(np.random.uniform(0.01, nh - h)))
img_A = shift_and_resize_image(img_A, shift_x, shift_y, nw, nh)
img_B = shift_and_resize_image(img_B, shift_x, shift_y, nw, nh)
img_A = normalize_image(img_A)
img_B = normalize_image(img_B)
img_A = np.expand_dims(img_A, axis=2)
img_B = np.expand_dims(img_B, axis=2)
return np.concatenate([img_A, img_B], axis=2)
finally:
img.close()
def handle_cns(cns_code):
cns_code_batch = []
seq_len = []
max_len = 28 # get max_len from check_cns_len.py
for cns in cns_code:
num = list(map(int, cns.split(',')))
seq_len.append(len(num))
num += [0] * (max_len - len(num))
cns_code_batch.append(num)
return cns_code_batch, seq_len
def batch_iter():
for i in range(0, len(padded), batch_size):
batch = padded[i:i + batch_size]
cns_code = [e[0] for e in batch]
labels = [e[1] for e in batch]
processed = [process(e[2]) for e in batch]
cns_code, seq_len = handle_cns(cns_code)
# stack into tensor
yield cns_code, seq_len, labels, np.array(processed).astype(
np.float32)
return batch_iter()
def get_x_train_batch(self, batch):
"""
将batch中的数据重整后输出
[char seq, word, char, char_pad_idx]
:param batch:
:return:
"""
x_batch = list()
x_char_seq, x_word, x_char, y_batch = zip(*batch)
x_char_seq = pad_seq_in_word(x_char_seq, MAX_LENGTH_CHARS)
x_batch.append(x_char_seq)
x_word = pad_seq_in_word(x_word, MAX_LENGTH_WORDS)
x_batch.append(x_word)
x_char, x_char_pad_idx = pad_seq(x_char, MAX_LENGTH_WORDS,
MAX_LENGTH_SUBWORDS, EMB_DIM)
x_batch.extend([x_char, x_char_pad_idx])
return x_batch, y_batch
def do_predict(self, urls):
"""
进行测试的主函数
:param urls:
:return:
"""
# 加载模型文件，测试数据预处理
data_obj = DataPreprocessTest()
data_obj.do_preprocess(urls)
batches = batch_iter(list(
zip(data_obj.ngramed_id_x, data_obj.worded_id_x,
data_obj.chared_id_x)),
BATCH_SIZE_TEST,
num_epochs=1,
shuffle=False)
all_predictions = []
all_scores = []
for index, batch in enumerate(batches):
if index % 1000 == 0:
print("Processing #batch {}".format(index))
x_char, x_word, x_char_seq = zip(*batch)
x_batch = []
x_char_seq = pad_seq_in_word(x_char_seq, MAX_LENGTH_CHARS)
x_batch.append(x_char_seq)
x_word = pad_seq_in_word(x_word, MAX_LENGTH_WORDS)
x_batch.append(x_word)
x_char, x_char_pad_idx = pad_seq(x_char, MAX_LENGTH_WORDS,
MAX_LENGTH_SUBWORDS, EMB_DIM)
x_batch.extend([x_char, x_char_pad_idx])
batch_predictions, batch_scores = self.test_step(x_batch)
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
all_scores.extend(batch_scores)
softmax_scores = [softmax(score) for score in all_scores]
return all_predictions, softmax_scores
def get_x_test(self, data_obj):
"""
构建验证集 x_test，将data_obj类中测试集数据重整后输出
[char seq, word, char, char_pad_idx]
:param data_obj:
:return:
"""
x_test = list()
x_test_char_seq = pad_seq_in_word(data_obj.x_test_char_seq,
MAX_LENGTH_CHARS)
x_test.append(x_test_char_seq)
x_test_word = pad_seq_in_word(data_obj.x_test_word, MAX_LENGTH_WORDS)
x_test.append(x_test_word)
x_test_char, x_test_char_pad_idx = pad_seq(data_obj.x_test_char,
MAX_LENGTH_WORDS,
MAX_LENGTH_SUBWORDS,
EMB_DIM)
x_test.extend([x_test_char, x_test_char_pad_idx])
y_test = data_obj.y_test
return x_test, y_test
def next_batch(self, interactions, current_pos):
"""generate the next batch data
:param interactions: list, element like [user, item, label]
:param current_pos: int
:returns items:
labels:
initial_entity: list, length: batch_size
"""
if current_pos + self.batch_size < len(interactions):
next_records = interactions[current_pos:current_pos +
self.batch_size]
current_pos += self.batch_size
else:
next_records = interactions[current_pos:]
current_pos = len(interactions)
h_list = []
t_list = []
r_list = []
items = []
labels = []
init_h = []
init_t = []
init_r = []
for interact in next_records:
user = interact[0]
if user in self.user_clicks:
user_click = self.user_clicks[
user] # the items set of the user history
else:
print("user %d not in the training dataset" % user)
assert 0 == 1
continue
items.append(interact[1])
labels.append(interact[2])
degree = min(self.user_click_limit, len(user_click))
user_click = user_click[:degree]
init_h_tmp = []
init_t_tmp = []
init_r_tmp = []
for click in user_click: # item
degree = max(
1,
int(
min(self.entity_limit, len(self.KG[click][0])) *
self.kg_ratio))
init_h_tmp += degree * [click]
init_t_tmp += self.KG[click][0][0:degree] # tail
init_r_tmp += self.KG[click][1][0:degree] # relation
init_h.append(init_h_tmp)
init_t.append(init_t_tmp)
init_r.append(init_r_tmp)
h_list.append(init_h)
t_list.append(init_t)
r_list.append(init_r)
for _ in range(self.n_hops - 1):
next_h, next_t, next_r = self.get_next_entity(t_list[-1])
h_list.append(next_h)
t_list.append(next_t)
r_list.append(next_r)
# get the memory length
mem_len_list = []
for i in range(self.n_hops): # n_hops
mem_len_batch = []
for j in range(len(h_list[0])): # batch_size
# padding
mem_len_batch.append(min(len(h_list[i][j]), self.n_memory))
h_list[i][j] = utils.pad_seq(h_list[i][j], self.n_memory)
r_list[i][j] = utils.pad_seq(r_list[i][j], self.n_memory)
t_list[i][j] = utils.pad_seq(t_list[i][j], self.n_memory)
mem_len_list.append(mem_len_batch)
return items, labels, h_list, t_list, r_list, mem_len_list, current_pos
def conversion(args, net, device='cuda'):
assert os.path.isdir(args.data_dir), 'Cannot found data dir : {}'.format(
args.data_dir)
all_spk_path = [
p for p in glob.glob(os.path.join(args.data_dir, '*'))
if os.path.isdir(p)
]
all_test_samples = [
glob.glob(os.path.join(p, 'test', '*.npz'))[0] for p in all_spk_path
]
os.makedirs(args.out_dir, exist_ok=True)
all_pair = itertools.product(all_test_samples, all_test_samples)
for src, trg in tqdm(all_pair, desc="converting voices"):
src_name = src.split('/')[-3]
trg_name = trg.split('/')[-3]
src_npz = np.load(src)
trg_npz = np.load(trg)
x = src_npz['mel']
p = src_npz['f0'][:, np.newaxis]
emb_src_np = make_onehot(src_npz['spk_label'].item(),
hparams.n_speakers)
emb_trg_np = make_onehot(trg_npz['spk_label'].item(),
hparams.n_speakers)
x_padded, pad_len = pad_seq(x, base=hparams.freq, constant_values=None)
p_padded, pad_len = pad_seq(p,
base=hparams.freq,
constant_values=-1e10)
quantized_p, _ = quantize_f0_numpy(p_padded[:, 0],
num_bins=hparams.pitch_bin)
x_src = torch.from_numpy(x_padded).unsqueeze(0).to(device)
p_src = torch.from_numpy(quantized_p).unsqueeze(0).to(device)
emb_src = torch.from_numpy(emb_src_np).unsqueeze(0).to(device)
emb_trg = torch.from_numpy(emb_trg_np).unsqueeze(0).to(device)
if args.model == 'autovc':
out, out_psnt, _ = net(x_src, emb_src, emb_trg)
elif args.model == 'autovc-f0':
out, out_psnt, _ = net(x_src, p_src, emb_src, emb_trg)
else:
print("Wrong model name : {}".format(args.model))
print(out_psnt)
if pad_len == 0:
out_mel = out_psnt.squeeze().detach().cpu().numpy()[:, :]
else:
out_mel = out_psnt.squeeze().detach().cpu().numpy()[:-pad_len, :]
src_mel = src_npz['mel']
trg_mel = trg_npz['mel']
np.save(
os.path.join(
args.out_dir, '{}-{}-feats.npy'.format(
src_name,
os.path.splitext(src.split('/')[-1])[0])), src_mel)
np.save(
os.path.join(
args.out_dir, '{}-{}-feats.npy'.format(
trg_name,
os.path.splitext(trg.split('/')[-1])[0])), trg_mel)
np.save(
os.path.join(
args.out_dir, '{}-to-{}-{}.npy'.format(
src_name, trg_name,
os.path.splitext(src.split('/')[-1])[0])), out_mel)
def predict(model, args, ckpt_args, test_seqs, test_labels, word2idx):
# switch to eval mode
model.eval()
meters = Meters()
fig, ax = None, None
if args.plt and (ckpt_args.proto_dim == 2 or ckpt_args.proto_dim == 3):
fig = plt.figure()
if ckpt_args.proto_dim == 2:
ax = fig.add_subplot()
else:
ax = fig.add_subplot(111, projection='3d')
# colors = np.arange(start=1, stop=ckpt_args.class_num + 1) / (ckpt_args.class_num + 1)
colors = [
'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple',
'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan',
]
if args.plt_show:
fig.show()
att_json = None
if args.att_json:
att_json = []
# eval loop in an epoch
batch_size = args.samples_per_class * ckpt_args.class_num
data_iter = create_data_iter(test_seqs, test_labels, batch_size)
for x_eval, y_eval, batch, batch_num in data_iter:
with torch.no_grad():
x_eval_len = [
len(s)
if args.max_seq_len <= 0 or len(s) <= args.max_seq_len
else args.max_seq_len
for s in x_eval
]
max_seq_len = max(x_eval_len)
x_eval_len = Variable(torch.FloatTensor(x_eval_len))
x_eval = [pad_seq(s, max_seq_len) for s in x_eval]
x_eval = Variable(torch.LongTensor(x_eval).t())
if args.cuda:
x_eval = x_eval.cuda()
x_eval_len = x_eval_len.cuda()
y_eval = Variable(torch.LongTensor(y_eval))
if args.cuda:
y_eval = y_eval.cuda(async=True)
# prototypical loss
prob, protos, feats, att = model(x_eval, x_eval_len)
# all samples are query sample
p_loss, p_acc = proto_loss(prob, y_eval, protos, ckpt_args.disable_reg)
meters.update('loss', p_loss)
meters.update('acc', p_acc)
if (batch + 1) % args.log_freq == 0 or batch + 1 == batch_num:
msg = ' TEST [{:3d}/{:3d}] {:.4f}'
log.info(msg.format(batch + 1, batch_num, meters))
if args.plt and (ckpt_args.proto_dim == 2 or ckpt_args.proto_dim == 3):
c = [colors[int(l.item())] for l in y_eval]
if args.cuda and torch.cuda.is_available():
feats = feats.cpu()
feats = feats.numpy()
if ckpt_args.proto_dim == 2:
ax.scatter(feats[:, 0], feats[:, 1], c=c, s=10, alpha=0.4)
else:
ax.scatter(feats[:, 0], feats[:, 2], feats[:, 1], c=c, s=10, alpha=0.4)
if args.plt_show:
fig.show()
if att_json is not None:
p, y_pred = torch.max(prob, dim=-1)
for i, pp in enumerate(p.tolist()):
if y_pred[i] == y_eval[i]:
seq = x_eval[:, i].tolist()
seq = [word2idx.get(w, '<unk>') for w in seq if w > 0]
seq_att = [a.tolist() for a in att[i]]
att_json.append({
"seq": seq,
"att": seq_att,
"y_pred": y_pred[i].item(),
"y_prob": pp,
"y": y_eval[i].item()
})
if args.plt and (ckpt_args.proto_dim == 2 or ckpt_args.proto_dim == 3):
protos = model.protos
if args.cuda and torch.cuda.is_available():
protos = protos.cpu()
protos = protos.numpy()
if ckpt_args.proto_dim == 2:
proto_x, proto_y = protos[:, 0], protos[:, 1]
ax.scatter(proto_x, proto_y, marker="P", c="black", s=50, alpha=0.9)
else:
proto_x, proto_y, proto_z = protos[:, 0], protos[:, 2], protos[:, 1]
ax.scatter(proto_x, proto_y, proto_z, marker="P", c="black", s=100, alpha=0.9)
fig.savefig(args.plt_name)
if args.plt_show:
fig.show()
if att_json is not None:
with open(args.att_json, 'w', encoding='utf-8') as f_json:
f_json.write("const att = ")
f_json.write(json.dumps(att_json, indent=2))
return meters.val('loss'), meters.val('acc')
