def apply_model(model_fn, subset, language):
# assert language is None # to-do
# Load the model options
model_dir = path.split(model_fn)[0]
options_dict_fn = path.join(model_dir, "options_dict.pkl")
print("Reading:", options_dict_fn)
with open(options_dict_fn, "rb") as f:
options_dict = pickle.load(f)
# Load data
npz_fn = path.join(options_dict["data_dir"], subset + ".npz")
if language is not None:
if "buckeye" in npz_fn:
npz_fn = npz_fn.replace("buckeye", language)
elif "xitsonga" in npz_fn:
npz_fn = npz_fn.replace("xitsonga", language)
x_data, labels, lengths, keys, speakers = data_io.load_data_from_npz(
npz_fn)
# Truncate and limit dimensionality
data_io.trunc_and_limit_dim(x_data, lengths, options_dict["n_input"],
options_dict["max_length"])
# Build model
x = tf.placeholder(TF_DTYPE, [None, None, options_dict["n_input"]])
x_lengths = tf.placeholder(TF_ITYPE, [None])
model = build_model(x, x_lengths, options_dict)
# Embed data
batch_iterator = batching.SimpleIterator(x_data, len(x_data), False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
for batch_x_padded, batch_x_lengths in batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
np_z = session.run([model["encoding"]],
feed_dict={
x: np_x,
x_lengths: np_x_lengths
})[0]
# np_y = session.run(
# [y], feed_dict={a: np_x, a_lengths: np_x_lengths, b_lengths: np_x_lengths}
# )[0]
break # single batch
embed_dict = {}
for i, utt_key in enumerate([keys[i] for i in batch_iterator.indices]):
embed_dict[utt_key] = np_z[i]
return embed_dict
def samediff_val(normalise=True):
# Embed validation
np.random.seed(options_dict["rnd_seed"])
val_batch_iterator = batching.SimpleIterator(val_x, len(val_x), False)
labels = [val_labels[i] for i in val_batch_iterator.indices]
speakers = [val_speakers[i] for i in val_batch_iterator.indices]
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, val_model_fn)
for batch_x_padded, batch_x_lengths in val_batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
np_z = session.run([encoding],
feed_dict={
x: np_x,
x_lengths: np_x_lengths
})[0]
break # single batch
embed_dict = {}
for i, utt_key in enumerate(
[val_keys[i] for i in val_batch_iterator.indices]):
embed_dict[utt_key] = np_z[i]
# Same-different
if normalise:
np_z_normalised = (np_z - np_z.mean(axis=0)) / np_z.std(axis=0)
distances = pdist(np_z_normalised, metric="cosine")
else:
distances = pdist(np_z, metric="cosine")
# matches = samediff.generate_matches_array(labels)
# ap, prb = samediff.average_precision(
# distances[matches == True], distances[matches == False]
# )
word_matches = samediff.generate_matches_array(labels)
speaker_matches = samediff.generate_matches_array(speakers)
sw_ap, sw_prb, swdp_ap, swdp_prb = samediff.average_precision_swdp(
distances[np.logical_and(word_matches, speaker_matches)],
distances[np.logical_and(word_matches, speaker_matches == False)],
distances[word_matches == False])
# return [sw_prb, -sw_ap, swdp_prb, -swdp_ap]
return [swdp_prb, -swdp_ap]
def samediff_val(normalise=True):
# Embed validation
np.random.seed(options_dict["rnd_seed"])
val_batch_iterator = batching.SimpleIterator(val_x, len(val_x), False)
labels = [val_labels[i] for i in val_batch_iterator.indices]
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, val_model_fn)
for batch_x_padded, batch_x_lengths in val_batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
# np_z = session.run(
# [z], feed_dict={a: np_x, a_lengths: np_x_lengths}
# )[0]
np_z = session.run(
[z_mean], feed_dict={a: np_x, a_lengths: np_x_lengths}
)[0]
# print(np_z)
break # single batch
embed_dict = {}
for i, utt_key in enumerate(
[val_keys[i] for i in val_batch_iterator.indices]):
embed_dict[utt_key] = np_z[i]
# Same-different
if normalise:
# print(np_z.shape)
np_z_normalised = (np_z - np_z.mean(axis=0))/np_z.std(axis=0)
distances = pdist(np_z_normalised, metric="cosine")
matches = samediff.generate_matches_array(labels)
ap, prb = samediff.average_precision(
distances[matches == True], distances[matches == False]
)
else:
distances = pdist(np_z, metric="cosine")
matches = samediff.generate_matches_array(labels)
ap, prb = samediff.average_precision(
distances[matches == True], distances[matches == False]
)
return [prb, -ap]
def apply_model(model_fn, subset, batch_size=None):
# assert language is None # to-do
# Load the model options
model_dir = path.split(model_fn)[0]
options_dict_fn = path.join(model_dir, "options_dict.pkl")
print("Reading:", options_dict_fn)
with open(options_dict_fn, "rb") as f:
options_dict = pickle.load(f)
# Load data
npz_fn = path.join(options_dict["data_dir"], subset + ".npz")
# if language is not None:
# if "buckeye" in npz_fn:
# npz_fn = npz_fn.replace("buckeye", language)
# elif "xitsonga" in npz_fn:
# npz_fn = npz_fn.replace("xitsonga", language)
x_data, labels, lengths, keys = data_io.load_data_from_npz(npz_fn)
if "cnn" in options_dict["script"]:
# Pad and flatten data
x_data, _ = data_io.pad_sequences(x_data, options_dict["max_length"],
True)
x_data = np.transpose(x_data, (0, 2, 1))
x_data = x_data.reshape((-1, options_dict["d_in"]))
# Build model
x = tf.placeholder(TF_DTYPE, [None, options_dict["d_in"]])
model = build_model(x, None, options_dict)
# Embed data
if batch_size is None:
batch_iterator = batching.LabelledIterator(x_data, None,
x_data.shape[0], False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
for batch_x in batch_iterator:
np_z = session.run([model["encoding"]],
feed_dict={x: batch_x})[0]
break # single batch
else:
assert False, "need to implement"
else: # rnn
# Truncate and limit dimensionality
data_io.trunc_and_limit_dim(x_data, lengths, options_dict["n_input"],
options_dict["max_length"])
# Build model
x = tf.placeholder(TF_DTYPE, [None, None, options_dict["n_input"]])
x_lengths = tf.placeholder(TF_ITYPE, [None])
model = build_model(x, x_lengths, options_dict)
# Embed data
if batch_size is None:
batch_iterator = batching.SimpleIterator(x_data, len(x_data),
False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
for batch_x_padded, batch_x_lengths in batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
np_z = session.run([model["encoding"]],
feed_dict={
x: np_x,
x_lengths: np_x_lengths
})[0]
break # single batch
else:
batch_iterator = batching.SimpleIterator(x_data, batch_size, False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
np_z = []
for batch_x_padded, batch_x_lengths in batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
cur_np_z = session.run([model["encoding"]],
feed_dict={
x: np_x,
x_lengths: np_x_lengths
})[0]
print("!", cur_np_z.shape)
np_z.append(cur_np_z)
np_z = np.vstack(np_z)
print("!", np_z.shape)
embed_dict = {}
for i, utt_key in enumerate([keys[i] for i in batch_iterator.indices]):
embed_dict[utt_key] = np_z[i]
return embed_dict
def apply_model(model_fn, npz_fn):
# Load the model options
model_dir = path.split(model_fn)[0]
options_dict_fn = path.join(model_dir, "options_dict.pkl")
print("Reading:", options_dict_fn)
with open(options_dict_fn, "rb") as f:
options_dict = pickle.load(f)
# Load data
x_data, labels, lengths, keys, speakers = data_io.load_data_from_npz(
npz_fn)
if "cnn" in options_dict["script"]:
# Pad and flatten data
x_data, _ = data_io.pad_sequences(x_data, options_dict["max_length"],
True)
x_data = np.transpose(x_data, (0, 2, 1))
x_data = x_data.reshape((-1, options_dict["d_in"]))
# Build model
x = tf.placeholder(TF_DTYPE, [None, options_dict["d_in"]])
model = build_model(x, None, options_dict)
# Embed data
batch_iterator = batching.LabelledIterator(x_data, None,
x_data.shape[0], False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
for batch_x in batch_iterator:
np_z = session.run([model["encoding"]], feed_dict={x:
batch_x})[0]
break # single batch
else: # rnn
# Truncate and limit dimensionality
data_io.trunc_and_limit_dim(x_data, lengths, options_dict["n_input"],
options_dict["max_length"])
# Build model
x = tf.placeholder(TF_DTYPE, [None, None, options_dict["n_input"]])
x_lengths = tf.placeholder(TF_ITYPE, [None])
model = build_model(x, x_lengths, options_dict)
# Embed data
batch_iterator = batching.SimpleIterator(x_data, len(x_data), False)
saver = tf.train.Saver()
with tf.Session() as session:
saver.restore(session, model_fn)
for batch_x_padded, batch_x_lengths in batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
np_z = session.run([model["encoding"]],
feed_dict={
x: np_x,
x_lengths: np_x_lengths
})[0]
break # single batch
embed_dict = {}
for i, utt_key in enumerate([keys[i] for i in batch_iterator.indices]):
embed_dict[utt_key] = np_z[i]
return embed_dict
def do_dist(self, quota):
data.trunc_and_limit_dim(x=self.quer_x,
lengths=self.quer_lengths,
d_frame=self.options_dict["n_input"],
max_length=self.options_dict["max_length"])
iterator = batching.SimpleIterator(self.quer_x, len(self.quer_x),
False)
saver = tf.train.Saver()
temp = path.join(
"models",
path.split(self.options_dict["data_dir"])[-1] + "." +
self.options_dict["train_tag"], "4cb3a05a07")
self.charg = path.join(temp, "ae.tmp.ckpt")
with tf.Session() as session:
# saver.restore(session, self.val_model_fn)
saver.restore(session, self.charg)
for batch_x_padded, batch_x_lengths in iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
emb_queries = session.run([self.z],
feed_dict={
self.a: np_x,
self.a_lengths: np_x_lengths
})[0]
embed_dict = {}
for i, utt_key in enumerate(
[self.train_labels[i] for i in iterator.indices]):
embed_dict[utt_key] = emb_queries[i]
# for z, w in enumerate(embed_dict):
# self.retrieved[w] = []
quers = np.vstack(emb_queries)
with open("./lex_vecs.pkl", mode="wb") as pk_lex:
pickle.dump(quers, pk_lex)
print(datetime.now())
segs, seg_ref = self.segment(self.corpus)
print("distances")
distances = cdist(segs, quers, metric="cosine")
ret = []
for i, seg in enumerate(distances):
for j, partof in enumerate(seg):
ret.append([partof, self.queries[j]["word"], i])
# mn = np.min(seg)
# w = np.where(seg == mn)
# ind = (w[0][0])
# ret.append([mn, self.queries[ind]["word"], i])
meilleur = []
verif = {}
sort_ret = sorted(ret, key=lambda x: x[0])
double = set()
for j, m in enumerate(sort_ret):
if m[1] not in verif:
verif[m[1]] = 0
if verif[m[1]] < quota:
code = (seg_ref[m[2]]["time"], m[1], seg_ref[m[2]]["file"])
if code not in double:
out = {}
out["dist"] = m[0]
out["word"] = m[1]
out["ref"] = seg_ref[m[2]]["file"]
out["time"] = seg_ref[m[2]]["time"]
out["duree"] = seg_ref[m[2]]["wind"]
meilleur.append(out)
verif[m[1]] += 1
double.add(code)
print(datetime.now())
# for i in meilleur:
# print(i["word"])
# with open("/home/leferrae/Desktop/These/mboshi/mboshi-french-parallel-corpus/full_corpus_newsplit/all/"+i["ref"].replace(".wav", ".mb")) as foo:
# print(foo.read())
# print(meilleur)
return meilleur
def segment(self, corp, step=3, wind=[100, 70, 40]):
"""
:param corp: corpus from which the segment are extracted
:param step:
:param wind: the lenghts of the differents segments extracted at a same time
:return: a stack of embeded segments and a reference file
"""
segs = []
seg_ref = {}
cpt = 0
utt_lengths = []
bar = Bar("Segmentation...", max=len(corp))
for i, u in enumerate(corp):
utt = u["data"]
i_start = 0
n_search = utt.shape[0]
seg_list = []
while i_start <= n_search - 1 or i_start == 0:
for ind in wind:
sref = {}
sref["file"] = os.path.basename(u["file"])
sref["wind"] = ind
sref["time"] = i_start / 100
seg_ref[cpt] = sref
seg = utt[i_start:i_start + ind]
utt_lengths.append(seg.shape[0])
i_start += step
cpt += 1
segs.append(seg)
bar.next()
bar.finish()
data.trunc_and_limit_dim(x=segs,
lengths=utt_lengths,
d_frame=self.options_dict["n_input"],
max_length=self.options_dict["max_length"])
batch_iterator = batching.SimpleIterator(segs, 128, False)
saver = tf.train.Saver()
gups = len(tf.config.experimental.list_physical_devices('GPU'))
with tf.Session() as session:
saver.restore(session, self.charg)
bar = Bar("Vectorization...", max=len(segs) // 128)
for batch_x_padded, batch_x_lengths in batch_iterator:
np_x = batch_x_padded
np_x_lengths = batch_x_lengths
np_z = session.run([self.z],
feed_dict={
self.a: np_x,
self.a_lengths: np_x_lengths
})[0]
seg_list.append(np_z)
bar.next()
bar.finish()
out = []
for x in seg_list:
out.append([y for y in x])
sent_emb = np.vstack(out)
return sent_emb, seg_ref