def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("scale", help="varをscale倍するよう、二次統計量をいじる", type=float)
ap.add_argument("dim_for_scaling",
help="33-52,3 (same as select-feats)",
type=str)
ap.add_argument("stats_rspecifier", help="in", type=str)
ap.add_argument("stats_wspecifier", help="out", type=str)
args = ap.parse_args()
dim_for_scaling = FeatureOperator.parse_dim_specifier(args.dim_for_scaling)
scale = args.scale
with kaldi_io.open_or_fd(args.stats_wspecifier, "wb") as stats_writer:
for spk, rstats in kaldi_io.read_mat_ark(args.stats_rspecifier):
T, D = rstats.shape
assert (T == 2)
count = rstats[0, -1]
wstats = np.array(rstats)
for d in dim_for_scaling:
x, x2 = rstats[0, d], rstats[1, d]
mean = x / count
y1 = scale * x2
y2 = (1 - scale) * count * mean * mean
y = y1 + y2
assert (y > 0)
wstats[1, d] = y
del x, x2, mean, y, y1, y2
kaldi_io.write_mat(stats_writer, wstats, key=spk)
def main(copy_stream, write_hdf5):
write_scp = write_hdf5.split(".")[0] + ".scp"
with h5py.File(write_hdf5, "w") as feature_store, open(write_scp,
"w") as key_store:
for key, feature in kaldi_io.read_mat_ark(copy_stream):
feature_store[key] = feature
key_store.write(key + "\n")
print("Successfully copy feature to {}".format(write_hdf5))
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
kaldi_io.write_mat(feature_writer, np.abs(feats), key=key)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("minmax_wspecifier", help="output feat")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.minmax_wspecifier, "wb") as minmax_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
min_value, max_value = feats.min(axis=0), feats.max(axis=0)
tmp = np.array([min_value, max_value], dtype=float)
kaldi_io.write_vec_flt(minmax_writer, tmp, key=key)
def main_():
ap = ArgumentParser()
ap.add_argument("feat_rspecifier", type=str)
ap.add_argument("ali_wspecifier", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.ali_wspecifier, mode="wb") as ali_writer:
for utt, feat in kaldi_io.read_mat_ark(args.feat_rspecifier):
T, dim = feat.shape
assert (dim == 1)
ali = feat.squeeze(1).astype(np.int)
kaldi_io.write_vec_int(ali_writer, ali, key=utt)
def main_():
ap = argparse.ArgumentParser()
ap.add_argument("--dim", default=1, type=int)
ap.add_argument("feat_rspecifier", type=str)
ap.add_argument("feat_wspecifier", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feat_wspecifier, "wb") as feature_writer:
for utt, feat in kaldi_io.read_mat_ark(args.feat_rspecifier):
length = feat.shape[0]
new_feats = np.zeros((length, args.dim))
kaldi_io.write_mat(feature_writer, new_feats, key=utt)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--base", type=float, default=math.e)
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
new_feats = args.base**feats
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
thresh = 0.5
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
new_feats = np.where(feats >= thresh, 1.0, 0.0)
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--decimate", default=2, type=int)
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
decimate = args.decimate
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
new_feats = signal.decimate(feats, decimate, axis=0)
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def __init__(self, kaldi_stream, caption_df, vocabulary, transform=None):
"""Dataloader for the SJTU Audiocaptioning dataset
Args:
kaldi_stream (string): Kaldi command to load the data (e.g., copy-feats ark:- ark:- |)
caption_df (pd.DataFrame): Captioning dataframe
vocab_file (string): Path to the vocabulary (preprocessed by build_vocab)
transform (function, optional): Defaults to None. Transformation onto the data (function)
"""
super(SJTUDataset, self).__init__()
self._dataset = {k: v for k, v in kaldi_io.read_mat_ark(kaldi_stream)}
self._transform = transform
self._caption_df = caption_df
self._vocabulary = vocabulary
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--min", type=float, default=-1)
ap.add_argument("--max", type=float, default=1)
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
new_feats = np.where(feats>=args.max, args.max, feats)
new_feats = np.where(feats<=args.min, args.min, new_feats)
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--window", default=3, type=int)
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
avg_mask = np.ones(args.window) / args.window
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
dim = feats.shape[1]
feats_avg = np.zeros(feats.shape, dtype=float)
for d in range(dim):
feats_avg[:, d] = np.convolve(feats[:, d], avg_mask, 'same')
kaldi_io.write_mat(feature_writer, feats_avg, key=key)
def main_():
ap = argparse.ArgumentParser()
ap.add_argument("feat_src_rspecifier", type=str)
ap.add_argument("feat_tgt_rspecifier", type=str)
ap.add_argument("tgt_start_dim", type=int)
ap.add_argument("feat_wspecifier", type=str)
args = ap.parse_args()
tgt_start_dim = args.tgt_start_dim
with kaldi_io.open_or_fd(args.feat_wspecifier, mode="wb") as feat_writer:
with kaldi_io.open_or_fd(args.feat_tgt_rspecifier, mode="rb") as feat_tgt_reader:
for utt, feat_src in kaldi_io.read_mat_ark(args.feat_src_rspecifier):
new_feat = feat_src.copy()
T1, _ = new_feat.shape
feat_tgt = FO.read_next_feat(feat_tgt_reader, key=utt, num_frame=T1)
_, D2 = feat_tgt.shape
new_feat[:, tgt_start_dim:tgt_start_dim+D2] = feat_tgt
kaldi_io.write_mat(feat_writer, new_feat, key=utt)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--write_int", type=str_to_bool, default=True)
ap.add_argument("--thresh", type=float, default=0.5)
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
new_feats = np.where(feats>=args.thresh, 1, feats)
new_feats = np.where(feats<args.thresh, 0, new_feats)
if args.write_int:
assert (new_feats.shape[1] == 1)
kaldi_io.write_vec_int(feature_writer, new_feats.astype("int32").reshape(new_feats.size), key=key)
else:
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("meanvar_rspecifier", help="in", type=str)
ap.add_argument("denormscale_wspecifier", help="out", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.denormscale_wspecifier, "wb") as denormscale_writer:
for spk, rstats in kaldi_io.read_mat_ark(args.meanvar_rspecifier):
T, D = rstats.shape
assert (T == 2)
mean, var = rstats[0, :], rstats[1, :]
stdv = np.sqrt(var)
wstats = np.empty_like(rstats)
wstats[0], wstats[1] = stdv, mean
kaldi_io.write_mat(denormscale_writer, wstats, key=spk)
def main_():
ap = argparse.ArgumentParser()
ap.add_argument("--trans", type=str_to_bool, default=True)
ap.add_argument("--save_pickle", type=str, default="")
ap.add_argument("feat_rspecifier", type=str)
ap.add_argument("npy_outdir", type=str)
args = ap.parse_args()
npy_dir = args.npy_outdir
uttids = []
for utt, feat in kaldi_io.read_mat_ark(args.feat_rspecifier):
uttids.append(utt)
new_feats = feats.copy()
if args.trans:
new_feats = new_feats.T
np.save(os.path.join(npy_dir, "{}.npy".format(utt)), new_feats)
if args.save_pickle != "":
with open(args.save_pickle, 'wb') as f:
pickle.dump(uttids, f)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--floor_value", default=0, type=float)
ap.add_argument("uv_rspecifier", help="input uv")
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
thresh = 0.5
floor = args.floor_value
with kaldi_io.open_or_fd(args.uv_rspecifier, "rb") as uv_reader:
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
assert (key == kaldi_io.read_key(uv_reader))
uv = kaldi_io.read_mat(uv_reader)
new_feats = np.where(uv>=thresh, feats, floor)
kaldi_io.write_mat(feature_writer, new_feats, key=key)
def main_():
ap = argparse.ArgumentParser(
"concat feats by using given utt2spk. spkname is used as new utt key")
ap.add_argument("utt2spk")
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat (concatinated)")
args = ap.parse_args()
utt2spk = read_utt2spk(args.utt2spk)
prev_spk, feat_buf = "", []
with kaldi_io.open_or_fd(args.feature_wspecifier, "wb") as writer:
for utt, feat in kaldi_io.read_mat_ark(args.feature_rspecifier):
spk = utt2spk.get(utt)
assert (spk is not None)
if prev_spk != "" and spk != prev_spk:
write_concated_feats(writer, feat_buf, prev_spk)
feat_buf.append(feat)
prev_spk = spk
write_concated_feats(writer, feat_buf, prev_spk)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("scalevec_rspecifier", help="in", type=str)
ap.add_argument("trans_wspecifier", help="out", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.trans_wspecifier, "wb") as trans_writer:
for spk, rstats in kaldi_io.read_mat_ark(args.scalevec_rspecifier):
T, D = rstats.shape
assert (T == 2)
trans = np.zeros([D, D+1]).astype(float)
scale, bias = rstats[0, :], rstats[1, :]
assert isinstance(scale, np.ndarray)
for d in range(0, D):
trans[d, d] = scale[d]
trans[:, -1] = bias
# print(trans)
kaldi_io.write_mat(trans_writer, trans, key=spk)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("--feat1_is_int", type=str_to_bool, default=False, help="")
ap.add_argument("--feat2_is_int", type=str_to_bool, default=False, help="")
ap.add_argument("feature1_rspecifier", help="input feat1")
ap.add_argument("feature2_rspecifier", help="input feat2")
ap.add_argument("feature1_wspecifier", help="output feat1")
ap.add_argument("feature2_wspecifier", help="output feat2")
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feature1_wspecifier,
"wb") as feature1_writer:
with kaldi_io.open_or_fd(args.feature2_wspecifier,
"wb") as feature2_writer:
with kaldi_io.open_or_fd(args.feature2_rspecifier,
"rb") as feature2_reader:
for utt, feats1 in kaldi_io.read_vec_int_ark(
args.feature1_rspecifier
) if args.feat1_is_int else kaldi_io.read_mat_ark(
args.feature1_rspecifier):
T1 = feats1.shape[0]
feats2 = FeatureOperator.read_next_feat(
feature2_reader,
utt,
num_frame=T1,
length_tolerance=True,
integer=args.feat2_is_int)
minT = min(T1, feats2.shape[0])
write_(feature1_writer,
feats1[:minT],
utt,
is_int=args.feat1_is_int)
write_(feature2_writer,
feats2[:minT],
utt,
is_int=args.feat2_is_int)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("meanvar_rspecifier", help="in", type=str)
ap.add_argument("normscale_wspecifier", help="out", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.normscale_wspecifier,
"wb") as normscale_writer:
for spk, rstats in kaldi_io.read_mat_ark(args.meanvar_rspecifier):
T, D = rstats.shape
assert (T == 2)
mean, var = rstats[0, :], rstats[1, :]
assert isinstance(var, np.ndarray)
stdv = np.sqrt(var)
scale = 1.0 / stdv
offset = -(mean * scale)
wstats = np.empty_like(rstats)
wstats[0], wstats[1] = scale, offset
kaldi_io.write_mat(normscale_writer, wstats, key=spk)
def main_():
ap = argparse.ArgumentParser(usage="")
# positional args
ap.add_argument("uv_rspecifier", help="input uv")
ap.add_argument("feature_rspecifier", help="input feat")
ap.add_argument("feature_wspecifier", help="output feat")
args = ap.parse_args()
thresh = 0.5
with kaldi_io.open_or_fd(args.uv_rspecifier, "rb") as uv_reader:
with kaldi_io.open_or_fd(args.feature_wspecifier,
"wb") as feature_writer:
for key, feats in kaldi_io.read_mat_ark(args.feature_rspecifier):
assert (key == kaldi_io.read_key(uv_reader))
# import pdb
# pdb.set_trace()
uv = kaldi_io.read_mat(uv_reader)
idx = np.where(uv < thresh)
new_feats = np.delete(feats, idx[0], axis=0)
if new_feats.shape[0] < 1:
continue
kaldi_io.write_mat(feature_writer, new_feats, key=key)
del uv, idx, new_feats
def __init__(self, kaldi_stream, transform=None):
super(SJTUDatasetEval, self).__init__()
self._kaldi_scp = scp_pattern.search(kaldi_stream).group()
self._data_generator = kaldi_io.read_mat_ark(kaldi_stream)
self._transform = transform
#!/usr/bin/env python
import argparse
import numpy as np
from utils import kaldi_io
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument("scale", type=float)
ap.add_argument("feat_rspecifier", type=str)
ap.add_argument("feat_wspecifier", type=str)
args = ap.parse_args()
with kaldi_io.open_or_fd(args.feat_wspecifier, "wb") as feats_writer:
for utt, feat in kaldi_io.read_mat_ark(args.feat_rspecifier):
feat2 = feat * args.scale
kaldi_io.write_mat(feats_writer, feat2, key=utt)
