def run(cls, args):
corpus_dir = cls._parse_corpus_dir(args.corpus)
log = utils.logger.get_log(verbose=True)
try:
Corpus.load(corpus_dir, validate=True, log=log)
log.info('corpus is valid')
sys.exit(0)
except IOError as err:
log.error(f'ERROR: {err}')
log.error('corpus is NOT valid')
sys.exit(1)
def run(cls, args):
# overload the default of use-energy to be false instead of
# true (if not already specified in options) TODO unify that
# with the overloaded_options above
if all('use-energy' not in c[0] for c in cls.parsed_options):
cls.parsed_options.append(('use-energy', 'false'))
corpus_dir, output_dir = cls._parse_io_dirs(args, 'features')
log = utils.logger.get_log(
os.path.join(output_dir, 'features.log'), verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
recipe = features.Features(corpus, output_dir, log=log)
recipe.type = cls.feat_name
recipe.use_pitch = utils.str2bool(args.pitch) # 'true' to True
recipe.use_cmvn = utils.str2bool(args.cmvn)
recipe.delta_order = args.delta_order
recipe.features_options = cls.parsed_options
recipe.njobs = args.njobs
recipe.delete_recipe = False if args.recipe else True
recipe.compute()
# export to h5features if asked for
if args.h5f:
recipe.log.info('exporting Kaldi ark features to h5features...')
kaldi.scp_to_h5f(
os.path.join(recipe.output_dir, 'feats.scp'),
os.path.join(recipe.output_dir, 'feats.h5f'))
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(
os.path.join(output_dir, 'decode.log'), verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
# get back the features, language and acoustic models directories
feat = cls._parse_aux_dir(corpus_dir, args.features, 'features')
lang = cls._parse_aux_dir(corpus_dir, args.language_model, 'language')
acou = cls._parse_aux_dir(corpus_dir, args.acoustic_model, 'acoustic')
# instanciate the kaldi recipe
recipe = decode.Decode(
corpus, lang, feat, acou, output_dir,
decode_type=cls.name, log=log)
recipe.njobs = args.njobs
recipe.delete_recipe = False if args.recipe else True
# setup the model options parsed from command line
for k, v in vars(args).iteritems():
try:
recipe.set_option(k.replace('_', '-'), v)
except KeyError:
pass
# finally decode the corpus
recipe.compute()
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(
os.path.join(output_dir, 'filter.log'), verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
# retrieve the test proportion
(subcorpus, not_kept_utterances) = corpus.create_filter(
output_dir,
function=args.function,
nb_speaker=args.nb_speaker,
new_speakers=args.new_speakers,
THCHS30=args.THCHS30)
if args.plot:
subcorpus.plot()
if args.trim:
print("trimming utterances")
subcorpus.save(
os.path.join(
output_dir, args.function, 'data'),
no_wavs=True, copy_wavs=False)
subcorpus.trim(
corpus_dir, output_dir,
args.function, not_kept_utterances)
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(os.path.join(output_dir, 'language.log'),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
if args.model_order == 0:
# we want a flat LM
recipe = FlatLanguageModel(
corpus,
output_dir,
log=log,
level=args.model_level,
position_dependent_phones=args.word_position_dependent,
silence_probability=args.silence_probability)
else:
# order >= 1, classical ngram
recipe = LanguageModel(
corpus,
output_dir,
log=log,
order=args.model_order,
level=args.model_level,
position_dependent_phones=args.word_position_dependent,
silence_probability=args.silence_probability)
recipe.delete_recipe = False if args.recipe else True
recipe.compute()
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(os.path.join(output_dir, 'merge_wavs.log'),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
corpus.merge_wavs(corpus_dir, output_dir)
corpus.save(os.path.join(output_dir, 'data'), no_wavs=True)
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(os.path.join(output_dir, 'filter.log'),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
corpus_plot = corpus.plot()
corpus_plot.show()
corpus_plot.savefig(os.path.join(output_dir, 'plot.png'))
def test_save_corpus(tmpdir, corpus, copy_wavs):
assert corpus.is_valid()
corpus_saved = str(tmpdir.mkdir('corpus-{}'.format(copy_wavs)))
corpus.save(corpus_saved, copy_wavs=copy_wavs)
assert os.path.isfile(os.path.join(corpus_saved, 'meta.txt'))
d = Corpus.load(corpus_saved)
assert d.is_valid()
assert corpus.lexicon == d.lexicon
assert corpus.segments == d.segments
assert corpus.phones == d.phones
assert corpus.wavs == d.wavs
def run(cls, args):
corpus_dir = cls._parse_corpus_dir(args.corpus)
log = utils.logger.get_log(verbose=True)
corpus = Corpus.load(corpus_dir, validate=False, log=log)
if corpus.is_valid():
log.info('corpus is valid')
sys.exit(0)
else:
log.error('corpus is NOT valid')
sys.exit(1)
def prepare_corpus(utts, tmpdir):
# first try to load any prepared buckeye
buckeye = os.path.join(utils.config.get('abkhazia', 'data-directory'),
'buckeye', 'data')
# first search buckeye in the environment variable, the in the data
# directory from the configuration file
if 'ABKHAZIA_BUCKEYE' in os.environ:
corpus = Corpus.load(os.environ['ABKHAZIA_BUCKEYE'])
elif os.path.isdir(buckeye):
corpus = Corpus.load(buckeye)
else: # prepare the whole buckeye corpus
buckeye = utils.config.get('corpus', 'buckeye-directory')
corpus = BuckeyePreparator(buckeye).prepare(tmpdir)
corpus = corpus.subcorpus(utts)
corpus.validate()
assert sorted(corpus.utts()) == utts
assert len(corpus.utts()) == len(utts)
assert len(corpus.spks()) == len(set(utt.split('-')[0] for utt in utts))
return corpus
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(
os.path.join(output_dir, '{}.log'.format(cls.name)),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
# get back the features directory TODO use cls._parse_aux_dir
feats = (os.path.join(os.path.dirname(corpus_dir), 'features')
if args.features is None
else os.path.abspath(args.features))
# pack the prepare_lang related arguments
lang_args = {
'level': args.lang_level,
'silence_probability': args.silence_probability,
'position_dependent_phones': args.word_position_dependent,
'keep_tmp_dirs': True if args.recipe else False}
# instanciate and setup the kaldi recipe with standard args
if cls.am_class is not acoustic.Monophone:
# get back the input directory
input_dir = (
os.path.join(os.path.dirname(corpus_dir),
'{}'.format(cls.prev_step[1]))
if args.input_dir is None
else os.path.abspath(args.input_dir))
recipe = cls.am_class(
corpus, feats, input_dir, output_dir, lang_args, log=log)
else:
recipe = cls.am_class(corpus, feats, output_dir, lang_args, log=log)
recipe.njobs = args.njobs
if args.recipe:
recipe.delete_recipe = False
# setup the model options parsed from command line
for k, v in vars(args).iteritems():
try:
recipe.set_option(k.replace('_', '-'), v)
except KeyError:
pass
# finally train the acoustic model
recipe.compute()
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(os.path.join(output_dir, 'language.log'),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
# instanciate the lm recipe and compute
recipe = LanguageModel(
corpus,
output_dir,
log=log,
order=args.model_order,
level=args.model_level,
position_dependent_phones=args.word_position_dependent,
silence_probability=args.silence_probability)
recipe.delete_recipe = False if args.recipe else True
recipe.compute()
def prepare_corpus(utts, tmpdir):
# first try to load any prepared buckeye
buckeye = os.path.join(utils.config.get('abkhazia', 'data-directory'),
'buckeye', 'data')
if os.path.isdir(buckeye):
corpus = Corpus.load(buckeye)
else: # prepare the whole buckeye corpus
buckeye = utils.config.get('corpus', 'buckeye-directory')
corpus = BuckeyePreparator(buckeye).prepare(tmpdir)
corpus = corpus.subcorpus(utts)
corpus.validate()
assert sorted(corpus.utts()) == utts
assert len(corpus.utts()) == len(utts)
assert len(corpus.spks()) == len(set(utt.split('-')[0] for utt in utts))
return corpus
def run(cls, args):
corpus_dir, output_dir = cls._parse_io_dirs(args)
log = utils.logger.get_log(os.path.join(output_dir, 'split.log'),
verbose=args.verbose)
corpus = Corpus.load(corpus_dir, validate=args.validate, log=log)
# retrieve the test proportion
if args.train_prop is None:
test_prop = (cls.default_test_proportion
if args.test_prop is None else args.test_prop)
else:
test_prop = (1 - args.train_prop
if args.test_prop is None else args.test_prop)
train, test = corpus.split(train_prop=args.train_prop,
test_prop=test_prop,
by_speakers=args.by_speakers,
random_seed=args.random_seed)
train.save(os.path.join(output_dir, 'train', 'data'))
test.save(os.path.join(output_dir, 'test', 'data'))
def test_empty():
c = Corpus()
assert not c.is_valid()
assert c.spk2utt() == {}
assert len(c.utts()) == 0
def test_spk2utt():
c = Corpus()
c.utt2spk = {'u1': 's1', 'u2': 's1', 'u3': 's2'}
assert c.spk2utt() == {'s1': ['u1', 'u2'], 's2': ['u3']}