def doit(line):
"""
Run *doit* with `task_creators` from all interactive variables
(IPython's global namespace).
Examples:
>>> %doit --help ## Show help for options and arguments.
>>> def task_foo():
return {'actions': ['echo hi IPython'],
'verbosity': 2}
>>> %doit list ## List any tasks discovered.
foo
>>> %doit ## Run any tasks.
. foo
hi IPython
"""
# Override db-files location inside ipython-profile dir,
# which is certainly writable.
prof_dir = ip.profile_dir.location
opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')}
commander = DoitMain(ModuleTaskLoader(ip.user_module),
extra_config={'GLOBAL': opt_vals})
commander.BIN_NAME = 'doit'
commander.run(line.split())
def run(self, doit_args=None, verbose=True):
'''Run the pipeline. Movees to the directory, loads the tasks into doit,
and executes that tasks that are not up-to-date.
Args:
doit_args (list): Args that would be passed to the doit shell
command. By default, just run.
verbose (bool): If True, print UI stuff.
Returns:
int: Exit status of the doit command.
'''
if verbose:
print(ui.header('Run Tasks', level=4))
if doit_args is None:
doit_args = ['run']
if self.n_threads > 1:
doit_args.extend(['-n', str(self.n_threads)])
runner = DoitMain(self)
with Move(self.directory):
if self.profile is True:
profile_fn = path.join(self.directory, 'profile.csv')
with StartProfiler(filename=profile_fn):
return runner.run(doit_args)
else:
return runner.run(doit_args)
def process_pipeline(tasks,
*args,
version=None,
workdir=None,
patterns=None,
num_processes=None,
output=None,
**kwargs):
doit_args = ['-n', num_processes, '--continue']
task_names = load_tasks(tasks)
if not task_names:
domains = get_domains(patterns)
task_names = load_tasks([
gen_task(workdir, domains)
for gen_task in (gen_dig, gen_host, gen_ssl)
])
def task_setup():
return {
'actions': [
f'rm -rf {workdir}',
],
}
globals()[task_setup.__name__] = task_setup
exitcode = DoitMain(ModuleTaskLoader(globals())).run(doit_args +
task_names)
create_result(workdir, output)
sys.exit(exitcode)
def test_help_plugin_name(self, capsys):
plugin = {'XXX': 'tests.sample_plugin:MyCmd'}
returned = DoitMain(extra_config={'COMMAND': plugin}).run(["help"])
assert returned == 0
out, err = capsys.readouterr()
assert "doit XXX " in out
assert "test extending doit commands" in out, out
def start(self):
"""Begin executing tasks."""
if self.log_filename:
print('Output will be logged to `%s`.' % self.log_filename)
start_time = time.strftime(self.timestamp_fmt)
print('Started %s' % start_time)
if self.log_filename:
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = self.log_file
sys.stderr = self.log_file
print('Started %s' % start_time)
doit_main = DoitMain(self)
doit_main.run(['run'])
stop_time = time.strftime(self.timestamp_fmt)
if self.log_filename:
print('Stopped %s' % stop_time)
print()
sys.stdout = orig_stdout
sys.stderr = orig_stderr
self.log_file.close()
print('Stopped %s' % stop_time)
def main():
# Fixes issues with multiprocessing with cx_freeze on windows
freeze_support()
""" Start doit.
"""
import sys
from doit.doit_cmd import DoitMain
sys.exit(DoitMain().run(sys.argv[1:]))
def test_doit_coverage(cookies):
result = cookies.bake()
with inside_dir(result.project):
with poetryenv_in_project():
importlib.reload(dodo)
dodo.webbrowser = mock.MagicMock()
assert DoitMain(ModuleTaskLoader(dodo)).run(["coverage"]) == 0
importlib.reload(dodo)
def execute(self, cmds=None, spec=None):
cmds = cmds or self.cmds
spec = spec or self.spec
if spec.otto.tasks:
loader = OttoTaskLoader(spec.otto.tasks)
else:
loader = OttoTaskLoader({'tasks': spec.otto})
if cmds:
sys.exit(DoitMain(loader).run(cmds))
def run_tasks(tasks, args, config={'verbosity': 2}):
tasks = list(tasks)
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def run_task(module, task):
"""
run_task - Have doit run the named task
:param module module: module containing tasks
:param str task: task to run
"""
start = time.time()
DoitMain(ModuleTaskLoader(module)).run([task])
print("%.2f seconds" % (time.time() - start))
def run(self, cmds):
tasks = {}
for v in self.loaders:
for name, l in v.list_tasks():
f = l
tasks[name] = f
ml = ModuleTaskLoader(tasks)
main = DoitMain(ml)
main.config['default_tasks'] = cmds
return main.run([])
def run(self, doit_args=None, move=False, profile_fn=None):
if doit_args is None:
doit_args = ['run']
runner = DoitMain(self)
print('\n--- Begin Task Execution ---')
if profile_fn is not False and doit_args[0] == 'run':
with StartProfiler(filename=profile_fn):
return runner.run(doit_args)
else:
return runner.run(doit_args)
def run(doit_args):
code = DoitMain(DoitLoader()).run(doit_args)
if code == 0:
print 'Doit: all tasks were executed.'
elif code == 1:
print 'Doit: some tasks were failed.'
elif code == 2:
print 'Doit: error when executing tasks.'
elif code == 3:
print 'Doit: error before task execution starts.'
return code
def run_tasks(tasks, args, config={'verbosity': 0}):
if type(tasks) is not list:
raise TypeError('tasks must be a list')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def run_doit(self, task_loader, configuration):
arguments = []
extra_configuration = {}
if 'threads' in configuration and configuration.get('threads') > 1:
arguments.extend(['-n', str(configuration.get('threads'))])
if 'doitConfig' in configuration:
extra_configuration = configuration.get('doitConfig')
return DoitMain(task_loader=task_loader,
extra_config=extra_configuration).run(arguments)
def test_doit_docs(cookies, docs_generator):
extra_context = {"docs_generator": docs_generator}
result = cookies.bake(extra_context=extra_context)
project = result.project
with inside_dir(project):
with poetryenv_in_project():
importlib.reload(dodo)
dodo.webbrowser = mock.MagicMock()
project.mkdir("docs", "htmlcov")
with project.join("docs", "htmlcov", "index.html").open("w") as fo:
fo.write("")
assert DoitMain(ModuleTaskLoader(dodo)).run(["docs"]) == 0
assert project.join("site", "htmlcov").check(dir=1)
importlib.reload(dodo)
def _doit_prepare(self, builder, task):
miner.dodo.builder = builder
# create build directory for storing doit database
if not os.path.exists(builder.build_dir):
os.makedirs(builder.build_dir)
opt_vals = {'dep_file': os.path.join(builder.build_dir, '.doit.db')}
commander = DoitMain(ModuleTaskLoader(miner.dodo),
extra_config={'GLOBAL': opt_vals})
commander.BIN_NAME = 'doit'
logging.info('Preparing LEDE build system...')
commander.run(['--verbosity', '2', task])
def run_tasks(tasks, args, config={'verbosity': 0}):
'''Given a list of `Task` objects, a list of arguments,
and a config dictionary, execute the tasks.
'''
if type(tasks) is not list:
raise TypeError('tasks must be of type list.')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def run_tasks(tasks, args, doit_config=None):
'''Given a list of `Task` objects, a list of arguments,
and a config dictionary, execute the tasks.
'''
doit_config = {} if doit_config is None else doit_config
doit_config.setdefault('verbosity', 0)
doit_config.setdefault('action_string_formatting', 'new')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, doit_config
return DoitMain(Loader()).run(args)
def main(output, build_dir, elm_path, mount_at, exclude_modules,
exclude_source_directories, force_exclusion, fake_user, fake_project,
fake_version, fake_summary, fake_license, validate, doit_args,
project_path, include_paths):
"""Generate static documentation for your Elm project"""
if not shutil.which('rsync'):
raise click.UsageError('this program requires rsync')
if not check_rsync_version():
raise click.UsageError(
'this program requires rsync version {} or greater'.format(
'.'.join(REQUIRED_RSYNC_VERSION)))
if not validate and output is None:
raise click.BadParameter('please specify --output directory')
resolved_include_paths = [_resolve_path(path) for path in include_paths]
exclude_modules = exclude_modules.split(',') if exclude_modules else []
exclude_source_directories = exclude_source_directories.split(
',') if exclude_source_directories else []
project_config = ProjectConfig(
include_paths=resolved_include_paths,
exclude_modules=exclude_modules,
exclude_source_directories=exclude_source_directories,
force_exclusion=force_exclusion,
fake_user=fake_user,
fake_project=fake_project,
fake_version=fake_version,
fake_summary=fake_summary,
fake_license=fake_license,
)
task_creators = build_task_creators(
_resolve_path(project_path),
project_config,
_resolve_path(elm_path) if elm_path else None,
_resolve_path(output) if output is not None else None,
build_path=_resolve_path(build_dir) if build_dir is not None else None,
mount_point=mount_at,
validate=validate)
extra_config = {'GLOBAL': {'outfile': LazyOutfile()}}
result = DoitMain(ModuleTaskLoader(task_creators),
extra_config=extra_config).run(
doit_args.split(' ') if doit_args else [])
if result is not None and result > 0:
raise DoitException('see output above', result)
def test_config_simple(capsys):
DOIT_CONFIG = doit_config(backend='r')
loader = ModuleTaskLoader(locals())
loader.setup({})
config = loader.load_doit_config()
assert config == {'backend': 'r'}
res = DoitMain(ModuleTaskLoader(locals())).run(())
assert res == 3
captured = capsys.readouterr()
with capsys.disabled():
assert "TypeError: 'NoneType' object is not callable" in captured.err
def run(clean=False, number_of_processes=None, db_file_name=None):
doit_args = []
if clean:
doit_args.append('clean')
if number_of_processes:
doit_args += ['-n', str(number_of_processes)]
if db_file_name:
doit_args += ['--db-file', db_file_name]
code = DoitMain(DoitLoader()).run(doit_args)
if code == 0:
logger.debug('Doit: all tasks were executed.')
elif code == 1:
logger.debug('Doit: some tasks were failed.')
elif code == 2:
logger.debug('Doit: error when executing tasks.')
elif code == 3:
logger.debub('Doit: error before task execution starts.')
return code
def _doIt(self, *args, **kwargs):
'''
DoIt wrapper
@param args [in] (list) arguments
@param kwargs [in] (dict) keyword arguments
'''
members = dict(kwargs.get('tasks', self._tasks))
members.update(
DOIT_CONFIG={
'backend': 'json',
'dep_file': self._dep_file,
'reporter': self._reporter,
'verbosity': self._verbosity,
'minversion': '0.27.0'
})
status = DoitMain(ModuleTaskLoader(members)).run(args)
if status:
sys.exit(status)
def task_report():
""" renders the template in reports\report_template.docx with all figures and numbers.yaml mapped"""
figure_files = glob.glob(r'reports\figures\*.png')
outfile = r'reports\report_analysis-double-pendulum.docx'
infile = r'reports\report_template.docx'
context_file = r'reports\numbers.yaml'
src_file = 'dodo_utils.py'
return {
'actions': [(template_renderer(figure_files,
context_file), (infile, outfile))],
'targets': [outfile],
'file_dep': [infile] + figure_files + [context_file, src_file],
'clean':
True
}
from doit_xtended.linkedtasks import _generated_linked_tasks
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
from doit.cmd_base import ModuleTaskLoader
from doit.doit_cmd import DoitMain
d = DoitMain(ModuleTaskLoader(globals()))
d.run(['-s', 'models:prediction'])
def cmd_main(args, extra_config=None):
if extra_config:
extra_config = {'GLOBAL': extra_config}
return DoitMain(extra_config=extra_config).run(args)
def cmd_main(args, extra_config=None, bin_name='doit'):
if extra_config:
extra_config = {'GLOBAL': extra_config}
main = DoitMain(extra_config=extra_config)
main.BIN_NAME = bin_name
return main.run(args)
#! /usr/bin/env python3
import sys
from doit.task import dict_to_task
from doit.cmd_base import TaskLoader
from doit.doit_cmd import DoitMain
my_builtin_task = {
'name': 'sample_task',
'actions': ['echo hello from built in'],
'doc': 'sample doc',
}
class MyLoader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
task_list = [dict_to_task(my_builtin_task)]
config = {'verbosity': 2}
return task_list, config
if __name__ == "__main__":
sys.exit(DoitMain(MyLoader()).run(sys.argv[1:]))
def cmd_main(args):
return DoitMain().run(args)
def run(task_creators):
"""run doit using task_creators
@param task_creators: module or dict containing task creators
"""
sys.exit(DoitMain(ModuleTaskLoader(task_creators)).run(sys.argv[1:]))
def train_profile(profile_dir: Path,
profile: Profile) -> Tuple[int, List[str]]:
# Compact
def ppath(query, default=None, write=False):
return utils_ppath(profile, profile_dir, query, default, write=write)
language = profile.get("language", "")
# Inputs
stt_system = profile.get("speech_to_text.system")
stt_prefix = f"speech_to_text.{stt_system}"
# intent_whitelist = ppath("training.intent-whitelist", "intent_whitelist")
sentences_ini = ppath("speech_to_text.sentences_ini", "sentences.ini")
sentences_dir = ppath("speech_to_text.sentences_dir", "sentences.dir")
base_dictionary = ppath(f"{stt_prefix}.base_dictionary",
"base_dictionary.txt")
base_language_model = ppath(f"{stt_prefix}.base_language_model",
"base_language_model.txt")
base_language_model_weight = float(
profile.get(f"{stt_prefix}.mix_weight", 0))
g2p_model = ppath(f"{stt_prefix}.g2p_model", "g2p.fst")
acoustic_model_type = stt_system
# Pocketsphinx
acoustic_model = ppath(f"{stt_prefix}.acoustic_model", "acoustic_model")
# Kaldi
kaldi_dir = Path(
os.path.expandvars(profile.get(f"{stt_prefix}.kaldi_dir",
"/opt/kaldi")))
kaldi_graph_dir = acoustic_model / profile.get(f"{stt_prefix}.graph",
"graph")
if acoustic_model_type == "kaldi":
# Kaldi acoustic models are inside model directory
acoustic_model = ppath(f"{stt_prefix}.model_dir", "model")
else:
_LOGGER.warning("Unsupported acoustic model type: %s",
acoustic_model_type)
# ignore/upper/lower
word_casing = profile.get("speech_to_text.dictionary_casing",
"ignore").lower()
# default/ignore/upper/lower
g2p_word_casing = profile.get("speech_to_text.g2p_casing",
word_casing).lower()
# all/first
dict_merge_rule = profile.get("speech_to_text.dictionary_merge_rule",
"all").lower()
# Outputs
dictionary = ppath(f"{stt_prefix}.dictionary",
"dictionary.txt",
write=True)
custom_words = ppath(f"{stt_prefix}.custom_words",
"custom_words.txt",
write=True)
language_model = ppath(f"{stt_prefix}.language_model",
"language_model.txt",
write=True)
base_language_model_fst = ppath(f"{stt_prefix}.base_language_model_fst",
"base_language_model.fst",
write=True)
intent_graph = ppath("intent.fsticiffs.intent_graph",
"intent.json",
write=True)
intent_fst = ppath("intent.fsticiffs.intent_fst", "intent.fst", write=True)
vocab = ppath(f"{stt_prefix}.vocabulary", "vocab.txt", write=True)
unknown_words = ppath(f"{stt_prefix}.unknown_words",
"unknown_words.txt",
write=True)
grammar_dir = ppath("speech_to_text.grammars_dir", "grammars", write=True)
fsts_dir = ppath("speech_to_text.fsts_dir", "fsts", write=True)
slots_dir = ppath("speech_to_text.slots_dir", "slots", write=True)
# -----------------------------------------------------------------------------
# Create cache directories
for dir_path in [grammar_dir, fsts_dir]:
dir_path.mkdir(parents=True, exist_ok=True)
# -----------------------------------------------------------------------------
ini_paths: List[Path] = get_ini_paths(sentences_ini, sentences_dir)
# Join ini files into a single combined file and parse
_LOGGER.debug("Parsing ini file(s): %s", [str(p) for p in ini_paths])
try:
intents = get_all_intents(ini_paths)
except Exception:
_LOGGER.exception("Failed to parse %s", ini_paths)
return (1, ["Failed to parse sentences"])
# -----------------------------------------------------------------------------
def get_slot_names(item):
"""Yield referenced slot names."""
if isinstance(item, jsgf.SlotReference):
yield item.slot_name
elif isinstance(item, jsgf.Sequence):
for sub_item in item.items:
for slot_name in get_slot_names(sub_item):
yield slot_name
elif isinstance(item, jsgf.Rule):
for slot_name in get_slot_names(item.rule_body):
yield slot_name
def number_transform(word):
"""Automatically transform numbers"""
if not isinstance(word, jsgf.Word):
# Skip anything besides words
return
try:
n = int(word.text)
# 75 -> (seventy five):75
number_text = num2words(n, lang=language).replace("-", " ").strip()
assert number_text, f"Empty num2words result for {n}"
number_words = number_text.split()
if len(number_words) == 1:
# Easy case, single word
word.text = number_text
word.substitution = str(n)
else:
# Hard case, split into mutliple Words
return jsgf.Sequence(
text=number_text,
type=jsgf.SequenceType.GROUP,
substitution=str(n),
items=[jsgf.Word(w) for w in number_words],
)
except ValueError:
# Not a number
pass
def do_intents_to_graph(intents, slot_names, targets):
sentences, replacements = ini_jsgf.split_rules(intents)
# Load slot values
for slot_name in slot_names:
slot_path = slots_dir / slot_name
assert slot_path.is_file(), f"Missing slot file at {slot_path}"
# Parse each non-empty line as a JSGF sentence
slot_values = []
with open(slot_path, "r") as slot_file:
for line in slot_file:
line = line.strip()
if line:
sentence = jsgf.Sentence.parse(line)
slot_values.append(sentence)
# Replace $slot with sentences
replacements[f"${slot_name}"] = slot_values
if profile.get("intent.replace_numbers", True):
# Replace numbers in parsed sentences
for intent_sentences in sentences.values():
for sentence in intent_sentences:
jsgf.walk_expression(sentence, number_transform,
replacements)
# Convert to directed graph
graph = intents_to_graph(intents, replacements)
# Write graph to JSON file
json_graph = graph_to_json(graph)
with open(targets[0], "w") as graph_file:
json.dump(json_graph, graph_file)
def task_ini_graph():
"""sentences.ini -> intent.json"""
slot_names = set()
for intent_name in intents:
for item in intents[intent_name]:
for slot_name in get_slot_names(item):
slot_names.add(slot_name)
# Add slot files as dependencies
deps = [(slots_dir / slot_name) for slot_name in slot_names]
# Add profile itself as a dependency
profile_json_path = profile_dir / "profile.json"
if profile_json_path.is_file():
deps.append(profile_json_path)
return {
"file_dep": ini_paths + deps,
"targets": [intent_graph],
"actions": [(do_intents_to_graph, [intents, slot_names])],
}
# -----------------------------------------------------------------------------
def do_graph_to_fst(intent_graph, targets):
with open(intent_graph, "r") as graph_file:
json_graph = json.load(graph_file)
graph = json_to_graph(json_graph)
graph_fst = graph_to_fst(graph)
# Create symbol tables
isymbols = fst.SymbolTable()
for symbol, number in graph_fst.input_symbols.items():
isymbols.add_symbol(symbol, number)
osymbols = fst.SymbolTable()
for symbol, number in graph_fst.output_symbols.items():
osymbols.add_symbol(symbol, number)
# Compile FST
compiler = fst.Compiler(isymbols=isymbols,
osymbols=osymbols,
keep_isymbols=True,
keep_osymbols=True)
compiler.write(graph_fst.intent_fst)
compiled_fst = compiler.compile()
# Write to file
compiled_fst.write(str(targets[0]))
def task_intent_fst():
"""intent.json -> intent.fst"""
return {
"file_dep": [intent_graph],
"targets": [intent_fst],
"actions": [(do_graph_to_fst, [intent_graph])],
}
# -----------------------------------------------------------------------------
@create_after(executed="intent_fst")
def task_language_model():
"""Creates an ARPA language model from intent.fst."""
if base_language_model_weight > 0:
yield {
"name": "base_lm_to_fst",
"file_dep": [base_language_model],
"targets": [base_language_model_fst],
"actions": ["ngramread --ARPA %(dependencies)s %(targets)s"],
}
# FST -> n-gram counts
intent_counts = str(intent_fst) + ".counts"
yield {
"name": "intent_counts",
"file_dep": [intent_fst],
"targets": [intent_counts],
"actions": ["ngramcount %(dependencies)s %(targets)s"],
}
# n-gram counts -> model
intent_model = str(intent_fst) + ".model"
yield {
"name": "intent_model",
"file_dep": [intent_counts],
"targets": [intent_model],
"actions": ["ngrammake %(dependencies)s %(targets)s"],
}
if base_language_model_weight > 0:
merged_model = Path(str(intent_model) + ".merge")
# merge
yield {
"name":
"lm_merge",
"file_dep": [base_language_model_fst, intent_model],
"targets": [merged_model],
"actions": [
f"ngrammerge --alpha={base_language_model_weight} %(dependencies)s %(targets)s"
],
}
intent_model = merged_model
# model -> ARPA
yield {
"name": "intent_arpa",
"file_dep": [intent_model],
"targets": [language_model],
"actions": ["ngramprint --ARPA %(dependencies)s > %(targets)s"],
}
# -----------------------------------------------------------------------------
def do_vocab(targets):
with open(targets[0], "w") as vocab_file:
input_symbols = fst.Fst.read(str(intent_fst)).input_symbols()
for i in range(input_symbols.num_symbols()):
# Critical that we use get_nth_key here when input symbols
# numbering is discontiguous.
key = input_symbols.get_nth_key(i)
symbol = input_symbols.find(key).decode().strip()
if symbol and not (symbol.startswith("__")
or symbol.startswith("<")):
print(symbol, file=vocab_file)
if base_language_model_weight > 0:
# Add all words from base dictionary
with open(base_dictionary, "r") as dict_file:
for word in read_dict(dict_file):
print(word, file=vocab_file)
@create_after(executed="language_model")
def task_vocab():
"""Writes all vocabulary words to a file from intent.fst."""
return {
"file_dep": [intent_fst],
"targets": [vocab],
"actions": [do_vocab]
}
# -----------------------------------------------------------------------------
def do_dict(dictionary_paths: Iterable[Path], targets):
with open(targets[0], "w") as dictionary_file:
if unknown_words.exists():
unknown_words.unlink()
dictionary_format = FORMAT_CMU
if acoustic_model_type == "julius":
dictionary_format = FORMAT_JULIUS
make_dict(
vocab,
dictionary_paths,
dictionary_file,
unknown_path=unknown_words,
dictionary_format=dictionary_format,
merge_rule=dict_merge_rule,
upper=(word_casing == "upper"),
lower=(word_casing == "lower"),
)
if unknown_words.exists() and g2p_model.exists():
# Generate single pronunciation guesses
_LOGGER.debug("Guessing pronunciations for unknown word(s)")
g2p_output = subprocess.check_output(
[
"phonetisaurus-apply",
"--model",
str(g2p_model),
"--word_list",
str(unknown_words),
"--nbest",
"1",
],
universal_newlines=True,
)
g2p_transform = lambda w: w
if g2p_word_casing == "upper":
g2p_transform = lambda w: w.upper()
elif g2p_word_casing == "lower":
g2p_transform = lambda w: w.lower()
# Append to dictionary and custom words
with open(custom_words, "a") as words_file:
with open(unknown_words, "w") as unknown_words_file:
for line in g2p_output.splitlines():
line = line.strip()
word, phonemes = re.split(r"\s+", line, maxsplit=1)
word = g2p_transform(word)
print(word, phonemes, file=dictionary_file)
print(word, phonemes, file=words_file)
print(word, phonemes, file=unknown_words_file)
@create_after(executed="vocab")
def task_vocab_dict():
"""Creates custom pronunciation dictionary based on desired vocabulary."""
dictionary_paths = [base_dictionary]
if custom_words.exists():
# Custom dictionary goes first so that the "first" dictionary merge
# rule will choose pronunciations from it.
dictionary_paths.insert(0, custom_words)
# Exclude dictionaries that don't exist
dictionary_paths = [p for p in dictionary_paths if p.exists()]
return {
"file_dep": [vocab] + dictionary_paths,
"targets": [dictionary],
"actions": [(do_dict, [dictionary_paths])],
}
# -----------------------------------------------------------------------------
@create_after(executed="vocab_dict")
def task_kaldi_train():
"""Creates HCLG.fst for a Kaldi nnet3 or gmm model."""
if acoustic_model_type == "kaldi":
return {
"file_dep": [dictionary, language_model],
"targets": [kaldi_graph_dir / "HCLG.fst"],
"actions": [[
"bash",
str(acoustic_model / "train.sh"),
str(kaldi_dir),
str(acoustic_model),
str(dictionary),
str(language_model),
]],
}
# -----------------------------------------------------------------------------
errors = []
class MyReporter(ConsoleReporter):
def add_failure(self, task, exception):
super().add_failure(task, exception)
errors.append(f"{task}: {exception}")
def runtime_error(self, msg):
super().runtime_error(msg)
errors.append(msg)
DOIT_CONFIG = {"action_string_formatting": "old", "reporter": MyReporter}
# Monkey patch inspect to make doit work inside Pyinstaller.
# It grabs the line numbers of functions probably for debugging reasons, but
# PyInstaller doesn't seem to keep that information around.
#
# This better thing to do would be to create a custom TaskLoader.
import inspect
inspect.getsourcelines = lambda obj: [0, 0]
# Run doit main
result = DoitMain(ModuleTaskLoader(locals())).run(sys.argv[1:])
return (result, errors)
