def test_multi_with_no_command_shows_help(capsys):
def f():
...
def g():
...
parser = multi_argparser(f, g, description="Awesome documentation")
with pytest.raises(SystemExit):
parse_and_call(parser, [])
out = capsys.readouterr().out
assert "Awesome documentation" in out
conf = Config.from_json(Path(config_base))
else:
raise ValueError(
f"Invalid value {config_base} for --config. "
f"Choose from ({', '.join(PREDEF_CONFIGS)}) or give an existing .json file."
)
conf = conf._replace(
**{k: v
for (k, v) in config_as_dict.items() if v is not None})
print(f"Will run cc_net.mine.main with the following config:", conf)
all_files = mine(conf)
if conf.will_split:
assert all_files
assert all(d.is_dir() for d in all_files)
all_dirs = all_files
if "split_by_lang" in conf.pipeline:
# Only try regrouping if we split the shards.
regroup(conf, all_dirs)
elif "split_by_segment" in conf.pipeline:
# If we split by segment then regrouping is trivial, since segments appear in only one shard.
move_segments(conf, all_dirs)
if conf.config_name == "test":
_validate_test(conf, conf.get_mined_dir(regroup=True))
if __name__ == "__main__":
func_argparse.parse_and_call(get_main_parser())
def main():
parser = func_argparse.multi_argparser(
mine=cc_net.mine.get_main_parser(),
reproduce=func_argparse.func_argparser(cc_net.minify.reproduce),
)
func_argparse.parse_and_call(parser)
def main():
func_argparse.parse_and_call(cc_net.mine.get_main_parser())