def test_multi_with_no_command_shows_help(capsys): def f(): ... def g(): ... parser = multi_argparser(f, g, description="Awesome documentation") with pytest.raises(SystemExit): parse_and_call(parser, []) out = capsys.readouterr().out assert "Awesome documentation" in out
conf = Config.from_json(Path(config_base)) else: raise ValueError( f"Invalid value {config_base} for --config. " f"Choose from ({', '.join(PREDEF_CONFIGS)}) or give an existing .json file." ) conf = conf._replace( **{k: v for (k, v) in config_as_dict.items() if v is not None}) print(f"Will run cc_net.mine.main with the following config:", conf) all_files = mine(conf) if conf.will_split: assert all_files assert all(d.is_dir() for d in all_files) all_dirs = all_files if "split_by_lang" in conf.pipeline: # Only try regrouping if we split the shards. regroup(conf, all_dirs) elif "split_by_segment" in conf.pipeline: # If we split by segment then regrouping is trivial, since segments appear in only one shard. move_segments(conf, all_dirs) if conf.config_name == "test": _validate_test(conf, conf.get_mined_dir(regroup=True)) if __name__ == "__main__": func_argparse.parse_and_call(get_main_parser())
def main(): parser = func_argparse.multi_argparser( mine=cc_net.mine.get_main_parser(), reproduce=func_argparse.func_argparser(cc_net.minify.reproduce), ) func_argparse.parse_and_call(parser)
def main(): func_argparse.parse_and_call(cc_net.mine.get_main_parser())