def add_include(path): """ Import tasks (and associated components) from the folder name. """ eprint("Including:", path) modules = glob.glob(os.path.join(path, "*.py")) all = [ os.path.basename(f)[:-3].replace("/", ".") for f in modules if os.path.isfile(f) and not f.endswith("__init__.py") ] for mod_name in all: mod_path = path + "." + mod_name eprint("... importing module:", mod_path) my_module = importlib.import_module(mod_path) for m in inspect.getmembers(my_module, inspect.isclass): if m[1].__module__ != mod_path: pass elif Task_Deprecated in m[1].__bases__ or NewTask in m[1].__bases__: eprint("... task:", m[1].__name__) register_tasks(m[1]) else: eprint("... importing:", m[1]) importlib.import_module(mod_path, m[1])
def gen_default_config(context, task_name, options): """ Generate a config for `task_name` with default values. Optionally, override the defaults by passing your desired components as `options`. """ try: cfg = gen_config_impl(task_name, options) except TypeError as ex: eprint( "ERROR - Cannot create this config", "because some fields don't have a default value:", ex, ) sys.exit(-1) print(json.dumps(cfg, sort_keys=True, indent=2))
def gen_config_impl(task_name, options): # import the classes required by parameters requested_classes = [locate(opt) for opt in options] + [locate(task_name)] register_tasks(requested_classes) task_class_set = find_config_class(task_name) if not task_class_set: raise Exception(f"Unknown task class: {task_name} " "(try fully qualified class name?)") elif len(task_class_set) > 1: raise Exception(f"Multiple tasks named {task_name}: {task_class_set}") task_class = next(iter(task_class_set)) task_config = getattr(task_class, "example_config", task_class.Config) root = PyTextConfig(task=task_config(), version=LATEST_VERSION) # Use components listed in options instead of defaults for opt in options: replace_class_set = find_config_class(opt) if not replace_class_set: raise Exception(f"Not a component class: {opt}") elif len(replace_class_set) > 1: raise Exception( f"Multiple component named {opt}: {replace_class_set}") replace_class = next(iter(replace_class_set)) found = replace_components(root, opt, get_subclasses(replace_class)) if found: eprint("INFO - Applying option:", "->".join(reversed(found)), "=", opt) obj = root for k in reversed(found[1:]): obj = getattr(obj, k) if hasattr(replace_class, "Config"): setattr(obj, found[0], replace_class.Config()) else: setattr(obj, found[0], replace_class()) else: raise Exception(f"Unknown option: {opt}") return config_to_json(PyTextConfig, root)
def gen_config_impl(task_name, options): # import the classes required by parameters requested_classes = [locate(opt) for opt in options] + [locate(task_name)] register_tasks(requested_classes) task_class_set = find_config_class(task_name) if not task_class_set: raise Exception(f"Unknown task class: {task_name} " "(try fully qualified class name?)") elif len(task_class_set) > 1: raise Exception(f"Multiple tasks named {task_name}: {task_class_set}") task_class = next(iter(task_class_set)) task_config = getattr(task_class, "example_config", task_class.Config) root = PyTextConfig(task=task_config(), version=LATEST_VERSION) eprint("INFO - Applying task option:", task_class.__name__) # Use components listed in options instead of defaults for opt in options: if "=" in opt: param_path, value = opt.split("=", 1) found = find_param(root, "." + param_path) if len(found) == 1: eprint("INFO - Applying parameter option to", found[0], ":", opt) replace_param(root, found[0].split("."), value) elif not found: raise Exception(f"Unknown parameter option: {opt}") else: raise Exception( f"Multiple possibilities for {opt}: {', '.join(found)}") else: replace_class_set = find_config_class(opt) if not replace_class_set: raise Exception(f"Not a component class: {opt}") elif len(replace_class_set) > 1: raise Exception( f"Multiple component named {opt}: {replace_class_set}") replace_class = next(iter(replace_class_set)) found = replace_components(root, opt, get_subclasses(replace_class)) if found: eprint( "INFO - Applying class option:", "->".join(reversed(found)), "=", opt, ) obj = root for k in reversed(found[1:]): obj = getattr(obj, k) if hasattr(replace_class, "Config"): setattr(obj, found[0], replace_class.Config()) else: setattr(obj, found[0], replace_class()) else: raise Exception(f"Unknown class option: {opt}") return root