def gen_default_config(context, task_name, options):
"""
Generate a config for `task_name` with default values.
Optionally, override the defaults by passing your desired
components as `options`.
"""
try:
cfg = gen_config_impl(task_name, *options)
except TypeError as ex:
eprint(
"ERROR - Cannot create this config",
"because some fields don't have a default value:",
ex,
)
sys.exit(-1)
# add the --include to the config generated
if context.obj.include:
if cfg.include_dirs is None:
cfg.include_dirs = []
for path in context.obj.include:
cfg.include_dirs.append(path.rstrip("/"))
cfg_json = config_to_json(PyTextConfig, cfg)
print(json.dumps(cfg_json, sort_keys=True, indent=2))
def _union_from_json(subclasses, json_obj):
if not _is_dict(json_obj):
raise IncorrectTypeError(
f"incorrect Union value {json_obj} for union {subclasses}"
)
subclasses_dict = {}
for subclass in subclasses:
if type(None) != subclass:
if getattr(subclass, "__EXPANSIBLE__", False):
children = Registry.subconfigs(subclass)
for child in children:
subclasses_dict[_canonical_typename(child).lower()] = child
else:
subclasses_dict[_canonical_typename(subclass).lower()] = subclass
type_name = list(json_obj)[0].lower()
if len(json_obj) == 1 and type_name in subclasses_dict:
json_obj = next(iter(json_obj.values()))
else:
type_name = next(iter(subclasses_dict))
eprint(
"WARNING - Can not find class type in json: "
f"trying with first class {type_name} in the union."
)
try:
return _value_from_json(subclasses_dict[type_name], json_obj)
except Exception as e:
raise UnionTypeError(
f"failed to parse union {subclasses} from json payload {json_obj}"
) from e
def _union_from_json(subclasses, json_obj):
if not _is_dict(json_obj):
raise IncorrectTypeError(
f"incorrect Union value {json_obj} for union {subclasses}"
)
subclasses_dict = build_subclass_dict(subclasses)
type_name = list(json_obj)[0].lower()
if len(json_obj) == 1 and type_name in subclasses_dict:
json_obj = next(iter(json_obj.values()))
else:
type_name = next(iter(subclasses_dict))
eprint(
"WARNING - Can not find class type in json: "
f"trying with first class {type_name} in the union."
)
try:
return _value_from_json(subclasses_dict[type_name], json_obj)
except Exception as e:
raise UnionTypeError(
(
f"failed to parse union {subclasses} from"
f"json payload {json_obj} \n"
f"Reason: {e}"
)
) from e
def upgrade_one_version(json_config):
current_version = json_config.get("version", 0)
adapter = ADAPTERS.get(current_version)
if not adapter:
raise Exception(f"no adapter found for version {current_version}")
json_config = adapter(json_config)
eprint(
f"WARNING - Applying old config adapter for version={current_version}. "
"Please consider migrating your old configs to the latest version.")
json_config["version"] = current_version + 1
return json_config
def config_from_json(cls, json_obj, ignore_fields=()):
if getattr(cls, "__EXPANSIBLE__", False):
component_config = _try_component_config_from_json(cls, json_obj)
if component_config:
return component_config
parsed_dict = {}
if not hasattr(cls, "_fields"):
raise IncorrectTypeError(f"{cls} is not a valid config class")
cls_name = getattr(cls, "__name__", cls)
# Non-EXPANSIBLE classes can be found in configs
cls_name_wo_config = cls_name.split(".")[0]
unknown_fields = (
set(json_obj)
- {f[0] for f in cls.__annotations__.items()}
- {cls_name_wo_config}
)
if unknown_fields:
cls_fields = {f[0] for f in cls.__annotations__.items()}
raise ConfigParseError(
f"Unknown fields for class {cls_name} with fields {cls_fields} \
detected in config json: {unknown_fields}"
)
for field, f_cls in cls.__annotations__.items():
if field in ignore_fields:
eprint(
f"Info - field: {field} in class: {cls_name} is skipped in",
"config_from_json because it's found in the ignore_fields.",
)
continue
value = None
is_optional = _is_optional(f_cls)
if field not in json_obj:
if field in cls._field_defaults:
# if using default value, no conversion is needed
value = cls._field_defaults.get(field)
else:
try:
value = _value_from_json(f_cls, json_obj[field])
except ConfigParseError:
raise
except Exception as e:
raise ConfigParseError(
f"failed to parse {field} to {f_cls} with json payload \
{json_obj[field]}"
) from e
# validate value
if value is None and not is_optional:
raise MissingValueError(
f"missing value for {field} in class {cls_name} with json {json_obj}"
)
parsed_dict[field] = value
return cls(**parsed_dict)
def downgrade_one_version(json_config):
current_version = json_config.get("version", 0)
downgrade_adapter = DOWNGRADE_ADAPTERS.get(current_version)
if not downgrade_adapter:
raise Exception(f"no downgrade adapter found for version {current_version}")
json_config = downgrade_adapter(json_config)
eprint(
f"WARNING - Downgrading your current config version={current_version}. "
"Please wait for next pytext pkg release to let new config take effect."
)
json_config["version"] = current_version - 1
return json_config
def add_include(path):
"""
Import tasks (and associated components) from the folder name.
"""
eprint("Including:", path)
modules = glob.glob(os.path.join(path, "*.py"))
all = [
os.path.basename(f)[:-3].replace("/", ".")
for f in modules
if PathManager.isfile(f) and not f.endswith("__init__.py")
]
for mod_name in all:
mod_path = path.replace("/", ".") + "." + mod_name
eprint("... importing module:", mod_path)
my_module = importlib.import_module(mod_path)
for m in inspect.getmembers(my_module, inspect.isclass):
if m[1].__module__ != mod_path:
pass
elif Task_Deprecated in m[1].__bases__ or NewTask in m[1].__bases__:
eprint("... task:", m[1].__name__)
register_tasks(m[1])
else:
eprint("... importing:", m[1])
importlib.import_module(mod_path, m[1])
def upgrade_one_version(json_config):
current_version = json_config.get("version", 0)
adapter = ADAPTERS.get(current_version)
if not adapter:
raise Exception(
f"no adapter found for version {current_version}."
"Make sure current revision is after pytext pkg's revision, and rebase if necessary"
)
json_config = adapter(json_config)
eprint(
f"WARNING - Applying old config adapter for version={current_version}. "
"Please consider migrating your old configs to the latest version.")
json_config["version"] = current_version + 1
return json_config
def load_config():
# Cache the config object so it can be accessed multiple times
if not hasattr(context.obj, "config"):
if config_module:
context.obj.config = import_module(config_module).config
else:
if config_file:
with open(config_file) as file:
config = json.load(file)
elif config_json:
config = json.loads(config_json)
else:
eprint("No config file specified, reading from stdin")
config = json.load(sys.stdin)
context.obj.config = parse_config(config)
return context.obj.config
def gen_default_config(context, task_name, options):
"""
Generate a config for `task_name` with default values.
Optionally, override the defaults by passing your desired
components as `options`.
"""
try:
cfg = gen_config_impl(task_name, *options)
except TypeError as ex:
eprint(
"ERROR - Cannot create this config",
"because some fields don't have a default value:",
ex,
)
sys.exit(-1)
cfg_json = config_to_json(PyTextConfig, cfg)
print(json.dumps(cfg_json, sort_keys=True, indent=2))
def load_config():
# Cache the config object so it can be accessed multiple times
if not hasattr(context.obj, "config"):
if config_module:
context.obj.config = import_module(config_module).config
else:
if config_file:
with PathManager.open(config_file) as file:
config = json.load(file)
elif config_json:
config = json.loads(config_json)
else:
eprint("No config file specified, reading from stdin")
config = json.load(sys.stdin)
# before parsing the config, include the custom components
for path in config.get("include_dirs", []):
add_include(path.rstrip("/"))
context.obj.config = parse_config(config)
return context.obj.config
def gen_config_impl(task_name, *args, **kwargs):
# import the classes required by parameters
requested_classes = [locate(opt) for opt in args] + [locate(task_name)]
register_tasks(requested_classes)
task_class_set = find_config_class(task_name)
if not task_class_set:
raise Exception(
f"Unknown task class: {task_name} " "(try fully qualified class name?)"
)
elif len(task_class_set) > 1:
raise Exception(f"Multiple tasks named {task_name}: {task_class_set}")
task_class = next(iter(task_class_set))
task_config = getattr(task_class, "example_config", task_class.Config)
root = PyTextConfig(task=task_config(), version=LATEST_VERSION)
eprint("INFO - Applying task option:", task_class.__name__)
# Use components in args instead of defaults
for opt in args:
if "=" in opt:
param_path, value = opt.split("=", 1)
kwargs[param_path] = value
continue
replace_class_set = find_config_class(opt)
if not replace_class_set:
raise Exception(f"Not a component class: {opt}")
elif len(replace_class_set) > 1:
raise Exception(f"Multiple component named {opt}: {replace_class_set}")
replace_class = next(iter(replace_class_set))
found = replace_components(root, opt, get_subclasses(replace_class))
if found:
eprint("INFO - Applying class option:", ".".join(reversed(found)), "=", opt)
obj = root
for k in reversed(found[1:]):
obj = getattr(obj, k)
if hasattr(replace_class, "Config"):
setattr(obj, found[0], replace_class.Config())
else:
setattr(obj, found[0], replace_class())
else:
raise Exception(f"Unknown class option: {opt}")
# Use parameters in kwargs instead of defaults
for param_path, value in kwargs.items():
found = find_param(root, "." + param_path)
if len(found) == 1:
eprint("INFO - Applying parameter option to", found[0], "=", value)
replace_param(root, found[0].split("."), value)
elif not found:
raise Exception(f"Unknown parameter option: {param_path}")
else:
raise Exception(
f"Multiple possibilities for {param_path}: {', '.join(found)}"
)
return root