def task(cls):
"""
A decorator which turns a class into a Zookeeper task, which is a Zookeeper
method with an argument-less `run` method.
Tasks are runnable through the CLI. Upon execution, the task is instantiated
and all component fields are configured using configuration passed as CLI
arguments of the form `field_name=field_value`, and then the `run` method is
called.
"""
cls = component(cls)
if not (hasattr(cls, "run") and callable(cls.run)):
raise TypeError("Classes decorated with @task must define a `run` method.")
# Enforce argument-less `run`.
call_args = inspect.signature(cls.run).parameters
if len(call_args) > 1 or len(call_args) == 1 and "self" not in call_args:
raise TypeError(
"A @task class must define a `run` method taking no arguments except "
f"`self`, which runs the task, but `{cls.__name__}.run` accepts arguments "
f"{tuple(name for name in call_args)}."
)
# Register a CLI command to run the task.
if convert_to_snake_case(cls.__name__) in (
convert_to_snake_case(c) for c in cli.commands
):
raise ValueError(
f"Task naming conflict. Task with name '{cls.__name__}' (or similar) "
"already registered. Note that the task name is the name of the class that "
"the @task decorator is applied to."
)
@cli.command(cls.__name__, context_settings=dict(ignore_unknown_options=True))
@click.option(
"-i",
"--interactive",
is_flag=True,
default=False,
help="Interactively configure task.",
)
@click.argument("config", type=ConfigParam(), nargs=-1)
def command(config, interactive):
config = {k: v for k, v in config}
task_instance = cls()
configure(task_instance, config, interactive=interactive)
task_instance.run()
return cls
def task(cls):
"""
A decorator which turns a class into a Zookeeper task, which is a Zookeeper
method with an argument-less `run` method.
Tasks are runnable through the CLI. Upon execution, the task is instantiated
and all component fields are configured using configuration passed as CLI
arguments of the form `field_name=field_value`, and then the `run` method is
called.
"""
cls = component(cls)
if not (hasattr(cls, "run") and callable(cls.run)):
raise ValueError("Classes decorated with @task must define a `run` method.")
# Enforce argument-less `run`
call_args = inspect.signature(cls.run).parameters
if len(call_args) > 1 or len(call_args) == 1 and "self" not in call_args:
raise ValueError(
"A task class must define a `run` method taking no arguments except "
f"`self`, which runs the task, but `{cls.__name__}.run` accepts arguments "
f"{call_args}."
)
# Register a CLI command to run the task.
task_name = convert_to_snake_case(cls.__name__)
if task_name in cli.commands:
raise ValueError(
f"Task naming conflict. Task with name '{task_name}' already registered. "
"Note that the task name is the name of the class that the @task decorator "
"is applied to, normalised to 'snake case', e.g. 'FooBarTask' -> "
"'foo_bar_task'."
)
@cli.command(task_name)
@click.argument("config", type=ConfigParam(), nargs=-1)
@click.option("-i", "--interactive", is_flag=True, default=False)
def command(config, interactive):
config = {k: v for k, v in config}
task_instance = cls()
configure(task_instance, config, interactive=interactive)
task_instance.run()
return cls
def component(cls):
"""A decorater which turns a class into a Zookeeper component."""
if not inspect.isclass(cls):
raise ValueError("Only classes can be decorated with @component.")
if inspect.isabstract(cls):
raise ValueError(
"Abstract classes cannot be decorated with @component.")
if is_component_class(cls):
raise ValueError(
f"The class {cls} is already a component; the @component decorator cannot "
"be applied again.")
cls.__component_name__ = convert_to_snake_case(cls.__name__)
cls.__component_parent__ = None
cls.__component_configured__ = False
cls.__component_fields__ = {}
# Override `__getattribute__`, `__setattr__`, and `__delattr__` to correctly
# manage getting, setting, and deleting component fields.
cls.__getattribute__ = getattribute_wrapper(
cls.__getattribute__) # type: ignore
cls.__setattr__ = setattr_wrapper(cls.__setattr__)
cls.__delattr__ = delattr_wrapper(cls.__delattr__)
# Override `__dir__` so that field names are included.
cls.__dir__ = dir_wrapper(cls.__dir__)
# Override `__init__` to perform component initialisation and (potentially)
# set key-word args as field values.
cls.__init__ = init_wrapper(cls.__init__)
# Components should have nice `__str__` and `__repr__` methods.
cls.__str__ = __component_str__
cls.__repr__ = __component_repr__
return cls
def configure(
instance,
conf: Dict[str, Any],
name: Optional[str] = None,
interactive: bool = False,
):
"""
Configure the component instance with parameters from the `conf` dict.
Configuration passed through `conf` takes precedence over and will
overwrite any values already set on the instance - either class defaults
or those set in `__init__`.
"""
# Configuration can only happen once.
if instance.__component_configured__:
raise ValueError(
f"Component '{instance.__component_name__}' has already been configured."
)
if name is not None:
instance.__component_name__ = name
# Set the correct value for each field.
for field_name, field in instance.__component_fields__.items():
full_name = f"{instance.__component_name__}.{field_name}"
field_type_name = (
field.annotated_type.__name__
if inspect.isclass(field.annotated_type)
else str(field.annotated_type)
)
component_subclasses = list(generate_component_subclasses(field.annotated_type))
if field_name in conf:
field_value = conf[field_name]
# The configuration value could be a string specifying a component
# class to instantiate.
if len(component_subclasses) > 0 and isinstance(field_value, str):
for subclass in component_subclasses:
if (
field_value == subclass.__name__
or field_value == subclass.__qualname__
or convert_to_snake_case(field_value)
== convert_to_snake_case(subclass.__name__)
):
field_value = subclass()
break
set_field_value(instance, field_name, field_value)
# If this is a 'raw' value, add a placeholder to `conf` so that it
# gets picked up correctly in sub-components.
if (
field_value != OVERRIDEN_CONF_VALUE
and field_value != NON_OVERRIDEN_CONF_VALUE
):
conf[field_name] = OVERRIDEN_CONF_VALUE
# If there's no config value but a value is already set on the instance,
# we only need to add a placeholder to `conf` to make sure that the
# value will be accessible from sub-components. `hasattr` isn't safe so
# we have to check membership directly.
elif field_name in object.__dir__(instance): # type: ignore
conf[field_name] = NON_OVERRIDEN_CONF_VALUE
# If there is only one concrete component subclass of the annotated
# type, we assume the user must intend to use that subclass, and so
# instantiate and use an instance automatically.
elif len(component_subclasses) == 1:
component_cls = list(component_subclasses)[0]
print_formatted_text(
f"'{type_name_str(component_cls)}' is the only concrete component "
f"class that satisfies the type of the annotated field '{full_name}'. "
"Using an instance of this class by default."
)
# This is safe because we don't allow `__init__` to have any
# positional arguments without defaults.
field_value = component_cls()
set_field_value(instance, field_name, field_value)
# Add a placeholder to `conf` to so that this value can be accessed
# by sub-components.
conf[field_name] = NON_OVERRIDEN_CONF_VALUE
# If we are running interactively, prompt for a value.
elif interactive:
if len(component_subclasses) > 0:
component_cls = prompt_for_component_subclass(
full_name, component_subclasses
)
# This is safe because we don't allow `__init__` to have any
# positional arguments without defaults.
field_value = component_cls()
else:
field_value = prompt_for_value(full_name, field.annotated_type)
set_field_value(instance, field_name, field_value)
# Add a placeholder to `conf` so that this value can be accessed by
# sub-components.
conf[field_name] = OVERRIDEN_CONF_VALUE
# Otherwise, raise an appropriate error.
else:
if len(component_subclasses) > 0:
raise ValueError(
f"Annotated field '{full_name}' of type '{field_type_name}' "
f"has no configured value. Please configure '{full_name}' with "
f"one of the following component subclasses of '{field_type_name}':"
+ "\n ".join(
[""] + list(type_name_str(c) for c in component_subclasses)
)
)
raise ValueError(
"No configuration value found for annotated field "
f"'{full_name}' of type '{field_type_name}'."
)
# Recursively configure any sub-components.
for field_name, field_type in instance.__component_fields__.items():
field_value = getattr(instance, field_name)
full_name = f"{instance.__component_name__}.{field_name}"
if (
is_component_class(field_value.__class__)
and not field_value.__component_configured__
):
# Set the component parent so that inherited fields function
# correctly.
field_value.__component_parent__ = instance
# Configure the nested sub-component. The configuration we use
# consists of all non-scoped keys and any keys scoped to
# `field_name`, where the keys scoped to `field_name` override the
# non-scoped keys.
non_scoped_conf = {a: b for a, b in conf.items() if "." not in a}
field_name_scoped_conf = {
a[len(f"{field_name}.") :]: b
for a, b in conf.items()
if a.startswith(f"{field_name}.")
}
nested_conf = {**non_scoped_conf, **field_name_scoped_conf}
configure(field_value, nested_conf, name=full_name, interactive=interactive)
# Type check all fields.
for field_name, field in instance.__component_fields__.items():
assert field_name in instance.__component_fields__
field_value = getattr(instance, field_name)
try:
check_type(field_name, field_value, field.annotated_type)
# Because boolean `True` and `False` are coercible to ints and
# floats, `typeguard.check_type` doesn't throw if we e.g. pass
# `True` to a value expecting a float. This would, however, likely
# be a user error, so explicitly check for this.
if field.annotated_type in [float, int] and isinstance(field_value, bool):
raise TypeError
except TypeError:
raise TypeError(
f"Attempting to set field '{instance.__component_name__}.{field_name}' "
f"which has annotated type '{type_name_str(field.annotated_type)}' "
f"with value '{field_value}'."
) from None
instance.__component_configured__ = True
def configure(
instance,
conf: Dict[str, Any],
name: Optional[str] = None,
interactive: bool = False,
):
"""
Configure the component instance with parameters from the `conf` dict.
Configuration passed through `conf` takes precedence over and will
overwrite any values already set on the instance - either class defaults
or those set in `__init__`.
"""
# Only component instances can be configured.
if not utils.is_component_instance(instance):
raise TypeError(
"Only @component, @factory, and @task instances can be configured. "
f"Received: {instance}.")
# Configuration can only happen once.
if instance.__component_configured__:
raise ValueError(
f"Component '{instance.__component_name__}' has already been configured."
)
if name is not None:
instance.__component_name__ = name
# Set the correct value for each field.
for field in instance.__component_fields__.values():
full_name = f"{instance.__component_name__}.{field.name}"
field_type_name = (field.type.__name__
if inspect.isclass(field.type) else str(field.type))
if isinstance(field, ComponentField):
# Create a list of all component subclasses of the field type, and
# add to the list all factory classes which can build the type (or
# any subclass of the type).
component_subclasses = list(
utils.generate_component_subclasses(field.type))
for type_subclass in utils.generate_subclasses(field.type):
component_subclasses.extend(
FACTORY_REGISTRY.get(type_subclass, []))
if field.name in conf:
conf_field_value = conf[field.name]
if isinstance(field, ComponentField):
# The configuration value could be a string specifying a component
# or factory class to instantiate.
if len(component_subclasses) > 0 and isinstance(
conf_field_value, str):
for subclass in component_subclasses:
if (conf_field_value == subclass.__name__
or conf_field_value == subclass.__qualname__ or
utils.convert_to_snake_case(conf_field_value)
== utils.convert_to_snake_case(
subclass.__name__)):
conf_field_value = subclass()
break
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name] = conf_field_value
# This value has now been 'consumed', so delete it from `conf`.
del conf[field.name]
# If there's a value in scope, we don't need to do anything.
elif field.name in instance.__component_fields_with_values_in_scope__:
pass
# If the field explicitly allows values to be missing, there's no need
# to do anything.
elif field.allow_missing:
pass
# If there is only one concrete component subclass of the annotated
# type, we assume the user must intend to use that subclass, and so
# instantiate and use an instance automatically.
elif isinstance(field,
ComponentField) and len(component_subclasses) == 1:
component_cls = list(component_subclasses)[0]
utils.warn(
f"'{utils.type_name_str(component_cls)}' is the only concrete component "
f"class that satisfies the type of the annotated field '{full_name}'. "
"Using an instance of this class by default.", )
# This is safe because we don't allow custom `__init__` methods.
conf_field_value = component_cls()
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name] = conf_field_value
# If we are running interactively, prompt for a value.
elif interactive:
if isinstance(field, ComponentField):
if len(component_subclasses) > 0:
component_cls = utils.prompt_for_component_subclass(
full_name, component_subclasses)
# This is safe because we don't allow custom `__init__` methods.
conf_field_value = component_cls()
else:
raise ValueError(
"No component or factory class is defined which satisfies the "
f"type {field_type_name} of field {full_name}. If such a class "
"has been defined, it must be imported before calling "
"`configure`.")
else:
conf_field_value = utils.prompt_for_value(
full_name, field.type)
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name] = conf_field_value
# Otherwise, raise an appropriate error.
else:
if isinstance(field, ComponentField):
if len(component_subclasses) > 0:
raise ValueError(
f"Component field '{full_name}' of type '{field_type_name}' "
f"has no default or configured class. Please configure "
f"'{full_name}' with one of the following @component or "
"@factory classes:" + "\n ".join([""] + list(
utils.type_name_str(c)
for c in component_subclasses)))
else:
raise ValueError(
f"Component field '{full_name}' of type '{field_type_name}' "
f"has no default or configured class. No defined @component "
"or @factory class satisfies this type. Please define an "
f"@component class subclassing '{field_type_name}', or an "
"@factory class with a `build()` method returning a "
f"'{field_type_name}' instance. This class must be imported "
"before invoking `configure()`.")
raise ValueError(
"No configuration value found for annotated field "
f"'{full_name}' of type '{field_type_name}'.")
# At this point we are certain that this field has has a value, so keep
# track of that fact.
instance.__component_fields_with_values_in_scope__.add(field.name)
# Check that all `conf` values are being used, and throw if we've been
# passed an un-used option.
for key in conf:
error = ValueError(
f"Key '{key}' does not correspond to any field of component "
f"'{instance.__component_name__}'."
"\n\n"
"If you have nested components as follows:\n\n"
"```\n"
"@component\n"
"class ChildComponent:\n"
" a: int = Field(0)\n"
"\n"
"@task\n"
"class SomeTask:\n"
" child: ChildComponent = ComponentField(ChildComponent)\n"
" def run(self):\n"
" print(self.child.a)\n"
"```\n\n"
"then trying to configure `a=<SOME_VALUE>` will fail. You instead need to "
"fully qualify the key name, and configure the value with "
"`child.a=<SOME_VALUE>`.")
if "." in key:
scoped_component_name = key.split(".")[0]
if not (scoped_component_name in instance.__component_fields__
and isinstance(
instance.__component_fields__[scoped_component_name],
ComponentField)):
raise error
elif key not in instance.__component_fields__:
raise error
# Recursively configure any sub-components.
for field in instance.__component_fields__.values():
if not isinstance(field, ComponentField):
continue
try:
sub_component_instance = base_getattr(instance, field.name)
except AttributeError as e:
if field.allow_missing:
continue
raise e from None
if not utils.is_component_instance(sub_component_instance):
continue
full_name = f"{instance.__component_name__}.{field.name}"
if not sub_component_instance.__component_configured__:
# Set the component parent so that inherited fields function
# correctly.
sub_component_instance.__component_parent__ = instance
# Extend the field names in scope. All fields with values defined in
# the scope of the parent are also accessible in the child.
sub_component_instance.__component_fields_with_values_in_scope__ |= (
instance.__component_fields_with_values_in_scope__)
# Configure the nested sub-component. The configuration we use
# consists of all any keys scoped to `field.name`.
field_name_scoped_conf = {
a[len(f"{field.name}."):]: b
for a, b in conf.items() if a.startswith(f"{field.name}.")
}
configure(
sub_component_instance,
field_name_scoped_conf,
name=full_name,
interactive=interactive,
)
instance.__component_configured__ = True
if hasattr(instance.__class__, "__post_configure__"):
instance.__post_configure__()
def configure_component_instance(
instance,
conf: Dict[str, Any],
name: str,
fields_in_scope: AbstractSet[str],
interactive: bool,
):
"""Configure the component instance with parameters from the `conf` dict.
This method is recursively called for each component instance in the component tree
by the exported `configure` function.
"""
if name is not None:
instance.__component_name__ = name
if hasattr(instance.__class__, "__pre_configure__"):
conf = instance.__pre_configure__({**conf})
if not isinstance(conf, dict):
raise ValueError(
"Expected the `__pre_configure__` method of component "
f"'{instance.__component_name__}' to return a dict of configuration, "
f"but received: {conf}"
)
# Extend the field names in scope.
instance.__component_fields_with_values_in_scope__ |= fields_in_scope
# Set the correct value for each field.
for field in instance.__component_fields__.values():
full_name = f"{instance.__component_name__}.{field.name}"
field_type_name = (
field.type.__name__ if inspect.isclass(field.type) else str(field.type)
)
if isinstance(field, ComponentField):
# Create a list of all component subclasses of the field type, and
# add to the list all factory classes which can build the type (or
# any subclass of the type).
component_subclasses = list(utils.generate_component_subclasses(field.type))
for type_subclass in utils.generate_subclasses(field.type):
component_subclasses.extend(FACTORY_REGISTRY.get(type_subclass, []))
if field.name in conf:
conf_field_value = conf[field.name]
if isinstance(field, ComponentField):
# The configuration value could be a string specifying a component
# or factory class to instantiate.
if len(component_subclasses) > 0 and isinstance(conf_field_value, str):
for subclass in component_subclasses:
if (
conf_field_value == subclass.__name__
or conf_field_value == subclass.__qualname__
or utils.convert_to_snake_case(conf_field_value)
== utils.convert_to_snake_case(subclass.__name__)
):
conf_field_value = subclass()
break
# If this isn't the case, then it's a user type error, but we don't
# throw here and instead let the run-time type-checking take care of
# it (which will provide a better error message).
if utils.is_component_instance(conf_field_value):
# Set the component parent so that field value inheritence will
# work correctly.
conf_field_value.__component_parent__ = instance
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name
] = conf_field_value
# If there's a value in scope, we don't need to do anything.
elif field.name in instance.__component_fields_with_values_in_scope__:
pass
# If the field explicitly allows values to be missing, there's no need
# to do anything.
elif field.allow_missing:
continue
# If there is only one concrete component subclass of the annotated
# type, we assume the user must intend to use that subclass, and so
# instantiate and use an instance automatically.
elif isinstance(field, ComponentField) and len(component_subclasses) == 1:
component_cls = list(component_subclasses)[0]
utils.warn(
f"'{utils.type_name_str(component_cls)}' is the only concrete component "
f"class that satisfies the type of the annotated field '{full_name}'. "
"Using an instance of this class by default.",
)
# This is safe because we don't allow custom `__init__` methods.
conf_field_value = component_cls()
# Set the component parent so that field value inheritence will work
# correctly.
conf_field_value.__component_parent__ = instance
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name
] = conf_field_value
# If we are running interactively, prompt for a value.
elif interactive:
if isinstance(field, ComponentField):
if len(component_subclasses) > 0:
component_cls = utils.prompt_for_component_subclass(
full_name, component_subclasses
)
# This is safe because we don't allow custom `__init__` methods.
conf_field_value = component_cls()
else:
raise ValueError(
"No component or factory class is defined which satisfies the "
f"type {field_type_name} of field {full_name}. If such a class "
"has been defined, it must be imported before calling "
"`configure`."
)
else:
conf_field_value = utils.prompt_for_value(full_name, field.type)
# Set the value on the instance.
instance.__component_configured_field_values__[
field.name
] = conf_field_value
# Otherwise, raise an appropriate error.
else:
if isinstance(field, ComponentField):
if len(component_subclasses) > 0:
raise ValueError(
f"Component field '{full_name}' of type '{field_type_name}' "
f"has no default or configured class. Please configure "
f"'{full_name}' with one of the following @component or "
"@factory classes:"
+ "\n ".join(
[""]
+ list(utils.type_name_str(c) for c in component_subclasses)
)
)
else:
raise ValueError(
f"Component field '{full_name}' of type '{field_type_name}' "
f"has no default or configured class. No defined @component "
"or @factory class satisfies this type. Please define an "
f"@component class subclassing '{field_type_name}', or an "
"@factory class with a `build()` method returning a "
f"'{field_type_name}' instance. This class must be imported "
"before invoking `configure()`."
)
raise ValueError(
"No configuration value found for annotated field "
f"'{full_name}' of type '{field_type_name}'."
)
# At this point we are certain that this field has has a value, so keep
# track of that fact.
instance.__component_fields_with_values_in_scope__.add(field.name)
# Check that all `conf` values are being used, and throw if we've been
# passed an un-used option.
for key in conf:
error = ValueError(
f"Key '{key}' does not correspond to any field of component "
f"'{instance.__component_name__}'."
"\n\n"
"If you have nested components as follows:\n\n"
"```\n"
"@component\n"
"class ChildComponent:\n"
" a: int = Field(0)\n"
"\n"
"@task\n"
"class SomeTask:\n"
" child: ChildComponent = ComponentField(ChildComponent)\n"
" def run(self):\n"
" print(self.child.a)\n"
"```\n\n"
"then trying to configure `a=<SOME_VALUE>` will fail. You instead need to "
"fully qualify the key name, and configure the value with "
"`child.a=<SOME_VALUE>`."
)
if "." in key:
scoped_component_name = key.split(".")[0]
if not (
scoped_component_name in instance.__component_fields__
and isinstance(
instance.__component_fields__[scoped_component_name], ComponentField
)
):
raise error
elif key not in instance.__component_fields__:
raise error
instance.__component_configured__ = True
if hasattr(instance.__class__, "__post_configure__"):
instance.__post_configure__()
def get_command(self, ctx, cmd_name):
cmd_name = convert_to_snake_case(cmd_name)
for c in self.list_commands(ctx):
if convert_to_snake_case(c) == cmd_name:
return click.Group.get_command(self, ctx, c)
return None