def __init__(
self,
task_type: str,
name: str,
task_config: T,
interface: Optional[Interface] = None,
environment: Optional[Dict[str, str]] = None,
**kwargs,
):
"""
Args:
task_type: a string that defines a unique task-type for every new extension. If a backend plugin is required
then this has to be done in-concert with the backend plugin identifier
name: A unique name for the task instantiation. This is unique for every instance of task.
task_config: Configuration for the task. This is used to configure the specific plugin that handles this task
interface: A python native typed interface ``(inputs) -> outputs`` that declares the signature of the task
environment: Any environment variables that should be supplied during the execution of the task. Supplied as
a dictionary of key/value pairs
"""
super().__init__(
task_type=task_type,
name=name,
interface=transform_interface_to_typed_interface(interface),
**kwargs,
)
self._python_interface = interface if interface else Interface()
self._environment = environment if environment else {}
self._task_config = task_config
def __init__(
self,
name: str,
image: str,
command: List[str],
inputs: Optional[Dict[str, Type]] = None,
metadata: Optional[TaskMetadata] = None,
arguments: List[str] = None,
outputs: Dict[str, Type] = None,
requests: Optional[Resources] = None,
limits: Optional[Resources] = None,
input_data_dir: str = None,
output_data_dir: str = None,
metadata_format: MetadataFormat = MetadataFormat.JSON,
io_strategy: IOStrategy = None,
**kwargs,
):
super().__init__(
task_type="raw-container",
name=name,
interface=Interface(inputs, outputs),
metadata=metadata,
task_config=None,
**kwargs,
)
self._image = image
self._cmd = command
self._args = arguments
self._input_data_dir = input_data_dir
self._output_data_dir = output_data_dir
self._md_format = metadata_format
self._io_strategy = io_strategy
self._resources = ResourceSpec(
requests=requests if requests else Resources(), limits=limits if limits else Resources()
)
def create(
cls,
name: str,
workflow: _annotated_workflow.WorkflowBase,
default_inputs: Dict[str, Any] = None,
fixed_inputs: Dict[str, Any] = None,
schedule: _schedule_model.Schedule = None,
notifications: List[_common_models.Notification] = None,
auth_role: _common_models.AuthRole = None,
) -> LaunchPlan:
ctx = FlyteContextManager.current_context()
default_inputs = default_inputs or {}
fixed_inputs = fixed_inputs or {}
# Default inputs come from two places, the original signature of the workflow function, and the default_inputs
# argument to this function. We'll take the latter as having higher precedence.
wf_signature_parameters = transform_inputs_to_parameters(
ctx, workflow.python_interface)
# Construct a new Interface object with just the default inputs given to get Parameters, maybe there's an
# easier way to do this, think about it later.
temp_inputs = {}
for k, v in default_inputs.items():
temp_inputs[k] = (workflow.python_interface.inputs[k], v)
temp_interface = Interface(inputs=temp_inputs, outputs={})
temp_signature = transform_inputs_to_parameters(ctx, temp_interface)
wf_signature_parameters._parameters.update(temp_signature.parameters)
# These are fixed inputs that cannot change at launch time. If the same argument is also in default inputs,
# it'll be taken out from defaults in the LaunchPlan constructor
fixed_literals = translate_inputs_to_literals(
ctx,
incoming_values=fixed_inputs,
flyte_interface_types=workflow.interface.inputs,
native_types=workflow.python_interface.inputs,
)
fixed_lm = _literal_models.LiteralMap(literals=fixed_literals)
lp = cls(
name=name,
workflow=workflow,
parameters=wf_signature_parameters,
fixed_inputs=fixed_lm,
schedule=schedule,
notifications=notifications,
auth_role=auth_role,
)
# This is just a convenience - we'll need the fixed inputs LiteralMap for when serializing the Launch Plan out
# to protobuf, but for local execution and such, why not save the original Python native values as well so
# we don't have to reverse it back every time.
default_inputs.update(fixed_inputs)
lp._saved_inputs = default_inputs
if name in cls.CACHE:
raise AssertionError(
f"Launch plan named {name} was already created! Make sure your names are unique."
)
cls.CACHE[name] = lp
return lp
def __init__(
self, task_type: str, name: str, task_config: T, interface: Optional[Interface] = None, **kwargs,
):
super().__init__(
task_type=task_type, name=name, interface=transform_interface_to_typed_interface(interface), **kwargs
)
self._python_interface = interface if interface else Interface()
self._environment = kwargs.get("environment", {})
self._task_config = task_config
def __init__(
self,
reference: Union[WorkflowReference, TaskReference,
LaunchPlanReference],
inputs: Optional[Dict[str, Union[Type[Any], Tuple[Type[Any], Any]]]],
outputs: Dict[str, Type],
):
if (not isinstance(reference, WorkflowReference)
and not isinstance(reference, TaskReference)
and not isinstance(reference, LaunchPlanReference)):
raise Exception("Must be one of task, workflow, or launch plan")
self._reference = reference
self._native_interface = Interface(inputs=inputs, outputs=outputs)
self._interface = transform_interface_to_typed_interface(
self._native_interface)
def __init__(
self,
name: str,
failure_policy: Optional[WorkflowFailurePolicy] = None,
interruptible: Optional[bool] = False,
):
metadata = WorkflowMetadata(on_failure=failure_policy or WorkflowFailurePolicy.FAIL_IMMEDIATELY)
workflow_metadata_defaults = WorkflowMetadataDefaults(interruptible)
self._compilation_state = CompilationState(prefix="")
self._inputs = {}
self._unbound_inputs = set()
super().__init__(
name=name,
workflow_metadata=metadata,
workflow_metadata_defaults=workflow_metadata_defaults,
python_interface=Interface(),
)
def test_create_native_named_tuple():
ctx = FlyteContextManager.current_context()
t = create_native_named_tuple(ctx,
promises=None,
entity_interface=Interface())
assert t is None
p1 = Promise(var="x",
val=TypeEngine.to_literal(
ctx, 1, int, LiteralType(simple=SimpleType.INTEGER)))
p2 = Promise(var="y",
val=TypeEngine.to_literal(
ctx, 2, int, LiteralType(simple=SimpleType.INTEGER)))
t = create_native_named_tuple(
ctx, promises=p1, entity_interface=Interface(outputs={"x": int}))
assert t
assert t == 1
t = create_native_named_tuple(ctx,
promises=[],
entity_interface=Interface())
assert t is None
t = create_native_named_tuple(ctx,
promises=[p1, p2],
entity_interface=Interface(outputs={
"x": int,
"y": int
}))
assert t
assert t == (1, 2)
t = create_native_named_tuple(ctx,
promises=[p1, p2],
entity_interface=Interface(
outputs={
"x": int,
"y": int
},
output_tuple_name="Tup"))
assert t
assert t == (1, 2)
assert t.__class__.__name__ == "Tup"
with pytest.raises(KeyError):
create_native_named_tuple(ctx,
promises=[p1, p2],
entity_interface=Interface(
outputs={"x": int},
output_tuple_name="Tup"))
def __init__(
self,
name: str,
query_template: str,
task_type="sql_task",
inputs: Optional[Dict[str, Type]] = None,
metadata: Optional[TaskMetadata] = None,
task_config: Optional[T] = None,
outputs: Dict[str, Type] = None,
**kwargs,
):
super().__init__(
task_type=task_type,
name=name,
interface=Interface(inputs=inputs or {}, outputs=outputs or {}),
metadata=metadata,
task_config=task_config,
**kwargs,
)
self._query_template = query_template
def __init__(
self,
name: str,
failure_policy: Optional[WorkflowFailurePolicy] = None,
interruptible: bool = False,
):
metadata = WorkflowMetadata(on_failure=failure_policy
or WorkflowFailurePolicy.FAIL_IMMEDIATELY)
workflow_metadata_defaults = WorkflowMetadataDefaults(interruptible)
self._compilation_state = CompilationState(prefix="")
self._inputs = {}
# This unbound inputs construct is just here to help workflow authors detect issues a bit earlier. It just
# keeps track of workflow inputs that you've declared with add_workflow_input but haven't yet consumed. This
# is an error that Admin would return at compile time anyways, but this allows flytekit to raise
# the error earlier.
self._unbound_inputs = set()
super().__init__(
name=name,
workflow_metadata=metadata,
workflow_metadata_defaults=workflow_metadata_defaults,
python_interface=Interface(),
)
def __init__(
self,
name: str,
query_template: str,
task_type="sql_task",
inputs: Optional[Dict[str, Type]] = None,
metadata: Optional[TaskMetadata] = None,
task_config: Optional[T] = None,
outputs: Dict[str, Type] = None,
**kwargs,
):
"""
This SQLTask should mostly just be used as a base class for other SQL task types and should not be used
directly. See :py:class:`flytekit.extras.sqlite3.task.SQLite3Task`
"""
super().__init__(
task_type=task_type,
name=name,
interface=Interface(inputs=inputs or {}, outputs=outputs or {}),
metadata=metadata,
task_config=task_config,
**kwargs,
)
self._query_template = query_template
def reset_interface(self, inputs: Dict[str, Type], outputs: Dict[str, Type]):
self._interface = Interface(inputs=inputs, outputs=outputs)
self._typed_interface = transform_interface_to_typed_interface(self._interface)
def __init__(
self,
name: str,
debug: bool = False,
script: typing.Optional[str] = None,
script_file: typing.Optional[str] = None,
task_config: T = None,
inputs: typing.Optional[typing.Dict[str, typing.Type]] = None,
output_locs: typing.Optional[typing.List[OutputLocation]] = None,
**kwargs,
):
"""
Args:
name: str Name of the Task. Should be unique in the project
debug: bool Print the generated script and other debugging information
script: The actual script specified as a string
script_file: A path to the file that contains the script (Only script or script_file) can be provided
task_config: T Configuration for the task, can be either a Pod (or coming soon, BatchJob) config
inputs: A Dictionary of input names to types
output_locs: A list of :py:class:`OutputLocations`
**kwargs: Other arguments that can be passed to :ref:class:`PythonInstanceTask`
"""
if script and script_file:
raise ValueError(
"Only either of script or script_file can be provided")
if not script and not script_file:
raise ValueError("Either a script or script_file is needed")
if script_file:
if not os.path.exists(script_file):
raise ValueError(
f"FileNotFound: the specified Script file at path {script_file} cannot be loaded"
)
script_file = os.path.abspath(script_file)
if task_config is not None:
if str(type(task_config)) != "flytekitplugins.pod.task.Pod":
raise ValueError(
"TaskConfig can either be empty - indicating simple container task or a PodConfig."
)
# Each instance of NotebookTask instantiates an underlying task with a dummy function that will only be used
# to run pre- and post- execute functions using the corresponding task plugin.
# We rename the function name here to ensure the generated task has a unique name and avoid duplicate task name
# errors.
# This seem like a hack. We should use a plugin_class that doesn't require a fake-function to make work.
plugin_class = TaskPlugins.find_pythontask_plugin(type(task_config))
self._config_task_instance = plugin_class(
task_config=task_config, task_function=_dummy_task_func)
# Rename the internal task so that there are no conflicts at serialization time. Technically these internal
# tasks should not be serialized at all, but we don't currently have a mechanism for skipping Flyte entities
# at serialization time.
self._config_task_instance._name = f"_bash.{name}"
self._script = script
self._script_file = script_file
self._debug = debug
self._output_locs = output_locs if output_locs else []
self._interpolizer = _PythonFStringInterpolizer()
outputs = self._validate_output_locs()
super().__init__(
name,
task_config,
task_type=self._config_task_instance.task_type,
interface=Interface(inputs=inputs, outputs=outputs),
**kwargs,
)
