def __init__(self, pod_spec: V1PodSpec, primary_container_name: str):
if not pod_spec:
raise _user_exceptions.FlyteValidationException("A pod spec cannot be undefined")
if not primary_container_name:
raise _user_exceptions.FlyteValidationException("A primary container name cannot be undefined")
self._pod_spec = pod_spec
self._primary_container_name = primary_container_name
def __init__(
self,
task_function,
task_type,
discovery_version,
retries,
interruptible,
deprecated,
storage_request,
cpu_request,
gpu_request,
memory_request,
storage_limit,
cpu_limit,
gpu_limit,
memory_limit,
discoverable,
timeout,
environment,
pod_spec=None,
primary_container_name=None,
):
"""
:param _sdk_runnable.SdkRunnableTask sdk_runnable_task:
:param generated_pb2.PodSpec pod_spec:
:param Text primary_container_name:
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
if not pod_spec:
raise _user_exceptions.FlyteValidationException(
"A pod spec cannot be undefined")
if not primary_container_name:
raise _user_exceptions.FlyteValidationException(
"A primary container name cannot be undefined")
super(SdkSidecarTask, self).__init__(
task_function,
task_type,
discovery_version,
retries,
interruptible,
deprecated,
storage_request,
cpu_request,
gpu_request,
memory_request,
storage_limit,
cpu_limit,
gpu_limit,
memory_limit,
discoverable,
timeout,
environment,
custom=None,
)
self.reconcile_partial_pod_spec_and_task(pod_spec,
primary_container_name)
def to_flyte_idl(self):
"""
:rtype: flyteidl.plugins.spark_pb2.SparkJob
"""
if self.spark_type == _spark_type.PYTHON:
application_type = _spark_task.SparkApplication.PYTHON
elif self.spark_type == _spark_type.JAVA:
application_type = _spark_task.SparkApplication.JAVA
elif self.spark_type == _spark_type.SCALA:
application_type = _spark_task.SparkApplication.SCALA
elif self.spark_type == _spark_type.R:
application_type = _spark_task.SparkApplication.R
else:
raise _user_exceptions.FlyteValidationException(
"Invalid Spark Application Type Specified")
return _spark_task.SparkJob(
applicationType=application_type,
mainApplicationFile=self.application_file,
mainClass=self.main_class,
executorPath=self.executor_path,
sparkConf=self.spark_conf,
hadoopConf=self.hadoop_conf,
)
def _validate_outputs(self, outputs):
"""
This method should be overridden in sub-classes that intend to do additional checks on outputs. If validation
fails, this function should raise an informative exception.
:param dict[Text, flytekit.models.interface.Variable] outputs: Output variables to validate
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
for k, v in _six.iteritems(outputs):
if k in self.interface.outputs:
raise _user_exceptions.FlyteValidationException(
"An output with name '{}' is already defined. Redefinition is not allowed."
.format(k))
if k in self.interface.inputs:
raise _user_exceptions.FlyteValidationException(
"An input with name '{}' is already defined. Therefore '{}' can't be defined as an "
"input".format(k, v))
def create_launch_plan(self, *args, **kwargs):
# TODO: Correct after implementing new launch plan
assumable_iam_role = _auth_config.ASSUMABLE_IAM_ROLE.get()
kubernetes_service_account = _auth_config.KUBERNETES_SERVICE_ACCOUNT.get(
)
if not (assumable_iam_role or kubernetes_service_account):
raise _user_exceptions.FlyteValidationException(
"No assumable role or service account found")
auth_role = _common_models.AuthRole(
assumable_iam_role=assumable_iam_role,
kubernetes_service_account=kubernetes_service_account,
)
return SdkLaunchPlan(
workflow_id=self.id,
entity_metadata=_launch_plan_models.LaunchPlanMetadata(
schedule=_schedule_models.Schedule(""),
notifications=[],
),
default_inputs=_interface_models.ParameterMap({}),
fixed_inputs=_literal_models.LiteralMap(literals={}),
labels=_common_models.Labels({}),
annotations=_common_models.Annotations({}),
auth_role=auth_role,
raw_output_data_config=_common_models.RawOutputDataConfig(""),
)
def _validate_inputs(self, inputs):
"""
This method should be overridden in sub-classes that intend to do additional checks on inputs. If validation
fails, this function should raise an informative exception.
:param dict[Text, flytekit.models.interface.Variable] inputs: Input variables to validate
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
super(SdkRunnableTask, self)._validate_inputs(inputs)
for k, v in _six.iteritems(inputs):
if not self._is_argname_in_function_definition(k):
raise _user_exceptions.FlyteValidationException(
"The input named '{}' was not specified in the task function. Therefore, this input cannot be "
"provided to the task.".format(k))
if _type_helpers.get_sdk_type_from_literal_type(
v.type) in type(self)._banned_inputs:
raise _user_exceptions.FlyteValidationException(
"The input '{}' is not an accepted input type.".format(v))
def __init__(
self,
pod_spec: V1PodSpec,
primary_container_name: str,
labels: Dict[str, str] = None,
annotations: Dict[str, str] = None,
):
if not pod_spec:
raise _user_exceptions.FlyteValidationException(
"A pod spec cannot be undefined")
if not primary_container_name:
raise _user_exceptions.FlyteValidationException(
"A primary container name cannot be undefined")
self._pod_spec = pod_spec
self._primary_container_name = primary_container_name
self._labels = labels
self._annotations = annotations
def test_flyte_validation_error():
try:
raise user.FlyteValidationException(
"I validated that your stuff was wrong.")
except user.FlyteValidationException as e:
assert str(e) == "I validated that your stuff was wrong."
assert isinstance(e, AssertionError)
assert type(e).error_code == "USER:ValidationError"
assert isinstance(e, user.FlyteUserException)
def _validate_inputs(self, inputs):
"""
:param dict[Text, flytekit.models.interface.Variable] inputs: Input variables to validate
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
for k, v in _six.iteritems(inputs):
sdk_type = _helpers.get_sdk_type_from_literal_type(v.type)
if sdk_type not in input_types_supported:
raise _user_exceptions.FlyteValidationException(
"Input Type '{}' not supported. Only Primitives are supported for Scala/Java Spark."
.format(sdk_type))
super(SdkGenericSparkTask, self)._validate_inputs(inputs)
def _validate_outputs(self, outputs):
"""
:param dict[Text, flytekit.models.interface.Variable] inputs: Input variables to validate
:raises: flytekit.common.exceptions.user.FlyteValidationException
"""
# Add output_notebook as an implicit output to the task.
for k, v in _six.iteritems(outputs):
if k == OUTPUT_NOTEBOOK:
raise ValueError(
"{} is a reserved output keyword. Please use a different output name."
.format(OUTPUT_NOTEBOOK))
sdk_type = _type_helpers.get_sdk_type_from_literal_type(v.type)
if sdk_type not in _notebook_types_map.values():
raise _user_exceptions.FlyteValidationException(
"Output Type '{}' not supported. Only Primitives are supported for notebook."
.format(sdk_type))
super(SdkNotebookTask, self)._validate_outputs(outputs)