def test_job_refs_params(self):
config_dict = {
"inputs": [
{
"name": "param1",
"type": types.INT
},
{
"name": "param9",
"type": types.FLOAT
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
params = {
"param1": {
"ref": "job.A",
"value": "outputs.foo"
},
"param9": {
"value": 13.1
},
}
config = V1Component.from_dict(config_dict)
# Validation outside the context of a pipeline
with self.assertRaises(ValidationError):
ops_params.validate_params(params=params,
inputs=config.inputs,
outputs=None,
is_template=False)
def test_incomplete_params(self):
config_dict = {
"inputs": [
{
"name": "param1",
"type": types.INT
},
{
"name": "param2",
"type": types.INT
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": 1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
config_dict = {
"outputs": [
{
"name": "param1",
"type": types.INT,
"value": 12,
"isOptional": True
},
{
"name": "param2",
"type": types.INT
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
ops_params.validate_params(
params={"param1": {
"value": 1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
def test_extra_params(self):
# inputs
config_dict = {
"inputs": [{
"name": "param1",
"type": types.INT
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={
"param1": {
"value": 1
},
"param2": {
"value": 2
}
},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# outputs
config_dict = {
"outputs": [{
"name": "param1",
"type": types.INT
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={
"param1": {
"value": 1
},
"param2": {
"value": 2
}
},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
def validate_params(self,
params=None,
context=None,
is_template=True,
check_runs=False) -> List[ParamSpec]:
return validate_params(
inputs=self.inputs,
outputs=self.outputs,
params=params,
context=context,
is_template=is_template,
check_runs=check_runs,
)
def test_experiment_and_job_refs_params(self):
config_dict = {
"inputs": [
{
"name": "param1",
"type": types.INT
},
{
"name": "param2",
"type": types.FLOAT
},
{
"name": "param9",
"type": types.WASB
},
{
"name": "param11",
"type": types.METRIC
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
op = V1CompiledOperation.from_dict(config_dict)
params = {
"param1": {
"ref": "runs.64332180bfce46eba80a65caf73c5396",
"value": "outputs.foo",
},
"param2": {
"ref": "runs.0de53b5bf8b04a219d12a39c6b92bcce",
"value": "outputs.foo",
},
"param9": {
"value": "wasbs://[email protected]/"
},
"param11": {
"ref": "runs.fcc462d764104eb698d3cca509f34154",
"value": "outputs.accuracy",
},
}
validated_params = ops_params.validate_params(params=params,
inputs=op.inputs,
outputs=None,
is_template=False)
assert {p.name: p.param.to_dict() for p in validated_params} == params
def test_param_validation_with_mismatched_outputs(self):
config_dict = {
"outputs": [{
"name": "param1",
"type": types.INT
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
# Passing correct param
ops_params.validate_params(
params={"param1": {
"value": 1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# Passing wrong type
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": "text"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": 12.1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": {
"foo": "bar"
}
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": "gs://bucket/path/to/blob/"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
config_dict = {
"outputs": [{
"name": "param2",
"type": types.STR
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
# Passing correct param
ops_params.validate_params(
params={"param2": {
"value": "text"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# Passing wrong type
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param2": {
"value": 1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param2": {
"value": False
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param2": {
"value": {
"foo": "bar"
}
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param2": {
"value": ["gs://bucket/path/to/blob/"]
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
config_dict = {
"outputs": [{
"name": "param7",
"type": types.WASB
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
# Passing correct param
ops_params.validate_params(
params={
"param7": {
"value": "wasbs://[email protected]/"
}
},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# Passing wrong param
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param7": {
"value": "gs://bucket/path/to/blob/"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param7": {
"value": "s3://test/this/is/bad/key.txt"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param7": {
"value": 1
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
def test_param_validation_with_inputs(self):
config_dict = {
"inputs": [
{
"name": "param1",
"type": types.STR
},
{
"name": "param2",
"type": types.INT
},
{
"name": "param3",
"type": types.FLOAT
},
{
"name": "param4",
"type": types.BOOL
},
{
"name": "param5",
"type": types.DICT
},
{
"name": "param6",
"type": types.LIST
},
{
"name": "param7",
"type": types.GCS
},
{
"name": "param8",
"type": types.S3
},
{
"name": "param9",
"type": types.WASB
},
{
"name": "param10",
"type": types.PATH
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
component = V1Component.from_dict(config_dict)
params = {
"param1": {
"value": "text"
},
"param2": {
"value": 12
},
"param3": {
"value": 13.3
},
"param4": {
"value": False
},
"param5": {
"value": {
"foo": "bar"
}
},
"param6": {
"value": [1, 3, 45, 5]
},
"param7": {
"value": "gs://bucket/path/to/blob/"
},
"param8": {
"value": "s3://test/this/is/bad/key.txt"
},
"param9": {
"value": "wasbs://[email protected]/"
},
"param10": {
"value": "/foo/bar"
},
}
validated_params = ops_params.validate_params(params=params,
inputs=component.inputs,
outputs=None,
is_template=False)
assert params == {
p.name: {
"value": p.param.value
}
for p in validated_params
}
# Passing missing params
params.pop("param1")
params.pop("param2")
with self.assertRaises(ValidationError):
ops_params.validate_params(params=params,
inputs=component.inputs,
outputs=None,
is_template=False)
def test_required_input_no_param_only_validated_on_run(self):
# Inputs
config_dict = {
"inputs": [
{
"name": "param1",
"type": types.STR
},
{
"name": "param10",
"type": types.PATH
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
with self.assertRaises(ValidationError):
ops_params.validate_params(
params={"param1": {
"value": "text"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# Outputs
config_dict = {
"outputs": [
{
"name": "param1",
"type": types.STR
},
{
"name": "param10",
"type": types.PATH
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
ops_params.validate_params(
params={"param1": {
"value": "text"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
# IO
config_dict = {
"inputs": [{
"name": "param1",
"type": types.STR
}],
"outputs": [{
"name": "param10",
"type": types.PATH
}],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
config = V1Component.from_dict(config_dict)
ops_params.validate_params(
params={"param1": {
"value": "text"
}},
inputs=config.inputs,
outputs=config.outputs,
is_template=False,
)
def process_components(self, inputs=None):
inputs = inputs or []
for _input in inputs:
self._context["dag.inputs.{}".format(_input.name)] = _input
if not self.operations:
raise PolyaxonSchemaError(
"Pipeline is not valid, it has no ops to validate components."
)
components = self.components or []
for component in components:
component_name = component.name
if component_name in self._components_by_names:
raise PolyaxonSchemaError(
"Pipeline has multiple components with the same name `{}`".format(
component_name
)
)
self._components_by_names[component_name] = component
for op in self.operations:
op_name = op.name
if op.has_component_reference:
outputs = op.template.outputs
inputs = op.template.inputs
elif op.has_dag_reference:
component_ref_name = op.template.name
if op_name in self._op_component_mapping:
raise PolyaxonSchemaError(
"Pipeline has multiple ops with the same name `{}`".format(
op_name
)
)
if component_ref_name not in self._components_by_names:
raise PolyaxonSchemaError(
"Pipeline op with name `{}` requires a component with name `{}`, "
"which is not defined on this pipeline.".format(
op_name, component_ref_name
)
)
self._op_component_mapping[op_name] = component_ref_name
outputs = self._components_by_names[component_ref_name].outputs
inputs = self._components_by_names[component_ref_name].inputs
elif op.has_hub_reference:
continue
elif op.has_url_reference:
continue
elif op.has_path_reference:
continue
else:
raise PolyaxonSchemaError(
"Pipeline op has no template field `{}`".format(op_name)
)
if outputs:
for output in outputs:
self._context[
"ops.{}.outputs.{}".format(op_name, output.name)
] = output
if inputs:
for cinput in inputs:
self._context[
"ops.{}.inputs.{}".format(op_name, cinput.name)
] = cinput
# We allow to resolve name, status, project, all outputs/inputs, iteration
self._context["ops.{}.inputs".format(op_name)] = V1IO(
name="inputs", iotype=types.DICT, value={}, is_optional=True
)
self._context["ops.{}.outputs".format(op_name)] = V1IO(
name="outputs", iotype=types.DICT, value={}, is_optional=True
)
self._context["ops.{}.status".format(op_name)] = V1IO(
name="status", iotype=types.STR, value="", is_optional=True
)
self._context["ops.{}.name".format(op_name)] = V1IO(
name="name", iotype=types.STR, value="", is_optional=True
)
self._context["ops.{}.uuid".format(op_name)] = V1IO(
name="uuid", iotype=types.STR, value="", is_optional=True
)
self._context["ops.{}.project_name".format(op_name)] = V1IO(
name="project_name", iotype=types.STR, value="", is_optional=True
)
self._context["ops.{}.project_uuid".format(op_name)] = V1IO(
name="project_uuid", iotype=types.STR, value="", is_optional=True
)
self._context["ops.{}.iteration".format(op_name)] = V1IO(
name="iteration", iotype=types.STR, value="", is_optional=True
)
for op in self.operations:
if op.has_component_reference:
component_ref = op.template.name
outputs = op.template.outputs
inputs = op.template.inputs
elif op.has_hub_reference:
component_ref = op.template.name
continue
elif op.has_url_reference:
component_ref = op.template.url
continue
elif op.has_path_reference:
component_ref = op.template.path
continue
elif op.has_dag_reference:
component_ref = op.template.name
outputs = self._components_by_names[component_ref].outputs
inputs = self._components_by_names[component_ref].inputs
else:
raise PolyaxonSchemaError(
"Pipeline op has no template field `{}`".format(op.name)
)
ops_params.validate_params(
params=op.params,
inputs=inputs,
outputs=outputs,
context=self._context,
is_template=False,
check_runs=False,
extra_info="<op {}>.<component {}>".format(op.name, component_ref),
)
def test_param_validation_with_outputs(self):
config_dict = {
"outputs": [
{
"name": "param1",
"type": types.STR
},
{
"name": "param2",
"type": types.INT
},
{
"name": "param3",
"type": types.FLOAT
},
{
"name": "param4",
"type": types.BOOL
},
{
"name": "param5",
"type": types.DICT
},
{
"name": "param6",
"type": types.LIST
},
{
"name": "param7",
"type": types.GCS
},
{
"name": "param8",
"type": types.S3
},
{
"name": "param9",
"type": types.WASB
},
{
"name": "param10",
"type": types.PATH
},
{
"name": "param11",
"type": types.METRIC
},
{
"name": "param12",
"type": types.METADATA
},
{
"name": "param13",
"type": types.METADATA
},
{
"name": "param14",
"type": types.METADATA
},
],
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
}
},
}
op = V1CompiledOperation.from_dict(config_dict)
params = {
"param1": {
"value": "text"
},
"param2": {
"value": 12
},
"param3": {
"value": 13.3
},
"param4": {
"value": False
},
"param5": {
"value": {
"foo": "bar"
}
},
"param6": {
"value": [1, 3, 45, 5]
},
"param7": {
"value": "gs://bucket/path/to/blob/"
},
"param8": {
"value": "s3://test/this/is/bad/key.txt"
},
"param9": {
"value": "wasbs://[email protected]/"
},
"param10": {
"value": "/foo/bar"
},
"param11": {
"value": 124.4
},
"param12": {
"value": {
"foo": 124.4
}
},
"param13": {
"value": {
"foo": "bar"
}
},
"param14": {
"value": {
"foo": ["foo", 124.4]
}
},
}
validated_params = ops_params.validate_params(params=params,
inputs=None,
outputs=op.outputs,
is_template=False)
assert params == {p.name: p.param.to_dict() for p in validated_params}
# Passing missing params
params.pop("param1")
params.pop("param2")
validated_params = ops_params.validate_params(params=params,
inputs=None,
outputs=op.outputs,
is_template=False)
params["param1"] = {"value": None}
params["param2"] = {"value": None}
assert params == {p.name: p.param.to_dict() for p in validated_params}