def parse_param(k, v) -> V1Param:
if not isinstance(v, V1Param):
v = V1Param.read(v, config_type=".yaml")
if v and k in contexts_by_keys:
v.context_only = True
if v.to_init is None:
v.to_init = contexts_by_keys[k].to_init
if v.connection is None:
v.connection = contexts_by_keys[k].connection
return v
def parse_param(v) -> V1Param:
if isinstance(v, V1Param):
return v
return V1Param.read(v, config_type=".yaml")
def validate_params(
params: Dict[str, Union[Dict, V1Param]],
inputs: List[V1IO],
outputs: List[V1IO],
parallel: V1Parallel = None,
context: Dict = None,
is_template: bool = True,
check_runs: bool = False,
extra_info: str = None,
) -> List[ParamSpec]:
"""
Validates Params given inputs, and an optional context.
Params can be:
* plain values: we check them against the inputs types
* job/experiment references: We postpone to server side validation.
* ops reference: in that case a context must be provided to validate that the reference exists.
and types are correct.
"""
if requires_params(inputs, outputs):
if not is_template and not params and not parallel:
message = ("The Polyaxonfile has non optional inputs/outputs, "
"you need to pass valid params.")
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
elif not accepts_params(inputs, outputs) and params:
message = "Received unexpected params `{}`.".format(params)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
def parse_param(v) -> V1Param:
if isinstance(v, V1Param):
return v
return V1Param.read(v, config_type=".yaml")
def validate_parallel(io: V1IO) -> bool:
if isinstance(parallel, V1Mapping):
return parallel.has_key(io.name)
elif io.name in parallel.params:
parallel.params[io.name].validate_io(io)
return True
return False
params = params or {}
params = {k: parse_param(params[k]) for k in params}
parallel = parallel or {}
inputs = inputs or []
outputs = outputs or []
processed_params = []
validated_params = []
for inp in inputs:
if inp.name in params:
param_value = params[inp.name]
param_spec = param_value.get_spec(
name=inp.name,
iotype=inp.iotype,
is_flag=inp.is_flag,
is_list=inp.is_list,
)
if param_spec.param.is_ref:
param_spec.validate_ref(context, is_template, check_runs)
else: # Plain value
inp.validate_value(param_value.value)
validated_params.append(param_spec)
processed_params.append(inp.name)
elif parallel and validate_parallel(inp):
pass
elif not inp.is_optional and not is_template:
message = "Input {} is required, no param was passed.".format(
inp.name)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
else:
validated_params.append(
ParamSpec(
name=inp.name,
param=V1Param(value=inp.value),
iotype=inp.iotype,
is_flag=inp.is_flag,
is_list=inp.is_list,
))
for out in outputs:
if out.name in params:
param_value = params[out.name]
param_spec = param_value.get_spec(
name=out.name,
iotype=out.iotype,
is_flag=out.is_flag,
is_list=out.is_list,
)
validated_params.append(param_spec)
if param_spec.param.is_ref:
param_spec.validate_ref(None,
is_template=False,
check_runs=check_runs)
else: # Plain value
out.validate_value(param_value.value)
validated_params.append(param_spec)
processed_params.append(out.name)
# No validation for outputs we assume that the op might populate a context or send a metric
else:
validated_params.append(
ParamSpec(
name=out.name,
param=V1Param(value=out.value),
iotype=out.iotype,
is_flag=out.is_flag,
is_list=out.is_list,
))
extra_params = set(params.keys()) - set(processed_params)
if extra_params:
message = "Received unexpected params `{}`.".format(extra_params)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
return validated_params
def validate_params(
params: Dict[str, Union[Dict, V1Param]],
inputs: List[V1IO],
outputs: List[V1IO],
contexts: List[V1IO] = None,
matrix: V1Matrix = None,
context: Dict = None,
is_template: bool = True,
check_runs: bool = False,
extra_info: str = None,
parse_values: bool = False,
) -> List[ParamSpec]:
"""
Validates Params given inputs, and an optional context.
Params can be:
* plain values: we check them against the inputs types
* job/experiment references: We postpone to server side validation.
* ops reference: in that case a context must be provided to validate that the reference exists.
and types are correct.
"""
contexts_by_keys = {k.name: k for k in contexts or []}
def parse_param(k, v) -> V1Param:
if not isinstance(v, V1Param):
v = V1Param.read(v, config_type=".yaml")
if v and k in contexts_by_keys:
v.context_only = True
if v.to_init is None:
v.to_init = contexts_by_keys[k].to_init
if v.connection is None:
v.connection = contexts_by_keys[k].connection
return v
def validate_matrix(io: V1IO) -> bool:
if isinstance(matrix, V1Mapping):
return matrix.has_key(io.name) # noqa
elif io.name in matrix.params:
matrix.params[io.name].validate_io(io)
return True
elif hasattr(matrix, "resource") and io.name == matrix.resource.name:
return True
return False
params = params or {}
params = {k: parse_param(k, params[k]) for k in params}
if requires_params(inputs, outputs):
if not is_template and not params and not matrix:
message = ("The Polyaxonfile has non optional inputs/outputs, "
"you need to pass valid params")
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
elif not accepts_params(inputs, outputs) and params:
extra_params = set(
params.keys()) - {p
for p in params if params[p].context_only}
if extra_params:
message = "Received unexpected params `{}`".format(extra_params)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
matrix = matrix or {}
inputs = inputs or []
outputs = outputs or []
processed_params = []
validated_params = []
for inp in inputs:
if inp.name in params:
param_value = params[inp.name]
param_spec = param_value.get_spec(
name=inp.name,
iotype=inp.iotype,
is_flag=inp.is_flag,
is_list=inp.is_list,
is_context=False,
arg_format=inp.arg_format,
)
if param_spec.param.is_ref:
param_spec.validate_ref(context, is_template, check_runs)
else: # Plain value
parsed_value = inp.validate_value(param_value.value)
if parse_values:
param_spec.param.value = parsed_value
if not param_spec.param.connection and param_value.connection:
param_spec.param.connection = inp.connection
if not param_spec.param.to_init and param_value.to_init:
param_spec.param.to_init = inp.to_init
validated_params.append(param_spec)
if not param_spec.param.context_only:
processed_params.append(inp.name)
elif matrix and validate_matrix(inp):
pass
elif not inp.is_optional and not is_template:
message = "Input {} is required, no param was passed.".format(
inp.name)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
else:
validated_params.append(
ParamSpec(
name=inp.name,
param=V1Param(value=inp.value,
connection=inp.connection,
to_init=inp.to_init),
iotype=inp.iotype,
is_flag=inp.is_flag,
is_list=inp.is_list,
is_context=False,
arg_format=inp.arg_format,
))
for out in outputs:
if out.name in params:
param_value = params[out.name]
param_spec = param_value.get_spec(
name=out.name,
iotype=out.iotype,
is_flag=out.is_flag,
is_list=out.is_list,
is_context=False,
arg_format=out.arg_format,
)
if param_spec.param.is_ref:
param_spec.validate_ref(None,
is_template=False,
check_runs=check_runs)
else: # Plain value
parsed_value = out.validate_value(param_value.value)
if parse_values:
param_spec.param.value = parsed_value
if not param_spec.param.connection and param_value.connection:
param_spec.param.connection = out.connection
if not param_spec.param.to_init and param_value.to_init:
param_spec.param.to_init = out.to_init
validated_params.append(param_spec)
if not param_spec.param.context_only:
processed_params.append(out.name)
# No validation for outputs we assume that the op might populate a context or send a metric
else:
validated_params.append(
ParamSpec(
name=out.name,
param=V1Param(value=out.value,
connection=out.connection,
to_init=out.to_init),
iotype=out.iotype,
is_flag=out.is_flag,
is_list=out.is_list,
is_context=False,
arg_format=out.arg_format,
))
extra_params = set(params.keys()) - set(processed_params)
context_params = {p for p in params if params[p].context_only}
extra_invalid_params = extra_params - context_params
if extra_invalid_params:
message = "Received unexpected params `{}`".format(
extra_invalid_params)
if extra_info:
message += " Please check: {}".format(extra_info)
raise ValidationError(message)
# Add all (extra) context params that were not processed during the IO check
for p in contexts_by_keys:
if p in extra_params:
param_value = params[p]
param_spec = param_value.get_spec(
name=p,
iotype=None,
is_flag=None,
is_list=None,
is_context=True,
arg_format=None,
)
validated_params.append(param_spec)
else:
context_io = contexts_by_keys[p]
validated_params.append(
ParamSpec(
name=p,
param=V1Param(
value=context_io.value,
connection=context_io.connection,
to_init=context_io.to_init,
),
iotype=None,
is_flag=None,
is_list=None,
is_context=True,
arg_format=None,
))
return validated_params