def _apply_connections_params(
cls,
connections: List[str],
init: List[V1Init],
artifact_store: str = None,
param_spec: Dict[str, ParamSpec] = None,
):
if connections:
connections = Parser.parse_section(connections,
param_spec=param_spec,
parse_params=True)
_init = []
if init:
for i in init:
if i.artifacts and not i.connection:
i.connection = artifact_store
resolved_i = V1Init.from_dict(
Parser.parse_section(i.to_dict(),
param_spec=param_spec,
parse_params=True))
_init.append(resolved_i)
# Prepend any param that has to_init after validation
init_params = [
v.to_init() for v in param_spec.values() if v.validate_to_init()
]
init_params = [v for v in init_params if v]
_init = init_params + _init
return _init, connections
def apply_run_connections_params(
cls,
config: V1CompiledOperation,
artifact_store: str = None,
contexts: Dict = None,
) -> V1CompiledOperation:
params = config.validate_params(is_template=False, check_runs=True)
params = {param.name: param for param in params}
params = cls._update_params_with_contexts(params, contexts)
if config.run.kind in {V1RunKind.JOB, V1RunKind.SERVICE}:
if config.run.connections:
config.run.connections = Parser.parse_section(
config.run.connections, params=params, parse_params=True)
if config.run.init:
init = []
for i in config.run.init:
if i.artifacts and not i.connection:
i.connection = artifact_store
resolved_i = V1Init.from_dict(
Parser.parse_section(i.to_dict(),
params=params,
parse_params=True))
init.append(resolved_i)
config.run.init = init
return config
def test_if_operator_config(self):
config_dict = {
"cond": "{{ i }} == 5",
"do": "It was True",
"elseDo": "It was False",
}
config = IfConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
assert "It was True" == config.parse(Parser(), {"i": 5})
assert "It was False" == config.parse(Parser(), {"i": 3})
def test_parse_base_expressions(self):
data = [
1,
10.0,
[1, 1],
(1, 1),
"string",
["str1", "str2"],
{1: 2, "a": "a", "dict": {1: 1}},
]
parser = Parser()
for d in data:
assert d == parser.parse_expression(d, {})
def parse_params(params, is_cli: bool = True):
if isinstance(params, Mapping):
return {
k: {
"value": Parser.parse_expression(v, {})
}
for k, v in params.items()
}
parsed_params = {}
for param in params:
index = param.find("=")
if index == -1:
message = (
"Invalid format for -P parameter: '%s'. Use -P name=value." %
param)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
name = param[:index]
value = param[index + 1:]
if name in parsed_params:
message = "Repeated parameter: '%s'" % name
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
parsed_params[name] = {"value": value}
return parsed_params
def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
def has_param(k: str):
if not compiled_operation.matrix:
return None
return not compiled_operation.matrix.has_param(k)
op_content = V1Operation.read(content)
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}),
context_only=has_param(k))
for (k, v) in suggestion.items()
}
op_spec = copy.deepcopy(op_content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
yield op_spec
def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
ops.append(op_spec)
return ops
def apply_run_contexts(cls, config: V1CompiledOperation, contexts=None):
if config.has_pipeline:
raise PolyaxonSchemaError(
"This method is not allowed on this specification.")
params = config.validate_params(is_template=False, check_runs=True)
params = {param.name: param for param in params}
params = cls._update_params_with_contexts(params, contexts)
parsed_data = Parser.parse_run(config.to_dict(), params)
return cls.CONFIG.read(parsed_data)
def apply_section_contexts(
cls,
config: V1CompiledOperation,
section,
contexts: Dict = None,
param_spec: Dict[str, ParamSpec] = None,
):
if not param_spec:
param_spec = cls.calculate_context_spec(config=config, contexts=contexts)
return Parser.parse_section(section, param_spec)
def as_arg(self):
if self.arg_format:
from polyaxon.polyaxonfile.specs.libs.parser import Parser
return (Parser.parse_expression(self.arg_format,
{self.name: self.param.value})
if self.param.value is not None else "")
if self.iotype == types.BOOL:
return "--{}".format(self.name) if self.param.value else ""
return ("--{}={}".format(self.name.replace("_", "-"), self.as_str())
if self.param.value is not None else "")
def _apply_runtime_contexts(
cls,
config: V1CompiledOperation,
contexts: Dict = None,
param_spec: Dict[str, ParamSpec] = None,
) -> V1CompiledOperation:
if not param_spec:
param_spec = cls.calculate_context_spec(
config=config, contexts=contexts, should_be_resolved=True
)
parsed_data = Parser.parse_runtime(config.to_dict(), param_spec)
return cls.CONFIG.read(parsed_data)
def apply_hooks_contexts(
cls,
config: V1CompiledOperation,
contexts: Dict = None,
param_spec: Dict[str, ParamSpec] = None,
) -> List[V1Hook]:
if not param_spec:
param_spec = cls.calculate_context_spec(config=config,
contexts=contexts,
should_be_resolved=True)
hooks = Parser.parse_hooks(config, param_spec)
return [V1Hook.read(hook) for hook in hooks]
def _apply_operation_contexts(
cls,
config: V1CompiledOperation,
param_spec: Dict[str, ParamSpec] = None,
contexts: Dict = None,
) -> V1CompiledOperation:
if not param_spec:
param_spec = cls.calculate_context_spec(config=config,
contexts=contexts)
parsed_data = Parser.parse_operation(config, param_spec or {})
return cls.CONFIG.read(parsed_data)
def test_for_operator_config(self):
config_dict = {"len": 5, "do": "Value at {{ i }}", "index": "i"}
config = ForConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
expected = [
"Value at 0",
"Value at 1",
"Value at 2",
"Value at 3",
"Value at 4",
]
assert expected == config.parse(Parser(), {})
config_dict = {
"len": 5,
"do": [
{"Conv2D": {"strides": ["{{ i }}", "{{ i }}"]}},
{"Pooling2D": {"strides": ["{{ i }}", "{{ i }}"]}},
],
"index": "i",
}
config = ForConfig.from_dict(config_dict)
assert config.to_dict() == config_dict
# Lists get flattened
expected = [
{"Conv2D": {"strides": [0, 0]}},
{"Pooling2D": {"strides": [0, 0]}},
{"Conv2D": {"strides": [1, 1]}},
{"Pooling2D": {"strides": [1, 1]}},
{"Conv2D": {"strides": [2, 2]}},
{"Pooling2D": {"strides": [2, 2]}},
{"Conv2D": {"strides": [3, 3]}},
{"Pooling2D": {"strides": [3, 3]}},
{"Conv2D": {"strides": [4, 4]}},
{"Pooling2D": {"strides": [4, 4]}},
]
assert expected == config.parse(Parser(), {})
def _apply_connections_params(
cls,
connections: List[str],
init: List[V1Init],
artifact_store: str = None,
param_spec: Dict[str, ParamSpec] = None,
):
if connections:
connections = Parser.parse_section(connections,
param_spec=param_spec,
parse_params=True)
_init = []
if init:
for i in init:
if i.artifacts and not i.connection:
i.connection = artifact_store
resolved_i = V1Init.from_dict(
Parser.parse_section(i.to_dict(),
param_spec=param_spec,
parse_params=True))
_init.append(resolved_i)
return _init, connections
def _apply_distributed_runtime_contexts(
cls,
config: V1CompiledOperation,
contexts: Dict = None,
param_spec: Dict[str, ParamSpec] = None,
) -> V1CompiledOperation:
if not param_spec:
# Calculate the param_space once with empty contexts
replica_param_spec = cls.calculate_context_spec(
config=config, contexts=None, should_be_resolved=True
)
param_spec = {}
for k in contexts:
param_spec[k] = copy.copy(replica_param_spec)
param_spec[k].update(cls.dict_to_param_spec(contexts=contexts[k]))
parsed_data = Parser.parse_distributed_runtime(config.to_dict(), param_spec)
return cls.CONFIG.read(parsed_data)
def get_ops_from_suggestions(
content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict]
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None # remove matrix
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
ops.append(op_spec)
return ops
def test_parse_context_expression(self):
parser = Parser()
assert parser.parse_expression("{{ something }}", {}) == ""
assert parser.parse_expression("{{ something }}",
{"something": 1}) == 1
def _parse(cls, config, params: Dict[str, ParamSpec]):
params = params or {}
parsed_data = Parser.parse(config, params)
return cls.CONFIG.read(parsed_data)
