def _apply_connections_params( cls, connections: List[str], init: List[V1Init], artifact_store: str = None, param_spec: Dict[str, ParamSpec] = None, ): if connections: connections = Parser.parse_section(connections, param_spec=param_spec, parse_params=True) _init = [] if init: for i in init: if i.artifacts and not i.connection: i.connection = artifact_store resolved_i = V1Init.from_dict( Parser.parse_section(i.to_dict(), param_spec=param_spec, parse_params=True)) _init.append(resolved_i) # Prepend any param that has to_init after validation init_params = [ v.to_init() for v in param_spec.values() if v.validate_to_init() ] init_params = [v for v in init_params if v] _init = init_params + _init return _init, connections
def apply_run_connections_params( cls, config: V1CompiledOperation, artifact_store: str = None, contexts: Dict = None, ) -> V1CompiledOperation: params = config.validate_params(is_template=False, check_runs=True) params = {param.name: param for param in params} params = cls._update_params_with_contexts(params, contexts) if config.run.kind in {V1RunKind.JOB, V1RunKind.SERVICE}: if config.run.connections: config.run.connections = Parser.parse_section( config.run.connections, params=params, parse_params=True) if config.run.init: init = [] for i in config.run.init: if i.artifacts and not i.connection: i.connection = artifact_store resolved_i = V1Init.from_dict( Parser.parse_section(i.to_dict(), params=params, parse_params=True)) init.append(resolved_i) config.run.init = init return config
def test_if_operator_config(self): config_dict = { "cond": "{{ i }} == 5", "do": "It was True", "elseDo": "It was False", } config = IfConfig.from_dict(config_dict) assert config.to_dict() == config_dict assert "It was True" == config.parse(Parser(), {"i": 5}) assert "It was False" == config.parse(Parser(), {"i": 3})
def test_parse_base_expressions(self): data = [ 1, 10.0, [1, 1], (1, 1), "string", ["str1", "str2"], {1: 2, "a": "a", "dict": {1: 1}}, ] parser = Parser() for d in data: assert d == parser.parse_expression(d, {})
def parse_params(params, is_cli: bool = True): if isinstance(params, Mapping): return { k: { "value": Parser.parse_expression(v, {}) } for k, v in params.items() } parsed_params = {} for param in params: index = param.find("=") if index == -1: message = ( "Invalid format for -P parameter: '%s'. Use -P name=value." % param) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) name = param[:index] value = param[index + 1:] if name in parsed_params: message = "Repeated parameter: '%s'" % name if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) parsed_params[name] = {"value": value} return parsed_params
def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict], ) -> List[V1Operation]: def has_param(k: str): if not compiled_operation.matrix: return None return not compiled_operation.matrix.has_param(k) op_content = V1Operation.read(content) for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {}), context_only=has_param(k)) for (k, v) in suggestion.items() } op_spec = copy.deepcopy(op_content) op_spec.matrix = None op_spec.conditions = None op_spec.schedule = None op_spec.events = None op_spec.dependencies = None op_spec.trigger = None op_spec.skip_on_upstream_skip = None op_spec.cache = compiled_operation.cache op_spec.queue = compiled_operation.queue op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs op_spec.component.contexts = compiled_operation.contexts yield op_spec
def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict], ) -> List[V1Operation]: ops = [] for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {})) for (k, v) in suggestion.items() } op_spec = V1Operation.read(content) op_spec.matrix = None op_spec.conditions = None op_spec.schedule = None op_spec.events = None op_spec.dependencies = None op_spec.trigger = None op_spec.skip_on_upstream_skip = None op_spec.cache = compiled_operation.cache op_spec.queue = compiled_operation.queue op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs op_spec.component.contexts = compiled_operation.contexts ops.append(op_spec) return ops
def apply_run_contexts(cls, config: V1CompiledOperation, contexts=None): if config.has_pipeline: raise PolyaxonSchemaError( "This method is not allowed on this specification.") params = config.validate_params(is_template=False, check_runs=True) params = {param.name: param for param in params} params = cls._update_params_with_contexts(params, contexts) parsed_data = Parser.parse_run(config.to_dict(), params) return cls.CONFIG.read(parsed_data)
def apply_section_contexts( cls, config: V1CompiledOperation, section, contexts: Dict = None, param_spec: Dict[str, ParamSpec] = None, ): if not param_spec: param_spec = cls.calculate_context_spec(config=config, contexts=contexts) return Parser.parse_section(section, param_spec)
def as_arg(self): if self.arg_format: from polyaxon.polyaxonfile.specs.libs.parser import Parser return (Parser.parse_expression(self.arg_format, {self.name: self.param.value}) if self.param.value is not None else "") if self.iotype == types.BOOL: return "--{}".format(self.name) if self.param.value else "" return ("--{}={}".format(self.name.replace("_", "-"), self.as_str()) if self.param.value is not None else "")
def _apply_runtime_contexts( cls, config: V1CompiledOperation, contexts: Dict = None, param_spec: Dict[str, ParamSpec] = None, ) -> V1CompiledOperation: if not param_spec: param_spec = cls.calculate_context_spec( config=config, contexts=contexts, should_be_resolved=True ) parsed_data = Parser.parse_runtime(config.to_dict(), param_spec) return cls.CONFIG.read(parsed_data)
def apply_hooks_contexts( cls, config: V1CompiledOperation, contexts: Dict = None, param_spec: Dict[str, ParamSpec] = None, ) -> List[V1Hook]: if not param_spec: param_spec = cls.calculate_context_spec(config=config, contexts=contexts, should_be_resolved=True) hooks = Parser.parse_hooks(config, param_spec) return [V1Hook.read(hook) for hook in hooks]
def _apply_operation_contexts( cls, config: V1CompiledOperation, param_spec: Dict[str, ParamSpec] = None, contexts: Dict = None, ) -> V1CompiledOperation: if not param_spec: param_spec = cls.calculate_context_spec(config=config, contexts=contexts) parsed_data = Parser.parse_operation(config, param_spec or {}) return cls.CONFIG.read(parsed_data)
def test_for_operator_config(self): config_dict = {"len": 5, "do": "Value at {{ i }}", "index": "i"} config = ForConfig.from_dict(config_dict) assert config.to_dict() == config_dict expected = [ "Value at 0", "Value at 1", "Value at 2", "Value at 3", "Value at 4", ] assert expected == config.parse(Parser(), {}) config_dict = { "len": 5, "do": [ {"Conv2D": {"strides": ["{{ i }}", "{{ i }}"]}}, {"Pooling2D": {"strides": ["{{ i }}", "{{ i }}"]}}, ], "index": "i", } config = ForConfig.from_dict(config_dict) assert config.to_dict() == config_dict # Lists get flattened expected = [ {"Conv2D": {"strides": [0, 0]}}, {"Pooling2D": {"strides": [0, 0]}}, {"Conv2D": {"strides": [1, 1]}}, {"Pooling2D": {"strides": [1, 1]}}, {"Conv2D": {"strides": [2, 2]}}, {"Pooling2D": {"strides": [2, 2]}}, {"Conv2D": {"strides": [3, 3]}}, {"Pooling2D": {"strides": [3, 3]}}, {"Conv2D": {"strides": [4, 4]}}, {"Pooling2D": {"strides": [4, 4]}}, ] assert expected == config.parse(Parser(), {})
def _apply_connections_params( cls, connections: List[str], init: List[V1Init], artifact_store: str = None, param_spec: Dict[str, ParamSpec] = None, ): if connections: connections = Parser.parse_section(connections, param_spec=param_spec, parse_params=True) _init = [] if init: for i in init: if i.artifacts and not i.connection: i.connection = artifact_store resolved_i = V1Init.from_dict( Parser.parse_section(i.to_dict(), param_spec=param_spec, parse_params=True)) _init.append(resolved_i) return _init, connections
def _apply_distributed_runtime_contexts( cls, config: V1CompiledOperation, contexts: Dict = None, param_spec: Dict[str, ParamSpec] = None, ) -> V1CompiledOperation: if not param_spec: # Calculate the param_space once with empty contexts replica_param_spec = cls.calculate_context_spec( config=config, contexts=None, should_be_resolved=True ) param_spec = {} for k in contexts: param_spec[k] = copy.copy(replica_param_spec) param_spec[k].update(cls.dict_to_param_spec(contexts=contexts[k])) parsed_data = Parser.parse_distributed_runtime(config.to_dict(), param_spec) return cls.CONFIG.read(parsed_data)
def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict] ) -> List[V1Operation]: ops = [] for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {})) for (k, v) in suggestion.items() } op_spec = V1Operation.read(content) op_spec.matrix = None # remove matrix op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs ops.append(op_spec) return ops
def test_parse_context_expression(self): parser = Parser() assert parser.parse_expression("{{ something }}", {}) == "" assert parser.parse_expression("{{ something }}", {"something": 1}) == 1
def _parse(cls, config, params: Dict[str, ParamSpec]): params = params or {} parsed_data = Parser.parse(config, params) return cls.CONFIG.read(parsed_data)