def _build_resolver_for_latest_model_blessing(
self,
model_blessing_channel_key: str) -> pipeline_pb2.PipelineTaskSpec:
"""Builds the resolver spec for latest valid ModelBlessing artifact."""
# 1. Build the task info.
result = pipeline_pb2.PipelineTaskSpec()
name = '{}{}'.format(self._name, _MODEL_BLESSING_RESOLVER_SUFFIX)
result.task_info.CopyFrom(pipeline_pb2.PipelineTaskInfo(name=name))
executor_label = _EXECUTOR_LABEL_PATTERN.format(name)
result.executor_label = executor_label
# 2. Specify the outputs of the task.
result.outputs.artifacts[model_blessing_channel_key].CopyFrom(
compiler_utils.build_output_artifact_spec(
self._outputs[model_blessing_channel_key]))
# 3. Build the resolver executor spec for latest valid ModelBlessing.
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
artifact_queries = {}
query_filter = ("artifact_type='{type}' and state={state}"
" and custom_properties['{key}']='{value}'").format(
type=compiler_utils.get_artifact_title(
standard_artifacts.ModelBlessing),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE),
key=constants.ARTIFACT_PROPERTY_BLESSED_KEY,
value=constants.BLESSED_VALUE)
artifact_queries[
model_blessing_channel_key] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
executor.resolver.CopyFrom(
ResolverSpec(output_artifact_queries=artifact_queries))
self._deployment_config.executors[executor_label].CopyFrom(executor)
return result
def _build_latest_artifact_resolver(
self) -> List[pipeline_pb2.PipelineTaskSpec]:
"""Builds a resolver spec for a latest artifact resolver.
Returns:
A list of two PipelineTaskSpecs. One represents the query for latest valid
ModelBlessing artifact. Another one represents the query for latest
blessed Model artifact.
Raises:
ValueError: when desired_num_of_artifacts != 1. 1 is the only supported
value currently.
"""
task_spec = pipeline_pb2.PipelineTaskSpec()
task_spec.task_info.CopyFrom(
pipeline_pb2.PipelineTaskInfo(name=self._name))
executor_label = _EXECUTOR_LABEL_PATTERN.format(self._name)
task_spec.executor_label = executor_label
# Fetch the init kwargs for the resolver.
resolver_config = self._exec_properties[resolver.RESOLVER_CONFIG]
if (isinstance(resolver_config, dict)
and resolver_config.get('desired_num_of_artifacts', 0) > 1):
raise ValueError(
'Only desired_num_of_artifacts=1 is supported currently.'
' Got {}'.format(
resolver_config.get('desired_num_of_artifacts')))
# Specify the outputs of the task.
for name, output_channel in self._outputs.items():
# Currently, we're working under the assumption that for tasks
# (those generated by BaseComponent), each channel contains a single
# artifact.
output_artifact_spec = compiler_utils.build_output_artifact_spec(
output_channel)
task_spec.outputs.artifacts[name].CopyFrom(output_artifact_spec)
# Specify the input parameters of the task.
for k, v in compiler_utils.build_input_parameter_spec(
self._exec_properties).items():
task_spec.inputs.parameters[k].CopyFrom(v)
artifact_queries = {}
# Buid the artifact query for each channel in the input dict.
for name, c in self._inputs.items():
query_filter = ("artifact_type='{type}' and state={state}").format(
type=compiler_utils.get_artifact_title(c.type),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE))
artifact_queries[name] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
resolver_spec = ResolverSpec(output_artifact_queries=artifact_queries)
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
executor.resolver.CopyFrom(resolver_spec)
self._deployment_config.executors[executor_label].CopyFrom(executor)
return [task_spec]
def _build_resolver_for_latest_blessed_model(
self, model_channel_key: str, model_blessing_resolver_name: str,
model_blessing_channel_key: str) -> pipeline_pb2.PipelineTaskSpec:
"""Builds the resolver spec for latest blessed Model artifact."""
name = '{}{}'.format(self._name, _MODEL_RESOLVER_SUFFIX)
# Component def.
component_def = pipeline_pb2.ComponentSpec()
executor_label = _EXECUTOR_LABEL_PATTERN.format(name)
component_def.executor_label = executor_label
input_artifact_spec = compiler_utils.build_input_artifact_spec(
self._outputs[model_blessing_channel_key])
component_def.input_definitions.artifacts[
_MODEL_RESOLVER_INPUT_KEY].CopyFrom(input_artifact_spec)
output_artifact_spec = compiler_utils.build_output_artifact_spec(
self._outputs[model_channel_key])
component_def.output_definitions.artifacts[model_channel_key].CopyFrom(
output_artifact_spec)
self._component_defs[name] = component_def
# Task spec.
task_spec = pipeline_pb2.PipelineTaskSpec()
task_spec.task_info.name = name
task_spec.component_ref.name = name
input_artifact_spec = pipeline_pb2.TaskInputsSpec.InputArtifactSpec()
input_artifact_spec.task_output_artifact.producer_task = model_blessing_resolver_name
input_artifact_spec.task_output_artifact.output_artifact_key = model_blessing_channel_key
task_spec.inputs.artifacts[_MODEL_RESOLVER_INPUT_KEY].CopyFrom(
input_artifact_spec)
# Resolver executor spec.
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
artifact_queries = {}
query_filter = (
'schema_title="{type}" AND '
'state={state} AND '
'name="{{{{$.inputs.artifacts[\'{input_key}\']'
'.metadata[\'{property_key}\']}}}}"').format(
type=compiler_utils.get_artifact_title(standard_artifacts.Model),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE),
input_key=_MODEL_RESOLVER_INPUT_KEY,
property_key=constants.ARTIFACT_PROPERTY_CURRENT_MODEL_ID_KEY)
artifact_queries[model_channel_key] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
executor.resolver.CopyFrom(
ResolverSpec(output_artifact_queries=artifact_queries))
self._deployment_config.executors[executor_label].CopyFrom(executor)
return task_spec
def _build_resolver_for_latest_blessed_model(
self, model_channel_key: str, model_blessing_resolver_name: str,
model_blessing_channel_key: str) -> pipeline_pb2.PipelineTaskSpec:
"""Builds the resolver spec for latest blessed Model artifact."""
# 1. Build the task info.
result = pipeline_pb2.PipelineTaskSpec()
name = '{}{}'.format(self._name, _MODEL_RESOLVER_SUFFIX)
result.task_info.CopyFrom(pipeline_pb2.PipelineTaskInfo(name=name))
executor_label = _EXECUTOR_LABEL_PATTERN.format(name)
result.executor_label = executor_label
# 2. Specify the input of the task. The output from model_blessing_resolver
# will be used as the input.
input_artifact_spec = pipeline_pb2.TaskInputsSpec.InputArtifactSpec(
producer_task=model_blessing_resolver_name,
output_artifact_key=model_blessing_channel_key)
result.inputs.artifacts[_MODEL_RESOLVER_INPUT_KEY].CopyFrom(
input_artifact_spec)
# 3. Specify the outputs of the task. model_resolver has one output for
# the latest blessed model.
result.outputs.artifacts[model_channel_key].CopyFrom(
compiler_utils.build_output_artifact_spec(
self._outputs[model_channel_key]))
# 4. Build the resolver executor spec for latest blessed Model.
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
artifact_queries = {}
query_filter = (
"artifact_type='{type}' and "
"state={state} and name={{$.inputs.artifacts['{input_key}']"
".custom_properties['{property_key}']}}").format(
type=compiler_utils.get_artifact_title(
standard_artifacts.Model),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE),
input_key=_MODEL_RESOLVER_INPUT_KEY,
property_key=constants.ARTIFACT_PROPERTY_CURRENT_MODEL_ID_KEY)
artifact_queries[model_channel_key] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
executor.resolver.CopyFrom(
ResolverSpec(output_artifact_queries=artifact_queries))
self._deployment_config.executors[executor_label].CopyFrom(executor)
return result
def _build_resolver_for_latest_model_blessing(
self,
model_blessing_channel_key: str) -> pipeline_pb2.PipelineTaskSpec:
"""Builds the resolver spec for latest valid ModelBlessing artifact."""
name = '{}{}'.format(self._name, _MODEL_BLESSING_RESOLVER_SUFFIX)
# Component def.
component_def = pipeline_pb2.ComponentSpec()
executor_label = _EXECUTOR_LABEL_PATTERN.format(name)
component_def.executor_label = executor_label
output_artifact_spec = compiler_utils.build_output_artifact_spec(
self._outputs[model_blessing_channel_key])
component_def.output_definitions.artifacts[
model_blessing_channel_key].CopyFrom(output_artifact_spec)
self._component_defs[name] = component_def
# Task spec.
task_spec = pipeline_pb2.PipelineTaskSpec()
task_spec.task_info.name = name
task_spec.component_ref.name = name
# Builds the resolver executor spec for latest valid ModelBlessing.
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
artifact_queries = {}
query_filter = ('artifact_type="{type}" and state={state}'
' and metadata.{key}.number_value={value}').format(
type=compiler_utils.get_artifact_title(
standard_artifacts.ModelBlessing),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE),
key=constants.ARTIFACT_PROPERTY_BLESSED_KEY,
value=constants.BLESSED_VALUE)
artifact_queries[
model_blessing_channel_key] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
executor.resolver.CopyFrom(
ResolverSpec(output_artifact_queries=artifact_queries))
self._deployment_config.executors[executor_label].CopyFrom(executor)
return task_spec
def _build_latest_artifact_resolver(
self) -> Dict[str, pipeline_pb2.PipelineTaskSpec]:
"""Builds a resolver spec for a latest artifact resolver.
Returns:
A list of two PipelineTaskSpecs. One represents the query for latest valid
ModelBlessing artifact. Another one represents the query for latest
blessed Model artifact.
Raises:
ValueError: when desired_num_of_artifacts != 1. 1 is the only supported
value currently.
"""
# Fetch the init kwargs for the resolver.
resolver_config = self._exec_properties[resolver.RESOLVER_CONFIG]
if (isinstance(resolver_config, dict) and
resolver_config.get('desired_num_of_artifacts', 0) > 1):
raise ValueError('Only desired_num_of_artifacts=1 is supported currently.'
' Got {}'.format(
resolver_config.get('desired_num_of_artifacts')))
component_def = pipeline_pb2.ComponentSpec()
executor_label = _EXECUTOR_LABEL_PATTERN.format(self._name)
component_def.executor_label = executor_label
task_spec = pipeline_pb2.PipelineTaskSpec()
task_spec.task_info.name = self._name
for name, output_channel in self._outputs.items():
output_artifact_spec = compiler_utils.build_output_artifact_spec(
output_channel)
component_def.output_definitions.artifacts[name].CopyFrom(
output_artifact_spec)
for name, value in self._exec_properties.items():
if value is None:
continue
parameter_type_spec = compiler_utils.build_parameter_type_spec(value)
component_def.input_definitions.parameters[name].CopyFrom(
parameter_type_spec)
if isinstance(value, data_types.RuntimeParameter):
parameter_utils.attach_parameter(value)
task_spec.inputs.parameters[name].component_input_parameter = value.name
else:
task_spec.inputs.parameters[name].CopyFrom(
pipeline_pb2.TaskInputsSpec.InputParameterSpec(
runtime_value=compiler_utils.value_converter(value)))
self._component_defs[self._name] = component_def
task_spec.component_ref.name = self._name
artifact_queries = {}
# Buid the artifact query for each channel in the input dict.
for name, c in self._inputs.items():
query_filter = ('artifact_type="{type}" and state={state}').format(
type=compiler_utils.get_artifact_title(c.type),
state=metadata_store_pb2.Artifact.State.Name(
metadata_store_pb2.Artifact.LIVE))
# Resolver's output dict has the same set of keys as its input dict.
artifact_queries[name] = ResolverSpec.ArtifactQuerySpec(
filter=query_filter)
resolver_spec = ResolverSpec(output_artifact_queries=artifact_queries)
executor = pipeline_pb2.PipelineDeploymentConfig.ExecutorSpec()
executor.resolver.CopyFrom(resolver_spec)
self._deployment_config.executors[executor_label].CopyFrom(executor)
return {self._name: task_spec}
