def testRunResolverSteps_NoneRaisesSignal(self):
config = pipeline_pb2.ResolverConfig()
text_format.Parse(
"""
resolver_steps {
class_path: "tfx.orchestration.portable.input_resolution.processor_test.NoneStrategy"
}
""", config)
with self.assertRaises(exceptions.InputResolutionError):
processor.run_resolver_steps(self._input_dict,
resolver_steps=config.resolver_steps,
store=self._store)
def resolve_input_artifacts_v2(
*,
pipeline_node: pipeline_pb2.PipelineNode,
metadata_handler: metadata.Metadata,
) -> Union[Trigger, Skip]:
"""Resolve input artifacts according to a pipeline node IR definition.
Input artifacts are resolved in the following steps:
1. An initial input dict (Mapping[str, Sequence[Artifact]]) is fetched from
the input channel definitions (in NodeInputs.inputs.channels).
2. Optionally input resolution logic is performed if specified (in
NodeInputs.resolver_config).
3. Filters input map with enough number of artifacts as specified in
NodeInputs.inputs.min_count.
Args:
pipeline_node: Current PipelineNode on which input resolution is running.
metadata_handler: MetadataHandler instance for MLMD access.
Raises:
InputResolutionError: If input resolution went wrong.
Returns:
Trigger: a non-empty list of input dicts. All resolved input dicts should be
executed.
Skip: an empty list. Should effectively skip the current component
execution.
"""
node_inputs = pipeline_node.inputs
initial_dict = _resolve_initial_dict(metadata_handler, node_inputs)
try:
resolved = processor.run_resolver_steps(
initial_dict,
resolver_steps=node_inputs.resolver_config.resolver_steps,
store=metadata_handler.store)
except exceptions.SkipSignal:
return Skip()
except exceptions.InputResolutionError:
raise
except Exception as e:
raise exceptions.InputResolutionError(
f'Error occurred during input resolution: {str(e)}.') from e
if typing_utils.is_artifact_multimap(resolved):
resolved = [resolved]
if not typing_utils.is_list_of_artifact_multimap(resolved):
raise exceptions.FailedPreconditionError(
'Invalid input resolution result; expected Sequence[ArtifactMultiMap] '
f'type but got {resolved}.')
resolved = [d for d in resolved if _is_sufficient(d, node_inputs)]
if not resolved:
raise exceptions.FailedPreconditionError('No valid inputs.')
return Trigger(resolved)
def resolve_input_artifacts(
metadata_handler: metadata.Metadata, node_inputs: pipeline_pb2.NodeInputs
) -> Optional[typing_utils.ArtifactMultiMap]:
"""Resolves input artifacts of a pipeline node.
Args:
metadata_handler: A metadata handler to access MLMD store.
node_inputs: A pipeline_pb2.NodeInputs message that instructs artifact
resolution for a pipeline node.
Returns:
If `min_count` for every input is met, returns a Dict[str, List[Artifact]].
Otherwise, return None.
"""
initial_dict = _resolve_initial_dict(metadata_handler, node_inputs)
if not _is_sufficient(initial_dict, node_inputs):
min_counts = {key: input_spec.min_count
for key, input_spec in node_inputs.inputs.items()}
logging.warning('Resolved inputs should have %r artifacts, but got %r.',
min_counts, initial_dict)
return None
try:
result = processor.run_resolver_steps(
initial_dict,
resolver_steps=node_inputs.resolver_config.resolver_steps,
store=metadata_handler.store)
except exceptions.InputResolutionError:
# If ResolverStrategy has returned None in the middle, InputResolutionError
# is raised. Legacy input resolution has returned None in this case.
return None
except exceptions.SkipSignal:
# SkipSignal is not fully representable return value in legacy input
# resolution, but None is the best effort.
return None
if typing_utils.is_list_of_artifact_multimap(result):
result = cast(Sequence[typing_utils.ArtifactMultiMap], result)
if len(result) != 1:
raise ValueError(
'Invalid number of resolved inputs; expected 1 but got '
f'{len(result)}: {result}')
return result[0]
elif typing_utils.is_artifact_multimap(result):
return cast(typing_utils.ArtifactMultiMap, result)
else:
raise TypeError(f'Invalid input resolution result: {result}. Should be '
'Mapping[str, Sequence[Artifact]].')
def testRunResolverSteps_ResolverOp_IgnoresInputKeys(self):
config = pipeline_pb2.ResolverConfig()
text_format.Parse(
r"""
resolver_steps {
class_path: "tfx.orchestration.portable.input_resolution.processor_test.RepeatOp"
config_json: "{\"num\": 2}"
input_keys: ["examples"]
}
""", config)
result = processor.run_resolver_steps(
self._input_dict,
resolver_steps=config.resolver_steps,
store=self._store)
self.assertLen(result['examples'], 2)
self.assertLen(result['model'], 2)
def testRunResolverSteps_ResolverStrategy(self):
config = pipeline_pb2.ResolverConfig()
text_format.Parse(
r"""
resolver_steps {
class_path: "tfx.orchestration.portable.input_resolution.processor_test.RepeatStrategy"
config_json: "{\"num\": 2}"
}
""", config)
result = processor.run_resolver_steps(
self._input_dict,
resolver_steps=config.resolver_steps,
store=self._store)
strategy = RepeatStrategy.last_created
self.assertIs(strategy.call_history[0][0], self._store)
self.assertLen(result['examples'], 2)
self.assertLen(result['model'], 2)
def testRunResolverSteps_ResolverOp(self):
config = pipeline_pb2.ResolverConfig()
text_format.Parse(
r"""
resolver_steps {
class_path: "tfx.orchestration.portable.input_resolution.processor_test.RepeatOp"
config_json: "{\"num\": 2}"
}
""", config)
result = processor.run_resolver_steps(
self._input_dict,
resolver_steps=config.resolver_steps,
store=self._store)
op = RepeatOp.last_created
self.assertIs(op.context.store, self._store)
self.assertEqual(op.num, 2)
self.assertLen(result['examples'], 2)
self.assertLen(result['model'], 2)