def Do(self, input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text, Any]) -> None:
executor_output = execution_result_pb2.ExecutorOutput()
outputs_utils.populate_output_artifact(executor_output, output_dict)
with fileio.open(self._context.executor_output_uri, 'wb') as f:
f.write(executor_output.SerializeToString())
def Do(
self, input_dict: Dict[Text, List[types.Artifact]],
output_dict: Dict[Text, List[types.Artifact]],
exec_properties: Dict[Text,
Any]) -> execution_result_pb2.ExecutorOutput:
executor_output = execution_result_pb2.ExecutorOutput()
outputs_utils.populate_output_artifact(executor_output, output_dict)
return executor_output
def testPopulateOutputArtifact(self):
executor_output = execution_result_pb2.ExecutorOutput()
output_dict = {'output_key': [standard_artifacts.Model()]}
outputs_utils.populate_output_artifact(executor_output, output_dict)
self.assertProtoEquals(
"""
output_artifacts {
key: "output_key"
value {
artifacts {
}
}
}
""", executor_output)
def run_with_executor(
execution_info: data_types.ExecutionInfo,
executor: base_executor.BaseExecutor
) -> execution_result_pb2.ExecutorOutput:
"""Invokes executors given an executor instance and input from the Launcher.
Args:
execution_info: A wrapper of the details of this execution.
executor: An executor instance.
Returns:
The output from executor.
"""
# In cases where output directories are not empty due to a previous or
# unrelated execution, clear the directories to ensure consistency.
outputs_utils.clear_output_dirs(execution_info.output_dict)
for _, artifact_list in execution_info.input_dict.items():
for artifact in artifact_list:
if isinstance(artifact, ValueArtifact):
# Read ValueArtifact into memory.
artifact.read()
output_dict = copy.deepcopy(execution_info.output_dict)
result = executor.Do(execution_info.input_dict, output_dict,
execution_info.exec_properties)
if not result:
# If result is not returned from the Do function, then try to
# read from the executor_output_uri.
if fileio.exists(execution_info.execution_output_uri):
result = execution_result_pb2.ExecutorOutput.FromString(
fileio.open(execution_info.execution_output_uri, 'rb').read())
else:
# Old style TFX executor doesn't return executor_output, but modify
# output_dict and exec_properties in place. For backward compatibility,
# we use their executor_output and exec_properties to construct
# ExecutorOutput.
result = execution_result_pb2.ExecutorOutput()
outputs_utils.populate_output_artifact(result, output_dict)
outputs_utils.populate_exec_properties(
result, execution_info.exec_properties)
return result