def _to_local_deployment(
input_config: pipeline_pb2.IntermediateDeploymentConfig
) -> local_deployment_config_pb2.LocalDeploymentConfig:
"""Turns IntermediateDeploymentConfig to LocalDeploymentConfig."""
result = local_deployment_config_pb2.LocalDeploymentConfig()
for k, v in input_config.executor_specs.items():
result.executor_specs[k].CopyFrom(_build_executable_spec(k, v))
for k, v in input_config.custom_driver_specs.items():
result.custom_driver_specs[k].CopyFrom(_build_executable_spec(k, v))
for k, v in input_config.node_level_platform_configs.items():
result.node_level_platform_configs[k].CopyFrom(
_build_local_platform_config(k, v))
if input_config.HasField('metadata_connection_config'):
if not input_config.metadata_connection_config.Unpack(
result.metadata_connection_config):
raise ValueError(
'metadata_connection_config is expected to be in type '
'ml_metadata.ConnectionConfig, but got type {}'.format(
input_config.metadata_connection_config.type_url))
else:
# Some users like Kubernetes choose to use their own MLMD connection.
# So their IR doesn't contain it.
logging.warning('metadata_connection_config is not provided by IR.')
return result
def _extract_deployment_config(
self, pipeline: pipeline_pb2.Pipeline
) -> local_deployment_config_pb2.LocalDeploymentConfig:
"""Extracts the proto.Any pipeline.deployment_config to IntermediateDeploymentConfig."""
if not pipeline.deployment_config:
raise ValueError(
'deployment_config is not available in the pipeline.')
result = local_deployment_config_pb2.LocalDeploymentConfig()
if pipeline.deployment_config.Unpack(result):
return result
raise ValueError("deployment_config's type {} is not supported".format(
type(pipeline.deployment_config)))
def extract_local_deployment_config(
pipeline: pipeline_pb2.Pipeline
) -> local_deployment_config_pb2.LocalDeploymentConfig:
"""Extracts the proto.Any pipeline.deployment_config to LocalDeploymentConfig."""
if not pipeline.deployment_config:
raise ValueError('deployment_config is not available in the pipeline.')
result = local_deployment_config_pb2.LocalDeploymentConfig()
if pipeline.deployment_config.Unpack(result):
return result
result = pipeline_pb2.IntermediateDeploymentConfig()
if pipeline.deployment_config.Unpack(result):
return _to_local_deployment(result)
raise ValueError('deployment_config {} of type {} is not supported'.format(
pipeline.deployment_config, type(pipeline.deployment_config)))
def _to_local_deployment(
self,
input_config: pipeline_pb2.IntermediateDeploymentConfig
) -> local_deployment_config_pb2.LocalDeploymentConfig:
"""Turns IntermediateDeploymentConfig to LocalDeploymentConfig."""
result = local_deployment_config_pb2.LocalDeploymentConfig()
for k, v in input_config.executor_specs.items():
result.executor_specs[k].CopyFrom(self._build_executable_spec(k, v))
for k, v in input_config.custom_driver_specs.items():
result.custom_driver_specs[k].CopyFrom(self._build_executable_spec(k, v))
if not input_config.metadata_connection_config.Unpack(
result.metadata_connection_config):
raise ValueError('metadata_connection_config is expected to be in type '
'ml_metadata.ConnectionConfig, but got type {}'.format(
input_config.metadata_connection_config.type_url))
return result
def _to_local_deployment(
input_config: pipeline_pb2.IntermediateDeploymentConfig
) -> local_deployment_config_pb2.LocalDeploymentConfig:
"""Turns IntermediateDeploymentConfig to LocalDeploymentConfig."""
result = local_deployment_config_pb2.LocalDeploymentConfig()
for k, v in input_config.executor_specs.items():
result.executor_specs[k].CopyFrom(_build_executable_spec(k, v))
for k, v in input_config.custom_driver_specs.items():
result.custom_driver_specs[k].CopyFrom(_build_executable_spec(k, v))
for k, v in input_config.node_level_platform_configs.items():
result.node_level_platform_configs[k].CopyFrom(
_build_local_platform_config(k, v))
result.metadata_connection_config.CopyFrom(
extract_mlmd_connection(input_config.metadata_connection_config))
if result.metadata_connection_config.WhichOneof(
'connection_config') is None:
# Some users like Kubernetes choose to use their own MLMD connection.
# So their IR doesn't contain it.
logging.warning('metadata_connection_config is not provided by IR.')
return result
class_path: "tfx.components.trainer_executor"
}
}
}
custom_driver_specs {
key: "my_example_gen"
value {
python_class_executable_spec {
class_path: "tfx.components.example_gen_driver"
}
}
}
metadata_connection_config {
fake_database {}
}
""", local_deployment_config_pb2.LocalDeploymentConfig())
_INTERMEDIATE_DEPLOYMENT_CONFIG = text_format.Parse(
"""
executor_specs {
key: "my_example_gen"
value {
[type.googleapis.com/tfx.orchestration.executable_spec.PythonClassExecutableSpec] {
class_path: "tfx.components.example_gen_executor"
}
}
}
executor_specs {
key: "my_transform"
value {
[type.googleapis.com/tfx.orchestration.executable_spec.PythonClassExecutableSpec] {
