def setUp(self):
super(ExecutorTest, self).setUp()
self._source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
self._output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
self.component_id = 'test_component'
# Create input dict.
self._examples = standard_artifacts.Examples(split='unlabelled')
self._examples.uri = os.path.join(self._source_data_dir,
'csv_example_gen/unlabelled/')
self._model_export = standard_artifacts.Model()
self._model_export.uri = os.path.join(self._source_data_dir,
'trainer/current/')
self._model_blessing = standard_artifacts.ModelBlessing()
self._model_blessing.uri = os.path.join(self._source_data_dir,
'model_validator/blessed')
self._model_blessing.set_int_custom_property('blessed', 1)
self._inference_result = standard_artifacts.InferenceResult()
self._prediction_log_dir = os.path.join(self._output_data_dir,
'prediction_logs')
self._inference_result.uri = self._prediction_log_dir
# Create context
self._tmp_dir = os.path.join(self._output_data_dir, '.temp')
self._context = executor.Executor.Context(tmp_dir=self._tmp_dir,
unique_id='2')
def setUp(self):
super().setUp()
self._source_data_dir = os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(__file__)))),
'components', 'testdata')
self._output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
self.component_id = 'test_component'
# Create input dict.
self._examples = standard_artifacts.Examples()
self._examples.uri = os.path.join(self._source_data_dir, 'csv_example_gen')
self._examples.split_names = artifact_utils.encode_split_names(
['unlabelled'])
self._model = standard_artifacts.Model()
self._model.uri = os.path.join(self._source_data_dir, 'trainer/current')
self._model_version = 'version_' + hashlib.sha256(
self._model.uri.encode()).hexdigest()
self._model_blessing = standard_artifacts.ModelBlessing()
self._model_blessing.uri = os.path.join(self._source_data_dir,
'model_validator/blessed')
self._model_blessing.set_int_custom_property('blessed', 1)
self._inference_result = standard_artifacts.InferenceResult()
self._prediction_log_dir = os.path.join(self._output_data_dir,
'prediction_logs')
self._inference_result.uri = self._prediction_log_dir
# Create context
self._tmp_dir = os.path.join(self._output_data_dir, '.temp')
self._context = executor.Executor.Context(
tmp_dir=self._tmp_dir, unique_id='2')
def __init__(self,
examples: types.Channel = None,
model: Optional[types.Channel] = None,
model_blessing: Optional[types.Channel] = None,
data_spec: Optional[Union[bulk_inferrer_pb2.DataSpec,
Dict[Text, Any]]] = None,
model_spec: Optional[Union[bulk_inferrer_pb2.ModelSpec,
Dict[Text, Any]]] = None,
inference_result: Optional[types.Channel] = None,
instance_name: Optional[Text] = None,
enable_cache: Optional[bool] = None):
"""Construct an BulkInferrer component.
Args:
examples: A Channel of type `standard_artifacts.Examples`, usually
produced by an ExampleGen component. _required_
model: A Channel of type `standard_artifacts.Model`, usually produced by
a Trainer component.
model_blessing: A Channel of type `standard_artifacts.ModelBlessing`,
usually produced by a ModelValidator component.
data_spec: bulk_inferrer_pb2.DataSpec instance that describes data
selection. If any field is provided as a RuntimeParameter, data_spec
should be constructed as a dict with the same field names as DataSpec
proto message.
model_spec: bulk_inferrer_pb2.ModelSpec instance that describes model
specification. If any field is provided as a RuntimeParameter,
model_spec should be constructed as a dict with the same field names as
ModelSpec proto message.
inference_result: Channel of type `standard_artifacts.InferenceResult`
to store the inference results.
instance_name: Optional name assigned to this specific instance of
BulkInferrer. Required only if multiple BulkInferrer components are
declared in the same pipeline.
enable_cache: Optional boolean to indicate if cache is enabled for the
BulkInferrer component. If not specified, defaults to the value
specified for pipeline's enable_cache parameter.
"""
inference_result = inference_result or types.Channel(
type=standard_artifacts.InferenceResult,
artifacts=[standard_artifacts.InferenceResult()])
spec = BulkInferrerSpec(examples=examples,
model=model,
model_blessing=model_blessing,
data_spec=data_spec
or bulk_inferrer_pb2.DataSpec(),
model_spec=model_spec
or bulk_inferrer_pb2.ModelSpec(),
inference_result=inference_result)
super(BulkInferrer, self).__init__(spec=spec,
instance_name=instance_name,
enable_cache=enable_cache)
def testConstructInferenceResultAndOutputExample(self):
with self.assertRaises(ValueError):
component.BulkInferrer(examples=self._examples,
model=self._model,
model_blessing=self._model_blessing,
output_examples=channel_utils.as_channel(
[standard_artifacts.Examples()]))
with self.assertRaises(ValueError):
component.BulkInferrer(
examples=self._examples,
model=self._model,
model_blessing=self._model_blessing,
output_example_spec=bulk_inferrer_pb2.OutputExampleSpec(),
inference_result=channel_utils.as_channel(
[standard_artifacts.InferenceResult()]))
def __init__(self,
examples: types.Channel = None,
model: Optional[types.Channel] = None,
model_blessing: Optional[types.Channel] = None,
data_spec: Optional[Union[bulk_inferrer_pb2.DataSpec,
Dict[Text, Any]]] = None,
custom_config: Dict[Text, Any] = None,
inference_result: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an BulkInferrer component.
Args:
examples: A Channel of type `standard_artifacts.Examples`, usually
produced by an ExampleGen component. _required_
model: A Channel of type `standard_artifacts.Model`, usually produced by
a Trainer component.
model_blessing: A Channel of type `standard_artifacts.ModelBlessing`,
usually produced by a ModelValidator component.
data_spec: bulk_inferrer_pb2.DataSpec instance that describes data
selection. If any field is provided as a RuntimeParameter, data_spec
should be constructed as a dict with the same field names as DataSpec
proto message.
custom_config: A dict which contains the deployment job parameters to be
passed to Google Cloud AI Platform.
custom_config.ai_platform_serving_args need to contain the serving job
parameters. For the full set of parameters, refer to
https://cloud.google.com/ml-engine/reference/rest/v1/projects.models
inference_result: Channel of type `standard_artifacts.InferenceResult`
to store the inference results.
instance_name: Optional name assigned to this specific instance of
BulkInferrer. Required only if multiple BulkInferrer components are
declared in the same pipeline.
"""
inference_result = inference_result or types.Channel(
type=standard_artifacts.InferenceResult,
artifacts=[standard_artifacts.InferenceResult()])
spec = CloudAIBulkInferrerComponentSpec(
examples=examples,
model=model,
model_blessing=model_blessing,
data_spec=data_spec or bulk_inferrer_pb2.DataSpec(),
custom_config=json_utils.dumps(custom_config),
inference_result=inference_result)
super(CloudAIBulkInferrerComponent, self).__init__(
spec=spec, instance_name=instance_name)
def __init__(self,
examples: types.Channel = None,
model_export: Optional[types.Channel] = None,
model_blessing: Optional[types.Channel] = None,
model_push: Optional[types.Channel] = None,
data_spec: Optional[bulk_inferrer_pb2.DataSpec] = None,
model_spec: Optional[bulk_inferrer_pb2.ModelSpec] = None,
output: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an BulkInferrer component.
Args:
examples: A Channel of 'ExamplesPath' type, usually produced by ExampleGen
component. _required_
model_export: A Channel of 'ModelExportPath' type, usually produced by
Trainer component.
model_blessing: A Channel of 'ModelBlessingPath' type, usually produced by
Model Validator component.
model_push: A Channel of 'PushedModel' type, usually produced by Pusher
component.
data_spec: bulk_inferrer_pb2.DataSpec instance that describes data
selection.
model_spec: bulk_inferrer_pb2.ModelSpec instance that describes model
specification.
output: Channel of `InferenceResult` to store the inference results.
instance_name: Optional name assigned to this specific instance of
BulkInferrer. Required only if multiple BulkInferrer components are
declared in the same pipeline.
"""
output = output or types.Channel(
type=standard_artifacts.InferenceResult,
artifacts=[standard_artifacts.InferenceResult()])
spec = BulkInferrerSpec(examples=examples,
model_export=model_export,
model_blessing=model_blessing,
model_push=model_push,
data_spec=data_spec
or bulk_inferrer_pb2.DataSpec(),
model_spec=model_spec
or bulk_inferrer_pb2.ModelSpec(),
output=output)
super(BulkInferrer, self).__init__(spec=spec,
instance_name=instance_name)
def setUp(self):
super(ExecutorTest, self).setUp()
self._source_data_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'testdata')
self._output_data_dir = os.path.join(
os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()),
self._testMethodName)
self.component_id = 'test_component'
# Create input dict.
self._examples = standard_artifacts.Examples()
unlabelled_path = os.path.join(self._source_data_dir,
'csv_example_gen', 'unlabelled')
self._examples.uri = os.path.join(self._output_data_dir,
'csv_example_gen')
io_utils.copy_dir(unlabelled_path,
os.path.join(self._examples.uri, 'unlabelled'))
io_utils.copy_dir(unlabelled_path,
os.path.join(self._examples.uri, 'unlabelled2'))
self._examples.split_names = artifact_utils.encode_split_names(
['unlabelled', 'unlabelled2'])
self._model = standard_artifacts.Model()
self._model.uri = os.path.join(self._source_data_dir,
'trainer/current')
self._model_blessing = standard_artifacts.ModelBlessing()
self._model_blessing.uri = os.path.join(self._source_data_dir,
'model_validator/blessed')
self._model_blessing.set_int_custom_property('blessed', 1)
self._input_dict = {
'examples': [self._examples],
'model': [self._model],
'model_blessing': [self._model_blessing],
}
# Create output dict.
self._inference_result = standard_artifacts.InferenceResult()
self._prediction_log_dir = os.path.join(self._output_data_dir,
'prediction_logs')
self._inference_result.uri = self._prediction_log_dir
self._output_examples = standard_artifacts.Examples()
self._output_examples_dir = os.path.join(self._output_data_dir,
'output_examples')
self._output_examples.uri = self._output_examples_dir
self._output_dict_ir = {
'inference_result': [self._inference_result],
}
self._output_dict_oe = {
'output_examples': [self._output_examples],
}
# Create exe properties.
self._exec_properties = {
'data_spec':
proto_utils.proto_to_json(bulk_inferrer_pb2.DataSpec()),
'model_spec':
proto_utils.proto_to_json(bulk_inferrer_pb2.ModelSpec()),
'component_id': self.component_id,
}
# Create context
self._tmp_dir = os.path.join(self._output_data_dir, '.temp')
self._context = executor.Executor.Context(tmp_dir=self._tmp_dir,
unique_id='2')
