def setUp(self): super(ExecutorTest, self).setUp() self._source_data_dir = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'testdata') self._output_data_dir = os.path.join( os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()), self._testMethodName) self.component_id = 'test_component' # Create input dict. self._examples = standard_artifacts.Examples(split='unlabelled') self._examples.uri = os.path.join(self._source_data_dir, 'csv_example_gen/unlabelled/') self._model_export = standard_artifacts.Model() self._model_export.uri = os.path.join(self._source_data_dir, 'trainer/current/') self._model_blessing = standard_artifacts.ModelBlessing() self._model_blessing.uri = os.path.join(self._source_data_dir, 'model_validator/blessed') self._model_blessing.set_int_custom_property('blessed', 1) self._inference_result = standard_artifacts.InferenceResult() self._prediction_log_dir = os.path.join(self._output_data_dir, 'prediction_logs') self._inference_result.uri = self._prediction_log_dir # Create context self._tmp_dir = os.path.join(self._output_data_dir, '.temp') self._context = executor.Executor.Context(tmp_dir=self._tmp_dir, unique_id='2')
def setUp(self): super().setUp() self._source_data_dir = os.path.join( os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'components', 'testdata') self._output_data_dir = os.path.join( os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()), self._testMethodName) self.component_id = 'test_component' # Create input dict. self._examples = standard_artifacts.Examples() self._examples.uri = os.path.join(self._source_data_dir, 'csv_example_gen') self._examples.split_names = artifact_utils.encode_split_names( ['unlabelled']) self._model = standard_artifacts.Model() self._model.uri = os.path.join(self._source_data_dir, 'trainer/current') self._model_version = 'version_' + hashlib.sha256( self._model.uri.encode()).hexdigest() self._model_blessing = standard_artifacts.ModelBlessing() self._model_blessing.uri = os.path.join(self._source_data_dir, 'model_validator/blessed') self._model_blessing.set_int_custom_property('blessed', 1) self._inference_result = standard_artifacts.InferenceResult() self._prediction_log_dir = os.path.join(self._output_data_dir, 'prediction_logs') self._inference_result.uri = self._prediction_log_dir # Create context self._tmp_dir = os.path.join(self._output_data_dir, '.temp') self._context = executor.Executor.Context( tmp_dir=self._tmp_dir, unique_id='2')
def __init__(self, examples: types.Channel = None, model: Optional[types.Channel] = None, model_blessing: Optional[types.Channel] = None, data_spec: Optional[Union[bulk_inferrer_pb2.DataSpec, Dict[Text, Any]]] = None, model_spec: Optional[Union[bulk_inferrer_pb2.ModelSpec, Dict[Text, Any]]] = None, inference_result: Optional[types.Channel] = None, instance_name: Optional[Text] = None, enable_cache: Optional[bool] = None): """Construct an BulkInferrer component. Args: examples: A Channel of type `standard_artifacts.Examples`, usually produced by an ExampleGen component. _required_ model: A Channel of type `standard_artifacts.Model`, usually produced by a Trainer component. model_blessing: A Channel of type `standard_artifacts.ModelBlessing`, usually produced by a ModelValidator component. data_spec: bulk_inferrer_pb2.DataSpec instance that describes data selection. If any field is provided as a RuntimeParameter, data_spec should be constructed as a dict with the same field names as DataSpec proto message. model_spec: bulk_inferrer_pb2.ModelSpec instance that describes model specification. If any field is provided as a RuntimeParameter, model_spec should be constructed as a dict with the same field names as ModelSpec proto message. inference_result: Channel of type `standard_artifacts.InferenceResult` to store the inference results. instance_name: Optional name assigned to this specific instance of BulkInferrer. Required only if multiple BulkInferrer components are declared in the same pipeline. enable_cache: Optional boolean to indicate if cache is enabled for the BulkInferrer component. If not specified, defaults to the value specified for pipeline's enable_cache parameter. """ inference_result = inference_result or types.Channel( type=standard_artifacts.InferenceResult, artifacts=[standard_artifacts.InferenceResult()]) spec = BulkInferrerSpec(examples=examples, model=model, model_blessing=model_blessing, data_spec=data_spec or bulk_inferrer_pb2.DataSpec(), model_spec=model_spec or bulk_inferrer_pb2.ModelSpec(), inference_result=inference_result) super(BulkInferrer, self).__init__(spec=spec, instance_name=instance_name, enable_cache=enable_cache)
def testConstructInferenceResultAndOutputExample(self): with self.assertRaises(ValueError): component.BulkInferrer(examples=self._examples, model=self._model, model_blessing=self._model_blessing, output_examples=channel_utils.as_channel( [standard_artifacts.Examples()])) with self.assertRaises(ValueError): component.BulkInferrer( examples=self._examples, model=self._model, model_blessing=self._model_blessing, output_example_spec=bulk_inferrer_pb2.OutputExampleSpec(), inference_result=channel_utils.as_channel( [standard_artifacts.InferenceResult()]))
def __init__(self, examples: types.Channel = None, model: Optional[types.Channel] = None, model_blessing: Optional[types.Channel] = None, data_spec: Optional[Union[bulk_inferrer_pb2.DataSpec, Dict[Text, Any]]] = None, custom_config: Dict[Text, Any] = None, inference_result: Optional[types.Channel] = None, instance_name: Optional[Text] = None): """Construct an BulkInferrer component. Args: examples: A Channel of type `standard_artifacts.Examples`, usually produced by an ExampleGen component. _required_ model: A Channel of type `standard_artifacts.Model`, usually produced by a Trainer component. model_blessing: A Channel of type `standard_artifacts.ModelBlessing`, usually produced by a ModelValidator component. data_spec: bulk_inferrer_pb2.DataSpec instance that describes data selection. If any field is provided as a RuntimeParameter, data_spec should be constructed as a dict with the same field names as DataSpec proto message. custom_config: A dict which contains the deployment job parameters to be passed to Google Cloud AI Platform. custom_config.ai_platform_serving_args need to contain the serving job parameters. For the full set of parameters, refer to https://cloud.google.com/ml-engine/reference/rest/v1/projects.models inference_result: Channel of type `standard_artifacts.InferenceResult` to store the inference results. instance_name: Optional name assigned to this specific instance of BulkInferrer. Required only if multiple BulkInferrer components are declared in the same pipeline. """ inference_result = inference_result or types.Channel( type=standard_artifacts.InferenceResult, artifacts=[standard_artifacts.InferenceResult()]) spec = CloudAIBulkInferrerComponentSpec( examples=examples, model=model, model_blessing=model_blessing, data_spec=data_spec or bulk_inferrer_pb2.DataSpec(), custom_config=json_utils.dumps(custom_config), inference_result=inference_result) super(CloudAIBulkInferrerComponent, self).__init__( spec=spec, instance_name=instance_name)
def __init__(self, examples: types.Channel = None, model_export: Optional[types.Channel] = None, model_blessing: Optional[types.Channel] = None, model_push: Optional[types.Channel] = None, data_spec: Optional[bulk_inferrer_pb2.DataSpec] = None, model_spec: Optional[bulk_inferrer_pb2.ModelSpec] = None, output: Optional[types.Channel] = None, instance_name: Optional[Text] = None): """Construct an BulkInferrer component. Args: examples: A Channel of 'ExamplesPath' type, usually produced by ExampleGen component. _required_ model_export: A Channel of 'ModelExportPath' type, usually produced by Trainer component. model_blessing: A Channel of 'ModelBlessingPath' type, usually produced by Model Validator component. model_push: A Channel of 'PushedModel' type, usually produced by Pusher component. data_spec: bulk_inferrer_pb2.DataSpec instance that describes data selection. model_spec: bulk_inferrer_pb2.ModelSpec instance that describes model specification. output: Channel of `InferenceResult` to store the inference results. instance_name: Optional name assigned to this specific instance of BulkInferrer. Required only if multiple BulkInferrer components are declared in the same pipeline. """ output = output or types.Channel( type=standard_artifacts.InferenceResult, artifacts=[standard_artifacts.InferenceResult()]) spec = BulkInferrerSpec(examples=examples, model_export=model_export, model_blessing=model_blessing, model_push=model_push, data_spec=data_spec or bulk_inferrer_pb2.DataSpec(), model_spec=model_spec or bulk_inferrer_pb2.ModelSpec(), output=output) super(BulkInferrer, self).__init__(spec=spec, instance_name=instance_name)
def setUp(self): super(ExecutorTest, self).setUp() self._source_data_dir = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'testdata') self._output_data_dir = os.path.join( os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()), self._testMethodName) self.component_id = 'test_component' # Create input dict. self._examples = standard_artifacts.Examples() unlabelled_path = os.path.join(self._source_data_dir, 'csv_example_gen', 'unlabelled') self._examples.uri = os.path.join(self._output_data_dir, 'csv_example_gen') io_utils.copy_dir(unlabelled_path, os.path.join(self._examples.uri, 'unlabelled')) io_utils.copy_dir(unlabelled_path, os.path.join(self._examples.uri, 'unlabelled2')) self._examples.split_names = artifact_utils.encode_split_names( ['unlabelled', 'unlabelled2']) self._model = standard_artifacts.Model() self._model.uri = os.path.join(self._source_data_dir, 'trainer/current') self._model_blessing = standard_artifacts.ModelBlessing() self._model_blessing.uri = os.path.join(self._source_data_dir, 'model_validator/blessed') self._model_blessing.set_int_custom_property('blessed', 1) self._input_dict = { 'examples': [self._examples], 'model': [self._model], 'model_blessing': [self._model_blessing], } # Create output dict. self._inference_result = standard_artifacts.InferenceResult() self._prediction_log_dir = os.path.join(self._output_data_dir, 'prediction_logs') self._inference_result.uri = self._prediction_log_dir self._output_examples = standard_artifacts.Examples() self._output_examples_dir = os.path.join(self._output_data_dir, 'output_examples') self._output_examples.uri = self._output_examples_dir self._output_dict_ir = { 'inference_result': [self._inference_result], } self._output_dict_oe = { 'output_examples': [self._output_examples], } # Create exe properties. self._exec_properties = { 'data_spec': proto_utils.proto_to_json(bulk_inferrer_pb2.DataSpec()), 'model_spec': proto_utils.proto_to_json(bulk_inferrer_pb2.ModelSpec()), 'component_id': self.component_id, } # Create context self._tmp_dir = os.path.join(self._output_data_dir, '.temp') self._context = executor.Executor.Context(tmp_dir=self._tmp_dir, unique_id='2')