def produce_pipeline( self, data_handler, fitted_pipeline_id: str, input_data_id: str, *, timeout: float = None, expose_outputs: bool = False ) -> PipelineResult: pipeline_result = PipelineResult(fitted_pipeline_id=fitted_pipeline_id) pipeline_result.status = "RUNNING" pipeline_result.method_called = "produce" pipeline_result.fitted_pipeline_id = fitted_pipeline_id request_id = data_handler.get_data.remote(input_data_id) input_data = ray.get(request_id) with d3m_utils.silence(): output, result = runtime_module.produce( fitted_pipeline=self.fitted_pipelines[fitted_pipeline_id], test_inputs=input_data, expose_produced_outputs=expose_outputs ) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" if self.store_results: pipeline_result.exposed_outputs = save_exposed_values(result.values, fitted_pipeline_id, self.scratch_dir) pipeline_result.output = save_exposed_values(output, fitted_pipeline_id, self.scratch_dir) else: pipeline_result.exposed_outputs = result.values pipeline_result.output = output if self.store_results: pipeline_result.pipeline_run = save_pipeline_run(result.pipeline_run, self.scratch_dir) return pipeline_result
def fit( self, pipeline: Pipeline, input_data: typing.Sequence[container.Dataset], expose_outputs: bool = False ) -> typing.Tuple[runtime_module.Runtime, PipelineResult]: pipeline_result = PipelineResult(pipeline=pipeline) runtime, output, result = runtime_module.fit( pipeline=pipeline, inputs=input_data, problem_description=self.problem_description, context=Context.TESTING, hyperparams=None, random_seed=self.random_seed, volumes_dir=self.volumes_dir, runtime_environment=self.runtime_environment, scratch_dir=self.scratch_dir, expose_produced_outputs=expose_outputs) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" pipeline_result.exposed_outputs = result.values pipeline_result.output = output return runtime, pipeline_result
def fit_pipeline( self, data_handler, problem_description: Problem, pipeline: Pipeline, input_data_id: str, *, timeout: float = None, expose_outputs: bool = False ) -> PipelineResult: pipeline_result = PipelineResult(pipeline=pipeline) pipeline_result.status = "RUNNING" pipeline_result.method_called = "fit" request_id = data_handler.get_data.remote(input_data_id) input_data = ray.get(request_id) is_standard_pipeline = False if len(input_data) == 1 and len(pipeline.outputs) == 1: is_standard_pipeline = True with d3m_utils.silence(): runtime, output, result = runtime_module.fit( pipeline=pipeline, inputs=input_data, problem_description=problem_description, context=Context.TESTING, hyperparams=None, random_seed=self.random_seed, volumes_dir=self.volumes_dir, scratch_dir=self.scratch_dir, runtime_environment=self.runtime_environment, is_standard_pipeline=is_standard_pipeline, expose_produced_outputs=expose_outputs ) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" fitted_pipeline_id = str(uuid.uuid4()) if self.store_results: pipeline_result.exposed_outputs = save_exposed_values(result.values, pipeline.id, self.scratch_dir) pipeline_result.output = save_exposed_values(output, pipeline.id, self.scratch_dir) else: pipeline_result.exposed_outputs = result.values pipeline_result.output = output pipeline_result.fitted_pipeline_id = fitted_pipeline_id self.fitted_pipelines[fitted_pipeline_id] = runtime if self.store_results: pipeline_result.pipeline_run = save_pipeline_run(result.pipeline_run, self.scratch_dir) return pipeline_result
def produce_pipeline_request(self, fitted_pipeline_id: str, input_data: typing.Sequence[ContainerType], *, timeout: float = None, expose_outputs: bool = False) -> str: """ A method that submit a produce pipeline request. Parameters ---------- fitted_pipeline_id : str The fitted pipeline if of the fitted pipeline to be use to produce results. input_data : typing.Sequence[ContainerType] A list of D3M containers. timeout : float A maximum amount of time that pipelines are going to be executed in seconds. expose_outputs : bool A variable that enable exposing every intermediate results based on the input_data Returns ------- str A request id. """ request_id = str(uuid.uuid4()) pipeline_result = PipelineResult(fitted_pipeline_id=fitted_pipeline_id) pipeline_result.status = "RUNNING" pipeline_result.method_called = "produce" pipeline_result.fitted_pipeline_id = fitted_pipeline_id output, result = runtime_module.produce( fitted_pipeline=self.fitted_pipelines[fitted_pipeline_id], test_inputs=input_data, expose_produced_outputs=expose_outputs) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" pipeline_result.output = output pipeline_result.exposed_outputs = result.values pipeline_result.pipeline_run = result.pipeline_run self.request_results[request_id] = pipeline_result return request_id
def produce(self, fitted_pipeline: runtime_module.Runtime, input_data: typing.Sequence[container.Dataset], expose_outputs: bool = False) -> PipelineResult: pipeline_result = PipelineResult(fitted_pipeline_id='') with d3m_utils.silence(): output, result = runtime_module.produce( fitted_pipeline=fitted_pipeline, test_inputs=input_data, expose_produced_outputs=expose_outputs) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" pipeline_result.exposed_outputs = result.values pipeline_result.output = output return pipeline_result
def fit_pipeline_request(self, problem_description: Problem, pipeline: Pipeline, input_data: typing.Sequence[ContainerType], *, timeout: float = None, expose_outputs: bool = False) -> str: """ A method that submit a fit_pipeline job. Parameters ---------- problem_description : Problem A problem description. pipeline : Pipeline The pipeline that is going to be fitted. input_data : typing.Sequence[ContainerType] A list of D3M containers. timeout : float A maximum amount of time that pipelines are going to be executed in seconds. expose_outputs : bool A variable that enable exposing every intermediate results based on the input_data Returns ------- str A request id. """ request_id = str(uuid.uuid4()) pipeline_result = PipelineResult(pipeline=pipeline) pipeline_result.status = "RUNNING" pipeline_result.method_called = "fit" is_standard_pipeline = False if len(input_data) == 1 and len(pipeline.outputs) == 1: is_standard_pipeline = True runtime, output, result = runtime_module.fit( pipeline=pipeline, inputs=input_data, problem_description=problem_description, context=Context.TESTING, hyperparams=None, random_seed=self.random_seed, volumes_dir=self.volumes_dir, scratch_dir=self.scratch_dir, runtime_environment=self.runtime_environment, is_standard_pipeline=is_standard_pipeline, expose_produced_outputs=expose_outputs) if result.has_error(): pipeline_result.status = "ERRORED" pipeline_result.error = result.error else: pipeline_result.status = "COMPLETED" pipeline_result.exposed_outputs = result.values pipeline_result.output = output fitted_pipeline_id = str(uuid.uuid4()) pipeline_result.fitted_pipeline_id = fitted_pipeline_id self.fitted_pipelines[fitted_pipeline_id] = runtime pipeline_result.pipeline_run = result.pipeline_run self.request_results[request_id] = pipeline_result return request_id