def execute_pipeline(pipeline: pipeline_pb2.PipelineDescription, dataset_filenames: List[str], static_resource_path: Optional[str] = None) -> Any: """ Executes a binary protobuf pipeline against a supplied d3m dataset using the d3m common runtime. Parameters ---------- pipeline: protobuf pipeline definition dataset_filenames: paths to folders containing input D3M datasets Returns ------- The result of the pipeline execution. """ # transform the pipeline to the internal d3m representation pipeline_d3m = utils.decode_pipeline_description( pipeline, metadata_pipeline.Resolver()) # load the data inputs = _load_inputs(dataset_filenames) # fit and produce fitted_pipeline, _ = _fit(pipeline_d3m, inputs, volumes_dir=static_resource_path) result = _produce(fitted_pipeline, inputs) return result
def test_pipeline(self): with open(os.path.join(TEST_PIPELINES_DIR, 'random-sample.yml'), 'r') as pipeline_file: pipeline = pipeline_module.Pipeline.from_yaml( pipeline_file, resolver=pipeline_module.Resolver(), strict_digest=True, ) with tempfile.TemporaryDirectory() as scratch_dir: def validate_uri(uri): utils.validate_uri(uri, [scratch_dir]) pipeline_message = utils.encode_pipeline_description( pipeline, [ utils.ValueType.RAW, utils.ValueType.CSV_URI, utils.ValueType.DATASET_URI, utils.ValueType.PICKLE_BLOB, utils.ValueType.PICKLE_URI, ], scratch_dir, validate_uri=validate_uri, ) decoded_pipeline = utils.decode_pipeline_description( pipeline_message, pipeline_module.Resolver(), validate_uri=validate_uri, strict_digest=True, ) self.assertEqual(pipeline.to_json_structure(nest_subpipelines=True), decoded_pipeline.to_json_structure(nest_subpipelines=True))
def convert_pipeline(input_file: str, output_file: str) -> pipeline_pb2.PipelineDescription: # load the protobuf infile = open(input_file, "rb") pipeline = pipeline_pb2.PipelineDescription() pipeline.ParseFromString(infile.read()) infile.close() # convert to json and save return ta3ta2.decode_pipeline_description(pipeline, pipeline_module.Resolver())
def do_describe(self, solution_id): pipeline_description = None try: pipeline_description = self.core.DescribeSolution( pb_core.DescribeSolutionRequest( solution_id=solution_id, )).pipeline except: logger.exception("Exception during describe %r", solution_id) with silence(): pipeline = decode_pipeline_description( pipeline_description, pipeline_module.NoResolver()) return pipeline.to_json_structure()
def SearchSolutions(self, request, context): user_agent = request.user_agent logger.info('method=SearchSolution, agent=%s', user_agent) # Checking version of protocol. if request.version != self.version: logger.info(' method=SearchSolution, info=Different api version%s', self.version) # Types allowed by client allowed_value_types = list(request.allowed_value_types) if not allowed_value_types: allowed_value_types = ALLOWED_VALUE_TYPES problem_description = utils.decode_problem_description(request.problem) # Parsing and storing Pipeline Template (store this to a file instead of passing it) with d3m_utils.silence(): template = utils.decode_pipeline_description( pipeline_description=request.template, resolver=Resolver( primitives_blocklist=PrimitivesList.BlockList)) time_bound_search = request.time_bound_search time_bound_search = time_bound_search * 60 input_data = [load_data(utils.decode_value(x)) for x in request.inputs] search = SearchWrappers.remote( search_class=DataDrivenSearch, problem_description=problem_description, backend=self.backend, primitives_blocklist=PrimitivesList.BlockList, ranking_function=dummy_ranking_function, n_workers=self.n_workers) request_id = search.get_search_id.remote() search_id = ray.get(request_id) # print('got search_id') self.searches[search_id] = search request_id = self.searches[search_id].search_request.remote( time_left=time_bound_search, input_data=input_data) self.request_mapping[search_id] = request_id self.solutions[search_id] = [] self.problem_descriptions[search_id] = problem_description response = core_pb2.SearchSolutionsResponse(search_id=search_id) return response