def execute_pipeline(pipeline: pipeline_pb2.PipelineDescription,
dataset_filenames: List[str],
static_resource_path: Optional[str] = None) -> Any:
"""
Executes a binary protobuf pipeline against a supplied d3m dataset using the d3m common runtime.
Parameters
----------
pipeline: protobuf pipeline definition
dataset_filenames: paths to folders containing input D3M datasets
Returns
-------
The result of the pipeline execution.
"""
# transform the pipeline to the internal d3m representation
pipeline_d3m = utils.decode_pipeline_description(
pipeline, metadata_pipeline.Resolver())
# load the data
inputs = _load_inputs(dataset_filenames)
# fit and produce
fitted_pipeline, _ = _fit(pipeline_d3m,
inputs,
volumes_dir=static_resource_path)
result = _produce(fitted_pipeline, inputs)
return result
def test_pipeline(self):
with open(os.path.join(TEST_PIPELINES_DIR, 'random-sample.yml'), 'r') as pipeline_file:
pipeline = pipeline_module.Pipeline.from_yaml(
pipeline_file,
resolver=pipeline_module.Resolver(),
strict_digest=True,
)
with tempfile.TemporaryDirectory() as scratch_dir:
def validate_uri(uri):
utils.validate_uri(uri, [scratch_dir])
pipeline_message = utils.encode_pipeline_description(
pipeline,
[
utils.ValueType.RAW,
utils.ValueType.CSV_URI,
utils.ValueType.DATASET_URI,
utils.ValueType.PICKLE_BLOB,
utils.ValueType.PICKLE_URI,
],
scratch_dir,
validate_uri=validate_uri,
)
decoded_pipeline = utils.decode_pipeline_description(
pipeline_message,
pipeline_module.Resolver(),
validate_uri=validate_uri,
strict_digest=True,
)
self.assertEqual(pipeline.to_json_structure(nest_subpipelines=True), decoded_pipeline.to_json_structure(nest_subpipelines=True))
def convert_pipeline(input_file: str,
output_file: str) -> pipeline_pb2.PipelineDescription:
# load the protobuf
infile = open(input_file, "rb")
pipeline = pipeline_pb2.PipelineDescription()
pipeline.ParseFromString(infile.read())
infile.close()
# convert to json and save
return ta3ta2.decode_pipeline_description(pipeline,
pipeline_module.Resolver())
def do_describe(self, solution_id):
pipeline_description = None
try:
pipeline_description = self.core.DescribeSolution(
pb_core.DescribeSolutionRequest(
solution_id=solution_id, )).pipeline
except:
logger.exception("Exception during describe %r", solution_id)
with silence():
pipeline = decode_pipeline_description(
pipeline_description, pipeline_module.NoResolver())
return pipeline.to_json_structure()
def SearchSolutions(self, request, context):
user_agent = request.user_agent
logger.info('method=SearchSolution, agent=%s', user_agent)
# Checking version of protocol.
if request.version != self.version:
logger.info(' method=SearchSolution, info=Different api version%s',
self.version)
# Types allowed by client
allowed_value_types = list(request.allowed_value_types)
if not allowed_value_types:
allowed_value_types = ALLOWED_VALUE_TYPES
problem_description = utils.decode_problem_description(request.problem)
# Parsing and storing Pipeline Template (store this to a file instead of passing it)
with d3m_utils.silence():
template = utils.decode_pipeline_description(
pipeline_description=request.template,
resolver=Resolver(
primitives_blocklist=PrimitivesList.BlockList))
time_bound_search = request.time_bound_search
time_bound_search = time_bound_search * 60
input_data = [load_data(utils.decode_value(x)) for x in request.inputs]
search = SearchWrappers.remote(
search_class=DataDrivenSearch,
problem_description=problem_description,
backend=self.backend,
primitives_blocklist=PrimitivesList.BlockList,
ranking_function=dummy_ranking_function,
n_workers=self.n_workers)
request_id = search.get_search_id.remote()
search_id = ray.get(request_id)
# print('got search_id')
self.searches[search_id] = search
request_id = self.searches[search_id].search_request.remote(
time_left=time_bound_search, input_data=input_data)
self.request_mapping[search_id] = request_id
self.solutions[search_id] = []
self.problem_descriptions[search_id] = problem_description
response = core_pb2.SearchSolutionsResponse(search_id=search_id)
return response