def fit_solution(self, solution_id, dataset):
request = core_pb2.FitSolutionRequest(
solution_id=solution_id,
inputs=[Value(dataset_uri=self._get_dataset_doc_path(dataset))],
expose_outputs=[
'outputs.0'
# 'steps.0.produce'
],
expose_value_types=[ValueType.Value('CSV_URI')],
# users=[
# core_pb2.SolutionRunUser(
# id='dummy',
# choosen=True,
# reason='dummy'
# )
# ]
)
LOGGER.debug("%s: %s", request.__class__.__name__, request)
response = self.stub.FitSolution(request)
LOGGER.debug("%s: %s", response.__class__.__name__, response)
return response
def do_search(self,
dataset_doc_path,
problem_doc_path,
ta2_id,
time_bound=30.0,
pipelines_limit=0,
pipeline_template=None):
# Search
search = self.core.SearchSolutions(
pb_core.SearchSolutionsRequest(
user_agent='D3M_TA2_Evaluation',
version=version,
time_bound_search=time_bound,
priority=10,
rank_solutions_limit=pipelines_limit,
allowed_value_types=['RAW', 'DATASET_URI', 'CSV_URI'],
template=pipeline_template,
problem=self._build_problem(problem_doc_path),
inputs=[Value(dataset_uri='file://%s' % dataset_doc_path)],
))
# Get request
getsearch_request = pb_core.GetSearchSolutionsResultsRequest()
getsearch_request.search_id = search.search_id
# Make the call (loop cause streaming)-- It makes client docker remain open untill complete
logger.warning('Solution Stream: ')
for getsearch_response in self.core.GetSearchSolutionsResults(
getsearch_request):
logger.warning(getsearch_response)
if getsearch_response.solution_id:
pipeline_id = getsearch_response.solution_id
yield {'id': pipeline_id, 'search_id': str(search.search_id)}
logger.warning('------------------------------------')
def score_solution(self, solution_id, dataset):
problem = self._build_problem(dataset)
request = core_pb2.ScoreSolutionRequest(
solution_id=solution_id,
inputs=[Value(dataset_uri=self._get_dataset_doc_path(dataset))],
performance_metrics=problem.problem.performance_metrics,
# users=[
# core_pb2.SolutionRunUser(
# id='dummy',
# choosen=True,
# reason='dummy'
# )
# ],
configuration=core_pb2.ScoringConfiguration(
method=core_pb2.EvaluationMethod.Value('K_FOLD'),
folds=3,
train_test_ratio=3,
shuffle=True,
random_seed=0,
stratified=False))
LOGGER.debug("%s: %s", request.__class__.__name__, request)
response = self.stub.ScoreSolution(request)
LOGGER.debug("%s: %s", response.__class__.__name__, response)
return response
def test_apiclient_search_solutions(search_solutions_request_mock,
logger_mock):
instance = TA3APIClient(port=9999)
dataset = 'test_dataset'
expected_value = 'response'
# instance mocks
instance._get_dataset_doc_path = MagicMock(return_value='dataset-doc-path')
instance._build_problem = MagicMock(return_value='build-problem')
instance.stub.SearchSolutions = MagicMock(return_value=expected_value)
value = instance.search_solutions(dataset)
assert value == expected_value
search_solutions_request_mock.assert_called_once_with(
user_agent='ta3_api_test.py',
version='2019.6.11',
time_bound_search=1.,
priority=0.,
allowed_value_types=[
ValueType.Value('RAW'),
ValueType.Value('DATASET_URI'),
ValueType.Value('CSV_URI'),
],
inputs=[Value(dataset_uri='dataset-doc-path')],
problem='build-problem')
assert logger_mock.call_count == 2
def searchSolutions(self, stub, problem):
_logger.info("Calling Search Solutions:")
metric = problem['problem']['performance_metrics'][0]
request = SearchSolutionsRequest(
user_agent="Test Client",
version="2019.1.22",
time_bound_search=self.time_bound,
priority=0,
allowed_value_types=[value_pb2.RAW],
problem=ProblemDescription(
problem=Problem(
# id=problem['id'],
# version="3.1.2",
# name=problem['id'],
# description=problem['id'],
task_type=problem['problem']['task_type'].value,
task_subtype=problem['problem']['task_subtype'].value,
performance_metrics=[
ProblemPerformanceMetric(
# metric=problem['problem']['performance_metrics'][0]['metric'].value,
metric=metric['metric'].value,
k=metric.get('params', {}).get('k', None),
pos_label=metric.get('params',
{}).get('pos_label', None),
)
]),
inputs=[
ProblemInput(
dataset_id=problem['inputs'][0]['dataset_id'],
targets=[
ProblemTarget(target_index=problem['inputs'][0]
['targets'][0]['target_index'],
resource_id=problem['inputs'][0]
['targets'][0]['resource_id'],
column_index=problem['inputs'][0]
['targets'][0]['column_index'],
column_name=problem['inputs'][0]
['targets'][0]['column_name'])
])
]),
template=PipelineDescription(
), # TODO: We will handle pipelines later D3M-61
inputs=[
Value(dataset_uri='file://' +
dataset_docs[problem['inputs'][0]['dataset_id']]),
],
time_bound_run=5, # in minutes
rank_solutions_limit=20)
print_request(request)
reply = stub.SearchSolutions(request)
log_msg(reply)
return reply
def fitSolution(self, stub, solution_id, problem):
_logger.info("Calling FitSolution with solution_id: " + solution_id)
request = FitSolutionRequest(
solution_id=solution_id,
inputs=[
Value(dataset_uri='file://' +
dataset_docs[problem['inputs'][0]['dataset_id']])
],
# expose_outputs = ['steps.7.produce'],
expose_outputs=['outputs.0'],
expose_value_types=[value_pb2.CSV_URI])
print_request(request)
reply = stub.FitSolution(request)
log_msg(reply)
return reply
def scoreSolutionRequest(self, stub, solution_id, problem):
_logger.info("Calling Score Solution Request:")
request = ScoreSolutionRequest(
solution_id=solution_id,
inputs=[
Value(dataset_uri='file://' +
dataset_docs[problem['inputs'][0]['dataset_id']])
],
performance_metrics=[
ProblemPerformanceMetric(metric=problem_pb2.ACCURACY)
],
users=[SolutionRunUser()], # Optional so pushing for now
configuration=None # For future implementation
)
print_request(request)
reply = stub.ScoreSolution(request)
return reply
def test_apiclient_fit_solution(fit_solution_request_mock, logger_mock):
instance = TA3APIClient(port=9999)
solution_id = 'solution-id'
dataset = 'test-dataset'
expected_response = 'response'
# mocks
fit_solution_request_mock.return_value = 'request'
instance._get_dataset_doc_path = MagicMock(return_value='dataset-doc-path')
instance.stub.FitSolution = MagicMock(return_value=expected_response)
return_value = instance.fit_solution(solution_id, dataset)
assert return_value == expected_response
assert logger_mock.call_count == 2
fit_solution_request_mock.assert_called_once_with(
solution_id=solution_id,
inputs=[Value(dataset_uri='dataset-doc-path')],
expose_outputs=['outputs.0'],
expose_value_types=[ValueType.Value('CSV_URI')])
instance.stub.FitSolution.called_once_with('request')
def search_solutions(self, dataset, time_bound_search=1.):
created_at = Timestamp()
created_at.FromDatetime(datetime.utcnow())
request = core_pb2.SearchSolutionsRequest(
user_agent='ta3_api_test.py',
version='2019.7.9',
time_bound_search=time_bound_search,
priority=0.,
allowed_value_types=[
ValueType.Value('RAW'),
ValueType.Value('DATASET_URI'),
ValueType.Value('CSV_URI'),
],
inputs=[Value(dataset_uri=self._get_dataset_doc_path(dataset))],
problem=self._build_problem(dataset)
# template=pipeline_pb2.PipelineDescription(
# id='dummy',
# source=pipeline_pb2.PipelineSource(
# name='dummy',
# contact='dummy',
# pipelines=['dummy'],
# ),
# created=created_at,
# context=pipeline_pb2.PipelineContext.Value('PIPELINE_CONTEXT_UNKNOWN'),
# name='dummy',
# description='dummy',
# users=[
# pipeline_pb2.PipelineDescriptionUser(
# id='dummy',
# reason='dummy',
# rationale='dummy'
# )
# ],
# inputs=[
# pipeline_pb2.PipelineDescriptionInput(
# name='dummy'
# )
# ],
# outputs=[
# pipeline_pb2.PipelineDescriptionOutput(
# name='dummy',
# data='dummy'
# )
# ],
# steps=[
# pipeline_pb2.PipelineDescriptionStep(
# primitive=pipeline_pb2.PrimitivePipelineDescriptionStep(
# primitive=Primitive(
# id='dummy',
# version='dummy',
# python_path='dummy',
# name='dummy',
# digest='dummy'
# ),
# arguments={
# 'dummy': pipeline_pb2.PrimitiveStepArgument(
# data=pipeline_pb2.DataArgument(
# data='dummy'
# )
# )
# },
# outputs=[
# pipeline_pb2.StepOutput(
# id='dummy'
# )
# ],
# hyperparams={
# 'dummy': pipeline_pb2.PrimitiveStepHyperparameter(
# data=pipeline_pb2.DataArgument(
# data='dummy'
# )
# )
# },
# users=[
# pipeline_pb2.PipelineDescriptionUser(
# id='dummy',
# reason='dummy',
# rationale='dummy'
# )
# ],
# )
# )
# ]
# ),
)
LOGGER.debug("%s: %s", request.__class__.__name__, request)
response = self.stub.SearchSolutions(request)
LOGGER.debug("%s: %s", response.__class__.__name__, response)
return response