def score_solution(self, solution_id, dataset):
problem = self._build_problem(dataset)
request = core_pb2.ScoreSolutionRequest(
solution_id=solution_id,
inputs=[Value(dataset_uri=self._get_dataset_doc_path(dataset))],
performance_metrics=problem.problem.performance_metrics,
# users=[
# core_pb2.SolutionRunUser(
# id='dummy',
# choosen=True,
# reason='dummy'
# )
# ],
configuration=core_pb2.ScoringConfiguration(
method=core_pb2.EvaluationMethod.Value('K_FOLD'),
folds=3,
train_test_ratio=3,
shuffle=True,
random_seed=0,
stratified=False))
LOGGER.debug("%s: %s", request.__class__.__name__, request)
response = self.stub.ScoreSolution(request)
LOGGER.debug("%s: %s", response.__class__.__name__, response)
return response
def score_solution_request(solution_id, test_paths):
inputs = [value_pb2.Value(dataset_uri=test_paths['SCORE']['dataset'])]
problem = problem_module.Problem.load(test_paths['SCORE']['problem'])
performance_metrics = []
for performance_metric in problem['problem'].get('performance_metrics',
[]):
performance_metrics.append(
utils.encode_performance_metric(performance_metric))
# TODO add support for more evaluation methods
users = []
evaluation_method = 'K_FOLD'
configuration = core_pb2.ScoringConfiguration(
method=evaluation_method,
folds=2,
# train_test_ratio
shuffle=True,
random_seed=42,
stratified=True,
)
request = core_pb2.ScoreSolutionRequest(
solution_id=solution_id,
inputs=inputs,
performance_metrics=performance_metrics,
users=users,
configuration=configuration)
return request
def do_score(self, solution_id, dataset_path, problem_path, ta2_id):
try:
problem = Problem.load(problem_uri=problem_path)
except:
logger.exception('Error parsing problem')
# Encode metric
metrics = []
for metric in problem['problem']['performance_metrics']:
metrics.append(encode_performance_metric(metric))
# Showing only the first metric
target_metric = problem['problem']['performance_metrics'][0]['metric']
logger.info('target_metric %s !', target_metric)
response = self.core.ScoreSolution(
pb_core.ScoreSolutionRequest(
solution_id=solution_id,
inputs=[
pb_value.Value(dataset_uri='file://%s' % dataset_path, )
],
performance_metrics=metrics,
users=[],
configuration=pb_core.ScoringConfiguration(
method='HOLDOUT',
train_test_ratio=0.75,
shuffle=True,
random_seed=0),
))
logger.info('ScoreSolution response %s !', response)
# Get Results
results = self.core.GetScoreSolutionResults(
pb_core.GetScoreSolutionResultsRequest(
request_id=response.request_id, ))
for result in results:
logger.info('result %s !', result)
if result.progress.state == pb_core.COMPLETED:
scores = []
for metric_score in result.scores:
metric = decode_performance_metric(
metric_score.metric)['metric']
if metric == target_metric:
score = decode_value(metric_score.value)['value']
scores.append(score)
if len(scores) > 0:
avg_score = round(sum(scores) / len(scores), 5)
normalized_score = PerformanceMetric[
target_metric.name].normalize(avg_score)
return {
'score': avg_score,
'normalized_score': normalized_score,
'metric': target_metric.name.lower()
}
def encode_scoring_configuration(scoring_configuration):
"""
Decode a scoring configuration from grpc
Parameters
----------
scoring_configuration: dict
A dictionary with the scoring configuration.
Returns
-------
scoring_configuration: core_pb2.ScoringConfiguration
A grpc ScoringConfiguration message.
"""
if scoring_configuration is None:
return core_pb2.ScoringConfiguration()
else:
method = scoring_configuration['method']
folds = scoring_configuration.get('number_of_folds', None)
if folds is not None:
folds = int(folds)
train_test_ratio = scoring_configuration.get('train_score_ratio', None)
if train_test_ratio is not None:
train_test_ratio = float(train_test_ratio)
shuffle = scoring_configuration.get('shuffle', None)
if shuffle is not None:
shuffle = json.loads(shuffle.lower())
random_seed = scoring_configuration.get('randomSeed', None)
if random_seed is not None:
random_seed = int(random_seed)
stratified = scoring_configuration.get('stratified', None)
if stratified is not None:
stratified = json.loads(stratified.lower())
return core_pb2.ScoringConfiguration(method=method,
folds=folds,
train_test_ratio=train_test_ratio,
shuffle=shuffle,
random_seed=random_seed,
stratified=stratified)
def get_default_scoring_config(self):
cfg = core_pb2.ScoringConfiguration(method=core_pb2.K_FOLD,
folds=10,
train_test_ratio=5,
shuffle=True)
return cfg