def score_solution(self, solution_id, dataset): problem = self._build_problem(dataset) request = core_pb2.ScoreSolutionRequest( solution_id=solution_id, inputs=[Value(dataset_uri=self._get_dataset_doc_path(dataset))], performance_metrics=problem.problem.performance_metrics, # users=[ # core_pb2.SolutionRunUser( # id='dummy', # choosen=True, # reason='dummy' # ) # ], configuration=core_pb2.ScoringConfiguration( method=core_pb2.EvaluationMethod.Value('K_FOLD'), folds=3, train_test_ratio=3, shuffle=True, random_seed=0, stratified=False)) LOGGER.debug("%s: %s", request.__class__.__name__, request) response = self.stub.ScoreSolution(request) LOGGER.debug("%s: %s", response.__class__.__name__, response) return response
def score_solution_request(solution_id, test_paths): inputs = [value_pb2.Value(dataset_uri=test_paths['SCORE']['dataset'])] problem = problem_module.Problem.load(test_paths['SCORE']['problem']) performance_metrics = [] for performance_metric in problem['problem'].get('performance_metrics', []): performance_metrics.append( utils.encode_performance_metric(performance_metric)) # TODO add support for more evaluation methods users = [] evaluation_method = 'K_FOLD' configuration = core_pb2.ScoringConfiguration( method=evaluation_method, folds=2, # train_test_ratio shuffle=True, random_seed=42, stratified=True, ) request = core_pb2.ScoreSolutionRequest( solution_id=solution_id, inputs=inputs, performance_metrics=performance_metrics, users=users, configuration=configuration) return request
def do_score(self, solution_id, dataset_path, problem_path, ta2_id): try: problem = Problem.load(problem_uri=problem_path) except: logger.exception('Error parsing problem') # Encode metric metrics = [] for metric in problem['problem']['performance_metrics']: metrics.append(encode_performance_metric(metric)) # Showing only the first metric target_metric = problem['problem']['performance_metrics'][0]['metric'] logger.info('target_metric %s !', target_metric) response = self.core.ScoreSolution( pb_core.ScoreSolutionRequest( solution_id=solution_id, inputs=[ pb_value.Value(dataset_uri='file://%s' % dataset_path, ) ], performance_metrics=metrics, users=[], configuration=pb_core.ScoringConfiguration( method='HOLDOUT', train_test_ratio=0.75, shuffle=True, random_seed=0), )) logger.info('ScoreSolution response %s !', response) # Get Results results = self.core.GetScoreSolutionResults( pb_core.GetScoreSolutionResultsRequest( request_id=response.request_id, )) for result in results: logger.info('result %s !', result) if result.progress.state == pb_core.COMPLETED: scores = [] for metric_score in result.scores: metric = decode_performance_metric( metric_score.metric)['metric'] if metric == target_metric: score = decode_value(metric_score.value)['value'] scores.append(score) if len(scores) > 0: avg_score = round(sum(scores) / len(scores), 5) normalized_score = PerformanceMetric[ target_metric.name].normalize(avg_score) return { 'score': avg_score, 'normalized_score': normalized_score, 'metric': target_metric.name.lower() }
def encode_scoring_configuration(scoring_configuration): """ Decode a scoring configuration from grpc Parameters ---------- scoring_configuration: dict A dictionary with the scoring configuration. Returns ------- scoring_configuration: core_pb2.ScoringConfiguration A grpc ScoringConfiguration message. """ if scoring_configuration is None: return core_pb2.ScoringConfiguration() else: method = scoring_configuration['method'] folds = scoring_configuration.get('number_of_folds', None) if folds is not None: folds = int(folds) train_test_ratio = scoring_configuration.get('train_score_ratio', None) if train_test_ratio is not None: train_test_ratio = float(train_test_ratio) shuffle = scoring_configuration.get('shuffle', None) if shuffle is not None: shuffle = json.loads(shuffle.lower()) random_seed = scoring_configuration.get('randomSeed', None) if random_seed is not None: random_seed = int(random_seed) stratified = scoring_configuration.get('stratified', None) if stratified is not None: stratified = json.loads(stratified.lower()) return core_pb2.ScoringConfiguration(method=method, folds=folds, train_test_ratio=train_test_ratio, shuffle=shuffle, random_seed=random_seed, stratified=stratified)
def get_default_scoring_config(self): cfg = core_pb2.ScoringConfiguration(method=core_pb2.K_FOLD, folds=10, train_test_ratio=5, shuffle=True) return cfg