def test_regression(self):
dataset_doc_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'datasets', 'database_dataset_1', 'datasetDoc.json'))
dataset = container.Dataset.load(utils.path_to_uri(dataset_doc_path))
# We set semantic types like runtime would.
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 4), 'https://metadata.datadrivendiscovery.org/types/Target')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 4), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
dataset.metadata = dataset.metadata.remove_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 4), 'https://metadata.datadrivendiscovery.org/types/Attribute')
random = numpy.random.RandomState(42)
# Create a synthetic prediction DataFrame.
d3mIndex = dataset['learningData'].iloc[:, 0].astype(int)
value = random.randn(len(d3mIndex))
predictions = container.DataFrame({'d3mIndex': d3mIndex, 'value': value}, generate_metadata=True)
shuffled_predictions = predictions.reindex(random.permutation(predictions.index)).reset_index(drop=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'MEAN_SQUARED_ERROR',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'ROOT_MEAN_SQUARED_ERROR',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'MEAN_ABSOLUTE_ERROR',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'R_SQUARED',
'pos_label': None,
'k': None,
})],
}))
for name, pred in zip(['predictions', 'shuffled_predictions'], [predictions, shuffled_predictions]):
scores = primitive.produce(inputs=pred, score_dataset=dataset).value
self.assertEqual(scores.values.tolist(), [
['MEAN_SQUARED_ERROR', 3112.184932446708, 0.08521485450672399],
['ROOT_MEAN_SQUARED_ERROR', 55.786960236660214, 0.9721137517700256],
['MEAN_ABSOLUTE_ERROR', 54.579668078204385, 0.9727169385086356],
['R_SQUARED', -22.62418041588221, 0.9881884591239001],
], name)
self.assertEqual(scores.metadata.query_column(0)['name'], 'metric', name)
self.assertEqual(scores.metadata.query_column(1)['name'], 'value', name)
def time_classification_scores(self, rows):
# This has been cut-and-paste from test_compute_scores.py
truth = container.DataFrame({
'd3mIndex': range(rows),
'col0': (1, ) * rows
})
truth_dataset = container.Dataset({'learningData': truth},
generate_metadata=True)
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(
('learningData', metadata_base.ALL_ELEMENTS, 0),
'https://metadata.datadrivendiscovery.org/types/PrimaryKey')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(
('learningData', metadata_base.ALL_ELEMENTS, 1),
'https://metadata.datadrivendiscovery.org/types/Target')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(
('learningData', metadata_base.ALL_ELEMENTS, 1),
'https://metadata.datadrivendiscovery.org/types/TrueTarget')
# predictions are identical to truth, should have no impact on performance.
predictions = truth
# configure primitive
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams(
)
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(
hyperparams=hyperparams_class.defaults().replace({
'metrics': [
metrics_class({
'metric': 'ACCURACY',
'pos_label': None,
'k': None,
}),
metrics_class({
'metric': 'F1_MICRO',
'pos_label': None,
'k': None,
}),
metrics_class({
'metric': 'F1_MACRO',
'pos_label': None,
'k': None,
})
],
}))
# run scoring.
scores = primitive.produce(inputs=predictions,
score_dataset=truth_dataset).value
def test_multivariate(self):
dataset_doc_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'datasets', 'multivariate_dataset_1', 'datasetDoc.json'))
dataset = container.Dataset.load(utils.path_to_uri(dataset_doc_path))
# We set semantic types like runtime would.
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 2), 'https://metadata.datadrivendiscovery.org/types/Target')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 2), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
dataset.metadata = dataset.metadata.remove_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 2), 'https://metadata.datadrivendiscovery.org/types/Attribute')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/Target')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
dataset.metadata = dataset.metadata.remove_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/Attribute')
random = numpy.random.RandomState(42)
# Create a synthetic prediction DataFrame.
d3mIndex = dataset['learningData'].iloc[:, 0].astype(int)
amplitude = random.randn(len(d3mIndex))
lengthscale = random.randn(len(d3mIndex))
predictions = container.DataFrame({'d3mIndex': d3mIndex, 'amplitude': amplitude, 'lengthscale': lengthscale}, generate_metadata=True)
shuffled_predictions = predictions.reindex(random.permutation(predictions.index)).reset_index(drop=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'MEAN_SQUARED_ERROR',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'ROOT_MEAN_SQUARED_ERROR',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'MEAN_ABSOLUTE_ERROR',
'pos_label': None,
'k': None,
})],
}))
for name, pred in zip(['predictions', 'shuffled_predictions'], [predictions, shuffled_predictions]):
scores = primitive.produce(inputs=pred, score_dataset=dataset).value
self.assertEqual(scores.values.tolist(), [
['MEAN_SQUARED_ERROR', 1.7627871219522482, 0.9991186066672619],
['ROOT_MEAN_SQUARED_ERROR', 1.3243591896125282, 0.9993378205019783],
['MEAN_ABSOLUTE_ERROR', 1.043095768817859, 0.9994784521628801],
], name)
self.assertEqual(scores.metadata.query_column(0)['name'], 'metric', name)
self.assertEqual(scores.metadata.query_column(1)['name'], 'value', name)
def test_object_detection_just_bounding_polygon(self):
dataset_doc_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'datasets', 'object_dataset_1', 'datasetDoc.json'))
dataset = container.Dataset.load(utils.path_to_uri(dataset_doc_path))
# We set semantic types like runtime would.
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/Target')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
dataset.metadata = dataset.metadata.remove_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 3), 'https://metadata.datadrivendiscovery.org/types/Attribute')
random = numpy.random.RandomState(42)
# Create a synthetic prediction DataFrame.
predictions = container.DataFrame([
[0, '330,463,330,505,387,505,387,463', 0.0739],
[0, '420,433,420,498,451,498,451,433', 0.091],
[0, '328,465,328,540,403,540,403,465', 0.1008],
[0, '480,477,480,522,508,522,508,477', 0.1012],
[0, '357,460,357,537,417,537,417,460', 0.1058],
[0, '356,456,356,521,391,521,391,456', 0.0843],
[1, '345,460,345,547,415,547,415,460', 0.0539],
[1, '381,362,381,513,455,513,455,362', 0.0542],
[1, '382,366,382,422,416,422,416,366', 0.0559],
[1, '730,463,730,583,763,583,763,463', 0.0588],
], columns=['d3mIndex', 'bounding_polygon_area', 'confidence'], generate_metadata=True)
shuffled_predictions = predictions.reindex(random.permutation(predictions.index)).reset_index(drop=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'OBJECT_DETECTION_AVERAGE_PRECISION',
'pos_label': None,
'k': None,
})],
}))
for name, pred in zip(['predictions', 'shuffled_predictions'], [predictions, shuffled_predictions]):
scores = primitive.produce(inputs=pred, score_dataset=dataset).value
self.assertEqual(scores.values.tolist(), [
['OBJECT_DETECTION_AVERAGE_PRECISION', 0.125, 0.125],
], name)
self.assertEqual(scores.metadata.query_column(0)['name'], 'metric')
self.assertEqual(scores.metadata.query_column(1)['name'], 'value')
self.assertEqual(scores.metadata.query_column(2)['name'], 'normalized')
def test_classification(self):
dataset_doc_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data', 'datasets', 'iris_dataset_1', 'datasetDoc.json'))
dataset = container.Dataset.load(utils.path_to_uri(dataset_doc_path))
# We set semantic types like runtime would.
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 5), 'https://metadata.datadrivendiscovery.org/types/Target')
dataset.metadata = dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 5), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
dataset.metadata = dataset.metadata.remove_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 5), 'https://metadata.datadrivendiscovery.org/types/Attribute')
random = numpy.random.RandomState(42)
# Create a synthetic prediction DataFrame.
d3mIndex = dataset['learningData'].iloc[:, 0].astype(int)
species = random.choice(['Iris-setosa', 'Iris-versicolor', 'Iris-virginica'], len(d3mIndex))
predictions = container.DataFrame({'d3mIndex': d3mIndex, 'species': species}, generate_metadata=True)
shuffled_predictions = predictions.reindex(random.permutation(predictions.index)).reset_index(drop=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'ACCURACY',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'F1_MICRO',
'pos_label': None,
'k': None,
}), metrics_class({
'metric': 'F1_MACRO',
'pos_label': None,
'k': None,
})],
}))
for name, pred in zip(['predictions', 'shuffled_predictions'], [predictions, shuffled_predictions]):
scores = primitive.produce(inputs=pred, score_dataset=dataset).value
self.assertEqual(scores.values.tolist(), [
['ACCURACY', 0.4066666666666667, 0.4066666666666667],
['F1_MICRO', 0.4066666666666667, 0.4066666666666667],
['F1_MACRO', 0.4051068540623797, 0.4051068540623797],
], name)
self.assertEqual(scores.metadata.query_column(0)['name'], 'metric', name)
self.assertEqual(scores.metadata.query_column(1)['name'], 'value', name)
def test_classification_non_d3mindex(self):
truth = container.DataFrame([
[1, 'happy-pleased'],
[2, 'amazed-suprised'],
[3, 'sad-lonely'],
[4, 'relaxing-calm'],
], columns=['non_d3mIndex', 'class_label']) # Score dataset has a non-d3mIndex
truth_dataset = container.Dataset({'learningData': truth}, generate_metadata=True)
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 0), 'https://metadata.datadrivendiscovery.org/types/PrimaryKey')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 1), 'https://metadata.datadrivendiscovery.org/types/Target')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 1), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
predictions = container.DataFrame([
[1, 'happy-pleased'],
[2, 'amazed-suprised'],
[3, 'relaxing-calm'],
[4, 'sad-lonely'],
], columns=['d3mIndex', 'class_label'], generate_metadata=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'ACCURACY',
'pos_label': None,
'k': None,
})],
}))
scores = primitive.produce(inputs=predictions, score_dataset=truth_dataset).value
self.assertEqual(scores.values.tolist(), [
['ACCURACY', 0.5, 0.5],
])
def test_all_labels(self):
truth = container.DataFrame([
[3, 'happy-pleased'],
[3, 'relaxing-calm'],
[7, 'amazed-suprised'],
[7, 'happy-pleased'],
[13, 'quiet-still'],
[13, 'sad-lonely'],
], columns=['d3mIndex', 'class_label'])
truth_dataset = container.Dataset({'learningData': truth}, generate_metadata=True)
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 0), 'https://metadata.datadrivendiscovery.org/types/PrimaryKey')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 1), 'https://metadata.datadrivendiscovery.org/types/Target')
truth_dataset.metadata = truth_dataset.metadata.add_semantic_type(('learningData', metadata_base.ALL_ELEMENTS, 1), 'https://metadata.datadrivendiscovery.org/types/TrueTarget')
predictions = container.DataFrame([
[3, 'happy-pleased'],
[3, 'sad-lonely'],
[7, 'amazed-suprised'],
[7, 'happy-pleased'],
[13, 'quiet-still'],
[13, 'happy-pleased'],
], columns=['d3mIndex', 'class_label'], generate_metadata=True)
hyperparams_class = compute_scores.ComputeScoresPrimitive.metadata.get_hyperparams()
metrics_class = hyperparams_class.configuration['metrics'].elements
all_labels_class = hyperparams_class.configuration['all_labels'].elements
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'HAMMING_LOSS',
'pos_label': None,
'k': None,
})],
}))
scores = primitive.produce(inputs=predictions, score_dataset=truth_dataset).value
self.assertEqual(scores.values.tolist(), [
['HAMMING_LOSS', 0.26666666666666666, 0.7333333333333334],
])
self.assertEqual(scores.metadata.query_column(0)['name'], 'metric')
self.assertEqual(scores.metadata.query_column(1)['name'], 'value')
self.assertEqual(scores.metadata.query_column(2)['name'], 'normalized')
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'HAMMING_LOSS',
'pos_label': None,
'k': None,
})],
'all_labels': [all_labels_class({
'column_name': 'class_label',
'labels': ['happy-pleased', 'relaxing-calm', 'amazed-suprised', 'quiet-still', 'sad-lonely', 'foobar'],
})],
}))
scores = primitive.produce(inputs=predictions, score_dataset=truth_dataset).value
self.assertEqual(scores.values.tolist(), [
['HAMMING_LOSS', 0.2222222222222222, 0.7777777777777778],
])
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'HAMMING_LOSS',
'pos_label': None,
'k': None,
})],
'all_labels': [all_labels_class({
'column_name': 'class_label',
'labels': ['happy-pleased', 'relaxing-calm', 'amazed-suprised'],
})],
}))
with self.assertRaisesRegex(exceptions.InvalidArgumentValueError, 'Truth contains extra labels'):
primitive.produce(inputs=predictions, score_dataset=truth_dataset)
truth_dataset.metadata = truth_dataset.metadata.update_column(1, {
'all_distinct_values': ['happy-pleased', 'relaxing-calm', 'amazed-suprised', 'quiet-still', 'sad-lonely', 'foobar'],
}, at=('learningData',))
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'HAMMING_LOSS',
'pos_label': None,
'k': None,
})],
}))
scores = primitive.produce(inputs=predictions, score_dataset=truth_dataset).value
self.assertEqual(scores.values.tolist(), [
['HAMMING_LOSS', 0.2222222222222222, 0.7777777777777778],
])
truth_dataset.metadata = truth_dataset.metadata.update_column(1, {
'all_distinct_values': ['happy-pleased', 'relaxing-calm', 'amazed-suprised'],
}, at=('learningData',))
primitive = compute_scores.ComputeScoresPrimitive(hyperparams=hyperparams_class.defaults().replace({
'metrics': [metrics_class({
'metric': 'HAMMING_LOSS',
'pos_label': None,
'k': None,
})],
}))
with self.assertRaisesRegex(exceptions.InvalidArgumentValueError, 'Truth contains extra labels'):
primitive.produce(inputs=predictions, score_dataset=truth_dataset)
