def setUp(self):
super(GetBenchmarkResultsTest, self).setUp()
config = metadata_store_pb2.ConnectionConfig()
config.fake_database.SetInParent()
self.store = metadata_store.MetadataStore(config)
self.exec_type_id = self._put_execution_type()
self.artifact_type_id = self._put_artifact_type()
def test_overview(self, mlmd_store_path, metric_aggregators, want_columns):
config = metadata_store_pb2.ConnectionConfig()
config.sqlite.filename_uri = mlmd_store_path
store = metadata_store.MetadataStore(config)
df = results.overview(store, metric_aggregators=metric_aggregators)
self.assertEqual(want_columns, df.columns.tolist())
def __init__(self,
phoenix_spec,
study_name,
study_owner,
optimization_goal="minimize",
optimization_metric="loss",
connection_config=None):
"""Initializes a new MLMD connection instance.
Args:
phoenix_spec: PhoenixSpec proto.
study_name: The name of the study.
study_owner: The owner (username) of the study.
optimization_goal: minimize or maximize (string).
optimization_metric: what metric are we optimizing (string).
connection_config: a metadata_store_pb2.ConnectionConfig() proto. If None,
we fall back on the flags above.
"""
self._study_name = study_name
self._study_owner = study_owner
self._phoenix_spec = phoenix_spec
self._optimization_goal = optimization_goal
self._optimization_metric = optimization_metric
self._connection_config = connection_config
if not FLAGS.is_parsed():
logging.error(
"Flags are not parsed. Using default in file mlmd database."
" Please run main with absl.app.run(main) to fix this. "
"If running in distributed mode, this means that the "
"trainers are not sharing information between one another.")
if self._connection_config is None:
if FLAGS.is_parsed() and FLAGS.mlmd_default_sqllite_filename:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.sqlite.filename_uri = (
FLAGS.mlmd_default_sqllite_filename)
self._connection_config.sqlite.connection_mode = 3
elif FLAGS.is_parsed() and FLAGS.mlmd_socket:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.mysql.socket = FLAGS.mlmd_socket
self._connection_config.mysql.database = FLAGS.mlmd_database
self._connection_config.mysql.user = FLAGS.mlmd_user
self._connection_config.mysql.password = FLAGS.mlmd_password
else:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.sqlite.filename_uri = (
"/tmp/filedb-%d" % random.randint(0, 1000000))
self._connection_config.sqlite.connection_mode = 3
self._store = metadata_store.MetadataStore(self._connection_config)
trial_type = metadata_store_pb2.ExecutionType()
trial_type.name = "Trial"
trial_type.properties["id"] = metadata_store_pb2.INT
trial_type.properties["state"] = metadata_store_pb2.STRING
trial_type.properties["serialized_data"] = metadata_store_pb2.STRING
trial_type.properties["model_dir"] = metadata_store_pb2.STRING
trial_type.properties["evaluation"] = metadata_store_pb2.STRING
self._trial_type_id = self._store.put_execution_type(trial_type)
self._trial_id_to_run_id = {}
def test(self, use_keras, enable_tuning):
with requests_mock.Mocker() as mocker:
dataset_id_list = [1, 2]
testing_utils.register_mock_urls(mocker, dataset_id_list)
self.run_benchmarks(
[openml_cc18_benchmark.OpenMLCC18Benchmark()],
data_dir=os.path.join(self.pipeline_root, 'openML_mock_data'),
mock_data=True,
use_keras=use_keras,
enable_tuning=enable_tuning,
)
instance_names = []
for did in dataset_id_list:
instance_name = '.'.join(
['OpenMLCC18Benchmark', 'benchmark', f'OpenML.mockdata_{did}'])
instance_names.append(instance_name)
if enable_tuning:
self.assertComponentExecutionCount(8 * len(dataset_id_list))
self.assertComponentSucceeded('.'.join(
['Tuner.AutoTrainer', instance_name]))
else:
self.assertComponentExecutionCount(7 * len(dataset_id_list))
self.assertComponentSucceeded('.'.join(
[f'CsvExampleGen.OpenML.mockdata_{did}', instance_name]))
self.assertComponentSucceeded('.'.join(
['SchemaGen.AutoData', instance_name]))
self.assertComponentSucceeded('.'.join(
['StatisticsGen.AutoData', instance_name]))
self.assertComponentSucceeded('.'.join(
['Transform.AutoData', instance_name]))
self.assertComponentSucceeded('.'.join(
['Trainer.AutoTrainer', instance_name]))
self.assertComponentSucceeded('.'.join(
['Evaluator.model', instance_name]))
self.assertComponentSucceeded('.'.join(
['BenchmarkResultPublisher.model', instance_name]))
# Load benchmark results.
store = metadata_store.MetadataStore(self.metadata_config)
df = results.overview(store)
# Check benchmark results overview values.
self.assertEqual(len(df.index), len(dataset_id_list))
self.assertContainsSubset([
'run_id',
'benchmark_fullname',
'benchmark',
'run',
'num_runs',
'accuracy',
'average_loss',
'example_count',
'weighted_example_count',
], df.columns.values.tolist())
self.assertSameElements([1], df['run'].tolist())
self.assertSameElements([1], df['num_runs'].tolist())
self.assertSameElements(instance_names, df.benchmark.unique())
def __init__(self,
exec_type_name: str = 'BenchmarkResultPublisher',
artifact_type: str = results._BENCHMARK_RESULT):
self.artifact_type = artifact_type
config = metadata_store_pb2.ConnectionConfig()
config.fake_database.SetInParent()
self.store = metadata_store.MetadataStore(config)
self.exec_type_id = self._put_execution_type(exec_type_name)
self.artifact_type_id = self._put_artifact_type()
def __init__(
self,
mlmd_connection_config: Optional[
metadata_store_pb2.MetadataStoreClientConfig] = None,
):
if mlmd_connection_config is None:
# default to value suitable for local testing
mlmd_connection_config = metadata_store_pb2.MetadataStoreClientConfig(
host='localhost',
port=8080,
)
self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config)
def __init__(self,
phoenix_spec,
study_name,
study_owner,
optimization_goal="minimize",
optimization_metric="loss",
connection_config=None):
"""Initializes a new MLMD connection instance.
Args:
phoenix_spec: PhoenixSpec proto.
study_name: The name of the study.
study_owner: The owner (username) of the study.
optimization_goal: minimize or maximize (string).
optimization_metric: what metric are we optimizing (string).
connection_config: a metadata_store_pb2.ConnectionConfig() proto. If None,
we fall back on the flags above.
"""
self._study_name = study_name
self._study_owner = study_owner
self._phoenix_spec = phoenix_spec
self._optimization_goal = optimization_goal
self._optimization_metric = optimization_metric
self._connection_config = connection_config
if self._connection_config is None:
if FLAGS.mlmd_default_sqllite_filename:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.sqlite.filename_uri = (
FLAGS.mlmd_default_sqllite_filename)
self._connection_config.sqlite.connection_mode = 3
elif FLAGS.mlmd_socket:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.mysql.socket = FLAGS.mlmd_socket
self._connection_config.mysql.database = FLAGS.mlmd_database
self._connection_config.mysql.user = FLAGS.mlmd_user
self._connection_config.mysql.password = FLAGS.mlmd_password
else:
self._connection_config = metadata_store_pb2.ConnectionConfig()
self._connection_config.sqlite.filename_uri = (
"/tmp/filedb-%d" % random.randint(0, 1000000))
self._connection_config.sqlite.connection_mode = 3
self._store = metadata_store.MetadataStore(self._connection_config)
trial_type = metadata_store_pb2.ExecutionType()
trial_type.name = "Trial"
trial_type.properties["id"] = metadata_store_pb2.INT
trial_type.properties["state"] = metadata_store_pb2.STRING
trial_type.properties["serialized_data"] = metadata_store_pb2.STRING
trial_type.properties["model_dir"] = metadata_store_pb2.STRING
trial_type.properties["evaluation"] = metadata_store_pb2.STRING
self._trial_type_id = self._store.put_execution_type(trial_type)
self._trial_id_to_run_id = {}
def __init__(
self,
mlmd_connection_config: Optional[
metadata_store_pb2.MetadataStoreClientConfig] = None,
):
if mlmd_connection_config is None:
# default to value suitable for local testing
mlmd_connection_config = metadata_store_pb2.MetadataStoreClientConfig(
host='localhost',
port=8080,
)
self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config)
self.dag_type = self.mlmd_store.get_execution_type(
type_name='system.DAGExecution')
def test(self, use_keras, enable_tuning):
self.run_benchmarks([titanic_benchmark.TitanicBenchmark()],
use_keras=use_keras,
enable_tuning=enable_tuning)
if enable_tuning:
self.assertComponentSucceeded(
"Tuner.AutoTrainer.TitanicBenchmark.benchmark")
self.assertComponentSucceeded("ImportExampleGen.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"SchemaGen.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"StatisticsGen.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"Transform.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"Trainer.AutoTrainer.TitanicBenchmark.benchmark")
self.assertComponentSucceeded("Evaluator.model.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"BenchmarkResultPublisher.model.TitanicBenchmark.benchmark")
# Load benchmark results.
store = metadata_store.MetadataStore(self.metadata_config)
df = results.overview(store)
# Check benchmark results overview values.
self.assertEqual(len(df.index), 1)
self.assertContainsSubset([
"run_id",
"benchmark_fullname",
"benchmark",
"run",
"num_runs",
"accuracy",
"auc_pr",
"auc_roc",
"example_count",
"precision",
"recall",
"weighted_example_count",
], df.columns.values.tolist())
self.assertSameElements([1], df["run"].tolist())
self.assertSameElements([1], df["num_runs"].tolist())
self.assertSameElements(["TitanicBenchmark.benchmark"],
df.benchmark.unique())
def test(self, use_keras, enable_tuning):
self.run_benchmarks([titanic_benchmark.TitanicBenchmark()],
use_keras=use_keras,
enable_tuning=enable_tuning)
if enable_tuning:
self.assertComponentExecutionCount(8)
self.assertComponentSucceeded("Tuner.TitanicBenchmark.benchmark")
else:
self.assertComponentExecutionCount(7)
self.assertComponentSucceeded(
"ImportExampleGen.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"SchemaGen.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"StatisticsGen.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"Transform.AutoData.TitanicBenchmark.benchmark")
self.assertComponentSucceeded("Trainer.TitanicBenchmark.benchmark")
self.assertComponentSucceeded("Evaluator.TitanicBenchmark.benchmark")
self.assertComponentSucceeded(
"BenchmarkResultPublisher.TitanicBenchmark.benchmark")
# Load benchmark results.
store = metadata_store.MetadataStore(self.metadata_config)
df = results.overview(store)
# Check benchmark results overview values.
self.assertEqual(len(df.index), 1)
self.assertContainsSubset([
"benchmark",
"run",
"num_runs",
"accuracy",
"average_loss",
"post_export_metrics/example_count",
], df.columns.values.tolist())
self.assertSameElements([1], df["run"].tolist())
self.assertSameElements([1], df["num_runs"].tolist())
self.assertSameElements(["TitanicBenchmark.benchmark"],
df.benchmark.unique())
def test(self, algorithm):
with requests_mock.Mocker() as mocker:
testing_utils.register_mock_urls(mocker)
self.run_benchmarks(
[metalearning_benchmark.MetaLearningBenchmark()],
data_dir=os.path.join(self.pipeline_root,
'mock_metalearning_openml'),
mock_data=True,
algorithm=algorithm)
instance_name = 'MetaLearningBenchmark.benchmark'
self.assertComponentSucceeded(
f'CsvExampleGen.OpenML.mockdata_1.{instance_name}')
self.assertComponentSucceeded(
f'AugmentedTuner.train.OpenML.mockdata_1.{instance_name}')
self.assertComponentSucceeded(
f'SchemaGen.AutoData.train.OpenML.mockdata_1.{instance_name}')
self.assertComponentSucceeded(
f'StatisticsGen.AutoData.train.OpenML.mockdata_1.{instance_name}')
self.assertComponentSucceeded(
f'Transform.AutoData.train.OpenML.mockdata_1.{instance_name}')
instance_name_2 = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2'
self.assertComponentSucceeded(
f'CsvExampleGen.OpenML.mockdata_2.{instance_name_2}')
self.assertComponentSucceeded(f'SchemaGen.AutoData.{instance_name_2}')
self.assertComponentSucceeded(
f'StatisticsGen.AutoData.{instance_name_2}')
self.assertComponentSucceeded(f'Transform.AutoData.{instance_name_2}')
self.assertComponentSucceeded(f'AugmentedTuner.{instance_name_2}')
self.assertComponentSucceeded(f'Trainer.{instance_name_2}')
self.assertComponentSucceeded(f'Evaluator.model.{instance_name_2}')
self.assertComponentSucceeded(
f'BenchmarkResultPublisher.model.{instance_name_2}')
# Load benchmark results.
store = metadata_store.MetadataStore(self.metadata_config)
df = results.overview(store)
# Check benchmark results overview values.
self.assertEqual(len(df.index), 1)
self.assertContainsSubset([
'run_id',
'benchmark_fullname',
'benchmark',
'run',
'num_runs',
'accuracy',
'average_loss',
'example_count',
'weighted_example_count',
], df.columns.values.tolist())
self.assertSameElements([1], df['run'].tolist())
self.assertSameElements([1], df['num_runs'].tolist())
self.assertSameElements([instance_name_2], df.benchmark.unique())
# TODO(b/178401753): Reimplement this test when name parsing is updated for
# IR-based runners.
# Load Analytics
analytics = mlmd_analytics.Analytics(store=store)
# Check test_pipeline run analysis
run = analytics.get_latest_pipeline_run()
self.assertEqual('test_pipeline', run.name)
want_components = {
f'CsvExampleGen.OpenML.mockdata_1.{instance_name}',
f'AugmentedTuner.train.OpenML.mockdata_1.{instance_name}',
f'SchemaGen.AutoData.train.OpenML.'
f'mockdata_1.{instance_name}',
f'StatisticsGen.AutoData.train.OpenML.'
f'mockdata_1.{instance_name}',
f'Transform.AutoData.train.OpenML.'
f'mockdata_1.{instance_name}',
f'CsvExampleGen.OpenML.mockdata_2.{instance_name_2}',
f'SchemaGen.AutoData.{instance_name_2}',
f'StatisticsGen.AutoData.{instance_name_2}',
f'Transform.AutoData.{instance_name_2}',
f'AugmentedTuner.{instance_name_2}',
f'Trainer.{instance_name_2}',
f'Evaluator.model.{instance_name_2}',
f'BenchmarkResultPublisher.model.{instance_name_2}',
}
self.assertContainsSubset(want_components, run.components.keys())
component = f'Transform.AutoData.{instance_name_2}'
self.assertIs(
type(run.components[component].outputs.transformed_examples.
to_dataframe('train', 5)), pd.DataFrame)
def __init__(
self,
mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig
):
self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config)
def test(self, algorithm):
with requests_mock.Mocker() as mocker:
testing_utils.register_mock_urls(mocker)
self.run_benchmarks(
[metalearning_benchmark.MetaLearningBenchmark()],
data_dir=os.path.join(self.pipeline_root,
'mock_metalearning_openml'),
mock_data=True,
algorithm=algorithm)
train_dataset_ids = [1]
for ix in train_dataset_ids:
instance_name = 'MetaLearningBenchmark.benchmark'
self.assertComponentSucceeded(
f'CsvExampleGen.OpenML.mockdata_{ix}.{instance_name}')
self.assertComponentSucceeded(
f'AugmentedTuner.train.OpenML.mockdata_{ix}.{instance_name}')
self.assertComponentSucceeded(
f'SchemaGen.AutoData.train.OpenML.mockdata_{ix}.{instance_name}'
)
self.assertComponentSucceeded(
f'StatisticsGen.AutoData.train.OpenML.mockdata_{ix}.{instance_name}'
)
self.assertComponentSucceeded(
f'Transform.AutoData.train.OpenML.mockdata_{ix}.{instance_name}'
)
test_dataset_ids = [2]
for ix in test_dataset_ids:
instance_name = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2'
self.assertComponentSucceeded(
f'CsvExampleGen.OpenML.mockdata_{ix}.{instance_name}')
self.assertComponentSucceeded(
f'SchemaGen.AutoData.test.OpenML.mockdata_{ix}.{instance_name}'
)
self.assertComponentSucceeded(
f'StatisticsGen.AutoData.test.OpenML.mockdata_{ix}.{instance_name}'
)
self.assertComponentSucceeded(
f'Transform.AutoData.test.OpenML.mockdata_{ix}.{instance_name}'
)
self.assertComponentSucceeded(
f'AugmentedTuner.test.OpenML.mockdata_{ix}.{instance_name}')
self.assertComponentSucceeded(
f'Trainer.test.OpenML.mockdata_{ix}.{instance_name}')
instance_name = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2'
self.assertComponentSucceeded(f'Evaluator.{instance_name}')
self.assertComponentSucceeded(
f'BenchmarkResultPublisher.{instance_name}')
# Load benchmark results.
store = metadata_store.MetadataStore(self.metadata_config)
df = results.overview(store)
# Check benchmark results overview values.
self.assertEqual(len(df.index), 1)
self.assertContainsSubset([
'benchmark',
'run',
'num_runs',
'accuracy',
'average_loss',
'post_export_metrics/example_count',
], df.columns.values.tolist())
self.assertSameElements([1], df['run'].tolist())
self.assertSameElements([1], df['num_runs'].tolist())
self.assertSameElements([instance_name], df.benchmark.unique())
def setUp(self):
super(GetHparamsTest, self).setUp()
config = metadata_store_pb2.ConnectionConfig()
config.fake_database.SetInParent()
self.store = metadata_store.MetadataStore(config)
self.exec_type_id = self._put_execution_type()
