def setUp(self): super(GetBenchmarkResultsTest, self).setUp() config = metadata_store_pb2.ConnectionConfig() config.fake_database.SetInParent() self.store = metadata_store.MetadataStore(config) self.exec_type_id = self._put_execution_type() self.artifact_type_id = self._put_artifact_type()
def test_overview(self, mlmd_store_path, metric_aggregators, want_columns): config = metadata_store_pb2.ConnectionConfig() config.sqlite.filename_uri = mlmd_store_path store = metadata_store.MetadataStore(config) df = results.overview(store, metric_aggregators=metric_aggregators) self.assertEqual(want_columns, df.columns.tolist())
def __init__(self, phoenix_spec, study_name, study_owner, optimization_goal="minimize", optimization_metric="loss", connection_config=None): """Initializes a new MLMD connection instance. Args: phoenix_spec: PhoenixSpec proto. study_name: The name of the study. study_owner: The owner (username) of the study. optimization_goal: minimize or maximize (string). optimization_metric: what metric are we optimizing (string). connection_config: a metadata_store_pb2.ConnectionConfig() proto. If None, we fall back on the flags above. """ self._study_name = study_name self._study_owner = study_owner self._phoenix_spec = phoenix_spec self._optimization_goal = optimization_goal self._optimization_metric = optimization_metric self._connection_config = connection_config if not FLAGS.is_parsed(): logging.error( "Flags are not parsed. Using default in file mlmd database." " Please run main with absl.app.run(main) to fix this. " "If running in distributed mode, this means that the " "trainers are not sharing information between one another.") if self._connection_config is None: if FLAGS.is_parsed() and FLAGS.mlmd_default_sqllite_filename: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.sqlite.filename_uri = ( FLAGS.mlmd_default_sqllite_filename) self._connection_config.sqlite.connection_mode = 3 elif FLAGS.is_parsed() and FLAGS.mlmd_socket: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.mysql.socket = FLAGS.mlmd_socket self._connection_config.mysql.database = FLAGS.mlmd_database self._connection_config.mysql.user = FLAGS.mlmd_user self._connection_config.mysql.password = FLAGS.mlmd_password else: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.sqlite.filename_uri = ( "/tmp/filedb-%d" % random.randint(0, 1000000)) self._connection_config.sqlite.connection_mode = 3 self._store = metadata_store.MetadataStore(self._connection_config) trial_type = metadata_store_pb2.ExecutionType() trial_type.name = "Trial" trial_type.properties["id"] = metadata_store_pb2.INT trial_type.properties["state"] = metadata_store_pb2.STRING trial_type.properties["serialized_data"] = metadata_store_pb2.STRING trial_type.properties["model_dir"] = metadata_store_pb2.STRING trial_type.properties["evaluation"] = metadata_store_pb2.STRING self._trial_type_id = self._store.put_execution_type(trial_type) self._trial_id_to_run_id = {}
def test(self, use_keras, enable_tuning): with requests_mock.Mocker() as mocker: dataset_id_list = [1, 2] testing_utils.register_mock_urls(mocker, dataset_id_list) self.run_benchmarks( [openml_cc18_benchmark.OpenMLCC18Benchmark()], data_dir=os.path.join(self.pipeline_root, 'openML_mock_data'), mock_data=True, use_keras=use_keras, enable_tuning=enable_tuning, ) instance_names = [] for did in dataset_id_list: instance_name = '.'.join( ['OpenMLCC18Benchmark', 'benchmark', f'OpenML.mockdata_{did}']) instance_names.append(instance_name) if enable_tuning: self.assertComponentExecutionCount(8 * len(dataset_id_list)) self.assertComponentSucceeded('.'.join( ['Tuner.AutoTrainer', instance_name])) else: self.assertComponentExecutionCount(7 * len(dataset_id_list)) self.assertComponentSucceeded('.'.join( [f'CsvExampleGen.OpenML.mockdata_{did}', instance_name])) self.assertComponentSucceeded('.'.join( ['SchemaGen.AutoData', instance_name])) self.assertComponentSucceeded('.'.join( ['StatisticsGen.AutoData', instance_name])) self.assertComponentSucceeded('.'.join( ['Transform.AutoData', instance_name])) self.assertComponentSucceeded('.'.join( ['Trainer.AutoTrainer', instance_name])) self.assertComponentSucceeded('.'.join( ['Evaluator.model', instance_name])) self.assertComponentSucceeded('.'.join( ['BenchmarkResultPublisher.model', instance_name])) # Load benchmark results. store = metadata_store.MetadataStore(self.metadata_config) df = results.overview(store) # Check benchmark results overview values. self.assertEqual(len(df.index), len(dataset_id_list)) self.assertContainsSubset([ 'run_id', 'benchmark_fullname', 'benchmark', 'run', 'num_runs', 'accuracy', 'average_loss', 'example_count', 'weighted_example_count', ], df.columns.values.tolist()) self.assertSameElements([1], df['run'].tolist()) self.assertSameElements([1], df['num_runs'].tolist()) self.assertSameElements(instance_names, df.benchmark.unique())
def __init__(self, exec_type_name: str = 'BenchmarkResultPublisher', artifact_type: str = results._BENCHMARK_RESULT): self.artifact_type = artifact_type config = metadata_store_pb2.ConnectionConfig() config.fake_database.SetInParent() self.store = metadata_store.MetadataStore(config) self.exec_type_id = self._put_execution_type(exec_type_name) self.artifact_type_id = self._put_artifact_type()
def __init__( self, mlmd_connection_config: Optional[ metadata_store_pb2.MetadataStoreClientConfig] = None, ): if mlmd_connection_config is None: # default to value suitable for local testing mlmd_connection_config = metadata_store_pb2.MetadataStoreClientConfig( host='localhost', port=8080, ) self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config)
def __init__(self, phoenix_spec, study_name, study_owner, optimization_goal="minimize", optimization_metric="loss", connection_config=None): """Initializes a new MLMD connection instance. Args: phoenix_spec: PhoenixSpec proto. study_name: The name of the study. study_owner: The owner (username) of the study. optimization_goal: minimize or maximize (string). optimization_metric: what metric are we optimizing (string). connection_config: a metadata_store_pb2.ConnectionConfig() proto. If None, we fall back on the flags above. """ self._study_name = study_name self._study_owner = study_owner self._phoenix_spec = phoenix_spec self._optimization_goal = optimization_goal self._optimization_metric = optimization_metric self._connection_config = connection_config if self._connection_config is None: if FLAGS.mlmd_default_sqllite_filename: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.sqlite.filename_uri = ( FLAGS.mlmd_default_sqllite_filename) self._connection_config.sqlite.connection_mode = 3 elif FLAGS.mlmd_socket: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.mysql.socket = FLAGS.mlmd_socket self._connection_config.mysql.database = FLAGS.mlmd_database self._connection_config.mysql.user = FLAGS.mlmd_user self._connection_config.mysql.password = FLAGS.mlmd_password else: self._connection_config = metadata_store_pb2.ConnectionConfig() self._connection_config.sqlite.filename_uri = ( "/tmp/filedb-%d" % random.randint(0, 1000000)) self._connection_config.sqlite.connection_mode = 3 self._store = metadata_store.MetadataStore(self._connection_config) trial_type = metadata_store_pb2.ExecutionType() trial_type.name = "Trial" trial_type.properties["id"] = metadata_store_pb2.INT trial_type.properties["state"] = metadata_store_pb2.STRING trial_type.properties["serialized_data"] = metadata_store_pb2.STRING trial_type.properties["model_dir"] = metadata_store_pb2.STRING trial_type.properties["evaluation"] = metadata_store_pb2.STRING self._trial_type_id = self._store.put_execution_type(trial_type) self._trial_id_to_run_id = {}
def __init__( self, mlmd_connection_config: Optional[ metadata_store_pb2.MetadataStoreClientConfig] = None, ): if mlmd_connection_config is None: # default to value suitable for local testing mlmd_connection_config = metadata_store_pb2.MetadataStoreClientConfig( host='localhost', port=8080, ) self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config) self.dag_type = self.mlmd_store.get_execution_type( type_name='system.DAGExecution')
def test(self, use_keras, enable_tuning): self.run_benchmarks([titanic_benchmark.TitanicBenchmark()], use_keras=use_keras, enable_tuning=enable_tuning) if enable_tuning: self.assertComponentSucceeded( "Tuner.AutoTrainer.TitanicBenchmark.benchmark") self.assertComponentSucceeded("ImportExampleGen.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "SchemaGen.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "StatisticsGen.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "Transform.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "Trainer.AutoTrainer.TitanicBenchmark.benchmark") self.assertComponentSucceeded("Evaluator.model.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "BenchmarkResultPublisher.model.TitanicBenchmark.benchmark") # Load benchmark results. store = metadata_store.MetadataStore(self.metadata_config) df = results.overview(store) # Check benchmark results overview values. self.assertEqual(len(df.index), 1) self.assertContainsSubset([ "run_id", "benchmark_fullname", "benchmark", "run", "num_runs", "accuracy", "auc_pr", "auc_roc", "example_count", "precision", "recall", "weighted_example_count", ], df.columns.values.tolist()) self.assertSameElements([1], df["run"].tolist()) self.assertSameElements([1], df["num_runs"].tolist()) self.assertSameElements(["TitanicBenchmark.benchmark"], df.benchmark.unique())
def test(self, use_keras, enable_tuning): self.run_benchmarks([titanic_benchmark.TitanicBenchmark()], use_keras=use_keras, enable_tuning=enable_tuning) if enable_tuning: self.assertComponentExecutionCount(8) self.assertComponentSucceeded("Tuner.TitanicBenchmark.benchmark") else: self.assertComponentExecutionCount(7) self.assertComponentSucceeded( "ImportExampleGen.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "SchemaGen.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "StatisticsGen.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "Transform.AutoData.TitanicBenchmark.benchmark") self.assertComponentSucceeded("Trainer.TitanicBenchmark.benchmark") self.assertComponentSucceeded("Evaluator.TitanicBenchmark.benchmark") self.assertComponentSucceeded( "BenchmarkResultPublisher.TitanicBenchmark.benchmark") # Load benchmark results. store = metadata_store.MetadataStore(self.metadata_config) df = results.overview(store) # Check benchmark results overview values. self.assertEqual(len(df.index), 1) self.assertContainsSubset([ "benchmark", "run", "num_runs", "accuracy", "average_loss", "post_export_metrics/example_count", ], df.columns.values.tolist()) self.assertSameElements([1], df["run"].tolist()) self.assertSameElements([1], df["num_runs"].tolist()) self.assertSameElements(["TitanicBenchmark.benchmark"], df.benchmark.unique())
def test(self, algorithm): with requests_mock.Mocker() as mocker: testing_utils.register_mock_urls(mocker) self.run_benchmarks( [metalearning_benchmark.MetaLearningBenchmark()], data_dir=os.path.join(self.pipeline_root, 'mock_metalearning_openml'), mock_data=True, algorithm=algorithm) instance_name = 'MetaLearningBenchmark.benchmark' self.assertComponentSucceeded( f'CsvExampleGen.OpenML.mockdata_1.{instance_name}') self.assertComponentSucceeded( f'AugmentedTuner.train.OpenML.mockdata_1.{instance_name}') self.assertComponentSucceeded( f'SchemaGen.AutoData.train.OpenML.mockdata_1.{instance_name}') self.assertComponentSucceeded( f'StatisticsGen.AutoData.train.OpenML.mockdata_1.{instance_name}') self.assertComponentSucceeded( f'Transform.AutoData.train.OpenML.mockdata_1.{instance_name}') instance_name_2 = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2' self.assertComponentSucceeded( f'CsvExampleGen.OpenML.mockdata_2.{instance_name_2}') self.assertComponentSucceeded(f'SchemaGen.AutoData.{instance_name_2}') self.assertComponentSucceeded( f'StatisticsGen.AutoData.{instance_name_2}') self.assertComponentSucceeded(f'Transform.AutoData.{instance_name_2}') self.assertComponentSucceeded(f'AugmentedTuner.{instance_name_2}') self.assertComponentSucceeded(f'Trainer.{instance_name_2}') self.assertComponentSucceeded(f'Evaluator.model.{instance_name_2}') self.assertComponentSucceeded( f'BenchmarkResultPublisher.model.{instance_name_2}') # Load benchmark results. store = metadata_store.MetadataStore(self.metadata_config) df = results.overview(store) # Check benchmark results overview values. self.assertEqual(len(df.index), 1) self.assertContainsSubset([ 'run_id', 'benchmark_fullname', 'benchmark', 'run', 'num_runs', 'accuracy', 'average_loss', 'example_count', 'weighted_example_count', ], df.columns.values.tolist()) self.assertSameElements([1], df['run'].tolist()) self.assertSameElements([1], df['num_runs'].tolist()) self.assertSameElements([instance_name_2], df.benchmark.unique()) # TODO(b/178401753): Reimplement this test when name parsing is updated for # IR-based runners. # Load Analytics analytics = mlmd_analytics.Analytics(store=store) # Check test_pipeline run analysis run = analytics.get_latest_pipeline_run() self.assertEqual('test_pipeline', run.name) want_components = { f'CsvExampleGen.OpenML.mockdata_1.{instance_name}', f'AugmentedTuner.train.OpenML.mockdata_1.{instance_name}', f'SchemaGen.AutoData.train.OpenML.' f'mockdata_1.{instance_name}', f'StatisticsGen.AutoData.train.OpenML.' f'mockdata_1.{instance_name}', f'Transform.AutoData.train.OpenML.' f'mockdata_1.{instance_name}', f'CsvExampleGen.OpenML.mockdata_2.{instance_name_2}', f'SchemaGen.AutoData.{instance_name_2}', f'StatisticsGen.AutoData.{instance_name_2}', f'Transform.AutoData.{instance_name_2}', f'AugmentedTuner.{instance_name_2}', f'Trainer.{instance_name_2}', f'Evaluator.model.{instance_name_2}', f'BenchmarkResultPublisher.model.{instance_name_2}', } self.assertContainsSubset(want_components, run.components.keys()) component = f'Transform.AutoData.{instance_name_2}' self.assertIs( type(run.components[component].outputs.transformed_examples. to_dataframe('train', 5)), pd.DataFrame)
def __init__( self, mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig ): self.mlmd_store = metadata_store.MetadataStore(mlmd_connection_config)
def test(self, algorithm): with requests_mock.Mocker() as mocker: testing_utils.register_mock_urls(mocker) self.run_benchmarks( [metalearning_benchmark.MetaLearningBenchmark()], data_dir=os.path.join(self.pipeline_root, 'mock_metalearning_openml'), mock_data=True, algorithm=algorithm) train_dataset_ids = [1] for ix in train_dataset_ids: instance_name = 'MetaLearningBenchmark.benchmark' self.assertComponentSucceeded( f'CsvExampleGen.OpenML.mockdata_{ix}.{instance_name}') self.assertComponentSucceeded( f'AugmentedTuner.train.OpenML.mockdata_{ix}.{instance_name}') self.assertComponentSucceeded( f'SchemaGen.AutoData.train.OpenML.mockdata_{ix}.{instance_name}' ) self.assertComponentSucceeded( f'StatisticsGen.AutoData.train.OpenML.mockdata_{ix}.{instance_name}' ) self.assertComponentSucceeded( f'Transform.AutoData.train.OpenML.mockdata_{ix}.{instance_name}' ) test_dataset_ids = [2] for ix in test_dataset_ids: instance_name = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2' self.assertComponentSucceeded( f'CsvExampleGen.OpenML.mockdata_{ix}.{instance_name}') self.assertComponentSucceeded( f'SchemaGen.AutoData.test.OpenML.mockdata_{ix}.{instance_name}' ) self.assertComponentSucceeded( f'StatisticsGen.AutoData.test.OpenML.mockdata_{ix}.{instance_name}' ) self.assertComponentSucceeded( f'Transform.AutoData.test.OpenML.mockdata_{ix}.{instance_name}' ) self.assertComponentSucceeded( f'AugmentedTuner.test.OpenML.mockdata_{ix}.{instance_name}') self.assertComponentSucceeded( f'Trainer.test.OpenML.mockdata_{ix}.{instance_name}') instance_name = 'MetaLearningBenchmark.benchmark.OpenML.mockdata_2' self.assertComponentSucceeded(f'Evaluator.{instance_name}') self.assertComponentSucceeded( f'BenchmarkResultPublisher.{instance_name}') # Load benchmark results. store = metadata_store.MetadataStore(self.metadata_config) df = results.overview(store) # Check benchmark results overview values. self.assertEqual(len(df.index), 1) self.assertContainsSubset([ 'benchmark', 'run', 'num_runs', 'accuracy', 'average_loss', 'post_export_metrics/example_count', ], df.columns.values.tolist()) self.assertSameElements([1], df['run'].tolist()) self.assertSameElements([1], df['num_runs'].tolist()) self.assertSameElements([instance_name], df.benchmark.unique())
def setUp(self): super(GetHparamsTest, self).setUp() config = metadata_store_pb2.ConnectionConfig() config.fake_database.SetInParent() self.store = metadata_store.MetadataStore(config) self.exec_type_id = self._put_execution_type()