def test_log_metric(tmpdir: py.path.local, names: List[str], values: List[float]) -> None: tracking_file_name = "file:{}".format(tmpdir) study_name = "my_study" mlflc = MLflowCallback(tracking_uri=tracking_file_name, metric_name=names) study = optuna.create_study(study_name=study_name) mlflc._initialize_experiment(study) with mlflow.start_run(): mlflc._log_metrics(values) mlfl_client = MlflowClient(tracking_file_name) experiments = mlfl_client.list_experiments() experiment = experiments[0] experiment_id = experiment.experiment_id run_infos = mlfl_client.list_run_infos(experiment_id) assert len(run_infos) == 1 first_run_id = run_infos[0].run_id first_run = mlfl_client.get_run(first_run_id) first_run_dict = first_run.to_dictionary() assert all(name in first_run_dict["data"]["metrics"] for name in names) assert all([ first_run_dict["data"]["metrics"][name] == val for name, val in zip(names, values) ])
def test_log_metric_none(tmpdir: py.path.local) -> None: tracking_file_name = "file:{}".format(tmpdir) metric_name = "my_metric_name" study_name = "my_study" metric_value = None mlflc = MLflowCallback(tracking_uri=tracking_file_name, metric_name=metric_name) study = optuna.create_study(study_name=study_name) mlflc._initialize_experiment(study) with mlflow.start_run(): mlflc._log_metrics(metric_value) mlfl_client = MlflowClient(tracking_file_name) experiments = mlfl_client.list_experiments() experiment = experiments[0] experiment_id = experiment.experiment_id run_infos = mlfl_client.list_run_infos(experiment_id) assert len(run_infos) == 1 first_run_id = run_infos[0].run_id first_run = mlfl_client.get_run(first_run_id) first_run_dict = first_run.to_dictionary() # when `values` is `None`, do not save values with metric names assert metric_name not in first_run_dict["data"]["metrics"]
def test_log_params(tmpdir: py.path.local) -> None: tracking_file_name = "file:{}".format(tmpdir) metric_name = "my_metric_name" study_name = "my_study" param1_name = "my_param1" param1_value = "a" param2_name = "my_param2" param2_value = 5 params = {param1_name: param1_value, param2_name: param2_value} mlflc = MLflowCallback(tracking_uri=tracking_file_name, metric_name=metric_name) study = optuna.create_study(study_name=study_name) mlflc._initialize_experiment(study) with mlflow.start_run(): trial = optuna.trial.create_trial( params=params, distributions={ param1_name: optuna.distributions.CategoricalDistribution(["a", "b"]), param2_name: optuna.distributions.UniformDistribution(0, 10), }, value=5.0, ) mlflc._log_params(trial.params) mlfl_client = MlflowClient(tracking_file_name) experiments = mlfl_client.list_experiments() experiment = experiments[0] experiment_id = experiment.experiment_id run_infos = mlfl_client.list_run_infos(experiment_id) assert len(run_infos) == 1 first_run_id = run_infos[0].run_id first_run = mlfl_client.get_run(first_run_id) first_run_dict = first_run.to_dictionary() assert param1_name in first_run_dict["data"]["params"] assert first_run_dict["data"]["params"][param1_name] == param1_value assert param2_name in first_run_dict["data"]["params"] assert first_run_dict["data"]["params"][param2_name] == str(param2_value)
def test_initialize_experiment(tmpdir: py.path.local) -> None: tracking_file_name = "file:{}".format(tmpdir) metric_name = "my_metric_name" study_name = "my_study" mlflc = MLflowCallback(tracking_uri=tracking_file_name, metric_name=metric_name) study = optuna.create_study(study_name=study_name) mlflc._initialize_experiment(study) mlfl_client = MlflowClient(tracking_file_name) experiments = mlfl_client.list_experiments() assert len(experiments) == 1 experiment = experiments[0] assert experiment.name == study_name