def test_log_metric(tmpdir: py.path.local, names: List[str],
values: List[float]) -> None:
tracking_file_name = "file:{}".format(tmpdir)
study_name = "my_study"
mlflc = MLflowCallback(tracking_uri=tracking_file_name, metric_name=names)
study = optuna.create_study(study_name=study_name)
mlflc._initialize_experiment(study)
with mlflow.start_run():
mlflc._log_metrics(values)
mlfl_client = MlflowClient(tracking_file_name)
experiments = mlfl_client.list_experiments()
experiment = experiments[0]
experiment_id = experiment.experiment_id
run_infos = mlfl_client.list_run_infos(experiment_id)
assert len(run_infos) == 1
first_run_id = run_infos[0].run_id
first_run = mlfl_client.get_run(first_run_id)
first_run_dict = first_run.to_dictionary()
assert all(name in first_run_dict["data"]["metrics"] for name in names)
assert all([
first_run_dict["data"]["metrics"][name] == val
for name, val in zip(names, values)
])
def test_log_metric_none(tmpdir: py.path.local) -> None:
tracking_file_name = "file:{}".format(tmpdir)
metric_name = "my_metric_name"
study_name = "my_study"
metric_value = None
mlflc = MLflowCallback(tracking_uri=tracking_file_name,
metric_name=metric_name)
study = optuna.create_study(study_name=study_name)
mlflc._initialize_experiment(study)
with mlflow.start_run():
mlflc._log_metrics(metric_value)
mlfl_client = MlflowClient(tracking_file_name)
experiments = mlfl_client.list_experiments()
experiment = experiments[0]
experiment_id = experiment.experiment_id
run_infos = mlfl_client.list_run_infos(experiment_id)
assert len(run_infos) == 1
first_run_id = run_infos[0].run_id
first_run = mlfl_client.get_run(first_run_id)
first_run_dict = first_run.to_dictionary()
# when `values` is `None`, do not save values with metric names
assert metric_name not in first_run_dict["data"]["metrics"]
def test_log_params(tmpdir: py.path.local) -> None:
tracking_file_name = "file:{}".format(tmpdir)
metric_name = "my_metric_name"
study_name = "my_study"
param1_name = "my_param1"
param1_value = "a"
param2_name = "my_param2"
param2_value = 5
params = {param1_name: param1_value, param2_name: param2_value}
mlflc = MLflowCallback(tracking_uri=tracking_file_name,
metric_name=metric_name)
study = optuna.create_study(study_name=study_name)
mlflc._initialize_experiment(study)
with mlflow.start_run():
trial = optuna.trial.create_trial(
params=params,
distributions={
param1_name:
optuna.distributions.CategoricalDistribution(["a", "b"]),
param2_name:
optuna.distributions.UniformDistribution(0, 10),
},
value=5.0,
)
mlflc._log_params(trial.params)
mlfl_client = MlflowClient(tracking_file_name)
experiments = mlfl_client.list_experiments()
experiment = experiments[0]
experiment_id = experiment.experiment_id
run_infos = mlfl_client.list_run_infos(experiment_id)
assert len(run_infos) == 1
first_run_id = run_infos[0].run_id
first_run = mlfl_client.get_run(first_run_id)
first_run_dict = first_run.to_dictionary()
assert param1_name in first_run_dict["data"]["params"]
assert first_run_dict["data"]["params"][param1_name] == param1_value
assert param2_name in first_run_dict["data"]["params"]
assert first_run_dict["data"]["params"][param2_name] == str(param2_value)
def test_initialize_experiment(tmpdir: py.path.local) -> None:
tracking_file_name = "file:{}".format(tmpdir)
metric_name = "my_metric_name"
study_name = "my_study"
mlflc = MLflowCallback(tracking_uri=tracking_file_name,
metric_name=metric_name)
study = optuna.create_study(study_name=study_name)
mlflc._initialize_experiment(study)
mlfl_client = MlflowClient(tracking_file_name)
experiments = mlfl_client.list_experiments()
assert len(experiments) == 1
experiment = experiments[0]
assert experiment.name == study_name