def test_run_local_experiment_specification(experiment_name):
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"name=test",
"--experiment-name",
experiment_name,
],
)
client = MlflowClient()
experiment_id = client.get_experiment_by_name(
experiment_name).experiment_id
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR, "-e", "greeter", "-P", "name=test",
"--experiment-id", experiment_id
],
)
def extract_run_meta_metrics(experiment, float_cols=[], filter_unfinished=True):
client = MlflowClient()
# pull runs for experiment
experiment_id = client.get_experiment_by_name(experiment).experiment_id
run_infos = client.list_run_infos(experiment_id)
runs = [client.get_run(run_info.run_id) for run_info in run_infos]
# extract run metrics and metadata from runs
run_metrics = []
run_metas = []
for run in runs:
# extract metadata from metrics
run_id = run.info.run_id
run_status = run.info.status
# filter out unfinished runs if specified
if filter_unfinished and run_status != "FINISHED":
continue
time_ms = (run.info.end_time - run.info.start_time
if run.info.status == "FINISHED" else np.nan)
meta = {
"artifact_uri": run.info.artifact_uri,
"time_ms": time_ms,
"run_id": run_id,
"status": run.info.status,
}
meta.update(run.data.params)
meta.update(run.data.tags)
meta.update(run.data.metrics)
run_metas.append(meta)
# metrics history have to be extracted separately as they have
# a separate epoch dimension
metric_names = run.data.metrics.keys()
histories = [client.get_metric_history(run_id, name) for name in metric_names]
for history in histories:
for metric in history:
metric_dict = {
"run_id": run_id,
"metric": metric.key,
"step": metric.step,
"value": metric.value,
}
run_metrics.append(metric_dict)
run_metrics.append(metric_dict)
meta_df = pd.DataFrame(run_metas).astype({
col: "float64" for col in float_cols
}).set_index("run_id")
metrics_df = pd.DataFrame(run_metrics)
return meta_df, metrics_df
def set_experiment(experiment_name):
"""
Set given experiment as active experiment. If experiment does not exist, create an experiment
with provided name.
:param experiment_name: Name of experiment to be activated.
"""
client = MlflowClient()
experiment = client.get_experiment_by_name(experiment_name)
exp_id = experiment.experiment_id if experiment else None
if exp_id is None: # id can be 0
print("INFO: '{}' does not exist. Creating a new experiment".format(experiment_name))
exp_id = client.create_experiment(experiment_name)
global _active_experiment_id
_active_experiment_id = exp_id
def set_experiment(experiment_name):
"""
Set given experiment as active experiment. If experiment does not exist, create an experiment
with provided name.
:param experiment_name: Case sensitive name of an experiment to be activated.
.. code-block:: python
:caption: Example
import mlflow
# Set an experiment name, which must be unique and case sensitive.
mlflow.set_experiment("Social NLP Experiments")
# Get Experiment Details
experiment = mlflow.get_experiment_by_name("Social NLP Experiments")
# Print the contents of Experiment data
print("Experiment_id: {}".format(experiment.experiment_id))
print("Artifact Location: {}".format(experiment.artifact_location))
print("Tags: {}".format(experiment.tags))
print("Lifecycle_stage: {}".format(experiment.lifecycle_stage))
.. code-block:: text
:caption: Output
Experiment_id: 1
Artifact Location: file:///.../mlruns/1
Tags: {}
Lifecycle_stage: active
"""
client = MlflowClient()
experiment = client.get_experiment_by_name(experiment_name)
exp_id = experiment.experiment_id if experiment else None
if exp_id is None: # id can be 0
print("INFO: '{}' does not exist. Creating a new experiment".format(
experiment_name))
exp_id = client.create_experiment(experiment_name)
elif experiment.lifecycle_stage == LifecycleStage.DELETED:
raise MlflowException(
"Cannot set a deleted experiment '%s' as the active experiment."
" You can restore the experiment, or permanently delete the "
" experiment to create a new one." % experiment.name)
global _active_experiment_id
_active_experiment_id = exp_id
def set_experiment(experiment_name):
"""
Set given experiment as active experiment. If experiment does not exist, create an experiment
with provided name.
:param experiment_name: Name of experiment to be activated.
"""
client = MlflowClient()
experiment = client.get_experiment_by_name(experiment_name)
exp_id = experiment.experiment_id if experiment else None
if exp_id is None: # id can be 0
print("INFO: '{}' does not exist. Creating a new experiment".format(experiment_name))
exp_id = client.create_experiment(experiment_name)
elif experiment.lifecycle_stage == LifecycleStage.DELETED:
raise MlflowException(
"Cannot set a deleted experiment '%s' as the active experiment."
" You can restore the experiment, or permanently delete the "
" experiment to create a new one." % experiment.name)
global _active_experiment_id
_active_experiment_id = exp_id
def test_run_local_experiment_specification(experiment_name,
tracking_uri_mock): # pylint: disable=unused-argument
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e", "greeter",
"-P", "name=test",
"--experiment-name", experiment_name,
])
client = MlflowClient()
experiment_id = client.get_experiment_by_name(experiment_name).experiment_id
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e", "greeter",
"-P", "name=test",
"--experiment-id", experiment_id,
])
import mlflow
from mlflow.tracking.client import MlflowClient
client = MlflowClient()
exp_name = "artifact_test_experiment"
if client.get_experiment_by_name(exp_name) is None:
client.create_experiment(exp_name,
artifact_location="sqlite:///testartifactdb")
mlflow.set_experiment(exp_name)
fpath = "test.txt"
with open(fpath, "w") as f:
f.write("TEST")
with mlflow.start_run():
mlflow.log_artifact(fpath, "")
class MLflowWriter(object):
def __init__(self, exp_name, save_dir, log_every, **mlflow_cfg):
mlflow.set_tracking_uri(save_dir)
self.client = MlflowClient(**mlflow_cfg)
mlflow.set_experiment(exp_name)
self.experiment_id = self.client.get_experiment_by_name(
exp_name).experiment_id
self.run_id = self.client.create_run(self.experiment_id).info.run_id
self.log_every = log_every
self.clear()
def log_params_from_omegaconf(self, params):
self._explore_recursive("", params)
def _explore_recursive(self, parent_name, element):
if isinstance(element, DictConfig):
iterator = element.items()
elif isinstance(element, ListConfig):
iterator = enumerate(element)
for k, v in iterator:
if isinstance(v, DictConfig) or isinstance(v, ListConfig):
self._explore_recursive(f"{parent_name}{k}.", v)
else:
self.client.log_param(
self.run_id, f"{parent_name}{k}", v)
def log_torch_model(self, model, epoch):
with mlflow.start_run(self.run_id):
mlflow.pytorch.log_model(model, "model_%04d" % epoch)
def log_metric(self, key, value, is_training):
if isinstance(value, torch.Tensor):
value = float(value.detach().cpu().numpy())
metric_name = "train/" if is_training else "valid/"
metric_name += str(key)
if key in self.metrics:
self.metrics[metric_name].append(value)
else:
self.metrics[metric_name] = [value]
def next_iteration(self):
self.iterations += 1
if self.iterations % self.log_every == 0:
self.toMlflow(nb_data=self.log_every)
def toMlflow(self, nb_data=0, step=0):
for key, value in self.metrics.items():
self.client.log_metric(
self.run_id, key,
np.mean(value[-nb_data:]), step=step)
def get_mean(self, key, is_training):
metric_name = "train/" if is_training else "valid/"
metric_name += str(key)
return np.mean(self.metrics[metric_name])
def clear(self):
self.metrics = {}
self.iterations = 0
def log_artifact(self, path):
self.client.log_artifact(self.run_id, local_path=path)
def terminate(self):
self.client.set_terminated(self.run_id)
# MAGIC This section creates a mock experiment. To analyze your own experiment data rather than these mocks, change the value in the **Experiment ID** widget at the top of this notebook.
# COMMAND ----------
from mlflow.tracking.client import MlflowClient
from datetime import datetime, timedelta
import math
import random
random.seed(42)
client = MlflowClient()
mock_experiment_file = f'/tmp/mock_experiment'
# Delete experiment if it already exists
if client.get_experiment_by_name(mock_experiment_file):
client.delete_experiment(
client.get_experiment_by_name(mock_experiment_file).experiment_id)
mock_experiment_id = client.create_experiment(mock_experiment_file)
def simulate_runs(dt, eid, mae, num_runs=1):
'''
Simulates a run(s) with a specified datetime, experiment, and mae.
:param dt: Timestamp used for both run start_time and log_metric time.
:param eid: Experiment id to log this run under.
:param mae: Mean absolute error metric to log.
:param num_runs: Number of runs to store.
'''
