def test_run_local_experiment_specification(experiment_name): invoke_cli_runner( cli.run, [ TEST_PROJECT_DIR, "-e", "greeter", "-P", "name=test", "--experiment-name", experiment_name, ], ) client = MlflowClient() experiment_id = client.get_experiment_by_name( experiment_name).experiment_id invoke_cli_runner( cli.run, [ TEST_PROJECT_DIR, "-e", "greeter", "-P", "name=test", "--experiment-id", experiment_id ], )
def extract_run_meta_metrics(experiment, float_cols=[], filter_unfinished=True): client = MlflowClient() # pull runs for experiment experiment_id = client.get_experiment_by_name(experiment).experiment_id run_infos = client.list_run_infos(experiment_id) runs = [client.get_run(run_info.run_id) for run_info in run_infos] # extract run metrics and metadata from runs run_metrics = [] run_metas = [] for run in runs: # extract metadata from metrics run_id = run.info.run_id run_status = run.info.status # filter out unfinished runs if specified if filter_unfinished and run_status != "FINISHED": continue time_ms = (run.info.end_time - run.info.start_time if run.info.status == "FINISHED" else np.nan) meta = { "artifact_uri": run.info.artifact_uri, "time_ms": time_ms, "run_id": run_id, "status": run.info.status, } meta.update(run.data.params) meta.update(run.data.tags) meta.update(run.data.metrics) run_metas.append(meta) # metrics history have to be extracted separately as they have # a separate epoch dimension metric_names = run.data.metrics.keys() histories = [client.get_metric_history(run_id, name) for name in metric_names] for history in histories: for metric in history: metric_dict = { "run_id": run_id, "metric": metric.key, "step": metric.step, "value": metric.value, } run_metrics.append(metric_dict) run_metrics.append(metric_dict) meta_df = pd.DataFrame(run_metas).astype({ col: "float64" for col in float_cols }).set_index("run_id") metrics_df = pd.DataFrame(run_metrics) return meta_df, metrics_df
def set_experiment(experiment_name): """ Set given experiment as active experiment. If experiment does not exist, create an experiment with provided name. :param experiment_name: Name of experiment to be activated. """ client = MlflowClient() experiment = client.get_experiment_by_name(experiment_name) exp_id = experiment.experiment_id if experiment else None if exp_id is None: # id can be 0 print("INFO: '{}' does not exist. Creating a new experiment".format(experiment_name)) exp_id = client.create_experiment(experiment_name) global _active_experiment_id _active_experiment_id = exp_id
def set_experiment(experiment_name): """ Set given experiment as active experiment. If experiment does not exist, create an experiment with provided name. :param experiment_name: Case sensitive name of an experiment to be activated. .. code-block:: python :caption: Example import mlflow # Set an experiment name, which must be unique and case sensitive. mlflow.set_experiment("Social NLP Experiments") # Get Experiment Details experiment = mlflow.get_experiment_by_name("Social NLP Experiments") # Print the contents of Experiment data print("Experiment_id: {}".format(experiment.experiment_id)) print("Artifact Location: {}".format(experiment.artifact_location)) print("Tags: {}".format(experiment.tags)) print("Lifecycle_stage: {}".format(experiment.lifecycle_stage)) .. code-block:: text :caption: Output Experiment_id: 1 Artifact Location: file:///.../mlruns/1 Tags: {} Lifecycle_stage: active """ client = MlflowClient() experiment = client.get_experiment_by_name(experiment_name) exp_id = experiment.experiment_id if experiment else None if exp_id is None: # id can be 0 print("INFO: '{}' does not exist. Creating a new experiment".format( experiment_name)) exp_id = client.create_experiment(experiment_name) elif experiment.lifecycle_stage == LifecycleStage.DELETED: raise MlflowException( "Cannot set a deleted experiment '%s' as the active experiment." " You can restore the experiment, or permanently delete the " " experiment to create a new one." % experiment.name) global _active_experiment_id _active_experiment_id = exp_id
def set_experiment(experiment_name): """ Set given experiment as active experiment. If experiment does not exist, create an experiment with provided name. :param experiment_name: Name of experiment to be activated. """ client = MlflowClient() experiment = client.get_experiment_by_name(experiment_name) exp_id = experiment.experiment_id if experiment else None if exp_id is None: # id can be 0 print("INFO: '{}' does not exist. Creating a new experiment".format(experiment_name)) exp_id = client.create_experiment(experiment_name) elif experiment.lifecycle_stage == LifecycleStage.DELETED: raise MlflowException( "Cannot set a deleted experiment '%s' as the active experiment." " You can restore the experiment, or permanently delete the " " experiment to create a new one." % experiment.name) global _active_experiment_id _active_experiment_id = exp_id
def test_run_local_experiment_specification(experiment_name, tracking_uri_mock): # pylint: disable=unused-argument invoke_cli_runner( cli.run, [ TEST_PROJECT_DIR, "-e", "greeter", "-P", "name=test", "--experiment-name", experiment_name, ]) client = MlflowClient() experiment_id = client.get_experiment_by_name(experiment_name).experiment_id invoke_cli_runner( cli.run, [ TEST_PROJECT_DIR, "-e", "greeter", "-P", "name=test", "--experiment-id", experiment_id, ])
import mlflow from mlflow.tracking.client import MlflowClient client = MlflowClient() exp_name = "artifact_test_experiment" if client.get_experiment_by_name(exp_name) is None: client.create_experiment(exp_name, artifact_location="sqlite:///testartifactdb") mlflow.set_experiment(exp_name) fpath = "test.txt" with open(fpath, "w") as f: f.write("TEST") with mlflow.start_run(): mlflow.log_artifact(fpath, "")
class MLflowWriter(object): def __init__(self, exp_name, save_dir, log_every, **mlflow_cfg): mlflow.set_tracking_uri(save_dir) self.client = MlflowClient(**mlflow_cfg) mlflow.set_experiment(exp_name) self.experiment_id = self.client.get_experiment_by_name( exp_name).experiment_id self.run_id = self.client.create_run(self.experiment_id).info.run_id self.log_every = log_every self.clear() def log_params_from_omegaconf(self, params): self._explore_recursive("", params) def _explore_recursive(self, parent_name, element): if isinstance(element, DictConfig): iterator = element.items() elif isinstance(element, ListConfig): iterator = enumerate(element) for k, v in iterator: if isinstance(v, DictConfig) or isinstance(v, ListConfig): self._explore_recursive(f"{parent_name}{k}.", v) else: self.client.log_param( self.run_id, f"{parent_name}{k}", v) def log_torch_model(self, model, epoch): with mlflow.start_run(self.run_id): mlflow.pytorch.log_model(model, "model_%04d" % epoch) def log_metric(self, key, value, is_training): if isinstance(value, torch.Tensor): value = float(value.detach().cpu().numpy()) metric_name = "train/" if is_training else "valid/" metric_name += str(key) if key in self.metrics: self.metrics[metric_name].append(value) else: self.metrics[metric_name] = [value] def next_iteration(self): self.iterations += 1 if self.iterations % self.log_every == 0: self.toMlflow(nb_data=self.log_every) def toMlflow(self, nb_data=0, step=0): for key, value in self.metrics.items(): self.client.log_metric( self.run_id, key, np.mean(value[-nb_data:]), step=step) def get_mean(self, key, is_training): metric_name = "train/" if is_training else "valid/" metric_name += str(key) return np.mean(self.metrics[metric_name]) def clear(self): self.metrics = {} self.iterations = 0 def log_artifact(self, path): self.client.log_artifact(self.run_id, local_path=path) def terminate(self): self.client.set_terminated(self.run_id)
# MAGIC This section creates a mock experiment. To analyze your own experiment data rather than these mocks, change the value in the **Experiment ID** widget at the top of this notebook. # COMMAND ---------- from mlflow.tracking.client import MlflowClient from datetime import datetime, timedelta import math import random random.seed(42) client = MlflowClient() mock_experiment_file = f'/tmp/mock_experiment' # Delete experiment if it already exists if client.get_experiment_by_name(mock_experiment_file): client.delete_experiment( client.get_experiment_by_name(mock_experiment_file).experiment_id) mock_experiment_id = client.create_experiment(mock_experiment_file) def simulate_runs(dt, eid, mae, num_runs=1): ''' Simulates a run(s) with a specified datetime, experiment, and mae. :param dt: Timestamp used for both run start_time and log_metric time. :param eid: Experiment id to log this run under. :param mae: Mean absolute error metric to log. :param num_runs: Number of runs to store. '''