def start_experiment(experiment_id=None,
hp_space: Dict = None,
objective: Callable = None,
max_evals=15,
mode: str = "random"):
"""
Start either a new Kiwi experiment, or continue on a pre-existing one.
"""
if not experiment_id:
experiment_id: int = _get_experiment_id() if _get_experiment_id() \
else _active_run_stack[-1].info.experiment_id if _active_run_stack[-1] else None
print("current experiment: " + experiment_id)
hyperopt_space = {
k: (hp.choice(k, hp_space[k][1])
if hp_space[k][0] == "choice" else hp.uniform(k, *hp_space[k][1]))
for k in hp_space
}
MlflowClient().set_experiment_tag(experiment_id,
"kiwi.training.hyperparameters",
json.dumps(hp_space))
MlflowClient().set_experiment_tag(experiment_id, "kiwi.training.mode",
mode)
best = fmin(objective,
hyperopt_space,
algo=rand.suggest if mode == "random" else tpe.suggest,
max_evals=max_evals)
print(best)
def delete_run(run_id):
"""
Deletes a run with the given ID.
:param run_id: Unique identifier for the run to delete.
"""
MlflowClient().delete_run(run_id)
def delete_experiment(experiment_id):
"""
Delete an experiment from the backend store.
:param experiment_id: The experiment ID returned from ``create_experiment``.
"""
MlflowClient().delete_experiment(experiment_id)
def test_get_artifact_uri_appends_to_uri_path_component_correctly(
artifact_location, expected_uri_format):
client = MlflowClient()
client.create_experiment("get-artifact-uri-test",
artifact_location=artifact_location)
kiwi.set_experiment("get-artifact-uri-test")
with kiwi.start_run():
run_id = kiwi.active_run().info.run_id
for artifact_path in [
"path/to/artifact", "/artifact/path", "arty.txt"
]:
artifact_uri = kiwi.get_artifact_uri(artifact_path)
assert artifact_uri == tracking.artifact_utils.get_artifact_uri(
run_id, artifact_path)
assert artifact_uri == expected_uri_format.format(
run_id=run_id, path=artifact_path.lstrip("/"))
def get_experiment(experiment_id):
"""
Retrieve an experiment by experiment_id from the backend store
:param experiment_id: The experiment ID returned from ``create_experiment``.
:return: :py:class:`mlflow.entities.Experiment`
"""
return MlflowClient().get_experiment(experiment_id)
def get_experiment_by_name(name):
"""
Retrieve an experiment by experiment name from the backend store
:param name: The experiment name.
:return: :py:class:`mlflow.entities.Experiment`
"""
return MlflowClient().get_experiment_by_name(name)
def end_run(status=RunStatus.to_string(RunStatus.FINISHED)):
"""End an active MLflow run (if there is one)."""
global _active_run_stack
if len(_active_run_stack) > 0:
# Clear out the global existing run environment variable as well.
env.unset_variable(_RUN_ID_ENV_VAR)
run = _active_run_stack.pop()
MlflowClient().set_terminated(run.info.run_id, status)
def test_delete_tag():
"""
Confirm that fluent API delete tags actually works
:return:
"""
kiwi.set_tag('a', 'b')
run = MlflowClient().get_run(kiwi.active_run().info.run_id)
print(run.info.run_id)
assert 'a' in run.data.tags
kiwi.delete_tag('a')
run = MlflowClient().get_run(kiwi.active_run().info.run_id)
assert 'a' not in run.data.tags
with pytest.raises(MlflowException):
kiwi.delete_tag('a')
with pytest.raises(MlflowException):
kiwi.delete_tag('b')
kiwi.end_run()
def delete_tag(key):
"""
Delete a tag from a run. This is irreversible. If no run is active, this method
will create a new active run.
:param key: Name of the tag
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().delete_tag(run_id, key)
def set_tag(key, value):
"""
Set a tag under the current run. If no run is active, this method will create a
new active run.
:param key: Tag name (string)
:param value: Tag value (string, but will be string-ified if not)
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().set_tag(run_id, key, value)
def log_param(key, value):
"""
Log a parameter under the current run. If no run is active, this method will create
a new active run.
:param key: Parameter name (string)
:param value: Parameter value (string, but will be string-ified if not)
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().log_param(run_id, key, value)
def test_search_runs_multiple_experiments():
experiment_ids = [
kiwi.create_experiment("exp__{}".format(exp_id))
for exp_id in range(1, 4)
]
for eid in experiment_ids:
with kiwi.start_run(experiment_id=eid):
kiwi.log_metric("m0", 1)
kiwi.log_metric("m_{}".format(eid), 2)
assert len(MlflowClient().search_runs(experiment_ids, "metrics.m0 > 0",
ViewType.ALL)) == 3
assert len(MlflowClient().search_runs(experiment_ids, "metrics.m_1 > 0",
ViewType.ALL)) == 1
assert len(MlflowClient().search_runs(experiment_ids, "metrics.m_2 = 2",
ViewType.ALL)) == 1
assert len(MlflowClient().search_runs(experiment_ids, "metrics.m_3 < 4",
ViewType.ALL)) == 1
def test_start_run_exp_id_0():
kiwi.set_experiment("some-experiment")
# Create a run and verify that the current active experiment is the one we just set
with kiwi.start_run() as active_run:
exp_id = active_run.info.experiment_id
assert exp_id != FileStore.DEFAULT_EXPERIMENT_ID
assert MlflowClient().get_experiment(exp_id).name == "some-experiment"
# Set experiment ID to 0 when creating a run, verify that the specified experiment ID is honored
with kiwi.start_run(experiment_id=0) as active_run:
assert active_run.info.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
def create_experiment(name, artifact_location=None):
"""
Create an experiment.
:param name: The experiment name. Must be unique.
:param artifact_location: The location to store run artifacts.
If not provided, the server picks an appropriate default.
:return: Integer ID of the created experiment.
"""
return MlflowClient().create_experiment(name, artifact_location)
def log_artifact(local_path, artifact_path=None):
"""
Log a local file or directory as an artifact of the currently active run. If no run is
active, this method will create a new active run.
:param local_path: Path to the file to write.
:param artifact_path: If provided, the directory in ``artifact_uri`` to write to.
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().log_artifact(run_id, local_path, artifact_path)
def log_artifacts(local_dir, artifact_path=None):
"""
Log all the contents of a local directory as artifacts of the run. If no run is active,
this method will create a new active run.
:param local_dir: Path to the directory of files to write.
:param artifact_path: If provided, the directory in ``artifact_uri`` to write to.
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().log_artifacts(run_id, local_dir, artifact_path)
def _register_split_dataset(split: str,
dataloader=None,
dataset_location=None,
experiment_id=None):
# Find experiment if not specified
if not experiment_id:
experiment_id: int = _get_experiment_id() if _get_experiment_id() \
else _active_run_stack[-1].info.experiment_id if _active_run_stack[-1] else None
print("current experiment: " + experiment_id)
if dataloader:
MlflowClient().set_experiment_tag(experiment_id, split + ".hash",
dataloader.__hash__())
MlflowClient().set_experiment_tag(experiment_id, split + ".length",
len(dataloader))
if dataset_location:
MlflowClient().set_experiment_tag(experiment_id, split + ".path",
dataset_location)
MlflowClient().set_experiment_tag(experiment_id, split + ".bytes",
os.path.getsize(dataset_location))
def set_experiment(experiment_name):
"""
Set given experiment as active experiment. If experiment does not exist, create an experiment
with provided name.
:param experiment_name: Name of experiment to be activated.
"""
client = MlflowClient()
experiment = client.get_experiment_by_name(experiment_name)
exp_id = experiment.experiment_id if experiment else None
if exp_id is None: # id can be 0
print("INFO: '{}' does not exist. Creating a new experiment".format(
experiment_name))
exp_id = client.create_experiment(experiment_name)
elif experiment.lifecycle_stage == LifecycleStage.DELETED:
raise MlflowException(
"Cannot set a deleted experiment '%s' as the active experiment."
" You can restore the experiment, or permanently delete the "
" experiment to create a new one." % experiment.name)
global _active_experiment_id
_active_experiment_id = exp_id
def _get_paginated_runs(experiment_ids, filter_string, run_view_type,
max_results, order_by):
all_runs = []
next_page_token = None
while (len(all_runs) < max_results):
runs_to_get = max_results - len(all_runs)
if runs_to_get < NUM_RUNS_PER_PAGE_PANDAS:
runs = MlflowClient().search_runs(experiment_ids, filter_string,
run_view_type, runs_to_get,
order_by, next_page_token)
else:
runs = MlflowClient().search_runs(experiment_ids, filter_string,
run_view_type,
NUM_RUNS_PER_PAGE_PANDAS,
order_by, next_page_token)
all_runs.extend(runs)
if hasattr(runs,
'token') and runs.token != '' and runs.token is not None:
next_page_token = runs.token
else:
break
return all_runs
def log_metric(key, value, step=None):
"""
Log a metric under the current run. If no run is active, this method will create
a new active run.
:param key: Metric name (string).
:param value: Metric value (float). Note that some special values such as +/- Infinity may be
replaced by other values depending on the store. For example, sFor example, the
SQLAlchemy store replaces +/- Inf with max / min float values.
:param step: Metric step (int). Defaults to zero if unspecified.
"""
run_id = _get_or_start_run().info.run_id
MlflowClient().log_metric(run_id, key, value, int(time.time() * 1000), step
or 0)
def get_run(run_id):
"""
Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>`
contains a collection of run metadata -- :py:class:`RunInfo <mlflow.entities.RunInfo>`,
as well as a collection of run parameters, tags, and metrics --
:py:class:`RunData <mlflow.entities.RunData>`. In the case where multiple metrics with the
same key are logged for the run, the :py:class:`RunData <mlflow.entities.RunData>` contains
the most recently logged value at the largest step for each metric.
:param run_id: Unique identifier for the run.
:return: A single :py:class:`mlflow.entities.Run` object, if the run exists. Otherwise,
raises an exception.
"""
return MlflowClient().get_run(run_id)
def log_params(params):
"""
Log a batch of params for the current run. If no run is active, this method will create a
new active run.
:param params: Dictionary of param_name: String -> value: (String, but will be string-ified if
not)
:returns: None
"""
run_id = _get_or_start_run().info.run_id
params_arr = [Param(key, str(value)) for key, value in params.items()]
MlflowClient().log_batch(run_id=run_id,
metrics=[],
params=params_arr,
tags=[])
def set_tags(tags):
"""
Log a batch of tags for the current run. If no run is active, this method will create a
new active run.
:param tags: Dictionary of tag_name: String -> value: (String, but will be string-ified if
not)
:returns: None
"""
run_id = _get_or_start_run().info.run_id
tags_arr = [RunTag(key, str(value)) for key, value in tags.items()]
MlflowClient().log_batch(run_id=run_id,
metrics=[],
params=[],
tags=tags_arr)
def test_run_local_experiment_specification(experiment_name):
invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"name=test",
"--experiment-name",
experiment_name,
])
client = MlflowClient()
experiment_id = client.get_experiment_by_name(
experiment_name).experiment_id
invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"name=test",
"--experiment-id",
experiment_id,
])
def log_metrics(metrics, step=None):
"""
Log multiple metrics for the current run. If no run is active, this method will create a new
active run.
:param metrics: Dictionary of metric_name: String -> value: Float. Note that some special values
such as +/- Infinity may be replaced by other values depending on the store.
For example, sql based store may replace +/- Inf with max / min float values.
:param step: A single integer step at which to log the specified
Metrics. If unspecified, each metric is logged at step zero.
:returns: None
"""
run_id = _get_or_start_run().info.run_id
timestamp = int(time.time() * 1000)
metrics_arr = [
Metric(key, value, timestamp, step or 0)
for key, value in metrics.items()
]
MlflowClient().log_batch(run_id=run_id,
metrics=metrics_arr,
params=[],
tags=[])
def run(training_data, max_runs, max_p, epochs, metric, seed):
train_metric = "train_{}".format(metric)
val_metric = "val_{}".format(metric)
test_metric = "test_{}".format(metric)
np.random.seed(seed)
tracking_client = kiwi.tracking.MlflowClient()
def new_eval(nepochs,
experiment_id,
null_train_loss=_inf,
null_val_loss=_inf,
null_test_loss=_inf):
def eval(parms):
lr, momentum = parms
with kiwi.start_run(nested=True) as child_run:
p = kiwi.projects.run(run_id=child_run.info.run_id,
uri=".",
entry_point="train",
parameters={
"training_data": training_data,
"epochs": str(nepochs),
"learning_rate": str(lr),
"momentum": str(momentum),
"seed": str(seed)
},
experiment_id=experiment_id,
synchronous=False)
succeeded = p.wait()
if succeeded:
training_run = tracking_client.get_run(p.run_id)
metrics = training_run.data.metrics
# cap the loss at the loss of the null model
train_loss = min(null_train_loss, metrics[train_metric])
val_loss = min(null_val_loss, metrics[val_metric])
test_loss = min(null_test_loss, metrics[test_metric])
else:
# run failed => return null loss
tracking_client.set_terminated(p.run_id, "FAILED")
train_loss = null_train_loss
val_loss = null_val_loss
test_loss = null_test_loss
kiwi.log_metrics({
"train_{}".format(metric): train_loss,
"val_{}".format(metric): val_loss,
"test_{}".format(metric): test_loss
})
return p.run_id, train_loss, val_loss, test_loss
return eval
with kiwi.start_run() as run:
experiment_id = run.info.experiment_id
_, null_train_loss, null_val_loss, null_test_loss = new_eval(
0, experiment_id)((0, 0))
runs = [(np.random.uniform(1e-5, 1e-1), np.random.uniform(0, 1.0))
for _ in range(max_runs)]
with ThreadPoolExecutor(max_workers=max_p) as executor:
_ = executor.map(
new_eval(epochs, experiment_id, null_train_loss, null_val_loss,
null_test_loss), runs)
# find the best run, log its metrics as the final metrics of this run.
client = MlflowClient()
runs = client.search_runs(
[experiment_id], "tags.mlflow.parentRunId = '{run_id}' ".format(
run_id=run.info.run_id))
best_val_train = _inf
best_val_valid = _inf
best_val_test = _inf
best_run = None
for r in runs:
if r.data.metrics["val_rmse"] < best_val_valid:
best_run = r
best_val_train = r.data.metrics["train_rmse"]
best_val_valid = r.data.metrics["val_rmse"]
best_val_test = r.data.metrics["test_rmse"]
kiwi.set_tag("best_run", best_run.info.run_id)
kiwi.log_metrics({
"train_{}".format(metric): best_val_train,
"val_{}".format(metric): best_val_valid,
"test_{}".format(metric): best_val_test
})
def start_run(run_id=None, experiment_id=None, run_name=None, nested=False):
"""
Start a new MLflow run, setting it as the active run under which metrics and parameters
will be logged. The return value can be used as a context manager within a ``with`` block;
otherwise, you must call ``end_run()`` to terminate the current run.
If you pass a ``run_id`` or the ``MLFLOW_RUN_ID`` environment variable is set,
``start_run`` attempts to resume a run with the specified run ID and
other parameters are ignored. ``run_id`` takes precedence over ``MLFLOW_RUN_ID``.
MLflow sets a variety of default tags on the run, as defined in
:ref:`MLflow system tags <system_tags>`.
:param run_id: If specified, get the run with the specified UUID and log parameters
and metrics under that run. The run's end time is unset and its status
is set to running, but the run's other attributes (``source_version``,
``source_type``, etc.) are not changed.
:param experiment_id: ID of the experiment under which to create the current run (applicable
only when ``run_id`` is not specified). If ``experiment_id`` argument
is unspecified, will look for valid experiment in the following order:
activated using ``set_experiment``, ``MLFLOW_EXPERIMENT_NAME``
environment variable, ``MLFLOW_EXPERIMENT_ID`` environment variable,
or the default experiment as defined by the tracking server.
:param run_name: Name of new run (stored as a ``mlflow.runName`` tag).
Used only when ``run_id`` is unspecified.
:param nested: Controls whether run is nested in parent run. ``True`` creates a nest run.
:return: :py:class:`mlflow.ActiveRun` object that acts as a context manager wrapping
the run's state.
"""
global _active_run_stack
# back compat for int experiment_id
experiment_id = str(experiment_id) if isinstance(experiment_id,
int) else experiment_id
if len(_active_run_stack) > 0 and not nested:
raise Exception((
"Run with UUID {} is already active. To start a new run, first end the "
+ "current run with mlflow.end_run(). To start a nested " +
"run, call start_run with nested=True").format(
_active_run_stack[0].info.run_id))
if run_id:
existing_run_id = run_id
elif _RUN_ID_ENV_VAR in os.environ:
existing_run_id = os.environ[_RUN_ID_ENV_VAR]
del os.environ[_RUN_ID_ENV_VAR]
else:
existing_run_id = None
if existing_run_id:
_validate_run_id(existing_run_id)
active_run_obj = MlflowClient().get_run(existing_run_id)
# Check to see if experiment_id from environment matches experiment_id from set_experiment()
if (_active_experiment_id is not None and
_active_experiment_id != active_run_obj.info.experiment_id):
raise MlflowException(
"Cannot start run with ID {} because active run ID "
"does not match environment run ID. Make sure --experiment-name "
"or --experiment-id matches experiment set with "
"set_experiment(), or just use command-line "
"arguments".format(existing_run_id))
# Check to see if current run isn't deleted
if active_run_obj.info.lifecycle_stage == LifecycleStage.DELETED:
raise MlflowException(
"Cannot start run with ID {} because it is in the "
"deleted state.".format(existing_run_id))
else:
if len(_active_run_stack) > 0:
parent_run_id = _active_run_stack[-1].info.run_id
else:
parent_run_id = None
exp_id_for_run = experiment_id if experiment_id is not None else _get_experiment_id(
)
user_specified_tags = {}
if parent_run_id is not None:
user_specified_tags[MLFLOW_PARENT_RUN_ID] = parent_run_id
if run_name is not None:
user_specified_tags[MLFLOW_RUN_NAME] = run_name
tags = context_registry.resolve_tags(user_specified_tags)
active_run_obj = MlflowClient().create_run(
experiment_id=exp_id_for_run, tags=tags)
_active_run_stack.append(ActiveRun(active_run_obj))
return _active_run_stack[-1]
def _get_experiment_id_from_env():
experiment_name = env.get_env(_EXPERIMENT_NAME_ENV_VAR)
if experiment_name is not None:
exp = MlflowClient().get_experiment_by_name(experiment_name)
return exp.experiment_id if exp else None
return env.get_env(_EXPERIMENT_ID_ENV_VAR)
def set_experiment_tag(experiment_id: int, key, value):
MlflowClient().set_experiment_tag(experiment_id, key, value)
def _record_logged_model(mlflow_model):
run_id = _get_or_start_run().info.run_id
MlflowClient()._record_logged_model(run_id, mlflow_model)