Esempio n. 1
0
 def _timed_log_batch(self):
     start = time.time()
     metrics_slices = [
         self.data[i:i + MAX_METRICS_PER_BATCH]
         for i in range(0, len(self.data), MAX_METRICS_PER_BATCH)
     ]
     for metrics_slice in metrics_slices:
         try_mlflow_log(MlflowClient().log_batch,
                        run_id=self.run_id,
                        metrics=metrics_slice)
     end = time.time()
     self.total_log_batch_time += end - start
Esempio n. 2
0
def set_tags(tags):
    """
    Log a batch of tags for the current run. If no run is active, this method will create a
    new active run.

    :param tags: Dictionary of tag_name: String -> value: (String, but will be string-ified if
                 not)
    :returns: None
    """
    run_id = _get_or_start_run().info.run_id
    tags_arr = [RunTag(key, str(value)) for key, value in tags.items()]
    MlflowClient().log_batch(run_id=run_id, metrics=[], params=[], tags=tags_arr)
Esempio n. 3
0
def log_params(params):
    """
    Log a batch of params for the current run. If no run is active, this method will create a
    new active run.

    :param params: Dictionary of param_name: String -> value: (String, but will be string-ified if
                   not)
    :returns: None
    """
    run_id = _get_or_start_run().info.run_id
    params_arr = [Param(key, str(value)) for key, value in params.items()]
    MlflowClient().log_batch(run_id=run_id, metrics=[], params=params_arr, tags=[])
def test_with_managed_run_with_non_throwing_class_exhibits_expected_behavior():
    client = MlflowClient()

    @with_managed_run
    class TestPatch(PatchFunction):
        def _patch_implementation(self, original, *args, **kwargs):
            return mlflow.active_run()

        def _on_exception(self, exception):
            pass

    run1 = TestPatch.call(lambda: "foo")
    run1_status = client.get_run(run1.info.run_id).info.status
    assert RunStatus.from_string(run1_status) == RunStatus.FINISHED

    with mlflow.start_run() as active_run:
        run2 = TestPatch.call(lambda: "foo")

    assert run2 == active_run
    run2_status = client.get_run(run2.info.run_id).info.status
    assert RunStatus.from_string(run2_status) == RunStatus.FINISHED
Esempio n. 5
0
def _get_paginated_runs(experiment_ids, filter_string, run_view_type,
                        max_results, order_by):
    all_runs = []
    next_page_token = None
    while (len(all_runs) < max_results):
        runs_to_get = max_results - len(all_runs)
        if runs_to_get < NUM_RUNS_PER_PAGE_PANDAS:
            runs = MlflowClient().search_runs(experiment_ids, filter_string,
                                              run_view_type, runs_to_get,
                                              order_by, next_page_token)
        else:
            runs = MlflowClient().search_runs(experiment_ids, filter_string,
                                              run_view_type,
                                              NUM_RUNS_PER_PAGE_PANDAS,
                                              order_by, next_page_token)
        all_runs.extend(runs)
        if hasattr(runs, 'token') and runs.token != '':
            next_page_token = runs.token
        else:
            break
    return all_runs
Esempio n. 6
0
def set_experiment(experiment_name):
    """
    Set given experiment as active experiment. If experiment does not exist, create an experiment
    with provided name.

    :param experiment_name: Name of experiment to be activated.
    """
    client = MlflowClient()
    experiment = client.get_experiment_by_name(experiment_name)
    exp_id = experiment.experiment_id if experiment else None
    if exp_id is None:  # id can be 0
        print("INFO: '{}' does not exist. Creating a new experiment".format(
            experiment_name))
        exp_id = client.create_experiment(experiment_name)
    elif experiment.lifecycle_stage == LifecycleStage.DELETED:
        raise MlflowException(
            "Cannot set a deleted experiment '%s' as the active experiment."
            " You can restore the experiment, or permanently delete the "
            " experiment to create a new one." % experiment.name)
    global _active_experiment_id
    _active_experiment_id = exp_id
Esempio n. 7
0
def log_metric(key, value, step=None):
    """
    Log a metric under the current run, creating a run if necessary.

    :param key: Metric name (string).
    :param value: Metric value (float). Note that some special values such as +/- Infinity may be
                  replaced by other values depending on the store. For example, sFor example, the
                  SQLAlchemy store replaces +/- Inf with max / min float values.
    :param step: Metric step (int). Defaults to zero if unspecified.
    """
    run_id = _get_or_start_run().info.run_id
    MlflowClient().log_metric(run_id, key, value, int(time.time() * 1000), step or 0)
Esempio n. 8
0
def test_client_logs_metric_steps_correctly():
    client = MlflowAutologgingQueueingClient()

    with mlflow.start_run() as run:
        for step in range(3):
            client.log_metrics(
                run_id=run.info.run_id, metrics={"a": 1}, step=step,
            )
        client.flush()

    metric_history = MlflowClient().get_metric_history(run_id=run.info.run_id, key="a")
    assert len(metric_history) == 3
    assert [metric.step for metric in metric_history] == list(range(3))
Esempio n. 9
0
def test_sklearn_api_autolog_registering_model():
    registered_model_name = "test_autolog_registered_model"
    mlflow.lightgbm.autolog(registered_model_name=registered_model_name)

    X, y = datasets.load_iris(return_X_y=True)
    params = {"n_estimators": 10, "reg_lambda": 1}
    model = lgb.LGBMClassifier(**params)

    with mlflow.start_run():
        model.fit(X, y)

        registered_model = MlflowClient().get_registered_model(registered_model_name)
        assert registered_model.name == registered_model_name
Esempio n. 10
0
def log_metrics(metrics, step=None):
    """
    Log multiple metrics for the current run, starting a run if no runs are active.
    :param metrics: Dictionary of metric_name: String -> value: Float
    :param step: A single integer step at which to log the specified
                 Metrics. If unspecified, each metric is logged at step zero.

    :returns: None
    """
    run_id = _get_or_start_run().info.run_id
    timestamp = int(time.time() * 1000)
    metrics_arr = [Metric(key, value, timestamp, step or 0) for key, value in metrics.items()]
    MlflowClient().log_batch(run_id=run_id, metrics=metrics_arr, params=[], tags=[])
Esempio n. 11
0
def test_autolog_registering_model(random_train_data, random_one_hot_labels):
    registered_model_name = "test_autolog_registered_model"
    mlflow.keras.autolog(registered_model_name=registered_model_name)

    data = random_train_data
    labels = random_one_hot_labels

    model = create_model()
    with mlflow.start_run():
        model.fit(data, labels, epochs=10)

        registered_model = MlflowClient().get_registered_model(registered_model_name)
        assert registered_model.name == registered_model_name
Esempio n. 12
0
def log_metric(key, value):
    """
    Log a metric under the current run, creating a run if necessary.

    :param key: Metric name (string).
    :param value: Metric value (float).
    """
    if not isinstance(value, numbers.Number):
        _logger.warning(
            "The metric %s=%s was not logged because the value is not a number.", key, value)
        return
    run_id = _get_or_start_run().info.run_uuid
    MlflowClient().log_metric(run_id, key, value, int(time.time()))
Esempio n. 13
0
def test_run_local_experiment_specification(experiment_name,
                                            tracking_uri_mock):  # pylint: disable=unused-argument
    invoke_cli_runner(
        cli.run,
        [
            TEST_PROJECT_DIR,
            "-e", "greeter",
            "-P", "name=test",
            "--experiment-name", experiment_name,
        ])

    client = MlflowClient()
    experiment_id = client.get_experiment_by_name(experiment_name).experiment_id

    invoke_cli_runner(
        cli.run,
        [
            TEST_PROJECT_DIR,
            "-e", "greeter",
            "-P", "name=test",
            "--experiment-id", experiment_id,
        ])
Esempio n. 14
0
def _create_child_runs_for_parameter_search(parent_estimator, parent_model, parent_run, child_tags):
    from itertools import zip_longest

    client = MlflowClient()
    # Use the start time of the parent parameter search run as a rough estimate for the
    # start time of child runs, since we cannot precisely determine when each point
    # in the parameter search space was explored
    child_run_start_time = parent_run.info.start_time
    child_run_end_time = int(time.time() * 1000)

    estimator_param_maps = parent_estimator.getEstimatorParamMaps()
    tuned_estimator = parent_estimator.getEstimator()

    metrics_dict, _ = _get_param_search_metrics_and_best_index(parent_estimator, parent_model)
    for i in range(len(estimator_param_maps)):
        child_estimator = tuned_estimator.copy(estimator_param_maps[i])
        tags_to_log = dict(child_tags) if child_tags else {}
        tags_to_log.update({MLFLOW_PARENT_RUN_ID: parent_run.info.run_id})
        tags_to_log.update(_get_estimator_info_tags(child_estimator))

        child_run = client.create_run(
            experiment_id=parent_run.info.experiment_id,
            start_time=child_run_start_time,
            tags=tags_to_log,
        )

        params_to_log = _get_instance_param_map(
            child_estimator, parent_estimator._autologging_metadata.uid_to_indexed_name_map
        )
        param_batches_to_log = _chunk_dict(params_to_log, chunk_size=MAX_PARAMS_TAGS_PER_BATCH)
        metrics_to_log = {k: v[i] for k, v in metrics_dict.items()}
        for params_batch, metrics_batch in zip_longest(
            param_batches_to_log, [metrics_to_log], fillvalue={}
        ):
            # Trim any parameter keys / values and metric keys that exceed the limits
            # imposed by corresponding MLflow Tracking APIs (e.g., LogParam, LogMetric)
            truncated_params_batch = _truncate_dict(
                params_batch, MAX_ENTITY_KEY_LENGTH, MAX_PARAM_VAL_LENGTH
            )
            truncated_metrics_batch = _truncate_dict(
                metrics_batch, max_key_length=MAX_ENTITY_KEY_LENGTH
            )
            client.log_batch(
                run_id=child_run.info.run_id,
                params=[
                    Param(str(key), str(value)) for key, value in truncated_params_batch.items()
                ],
                metrics=[
                    Metric(key=str(key), value=value, timestamp=child_run_end_time, step=0)
                    for key, value in truncated_metrics_batch.items()
                ],
            )
        client.set_terminated(run_id=child_run.info.run_id, end_time=child_run_end_time)
Esempio n. 15
0
    def evaluate_model(self, item, model, run_hash):
        from mlflow.tracking.client import MlflowClient
        from mlflow.entities import ViewType

        exp_name = self.runner.analysis_name + ':' + self.runner.current_pipeline_name + ':'
        exp_name += str(item.get('base', 'None')) + ':' + str(
            item['func']) + ':' + item['hash']

        client = MlflowClient()
        experiments = [
            exp for exp in client.list_experiments() if exp.name == exp_name
        ]
        if len(experiments) == 0 or len(experiments) > 1:
            raise ValueError('Unable to find the experiment.')
        experiment = experiments[0]

        run = client.search_runs(
            experiment_ids=experiment.experiment_id,
            filter_string='tags."mlflow.runName" = ' + "'" + run_hash + "'",
            run_view_type=ViewType.ACTIVE_ONLY,
            max_results=1,
        )[0]

        run_id = run.info.run_id
        Tracker.resume_run(run_id)

        process = self.get_process(item)

        for source in model.sources:
            source.load_files()
            source.load()
        model.load()

        print('Evaluating run ' + run_hash + '...')
        process.run_id = run_hash
        process.evaluate(model)

        Tracker.end_run()
Esempio n. 16
0
def test_client_run_creation_and_termination_are_successful():
    experiment_name = "test_run_creation_termination"
    MlflowClient().create_experiment(experiment_name)
    experiment_id = MlflowClient().get_experiment_by_name(
        experiment_name).experiment_id

    client = MlflowAutologgingQueueingClient()
    pending_run_id = client.create_run(
        experiment_id=experiment_id,
        start_time=5,
        tags={"a": "b"},
    )
    client.set_terminated(run_id=pending_run_id, status="FINISHED", end_time=6)
    client.flush()

    runs = mlflow.search_runs(experiment_ids=[experiment_id],
                              output_format="list")
    assert len(runs) == 1
    run = runs[0]
    assert run.info.start_time == 5
    assert run.info.end_time == 6
    assert run.info.status == "FINISHED"
    assert {"a": "b"}.items() <= run.data.tags.items()
def test_with_managed_run_sets_specified_run_tags():
    client = MlflowClient()
    tags_to_set = {
        "foo": "bar",
        "num_layers": "7",
    }

    patch_function_1 = with_managed_run(
        "test_integration", lambda original, *args, **kwargs: mlflow.active_run(), tags=tags_to_set
    )
    run1 = patch_function_1(lambda: "foo")
    assert tags_to_set.items() <= client.get_run(run1.info.run_id).data.tags.items()

    class PatchFunction2(PatchFunction):
        def _patch_implementation(self, original, *args, **kwargs):
            return mlflow.active_run()

        def _on_exception(self, exception):
            pass

    patch_function_2 = with_managed_run("test_integration", PatchFunction2, tags=tags_to_set)
    run2 = patch_function_2.call(lambda: "foo")
    assert tags_to_set.items() <= client.get_run(run2.info.run_id).data.tags.items()
Esempio n. 18
0
def run(details):
    params = json.loads(details["params"])
    params["server_path"] = os.getcwd()
    project_uri = details["project_uri"]
    if not(project_uri.startswith("http://") or project_uri.startswith("https://")):
        assert project_uri.replace(".", "").replace("/", "").startswith("modules"), "Only support modules dir"

        project_uri = os.path.join(os.path.dirname(__file__), project_uri)

    submitted_run = mlflow.projects.run(project_uri,
                                        parameters=params,
                                        use_conda=False)
    out = MlflowClient().get_run(submitted_run.run_id).to_dictionary()
    return {"result": json.dumps(out)}
Esempio n. 19
0
def load_azure_workspace():
    """
    Load existing Azure Workspace from Tracking Store
    :rtype: AzureML Workspace object
    """
    from .store import AzureMLRestStore
    from mlflow.exceptions import ExecutionException
    from mlflow.tracking.client import MlflowClient

    try:
        def_store = MlflowClient()._tracking_client.store
    except ExecutionException:
        logger.warning(
            VERSION_WARNING.format("MlflowClient()._tracking_client.store"))
        def_store = MlflowClient().store
    if isinstance(def_store, AzureMLRestStore):
        workspace = Workspace._from_service_context(def_store.service_context,
                                                    _location=None)
        return workspace
    else:
        raise ExecutionException(
            "Workspace not found, please set the tracking URI in your script to AzureML."
        )
Esempio n. 20
0
def test_autolog_registering_model():
    registered_model_name = "test_autolog_registered_model"
    mlflow.pytorch.autolog(registered_model_name=registered_model_name)
    model = IrisClassification()
    dm = IrisDataModule()
    dm.setup(stage="fit")
    trainer = pl.Trainer(max_epochs=NUM_EPOCHS)

    with mlflow.start_run():
        trainer.fit(model, dm)

        registered_model = MlflowClient().get_registered_model(
            registered_model_name)
        assert registered_model.name == registered_model_name
Esempio n. 21
0
def search(data, max_runs, metric, algo):
    tracking_client = mlflow.tracking.MlflowClient()
    _inf = np.finfo(np.float64).max
    
    space = [
        hp.quniform('max_depth', 2, 12, 1)
        hp.quniform('min_samples_leaf', 2, 20, 1)
    ]
    
    with mlflow.start_run() as run:
        exp_id = run.info.experiment_id

        best = fmin(
          fn=train_fn(exp_id, _inf, _inf),
          space=space,
          algo=tpe.suggest if algo == "tpe.suggest" else rand.suggest,
          max_evals=max_runs
          )
        mlflow.set_tag("best params", str(best))
        # find all runs generated by this search
        client = MlflowClient()
        query = "tags.mlflow.parentRunId = '{run_id}' ".format(run_id=run.info.run_id)
        runs = client.search_runs([exp_id], query)
        # iterate over all runs to find best one
        best_train, best_valid = _inf, _inf
        best_run = None
        for r in runs:
            if r.data.metrics["val_auc"] < best_val_valid:
                best_run = r
                best_train = r.data.metrics["train_auc"]
                best_valid = r.data.metrics["val_auc"]
        # log best run metrics as the final metrics of this run.
        mlflow.set_tag("best_run", best_run.info.run_id)
        mlflow.log_metrics({
          "train_{}".format(metric): best_train,
          "val_{}".format(metric): best_valid
          })
Esempio n. 22
0
def test_with_managed_run_with_throwing_function_exhibits_expected_behavior():
    client = MlflowClient()
    patch_function_active_run = None

    @with_managed_run
    def patch_function(original, *args, **kwargs):
        nonlocal patch_function_active_run
        patch_function_active_run = mlflow.active_run()
        raise Exception("bad implementation")

    with pytest.raises(Exception):
        patch_function(lambda: "foo")

    assert patch_function_active_run is not None
    status1 = client.get_run(patch_function_active_run.info.run_id).info.status
    assert RunStatus.from_string(status1) == RunStatus.FAILED

    with mlflow.start_run() as active_run, pytest.raises(Exception):
        patch_function(lambda: "foo")
        assert patch_function_active_run == active_run
        # `with_managed_run` should not terminate a preexisting MLflow run,
        # even if the patch function throws
        status2 = client.get_run(active_run.info.run_id).info.status
        assert RunStatus.from_string(status2) == RunStatus.FINISHED
Esempio n. 23
0
def log_metric(key, value):
    """
    Log a metric under the current run, creating a run if necessary.

    :param key: Metric name (string).
    :param value: Metric value (float).
    """
    if not isinstance(value, numbers.Number):
        print(
            "WARNING: The metric {}={} was not logged because the value is not a number."
            .format(key, value),
            file=sys.stderr)
        return
    run_id = _get_or_start_run().info.run_uuid
    MlflowClient().log_metric(run_id, key, value, int(time.time()))
Esempio n. 24
0
def get_run(run_id):
    """
    Fetch the run from backend store. The resulting :py:class:`Run <mlflow.entities.Run>`
    contains a collection of run metadata -- :py:class:`RunInfo <mlflow.entities.RunInfo>`,
    as well as a collection of run parameters, tags, and metrics --
    :py:class:`RunData <mlflow.entities.RunData>`. In the case where multiple metrics with the
    same key are logged for the run, the :py:class:`RunData <mlflow.entities.RunData>` contains
    the most recently logged value at the largest step for each metric.

    :param run_id: Unique identifier for the run.

    :return: A single :py:class:`mlflow.entities.Run` object, if the run exists. Otherwise,
                raises an exception.
    """
    return MlflowClient().get_run(run_id)
Esempio n. 25
0
def check_finish(hparams, experiment_name):
    # NOTE : This is not a perfect logic. For example, the aborted run is also couted as completed for now.
    logging.info("checking status")
    query = ' and '.join(
        ['params.{}="{}"'.format(k, str(v)) for k, v in vars(hparams).items()])
    experiment = mlflow.get_experiment_by_name(experiment_name)
    if experiment is None:
        return False
    finished_runs = MlflowClient().search_runs(experiment_ids=[
        mlflow.get_experiment_by_name(experiment_name).experiment_id
    ],
                                               filter_string=query,
                                               run_view_type=ViewType.ALL)
    logging.info("done")
    return len(finished_runs) > 0
Esempio n. 26
0
def log_metrics(metrics):
    """
    Log multiple metrics for the current run, starting a run if no runs are active.
    :param metrics: Dictionary of metric_name: String -> value: Float
    :returns: None
    """
    run_id = _get_or_start_run().info.run_uuid
    timestamp = int(time.time())
    metrics_arr = [
        Metric(key, value, timestamp) for key, value in metrics.items()
    ]
    MlflowClient().log_batch(run_id=run_id,
                             metrics=metrics_arr,
                             params=[],
                             tags=[])
Esempio n. 27
0
def log_metrics(metrics, step=None):
    """
    Log multiple metrics for the current run, starting a run if no runs are active.
    :param metrics: Dictionary of metric_name: String -> value: Float. Note that some special values
                    such as +/- Infinity may be replaced by other values depending on the store.
                    For example, sql based store may replace +/- Inf with max / min float values.
    :param step: A single integer step at which to log the specified
                 Metrics. If unspecified, each metric is logged at step zero.

    :returns: None
    """
    run_id = _get_or_start_run().info.run_id
    timestamp = int(time.time() * 1000)
    metrics_arr = [Metric(key, value, timestamp, step or 0) for key, value in metrics.items()]
    MlflowClient().log_batch(run_id=run_id, metrics=metrics_arr, params=[], tags=[])
Esempio n. 28
0
def _log_parameter_search_results_as_artifact(cv_results_df, run_id):
    """
    Records a collection of parameter search results as an MLflow artifact
    for the specified run.

    :param cv_results_df: A Pandas DataFrame containing the results of a parameter search
                          training session, which may be obtained by parsing the `cv_results_`
                          attribute of a trained parameter search estimator such as
                          `GridSearchCV`.
    :param run_id: The ID of the MLflow Run to which the artifact should be recorded.
    """
    with TempDir() as t:
        results_path = t.path("cv_results.csv")
        cv_results_df.to_csv(results_path, index=False)
        try_mlflow_log(MlflowClient().log_artifact, run_id, results_path)
def test_with_managed_run_ends_run_on_keyboard_interrupt():
    client = MlflowClient()
    run = None

    def original():
        nonlocal run
        run = mlflow.active_run()
        raise KeyboardInterrupt

    patch_function_1 = with_managed_run(
        "test_integration", lambda original, *args, **kwargs: original(*args, **kwargs)
    )

    with pytest.raises(KeyboardInterrupt):
        patch_function_1(original)

    assert not mlflow.active_run()
    run_status_1 = client.get_run(run.info.run_id).info.status
    assert RunStatus.from_string(run_status_1) == RunStatus.FAILED

    class PatchFunction2(PatchFunction):
        def _patch_implementation(self, original, *args, **kwargs):
            return original(*args, **kwargs)

        def _on_exception(self, exception):
            pass

    patch_function_2 = with_managed_run("test_integration", PatchFunction2)

    with pytest.raises(KeyboardInterrupt):

        patch_function_2.call(original)

    assert not mlflow.active_run()
    run_status_2 = client.get_run(run.info.run_id).info.status
    assert RunStatus.from_string(run_status_2) == RunStatus.FAILED
def test_safe_patch_manages_run_if_specified_and_sets_expected_run_tags(
        patch_destination, test_autologging_integration):
    client = MlflowClient()
    active_run = None

    def patch_impl(original, *args, **kwargs):
        nonlocal active_run
        active_run = mlflow.active_run()
        return original(*args, **kwargs)

    with mock.patch("mlflow.utils.autologging_utils.with_managed_run",
                    wraps=with_managed_run) as managed_run_mock:
        safe_patch(test_autologging_integration,
                   patch_destination,
                   "fn",
                   patch_impl,
                   manage_run=True)
        patch_destination.fn()
        assert managed_run_mock.call_count == 1
        assert active_run is not None
        assert active_run.info.run_id is not None
        assert (client.get_run(
            active_run.info.run_id).data.tags[MLFLOW_AUTOLOGGING] ==
                "test_integration")