def import_run_data_batch(run_dct, run_id): import time from mlflow.entities import Metric, Param, RunTag now = int(time.time() + .5) params = [Param(k, v) for k, v in run_dct['params'].items()] metrics = [Metric(k, v, now, 0) for k, v in run_dct['metrics'].items() ] # TODO: check timestamp and step semantics tags = [RunTag(k, v) for k, v in run_dct['tags'].items()] client.log_batch(run_id, metrics, params, tags)
def _add_to_queue(key, value, step, time, run_id): """ Add a metric to the metric queue. Flush the queue if it exceeds max size. """ met = Metric(key=key, value=value, timestamp=time, step=step) _metric_queue.append((run_id, met)) if len(_metric_queue) > _MAX_METRIC_QUEUE_SIZE: _flush_queue()
def _log_metric(): request_message = _get_request_message(LogMetric()) metric = Metric(request_message.key, request_message.value, request_message.timestamp) _get_store().log_metric(request_message.run_uuid, metric) response_message = LogMetric.Response() response = Response(mimetype='application/json') response.set_data(message_to_json(response_message)) return response
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234)) metric = fs.get_metric(run_uuid, WEIRD_METRIC_NAME) assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def copy_run_data_batch(src_client, src_run, log_source_info, dst_client, dst_run_id): import time from mlflow.entities import Metric, Param, RunTag now = int(time.time()+.5) params = [ Param(k,v) for k,v in src_run.data.params.items() ] metrics = [ Metric(k,v,now,0) for k,v in src_run.data.metrics.items() ] # TODO: check timestamp and step semantics tags = utils.create_tags(src_client, src_run, log_source_info) tags = [ RunTag(k,v) for k,v in tags.items() ] dst_client.log_batch(dst_run_id, metrics, params, tags)
def log_metric(self, run_id, key, value, timestamp=None): """ Log a metric against the run ID. If timestamp is not provided, uses the current timestamp. """ _validate_metric_name(key) timestamp = timestamp if timestamp is not None else int(time.time()) metric = Metric(key, value, timestamp) self.store.log_metric(run_id, metric)
def _create(): metrics = [Metric(key=random_str(10), value=random_int(0, 1000), timestamp=int(time.time()) + random_int(-1e4, 1e4), step=random_int())] params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa rd = RunData(metrics=metrics, params=params, tags=tags) return rd, metrics, params, tags
def _get_metric_from_file(parent_path, metric_name): _validate_metric_name(metric_name) metric_data = read_file_lines(parent_path, metric_name) if len(metric_data) == 0: raise Exception("Metric '%s' is malformed. No data found." % metric_name) last_line = metric_data[-1] timestamp, val = last_line.strip().split(" ") return Metric(metric_name, float(val), int(timestamp))
def test_search_runs_data(): import numpy as np import pandas as pd runs = [ create_run( metrics=[Metric("mse", 0.2, 0, 0)], params=[Param("param", "value")], tags=[RunTag("tag", "value")], start=1564675200000, end=1564683035000, ), create_run( metrics=[Metric("mse", 0.6, 0, 0), Metric("loss", 1.2, 0, 5)], params=[Param("param2", "val"), Param("k", "v")], tags=[RunTag("tag2", "v2")], start=1564765200000, end=1564783200000, ), ] with mock.patch("mlflow.tracking.fluent._paginate", return_value=runs): pdf = search_runs() data = { "status": [RunStatus.FINISHED] * 2, "artifact_uri": [None] * 2, "run_id": [""] * 2, "experiment_id": [""] * 2, "metrics.mse": [0.2, 0.6], "metrics.loss": [np.nan, 1.2], "params.param": ["value", None], "params.param2": [None, "val"], "params.k": [None, "v"], "tags.tag": ["value", None], "tags.tag2": [None, "v2"], "start_time": [ pd.to_datetime(1564675200000, unit="ms", utc=True), pd.to_datetime(1564765200000, unit="ms", utc=True), ], "end_time": [ pd.to_datetime(1564683035000, unit="ms", utc=True), pd.to_datetime(1564783200000, unit="ms", utc=True), ], } validate_search_runs(pdf, data, "pandas")
def test_search_runs_data(): runs = [ create_run(metrics=[Metric("mse", 0.2, 0, 0)], params=[Param("param", "value")], tags=[RunTag("tag", "value")], start=1564675200000, end=1564683035000), create_run( metrics=[Metric("mse", 0.6, 0, 0), Metric("loss", 1.2, 0, 5)], params=[Param("param2", "val"), Param("k", "v")], tags=[RunTag("tag2", "v2")], start=1564765200000, end=1564783200000) ] with mock.patch('mlflow.tracking.fluent._paginate', return_value=runs): pdf = search_runs() data = { 'status': [RunStatus.FINISHED] * 2, 'artifact_uri': [None] * 2, 'run_id': [''] * 2, 'experiment_id': [""] * 2, 'metrics.mse': [0.2, 0.6], 'metrics.loss': [np.nan, 1.2], 'params.param': ["value", None], 'params.param2': [None, "val"], 'params.k': [None, "v"], 'tags.tag': ["value", None], 'tags.tag2': [None, "v2"], 'start_time': [ pd.to_datetime(1564675200000, unit="ms", utc=True), pd.to_datetime(1564765200000, unit="ms", utc=True) ], 'end_time': [ pd.to_datetime(1564683035000, unit="ms", utc=True), pd.to_datetime(1564783200000, unit="ms", utc=True) ] } expected_df = pd.DataFrame(data) pd.testing.assert_frame_equal(pdf, expected_df, check_like=True, check_frame_type=False)
def test_correct_sorting(order_bys, matching_runs): runs = [ Run(run_info=RunInfo(run_uuid="9", run_id="9", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=0, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[Metric("key1", 121, 1, 0)], params=[Param("my_param", "A")], tags=[])), Run(run_info=RunInfo(run_uuid="8", run_id="8", experiment_id=0, user_id="user-id", status=RunStatus.to_string(RunStatus.FINISHED), start_time=1, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[Metric("key1", 123, 1, 0)], params=[Param("my_param", "A")], tags=[RunTag("tag1", "C")])), Run(run_info=RunInfo(run_uuid="7", run_id="7", experiment_id=1, user_id="user-id", status=RunStatus.to_string(RunStatus.FAILED), start_time=1, end_time=1, lifecycle_stage=LifecycleStage.ACTIVE), run_data=RunData(metrics=[Metric("key1", 125, 1, 0)], params=[Param("my_param", "B")], tags=[RunTag("tag1", "D")])), ] sorted_runs = SearchUtils.sort(runs, order_bys) sorted_run_indices = [] for run in sorted_runs: for i, r in enumerate(runs): if r == run: sorted_run_indices.append(i) break assert sorted_run_indices == matching_runs
def _log_specialized_estimator_content(fitted_estimator, run_id, fit_args, fit_kwargs): import sklearn mlflow_client = MlflowClient() name_metric_dict = {} try: if sklearn.base.is_classifier(fitted_estimator): name_metric_dict = _get_classifier_metrics(fitted_estimator, fit_args, fit_kwargs) elif sklearn.base.is_regressor(fitted_estimator): name_metric_dict = _get_regressor_metrics(fitted_estimator, fit_args, fit_kwargs) except Exception as err: # pylint: disable=broad-except msg = ("Failed to autolog metrics for " + fitted_estimator.__class__.__name__ + ". Logging error: " + str(err)) _logger.warning(msg) else: # batch log all metrics try_mlflow_log( mlflow_client.log_batch, run_id, metrics=[ Metric(key=str(key), value=value, timestamp=int(time.time() * 1000), step=0) for key, value in name_metric_dict.items() ], ) if sklearn.base.is_classifier(fitted_estimator): try: artifacts = _get_classifier_artifacts(fitted_estimator, fit_args, fit_kwargs) except Exception as e: # pylint: disable=broad-except msg = ("Failed to autolog artifacts for " + fitted_estimator.__class__.__name__ + ". Logging error: " + str(e)) _logger.warning(msg) return with TempDir() as tmp_dir: for artifact in artifacts: try: display = artifact.function(**artifact.arguments) display.ax_.set_title(artifact.title) filepath = tmp_dir.path("{}.png".format(artifact.name)) display.figure_.savefig(filepath) plt.close(display.figure_) except Exception as e: # pylint: disable=broad-except _log_warning_for_artifacts(artifact.name, artifact.function, e) try_mlflow_log(mlflow_client.log_artifacts, run_id, tmp_dir.path())
def log_metric(self, run_id, key, value, timestamp=None, step=None): """ Log a metric against the run ID. The timestamp defaults to the current timestamp. The step defaults to 0. """ timestamp = timestamp if timestamp is not None else int(time.time()) step = step if step is not None else 0 _validate_metric(key, value, timestamp, step) metric = Metric(key, value, timestamp, step) self.store.log_metric(run_id, metric)
def _dict_to_run_metric_history(rm): metrics = rm["metrics"][::-1] return [ Metric( key=rm["key"], value=float(m["value"]), timestamp=int(m["timestamp"]), step=int(m.get("step", i)), ) for (i, m) in enumerate(metrics) ]
def log_metrics(metrics): """ Log multiple metrics for the current run, starting a run if no runs are active. :param metrics: Dictionary of metric_name: String -> value: Float :returns: None """ run_id = _get_or_start_run().info.run_uuid timestamp = int(time.time()) metrics_arr = [Metric(key, value, timestamp, 0) for key, value in metrics.items()] MlflowClient().log_batch(run_id=run_id, metrics=metrics_arr, params=[], tags=[])
def import_run_data(self, run_dct, run, src_user_id): from mlflow.entities import Metric, Param, RunTag now = round(time.time()) params = [ Param(k,v) for k,v in run_dct['params'].items() ] metrics = [ Metric(k,v,now,0) for k,v in run_dct['metrics'].items() ] # TODO: missing timestamp and step semantics? tags = self._create_tags_for_metadata(run_dct['tags']) tags = utils.create_tags_for_mlflow_tags(tags, self.import_mlflow_tags) #utils.dump_tags("RunImporter.import_run_data",tags) #utils.set_dst_user_id(tags, src_user_id, self.use_src_user_id) self.client.log_batch(run.info.run_id, metrics, params, tags)
def _copy_run_data(self, src_run, dst_run_id): from mlflow.entities import Metric, Param, RunTag now = int(time.time()+.5) params = [ Param(k,v) for k,v in src_run.data.params.items() ] metrics = [ Metric(k,v,now,0) for k,v in src_run.data.metrics.items() ] # TODO: timestamp and step semantics? tags = utils.create_tags_for_metadata(self.src_client, src_run, self.export_metadata_tags) #tags = [ RunTag(k,v) for k,v in tags.items() ] tags = utils.create_tags_for_mlflow_tags(tags, self.import_mlflow_tags) # XX utils.set_dst_user_id(tags, src_run.info.user_id, self.use_src_user_id) self.dst_client.log_batch(dst_run_id, metrics, params, tags)
def to_mlflow_entity(self): """ Convert DB model to corresponding MLflow entity. :return: :py:class:`mlflow.entities.Metric`. """ return Metric(key=self.key, value=self.value if not self.is_nan else float("nan"), timestamp=self.timestamp, step=self.step)
def _log_metric(): request_message = _get_request_message(LogMetric()) metric = Metric(request_message.key, request_message.value, request_message.timestamp, request_message.step) run_id = request_message.run_id or request_message.run_uuid _get_tracking_store().log_metric(run_id, metric) response_message = LogMetric.Response() response = Response(mimetype="application/json") response.set_data(message_to_json(response_message)) return response
def test_log_batch(self): fs = FileStore(self.test_root) run = fs.create_run(experiment_id=FileStore.DEFAULT_EXPERIMENT_ID, user_id='user', start_time=0, tags=[]) run_id = run.info.run_id metric_entities = [ Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, 0) ] param_entities = [Param("p1", "p1val"), Param("p2", "p2val")] tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")] fs.log_batch(run_id=run_id, metrics=metric_entities, params=param_entities, tags=tag_entities) self._verify_logged(fs, run_id, metric_entities, param_entities, tag_entities)
def _get_metric_from_line(metric_name, metric_line): metric_parts = metric_line.strip().split(" ") if len(metric_parts) != 2 and len(metric_parts) != 3: raise MlflowException("Metric '%s' is malformed; persisted metric data contained %s " "fields. Expected 2 or 3 fields." % (metric_name, len(metric_parts)), databricks_pb2.INTERNAL_ERROR) ts = int(metric_parts[0]) val = float(metric_parts[1]) step = int(metric_parts[2]) if len(metric_parts) == 3 else 0 return Metric(key=metric_name, value=val, timestamp=ts, step=step)
def log_metric(self, run_id, key, value, timestamp=None, step=None): """ Log a metric against the run ID. If timestamp is not provided, uses the current timestamp. The metric's step defaults to 0 if unspecified. """ timestamp = timestamp if timestamp is not None else int(time.time()) step = step if step is not None else 0 _validate_metric(key, value, timestamp, step) metric = Metric(key, value, timestamp, step) self.store.log_metric(run_id, metric)
def test_log_batch(tracking_uri_mock, tmpdir): expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0} expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"} exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"} approx_expected_tags = set([MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE]) t = int(time.time()) sorted_expected_metrics = sorted(expected_metrics.items(), key=lambda kv: kv[0]) metrics = [ Metric(key=key, value=value, timestamp=t, step=i) for i, (key, value) in enumerate(sorted_expected_metrics) ] params = [ Param(key=key, value=value) for key, value in expected_params.items() ] tags = [ RunTag(key=key, value=value) for key, value in exact_expected_tags.items() ] with start_run() as active_run: run_uuid = active_run.info.run_uuid mlflow.tracking.MlflowClient().log_batch(run_id=run_uuid, metrics=metrics, params=params, tags=tags) finished_run = tracking.MlflowClient().get_run(run_uuid) # Validate metrics assert len(finished_run.data.metrics) == 2 for key, value in finished_run.data.metrics.items(): assert expected_metrics[key] == value # TODO: use client get_metric_history API here instead once it exists fs = FileStore(os.path.join(tmpdir.strpath, "mlruns")) metric_history0 = fs.get_metric_history(run_uuid, "metric-key0") assert set([(m.value, m.timestamp, m.step) for m in metric_history0]) == set([ (1.0, t, 0), ]) metric_history1 = fs.get_metric_history(run_uuid, "metric-key1") assert set([(m.value, m.timestamp, m.step) for m in metric_history1]) == set([ (4.0, t, 1), ]) # Validate tags (for automatically-set tags) assert len(finished_run.data.tags ) == len(exact_expected_tags) + len(approx_expected_tags) for tag_key, tag_value in finished_run.data.tags.items(): if tag_key in approx_expected_tags: pass else: assert exact_expected_tags[tag_key] == tag_value # Validate params assert finished_run.data.params == expected_params
def test_log_batch_validates_entity_names_and_values(): with start_run() as active_run: run_id = active_run.info.run_id metrics = [ Metric(key="../bad/metric/name", value=0.3, timestamp=3, step=0) ] with pytest.raises(MlflowException, match="Invalid metric name") as e: tracking.MlflowClient().log_batch(run_id, metrics=metrics) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE) metrics = [ Metric(key="ok-name", value="non-numerical-value", timestamp=3, step=0) ] with pytest.raises(MlflowException, match="Got invalid value") as e: tracking.MlflowClient().log_batch(run_id, metrics=metrics) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE) metrics = [ Metric(key="ok-name", value=0.3, timestamp="non-numerical-timestamp", step=0) ] with pytest.raises(MlflowException, match="Got invalid timestamp") as e: tracking.MlflowClient().log_batch(run_id, metrics=metrics) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE) params = [Param(key="../bad/param/name", value="my-val")] with pytest.raises(MlflowException, match="Invalid parameter name") as e: tracking.MlflowClient().log_batch(run_id, params=params) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE) tags = [Param(key="../bad/tag/name", value="my-val")] with pytest.raises(MlflowException, match="Invalid tag name") as e: tracking.MlflowClient().log_batch(run_id, tags=tags) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def run(alpha, run_origin, log_artifact): with mlflow.start_run(run_name=run_origin) as run: print("runId:",run.info.run_uuid) print("artifact_uri:",mlflow.get_artifact_uri()) print("alpha:",alpha) print("log_artifact:",log_artifact) print("run_origin:",run_origin) mlflow.log_param("alpha", alpha) mlflow.log_metric("rmse", 0.789) mlflow.set_tag("run_origin", run_origin) mlflow.set_tag("log_artifact", log_artifact) if log_artifact: with open("info.txt", "w") as f: f.write("Hi artifact") mlflow.log_artifact("info.txt") params = [ Param("p1","0.1"), Param("p2","0.2") ] metrics = [ Metric("m1",0.1,now), Metric("m2",0.2,now) ] tags = [ RunTag("t1","hi1"), RunTag("t2","hi2") ] client.log_batch(run.info.run_uuid, metrics, params, tags)
def test_log_batch(self): fs = FileStore(self.test_root) run = fs.create_run( experiment_id=Experiment.DEFAULT_EXPERIMENT_ID, user_id='user', run_name=None, source_type='source_type', source_name='source_name', entry_point_name='entry_point_name', start_time=0, source_version=None, tags=[], parent_run_id=None) run_uuid = run.info.run_uuid metric_entities = [Metric("m1", 0.87, 12345), Metric("m2", 0.49, 12345)] param_entities = [Param("p1", "p1val"), Param("p2", "p2val")] tag_entities = [RunTag("t1", "t1val"), RunTag("t2", "t2val")] fs.log_batch( run_id=run_uuid, metrics=metric_entities, params=param_entities, tags=tag_entities) run = fs.get_run(run_uuid) tags = [(t.key, t.value) for t in run.data.tags] metrics = [(m.key, m.value, m.timestamp) for m in run.data.metrics] params = [(p.key, p.value) for p in run.data.params] assert set(tags) == set([("t1", "t1val"), ("t2", "t2val")]) assert set(metrics) == set([("m1", 0.87, 12345), ("m2", 0.49, 12345)]) assert set(params) == set([("p1", "p1val"), ("p2", "p2val")])
def test_log_batch_limits(self): # Test that log batch at the maximum allowed request size succeeds (i.e doesn't hit # SQL limitations, etc) experiment_id = self._experiment_factory('log_batch_limits') run_uuid = self._run_factory(self._get_run_configs('r1', experiment_id)).info.run_uuid metric_tuples = [("m%s" % i, i, 12345) for i in range(1000)] metric_entities = [Metric(*metric_tuple) for metric_tuple in metric_tuples] self.store.log_batch(run_id=run_uuid, metrics=metric_entities, params=[], tags=[]) run = self.store.get_run(run_uuid) metrics = [(m.key, m.value, m.timestamp) for m in run.data.metrics] assert set(metrics) == set(metric_tuples)
def test_log_batch_validates_entity_names_and_values(tracking_uri_mock): bad_kwargs = { "metrics": [ [Metric(key="../bad/metric/name", value=0.3, timestamp=3, step=0)], [Metric(key="ok-name", value="non-numerical-value", timestamp=3, step=0)], [Metric(key="ok-name", value=0.3, timestamp="non-numerical-timestamp", step=0)], ], "params": [[Param(key="../bad/param/name", value="my-val")]], "tags": [[Param(key="../bad/tag/name", value="my-val")]], } with start_run() as active_run: for kwarg, bad_values in bad_kwargs.items(): for bad_kwarg_value in bad_values: final_kwargs = { "run_id": active_run.info.run_id, "metrics": [], "params": [], "tags": [], } final_kwargs[kwarg] = bad_kwarg_value with pytest.raises(MlflowException) as e: tracking.MlflowClient().log_batch(**final_kwargs) assert e.value.error_code == ErrorCode.Name(INVALID_PARAMETER_VALUE)
def test_weird_metric_names(self): WEIRD_METRIC_NAME = "this is/a weird/but valid metric" fs = FileStore(self.test_root) run_uuid = self.exp_data[0]["runs"][0] fs.log_metric(run_uuid, Metric(WEIRD_METRIC_NAME, 10, 1234)) run = fs.get_run(run_uuid) my_metrics = [m for m in run.data.metrics if m.key == WEIRD_METRIC_NAME] assert len(my_metrics) == 1 metric = my_metrics[0] assert metric.key == WEIRD_METRIC_NAME assert metric.value == 10 assert metric.timestamp == 1234
def _create_child_runs_for_parameter_search(parent_estimator, parent_model, parent_run, child_tags): from itertools import zip_longest client = MlflowClient() # Use the start time of the parent parameter search run as a rough estimate for the # start time of child runs, since we cannot precisely determine when each point # in the parameter search space was explored child_run_start_time = parent_run.info.start_time child_run_end_time = int(time.time() * 1000) estimator_param_maps = parent_estimator.getEstimatorParamMaps() tuned_estimator = parent_estimator.getEstimator() metrics_dict, _ = _get_param_search_metrics_and_best_index(parent_estimator, parent_model) for i in range(len(estimator_param_maps)): child_estimator = tuned_estimator.copy(estimator_param_maps[i]) tags_to_log = dict(child_tags) if child_tags else {} tags_to_log.update({MLFLOW_PARENT_RUN_ID: parent_run.info.run_id}) tags_to_log.update(_get_estimator_info_tags(child_estimator)) child_run = client.create_run( experiment_id=parent_run.info.experiment_id, start_time=child_run_start_time, tags=tags_to_log, ) params_to_log = _get_instance_param_map( child_estimator, parent_estimator._autologging_metadata.uid_to_indexed_name_map ) param_batches_to_log = _chunk_dict(params_to_log, chunk_size=MAX_PARAMS_TAGS_PER_BATCH) metrics_to_log = {k: v[i] for k, v in metrics_dict.items()} for params_batch, metrics_batch in zip_longest( param_batches_to_log, [metrics_to_log], fillvalue={} ): # Trim any parameter keys / values and metric keys that exceed the limits # imposed by corresponding MLflow Tracking APIs (e.g., LogParam, LogMetric) truncated_params_batch = _truncate_dict( params_batch, MAX_ENTITY_KEY_LENGTH, MAX_PARAM_VAL_LENGTH ) truncated_metrics_batch = _truncate_dict( metrics_batch, max_key_length=MAX_ENTITY_KEY_LENGTH ) client.log_batch( run_id=child_run.info.run_id, params=[ Param(str(key), str(value)) for key, value in truncated_params_batch.items() ], metrics=[ Metric(key=str(key), value=value, timestamp=child_run_end_time, step=0) for key, value in truncated_metrics_batch.items() ], ) client.set_terminated(run_id=child_run.info.run_id, end_time=child_run_end_time)