def test_log_batch_validates_entity_names_and_values():
bad_kwargs = {
"metrics": [
[Metric(key="../bad/metric/name", value=0.3, timestamp=3, step=0)],
[
Metric(key="ok-name",
value="non-numerical-value",
timestamp=3,
step=0)
],
[
Metric(key="ok-name",
value=0.3,
timestamp="non-numerical-timestamp",
step=0)
],
],
"params": [[Param(key="../bad/param/name", value="my-val")]],
"tags": [[Param(key="../bad/tag/name", value="my-val")]],
}
with start_run() as active_run:
for kwarg, bad_values in bad_kwargs.items():
for bad_kwarg_value in bad_values:
final_kwargs = {
"run_id": active_run.info.run_id,
"metrics": [],
"params": [],
"tags": [],
}
final_kwargs[kwarg] = bad_kwarg_value
with pytest.raises(MlflowException) as e:
tracking.MlflowClient().log_batch(**final_kwargs)
assert e.value.error_code == ErrorCode.Name(
INVALID_PARAMETER_VALUE)
def test_validate_batch_log_limits():
too_many_metrics = [
Metric("metric-key-%s" % i, 1, 0, i * 2) for i in range(1001)
]
too_many_params = [Param("param-key-%s" % i, "b") for i in range(101)]
too_many_tags = [RunTag("tag-key-%s" % i, "b") for i in range(101)]
good_kwargs = {"metrics": [], "params": [], "tags": []}
bad_kwargs = {
"metrics": [too_many_metrics],
"params": [too_many_params],
"tags": [too_many_tags],
}
for arg_name, arg_values in bad_kwargs.items():
for arg_value in arg_values:
final_kwargs = copy.deepcopy(good_kwargs)
final_kwargs[arg_name] = arg_value
with pytest.raises(MlflowException):
_validate_batch_log_limits(**final_kwargs)
# Test the case where there are too many entities in aggregate
with pytest.raises(MlflowException):
_validate_batch_log_limits(too_many_metrics[:900],
too_many_params[:51], too_many_tags[:50])
# Test that we don't reject entities within the limit
_validate_batch_log_limits(too_many_metrics[:1000], [], [])
_validate_batch_log_limits([], too_many_params[:100], [])
_validate_batch_log_limits([], [], too_many_tags[:100])
def test_search_runs_data():
runs = [
create_run(metrics=[Metric("mse", 0.2, 0, 0)],
params=[Param("param", "value")],
tags=[RunTag("tag", "value")],
start=1564675200000,
end=1564683035000),
create_run(
metrics=[Metric("mse", 0.6, 0, 0),
Metric("loss", 1.2, 0, 5)],
params=[Param("param2", "val"),
Param("k", "v")],
tags=[RunTag("tag2", "v2")],
start=1564765200000,
end=1564783200000)
]
with mock.patch('mlflow.tracking.fluent._get_paginated_runs',
return_value=runs):
pdf = search_runs()
data = {
'status': [RunStatus.FINISHED] * 2,
'artifact_uri': [None] * 2,
'run_id': [''] * 2,
'experiment_id': [""] * 2,
'metrics.mse': [0.2, 0.6],
'metrics.loss': [np.nan, 1.2],
'params.param': ["value", None],
'params.param2': [None, "val"],
'params.k': [None, "v"],
'tags.tag': ["value", None],
'tags.tag2': [None, "v2"],
'start_time': [
pd.to_datetime(1564675200000, unit="ms", utc=True),
pd.to_datetime(1564765200000, unit="ms", utc=True)
],
'end_time': [
pd.to_datetime(1564683035000, unit="ms", utc=True),
pd.to_datetime(1564783200000, unit="ms", utc=True)
]
}
expected_df = pd.DataFrame(data)
pd.testing.assert_frame_equal(pdf,
expected_df,
check_like=True,
check_frame_type=False)
def _create():
metrics = [Metric(key=random_str(10),
value=random_int(0, 1000),
timestamp=int(time.time()) + random_int(-1e4, 1e4),
step=random_int())]
params = [Param(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa
tags = [RunTag(random_str(10), random_str(random_int(10, 35))) for _ in range(10)] # noqa
rd = RunData(metrics=metrics, params=params, tags=tags)
return rd, metrics, params, tags
def _add_to_queue(key, value, step, time, run_id):
"""
Add a metric to the metric queue. Flush the queue if it exceeds
max size.
"""
met = Metric(key=key, value=value, timestamp=time, step=step)
_metric_queue.append((run_id, met))
if len(_metric_queue) > _MAX_METRIC_QUEUE_SIZE:
_flush_queue()
def test_creation_and_hydration():
key = random_str()
value = 10000
ts = int(1000 * time.time())
step = random_int()
metric = Metric(key, value, ts, step)
_check(metric, key, value, ts, step)
as_dict = {"key": key, "value": value, "timestamp": ts, "step": step}
assert dict(metric) == as_dict
proto = metric.to_proto()
metric2 = metric.from_proto(proto)
_check(metric2, key, value, ts, step)
metric3 = Metric.from_dictionary(as_dict)
_check(metric3, key, value, ts, step)
def to_mlflow_entity(self):
"""
Convert DB model to corresponding MLflow entity.
:return: :py:class:`mlflow.entities.Metric`.
"""
return Metric(key=self.key,
value=self.value if not self.is_nan else float("nan"),
timestamp=self.timestamp,
step=self.step)
def _log_metric():
request_message = _get_request_message(LogMetric())
metric = Metric(request_message.key, request_message.value,
request_message.timestamp, request_message.step)
run_id = request_message.run_id or request_message.run_uuid
_get_tracking_store().log_metric(run_id, metric)
response_message = LogMetric.Response()
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def _get_metric_from_line(metric_name, metric_line):
metric_parts = metric_line.strip().split(" ")
if len(metric_parts) != 2 and len(metric_parts) != 3:
raise MlflowException(
"Metric '%s' is malformed; persisted metric data contained %s "
"fields. Expected 2 or 3 fields." %
(metric_name, len(metric_parts)),
databricks_pb2.INTERNAL_ERROR)
ts = int(metric_parts[0])
val = float(metric_parts[1])
step = int(metric_parts[2]) if len(metric_parts) == 3 else 0
return Metric(key=metric_name, value=val, timestamp=ts, step=step)
def get_metric_history(self, run_id, metric_key):
"""
Return all logged values for a given metric.
:param run_id: Unique identifier for run
:param metric_key: Metric name within the run
:return: A list of :py:class:`mlflow.entities.Metric` entities if logged, else empty list
"""
req_body = message_to_json(
GetMetricHistory(run_uuid=run_id,
run_id=run_id,
metric_key=metric_key))
response_proto = self._call_endpoint(GetMetricHistory, req_body)
return [Metric.from_proto(metric) for metric in response_proto.metrics]
def test_validate_batch_log_data():
metrics_with_bad_key = [
Metric("good-metric-key", 1.0, 0, 0),
Metric("super-long-bad-key" * 1000, 4.0, 0, 0)
]
metrics_with_bad_val = [
Metric("good-metric-key", "not-a-double-val", 0, 0)
]
metrics_with_bad_ts = [
Metric("good-metric-key", 1.0, "not-a-timestamp", 0)
]
metrics_with_neg_ts = [Metric("good-metric-key", 1.0, -123, 0)]
metrics_with_bad_step = [Metric("good-metric-key", 1.0, 0, "not-a-step")]
params_with_bad_key = [
Param("good-param-key", "hi"),
Param("super-long-bad-key" * 1000, "but-good-val")
]
params_with_bad_val = [
Param("good-param-key", "hi"),
Param("another-good-key", "but-bad-val" * 1000)
]
tags_with_bad_key = [
RunTag("good-tag-key", "hi"),
RunTag("super-long-bad-key" * 1000, "but-good-val")
]
tags_with_bad_val = [
RunTag("good-tag-key", "hi"),
RunTag("another-good-key", "but-bad-val" * 1000)
]
bad_kwargs = {
"metrics": [
metrics_with_bad_key, metrics_with_bad_val, metrics_with_bad_ts,
metrics_with_neg_ts, metrics_with_bad_step
],
"params": [params_with_bad_key, params_with_bad_val],
"tags": [tags_with_bad_key, tags_with_bad_val],
}
good_kwargs = {"metrics": [], "params": [], "tags": []}
for arg_name, arg_values in bad_kwargs.items():
for arg_value in arg_values:
final_kwargs = copy.deepcopy(good_kwargs)
final_kwargs[arg_name] = arg_value
with pytest.raises(MlflowException):
_validate_batch_log_data(**final_kwargs)
# Test that we don't reject entities within the limit
_validate_batch_log_data(metrics=[Metric("metric-key", 1.0, 0, 0)],
params=[Param("param-key", "param-val")],
tags=[RunTag("tag-key", "tag-val")])
def log_metric(self, run_id, key, value, timestamp=None, step=None):
"""
Log a metric against the run ID.
:param run_id: The run id to which the metric should be logged.
:param key: Metric name.
:param value: Metric value (float). Note that some special values such
as +/- Infinity may be replaced by other values depending on the store. For
example, the SQLAlchemy store replaces +/- Inf with max / min float values.
:param timestamp: Time when this metric was calculated. Defaults to the current system time.
:param step: Training step (iteration) at which was the metric calculated. Defaults to 0.
"""
timestamp = timestamp if timestamp is not None else int(time.time() *
1000)
step = step if step is not None else 0
_validate_metric(key, value, timestamp, step)
metric = Metric(key, value, timestamp, step)
self.store.log_metric(run_id, metric)
def _log_batch():
_validate_batch_log_api_req(_get_request_json())
request_message = _get_request_message(LogBatch())
metrics = [
Metric.from_proto(proto_metric)
for proto_metric in request_message.metrics
]
params = [
Param.from_proto(proto_param) for proto_param in request_message.params
]
tags = [RunTag.from_proto(proto_tag) for proto_tag in request_message.tags]
_get_tracking_store().log_batch(run_id=request_message.run_id,
metrics=metrics,
params=params,
tags=tags)
response_message = LogBatch.Response()
response = Response(mimetype='application/json')
response.set_data(message_to_json(response_message))
return response
def test_log_batch(mlflow_client, backend_store_uri):
experiment_id = mlflow_client.create_experiment('Batch em up')
created_run = mlflow_client.create_run(experiment_id)
run_id = created_run.info.run_id
mlflow_client.log_batch(run_id=run_id,
metrics=[Metric("metric", 123.456, 789, 3)],
params=[Param("param", "value")],
tags=[RunTag("taggity", "do-dah")])
run = mlflow_client.get_run(run_id)
assert run.data.metrics.get('metric') == 123.456
assert run.data.params.get('param') == 'value'
assert run.data.tags.get('taggity') == 'do-dah'
metric_history = mlflow_client.get_metric_history(run_id, "metric")
assert len(metric_history) == 1
metric = metric_history[0]
assert metric.key == "metric"
assert metric.value == 123.456
assert metric.timestamp == 789
assert metric.step == 3
def test_string_repr(self):
run_info = RunInfo(run_uuid="hi",
run_id="hi",
experiment_id=0,
user_id="user-id",
status=RunStatus.FAILED,
start_time=0,
end_time=1,
lifecycle_stage=LifecycleStage.ACTIVE)
metrics = [
Metric(key="key-%s" % i, value=i, timestamp=0, step=i)
for i in range(3)
]
run_data = RunData(metrics=metrics, params=[], tags=[])
run1 = Run(run_info, run_data)
expected = (
"<Run: data=<RunData: metrics={'key-0': 0, 'key-1': 1, 'key-2': 2}, "
"params={}, tags={}>, info=<RunInfo: artifact_uri=None, end_time=1, "
"experiment_id=0, "
"lifecycle_stage='active', run_id='hi', run_uuid='hi', "
"start_time=0, status=4, user_id='user-id'>>")
assert str(run1) == expected
def log_metrics(metrics, step=None):
"""
Log multiple metrics for the current run. If no run is active, this method will create a new
active run.
:param metrics: Dictionary of metric_name: String -> value: Float. Note that some special values
such as +/- Infinity may be replaced by other values depending on the store.
For example, sql based store may replace +/- Inf with max / min float values.
:param step: A single integer step at which to log the specified
Metrics. If unspecified, each metric is logged at step zero.
:returns: None
"""
run_id = _get_or_start_run().info.run_id
timestamp = int(time.time() * 1000)
metrics_arr = [
Metric(key, value, timestamp, step or 0)
for key, value in metrics.items()
]
MlflowClient().log_batch(run_id=run_id,
metrics=metrics_arr,
params=[],
tags=[])
def test_parse_dict_int_as_string_backcompat():
in_json = {"timestamp": "123"}
message = ProtoMetric()
parse_dict(in_json, message)
experiment = Metric.from_proto(message)
assert experiment.timestamp == 123
def test_log_batch():
expected_metrics = {"metric-key0": 1.0, "metric-key1": 4.0}
expected_params = {"param-key0": "param-val0", "param-key1": "param-val1"}
exact_expected_tags = {"tag-key0": "tag-val0", "tag-key1": "tag-val1"}
approx_expected_tags = set(
[MLFLOW_USER, MLFLOW_SOURCE_NAME, MLFLOW_SOURCE_TYPE])
t = int(time.time())
sorted_expected_metrics = sorted(expected_metrics.items(),
key=lambda kv: kv[0])
metrics = [
Metric(key=key, value=value, timestamp=t, step=i)
for i, (key, value) in enumerate(sorted_expected_metrics)
]
params = [
Param(key=key, value=value) for key, value in expected_params.items()
]
tags = [
RunTag(key=key, value=value)
for key, value in exact_expected_tags.items()
]
with start_run() as active_run:
run_id = active_run.info.run_id
kiwi.tracking.MlflowClient().log_batch(run_id=run_id,
metrics=metrics,
params=params,
tags=tags)
client = tracking.MlflowClient()
finished_run = client.get_run(run_id)
# Validate metrics
assert len(finished_run.data.metrics) == 2
for key, value in finished_run.data.metrics.items():
assert expected_metrics[key] == value
metric_history0 = client.get_metric_history(run_id, "metric-key0")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history0]) == set([
(1.0, t, 0),
])
metric_history1 = client.get_metric_history(run_id, "metric-key1")
assert set([(m.value, m.timestamp, m.step)
for m in metric_history1]) == set([
(4.0, t, 1),
])
# Validate tags (for automatically-set tags)
assert len(finished_run.data.tags
) == len(exact_expected_tags) + len(approx_expected_tags)
for tag_key, tag_value in finished_run.data.tags.items():
if tag_key in approx_expected_tags:
pass
else:
assert exact_expected_tags[tag_key] == tag_value
# Validate params
assert finished_run.data.params == expected_params
# test that log_batch works with fewer params
new_tags = {"1": "2", "3": "4", "5": "6"}
tags = [RunTag(key=key, value=value) for key, value in new_tags.items()]
client.log_batch(run_id=run_id, tags=tags)
finished_run_2 = client.get_run(run_id)
# Validate tags (for automatically-set tags)
assert len(finished_run_2.data.tags) == len(finished_run.data.tags) + 3
for tag_key, tag_value in finished_run_2.data.tags.items():
if tag_key in new_tags:
assert new_tags[tag_key] == tag_value
def test_requestor(self, request):
response = mock.MagicMock
response.status_code = 200
response.text = '{}'
request.return_value = response
creds = MlflowHostCreds('https://hello')
store = RestStore(lambda: creds)
user_name = "mock user"
source_name = "rest test"
source_name_patch = mock.patch(
"mlflow.tracking.context.default_context._get_source_name", return_value=source_name
)
source_type_patch = mock.patch(
"mlflow.tracking.context.default_context._get_source_type",
return_value=SourceType.LOCAL
)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http, \
mock.patch('mlflow.tracking._tracking_service.utils._get_store',
return_value=store), \
mock.patch('mlflow.tracking.context.default_context._get_user',
return_value=user_name), \
mock.patch('time.time', return_value=13579), \
source_name_patch, source_type_patch:
with kiwi.start_run(experiment_id="43"):
cr_body = message_to_json(CreateRun(experiment_id="43",
user_id=user_name, start_time=13579000,
tags=[ProtoRunTag(key='mlflow.source.name',
value=source_name),
ProtoRunTag(key='mlflow.source.type',
value='LOCAL'),
ProtoRunTag(key='mlflow.user',
value=user_name)]))
expected_kwargs = self._args(creds, "runs/create", "POST", cr_body)
assert mock_http.call_count == 1
actual_kwargs = mock_http.call_args[1]
# Test the passed tag values separately from the rest of the request
# Tag order is inconsistent on Python 2 and 3, but the order does not matter
expected_tags = expected_kwargs['json'].pop('tags')
actual_tags = actual_kwargs['json'].pop('tags')
assert (
sorted(expected_tags, key=lambda t: t['key']) ==
sorted(actual_tags, key=lambda t: t['key'])
)
assert expected_kwargs == actual_kwargs
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.log_param("some_uuid", Param("k1", "v1"))
body = message_to_json(LogParam(
run_uuid="some_uuid", run_id="some_uuid", key="k1", value="v1"))
self._verify_requests(mock_http, creds,
"runs/log-parameter", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.set_experiment_tag("some_id", ExperimentTag("t1", "abcd"*1000))
body = message_to_json(SetExperimentTag(
experiment_id="some_id",
key="t1",
value="abcd"*1000))
self._verify_requests(mock_http, creds,
"experiments/set-experiment-tag", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.set_tag("some_uuid", RunTag("t1", "abcd"*1000))
body = message_to_json(SetTag(
run_uuid="some_uuid", run_id="some_uuid", key="t1", value="abcd"*1000))
self._verify_requests(mock_http, creds,
"runs/set-tag", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.delete_tag("some_uuid", "t1")
body = message_to_json(DeleteTag(run_id="some_uuid", key="t1"))
self._verify_requests(mock_http, creds,
"runs/delete-tag", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.log_metric("u2", Metric("m1", 0.87, 12345, 3))
body = message_to_json(LogMetric(
run_uuid="u2", run_id="u2", key="m1", value=0.87, timestamp=12345, step=3))
self._verify_requests(mock_http, creds,
"runs/log-metric", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
metrics = [Metric("m1", 0.87, 12345, 0), Metric("m2", 0.49, 12345, -1),
Metric("m3", 0.58, 12345, 2)]
params = [Param("p1", "p1val"), Param("p2", "p2val")]
tags = [RunTag("t1", "t1val"), RunTag("t2", "t2val")]
store.log_batch(run_id="u2", metrics=metrics, params=params, tags=tags)
metric_protos = [metric.to_proto() for metric in metrics]
param_protos = [param.to_proto() for param in params]
tag_protos = [tag.to_proto() for tag in tags]
body = message_to_json(LogBatch(run_id="u2", metrics=metric_protos,
params=param_protos, tags=tag_protos))
self._verify_requests(mock_http, creds,
"runs/log-batch", "POST", body)
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.delete_run("u25")
self._verify_requests(mock_http, creds,
"runs/delete", "POST",
message_to_json(DeleteRun(run_id="u25")))
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.restore_run("u76")
self._verify_requests(mock_http, creds,
"runs/restore", "POST",
message_to_json(RestoreRun(run_id="u76")))
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.delete_experiment("0")
self._verify_requests(mock_http, creds,
"experiments/delete", "POST",
message_to_json(DeleteExperiment(experiment_id="0")))
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
store.restore_experiment("0")
self._verify_requests(mock_http, creds,
"experiments/restore", "POST",
message_to_json(RestoreExperiment(experiment_id="0")))
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
response = mock.MagicMock
response.text = '{"runs": ["1a", "2b", "3c"], "next_page_token": "67890fghij"}'
mock_http.return_value = response
result = store.search_runs(["0", "1"], "params.p1 = 'a'", ViewType.ACTIVE_ONLY,
max_results=10, order_by=["a"], page_token="12345abcde")
expected_message = SearchRuns(experiment_ids=["0", "1"], filter="params.p1 = 'a'",
run_view_type=ViewType.to_proto(ViewType.ACTIVE_ONLY),
max_results=10, order_by=["a"], page_token="12345abcde")
self._verify_requests(mock_http, creds,
"runs/search", "POST",
message_to_json(expected_message))
assert result.token == "67890fghij"
with mock.patch('mlflow.utils.rest_utils.http_request') as mock_http:
run_id = "run_id"
m = Model(artifact_path="model/path", run_id="run_id", flavors={"tf": "flavor body"})
result = store.record_logged_model("run_id", m)
expected_message = LogModel(run_id=run_id,
model_json=m.to_json())
self._verify_requests(mock_http, creds,
"runs/log-model", "POST",
message_to_json(expected_message))
