def log_metric(self, run_id, key, value, timestamp=None):
"""Logs a metric against the given run id. If timestamp is not provided, we will
use the current timestamp.
"""
_validate_metric_name(key)
timestamp = timestamp if timestamp is not None else int(time.time())
metric = Metric(key, value, timestamp)
self.store.log_metric(run_id, metric)
def log_metric(self, run_uuid, metric):
_validate_run_id(run_uuid)
_validate_metric_name(metric.key)
run = self.get_run(run_uuid)
check_run_is_active(run.info)
metric_path = self._get_metric_path(run.info.experiment_id, run_uuid,
metric.key)
make_containing_dirs(metric_path)
append_to(metric_path, "%s %s\n" % (metric.timestamp, metric.value))
def get_metric(self, run_uuid, metric_key):
_validate_run_id(run_uuid)
_validate_metric_name(metric_key)
parent_path, metric_files = self._get_run_files(run_uuid, "metric")
if metric_key not in metric_files:
raise MlflowException(
"Metric '%s' not found under run '%s'" %
(metric_key, run_uuid), databricks_pb2.RESOURCE_DOES_NOT_EXIST)
return self._get_metric_from_file(parent_path, metric_key)
def _get_metric_from_file(parent_path, metric_name):
_validate_metric_name(metric_name)
metric_data = read_file_lines(parent_path, metric_name)
if len(metric_data) == 0:
raise Exception("Metric '%s' is malformed. No data found." %
metric_name)
last_line = metric_data[-1]
timestamp, val = last_line.strip().split(" ")
return Metric(metric_name, float(val), int(timestamp))
def get_metric_history(self, run_id, metric_key):
_validate_run_id(run_id)
_validate_metric_name(metric_key)
parent_path, metric_files = self._get_run_files(run_id, "metric")
if metric_key not in metric_files:
raise MlflowException("Metric '%s' not found under run '%s'" % (metric_key, run_id),
databricks_pb2.RESOURCE_DOES_NOT_EXIST)
return [FileStore._get_metric_from_line(metric_key, line)
for line in read_file_lines(parent_path, metric_key)]
def get_metric(self, run_id, metric_key):
_validate_run_id(run_id)
_validate_metric_name(metric_key)
metrics = self._get_run_metrics(run_id, metric_key)
if not metrics:
raise MlflowException(
"Metric '%s' not found under run '%s'" % (metric_key, run_id),
RESOURCE_DOES_NOT_EXIST,
)
return _list_to_run_metric(metrics)[0]
def _get_metric_from_file(parent_path, metric_name):
_validate_metric_name(metric_name)
metric_objs = [FileStore._get_metric_from_line(metric_name, line)
for line in read_file_lines(parent_path, metric_name)]
if len(metric_objs) == 0:
raise ValueError("Metric '%s' is malformed. No data found." % metric_name)
# Python performs element-wise comparison of equal-length tuples, ordering them
# based on their first differing element. Therefore, we use max() operator to find the
# largest value at the largest timestamp. For more information, see
# https://docs.python.org/3/reference/expressions.html#value-comparisons
return max(metric_objs, key=lambda m: (m.step, m.timestamp, m.value))
def get_metric_history(self, run_uuid, metric_key):
_validate_run_id(run_uuid)
_validate_metric_name(metric_key)
parent_path, metric_files = self._get_run_files(run_uuid, "metric")
if metric_key not in metric_files:
raise Exception("Metric '%s' not found under run '%s'" %
(metric_key, run_uuid))
metric_data = read_file_lines(parent_path, metric_key)
rsl = []
for pair in metric_data:
ts, val = pair.strip().split(" ")
rsl.append(Metric(metric_key, float(val), int(ts)))
return rsl
def _get_metric_from_file(parent_path, metric_name):
_validate_metric_name(metric_name)
metric_data = []
for line in read_file_lines(parent_path, metric_name):
metric_timestamp, metric_value = line.split()
metric_data.append((int(metric_timestamp), float(metric_value)))
if len(metric_data) == 0:
raise ValueError("Metric '%s' is malformed. No data found." %
metric_name)
# Python performs element-wise comparison of equal-length tuples, ordering them
# based on their first differing element. Therefore, we use max() operator to find the
# largest value at the largest timestamp. For more information, see
# https://docs.python.org/3/reference/expressions.html#value-comparisons
max_timestamp, max_value = max(metric_data)
return Metric(metric_name, max_value, max_timestamp)
def get_metric_history(self, run_id, metric_key):
_validate_run_id(run_id)
_validate_metric_name(metric_key)
dynamodb = self._get_dynamodb_resource()
table_name = "_".join([self.table_prefix, DynamodbStore.METRICS_TABLE])
table = dynamodb.Table(table_name)
response = table.get_item(
Key={
"run_id": run_id,
"key": metric_key
},
ReturnConsumedCapacity="TOTAL",
)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
if "Item" in response:
return _dict_to_run_metric_history(response["Item"])
def log_metric(self, run_id, metric):
"""
Log a metric for the specified run
:param run_id: String id for the run
:param metric: :py:class:`mlflow.entities.Metric` instance to log
"""
_validate_run_id(run_id)
_validate_metric_name(metric.key)
run_info = self._get_run_info(run_id)
check_run_is_active(run_info)
dynamodb = self._get_dynamodb_resource()
table_name = "_".join([self.table_prefix, DynamodbStore.METRICS_TABLE])
table = dynamodb.Table(table_name)
# Append metrics to head of list, so the first element is most recent
response = table.update_item(
Key={
"run_id": run_id,
"key": metric.key
},
UpdateExpression="SET #m = list_append(:m, if_not_exists(#m, :e))",
ExpressionAttributeNames={"#m": "metrics"},
ExpressionAttributeValues={
":e": [],
":m": [{
"value": Decimal(str(metric.value)),
"timestamp": metric.timestamp,
"step": metric.step,
}],
},
ReturnValues="NONE",
ReturnConsumedCapacity="TOTAL",
)
if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
raise MlflowException("DynamoDB connection error")
return True
def _get_metric_path(self, experiment_id, run_uuid, metric_key):
_validate_run_id(run_uuid)
_validate_metric_name(metric_key)
return os.path.join(self._get_run_dir(experiment_id, run_uuid),
FileStore.METRICS_FOLDER_NAME, metric_key)
def log_metric(self, metric):
_validate_metric_name(metric.key)
self.store.log_metric(self.run_info.run_uuid, metric)
def log_metric(self, run_id, metric):
_validate_run_id(run_id)
_validate_metric_name(metric.key)
run_info = self._get_run_info(run_id)
check_run_is_active(run_info)
self._log_run_metric(run_info, metric)
def get_metric_history(self, run_id, metric_key):
_validate_run_id(run_id)
_validate_metric_name(metric_key)
run_info = self._get_run_info(run_id)
return self._get_metric_history(run_info, metric_key)
def test_validate_metric_name():
for good_name in GOOD_METRIC_OR_PARAM_NAMES:
_validate_metric_name(good_name)
for bad_name in BAD_METRIC_OR_PARAM_NAMES:
with pytest.raises(Exception, match="Invalid metric name"):
_validate_metric_name(bad_name)
