def log_metric(self, run_id, key, value, timestamp=None): """Logs a metric against the given run id. If timestamp is not provided, we will use the current timestamp. """ _validate_metric_name(key) timestamp = timestamp if timestamp is not None else int(time.time()) metric = Metric(key, value, timestamp) self.store.log_metric(run_id, metric)
def log_metric(self, run_uuid, metric): _validate_run_id(run_uuid) _validate_metric_name(metric.key) run = self.get_run(run_uuid) check_run_is_active(run.info) metric_path = self._get_metric_path(run.info.experiment_id, run_uuid, metric.key) make_containing_dirs(metric_path) append_to(metric_path, "%s %s\n" % (metric.timestamp, metric.value))
def get_metric(self, run_uuid, metric_key): _validate_run_id(run_uuid) _validate_metric_name(metric_key) parent_path, metric_files = self._get_run_files(run_uuid, "metric") if metric_key not in metric_files: raise MlflowException( "Metric '%s' not found under run '%s'" % (metric_key, run_uuid), databricks_pb2.RESOURCE_DOES_NOT_EXIST) return self._get_metric_from_file(parent_path, metric_key)
def _get_metric_from_file(parent_path, metric_name): _validate_metric_name(metric_name) metric_data = read_file_lines(parent_path, metric_name) if len(metric_data) == 0: raise Exception("Metric '%s' is malformed. No data found." % metric_name) last_line = metric_data[-1] timestamp, val = last_line.strip().split(" ") return Metric(metric_name, float(val), int(timestamp))
def get_metric_history(self, run_id, metric_key): _validate_run_id(run_id) _validate_metric_name(metric_key) parent_path, metric_files = self._get_run_files(run_id, "metric") if metric_key not in metric_files: raise MlflowException("Metric '%s' not found under run '%s'" % (metric_key, run_id), databricks_pb2.RESOURCE_DOES_NOT_EXIST) return [FileStore._get_metric_from_line(metric_key, line) for line in read_file_lines(parent_path, metric_key)]
def get_metric(self, run_id, metric_key): _validate_run_id(run_id) _validate_metric_name(metric_key) metrics = self._get_run_metrics(run_id, metric_key) if not metrics: raise MlflowException( "Metric '%s' not found under run '%s'" % (metric_key, run_id), RESOURCE_DOES_NOT_EXIST, ) return _list_to_run_metric(metrics)[0]
def _get_metric_from_file(parent_path, metric_name): _validate_metric_name(metric_name) metric_objs = [FileStore._get_metric_from_line(metric_name, line) for line in read_file_lines(parent_path, metric_name)] if len(metric_objs) == 0: raise ValueError("Metric '%s' is malformed. No data found." % metric_name) # Python performs element-wise comparison of equal-length tuples, ordering them # based on their first differing element. Therefore, we use max() operator to find the # largest value at the largest timestamp. For more information, see # https://docs.python.org/3/reference/expressions.html#value-comparisons return max(metric_objs, key=lambda m: (m.step, m.timestamp, m.value))
def get_metric_history(self, run_uuid, metric_key): _validate_run_id(run_uuid) _validate_metric_name(metric_key) parent_path, metric_files = self._get_run_files(run_uuid, "metric") if metric_key not in metric_files: raise Exception("Metric '%s' not found under run '%s'" % (metric_key, run_uuid)) metric_data = read_file_lines(parent_path, metric_key) rsl = [] for pair in metric_data: ts, val = pair.strip().split(" ") rsl.append(Metric(metric_key, float(val), int(ts))) return rsl
def _get_metric_from_file(parent_path, metric_name): _validate_metric_name(metric_name) metric_data = [] for line in read_file_lines(parent_path, metric_name): metric_timestamp, metric_value = line.split() metric_data.append((int(metric_timestamp), float(metric_value))) if len(metric_data) == 0: raise ValueError("Metric '%s' is malformed. No data found." % metric_name) # Python performs element-wise comparison of equal-length tuples, ordering them # based on their first differing element. Therefore, we use max() operator to find the # largest value at the largest timestamp. For more information, see # https://docs.python.org/3/reference/expressions.html#value-comparisons max_timestamp, max_value = max(metric_data) return Metric(metric_name, max_value, max_timestamp)
def get_metric_history(self, run_id, metric_key): _validate_run_id(run_id) _validate_metric_name(metric_key) dynamodb = self._get_dynamodb_resource() table_name = "_".join([self.table_prefix, DynamodbStore.METRICS_TABLE]) table = dynamodb.Table(table_name) response = table.get_item( Key={ "run_id": run_id, "key": metric_key }, ReturnConsumedCapacity="TOTAL", ) if response["ResponseMetadata"]["HTTPStatusCode"] != 200: raise MlflowException("DynamoDB connection error") if "Item" in response: return _dict_to_run_metric_history(response["Item"])
def log_metric(self, run_id, metric): """ Log a metric for the specified run :param run_id: String id for the run :param metric: :py:class:`mlflow.entities.Metric` instance to log """ _validate_run_id(run_id) _validate_metric_name(metric.key) run_info = self._get_run_info(run_id) check_run_is_active(run_info) dynamodb = self._get_dynamodb_resource() table_name = "_".join([self.table_prefix, DynamodbStore.METRICS_TABLE]) table = dynamodb.Table(table_name) # Append metrics to head of list, so the first element is most recent response = table.update_item( Key={ "run_id": run_id, "key": metric.key }, UpdateExpression="SET #m = list_append(:m, if_not_exists(#m, :e))", ExpressionAttributeNames={"#m": "metrics"}, ExpressionAttributeValues={ ":e": [], ":m": [{ "value": Decimal(str(metric.value)), "timestamp": metric.timestamp, "step": metric.step, }], }, ReturnValues="NONE", ReturnConsumedCapacity="TOTAL", ) if response["ResponseMetadata"]["HTTPStatusCode"] != 200: raise MlflowException("DynamoDB connection error") return True
def _get_metric_path(self, experiment_id, run_uuid, metric_key): _validate_run_id(run_uuid) _validate_metric_name(metric_key) return os.path.join(self._get_run_dir(experiment_id, run_uuid), FileStore.METRICS_FOLDER_NAME, metric_key)
def log_metric(self, metric): _validate_metric_name(metric.key) self.store.log_metric(self.run_info.run_uuid, metric)
def log_metric(self, run_id, metric): _validate_run_id(run_id) _validate_metric_name(metric.key) run_info = self._get_run_info(run_id) check_run_is_active(run_info) self._log_run_metric(run_info, metric)
def get_metric_history(self, run_id, metric_key): _validate_run_id(run_id) _validate_metric_name(metric_key) run_info = self._get_run_info(run_id) return self._get_metric_history(run_info, metric_key)
def test_validate_metric_name(): for good_name in GOOD_METRIC_OR_PARAM_NAMES: _validate_metric_name(good_name) for bad_name in BAD_METRIC_OR_PARAM_NAMES: with pytest.raises(Exception, match="Invalid metric name"): _validate_metric_name(bad_name)