def _show_metrics( metrics, markdown=False, all_branches=False, all_tags=False, all_commits=False, precision=None, ): from dvc.utils.diff import format_dict, table from dvc.utils.flatten import flatten # When `metrics` contains a `None` key, it means that some files # specified as `targets` in `repo.metrics.show` didn't contain any metrics. missing = metrics.pop(None, None) with_rev = any([all_branches, all_tags, all_commits]) header_set = set() rows = [] if precision is None: precision = DEFAULT_PRECISION def _round(val): if isinstance(val, float): return round(val, precision) return val for _branch, val in metrics.items(): for _fname, metric in val.items(): if not isinstance(metric, dict): header_set.add("") continue for key, _val in flatten(format_dict(metric)).items(): header_set.add(key) header = sorted(header_set) for branch, val in metrics.items(): for fname, metric in val.items(): row = [] if with_rev: row.append(branch) row.append(fname) if not isinstance(metric, dict): row.append(str(metric)) rows.append(row) continue flattened_val = flatten(format_dict(metric)) for i in header: row.append(_round(flattened_val.get(i))) rows.append(row) header.insert(0, "Path") if with_rev: header.insert(0, "Revision") if missing: raise BadMetricError(missing) return table(header, rows, markdown)
def _show_metrics(metrics, all_branches=False, all_tags=False, all_commits=False): from dvc.utils.diff import format_dict from dvc.utils.flatten import flatten # When `metrics` contains a `None` key, it means that some files # specified as `targets` in `repo.metrics.show` didn't contain any metrics. missing = metrics.pop(None, None) lines = [] for branch, val in metrics.items(): if all_branches or all_tags or all_commits: lines.append(f"{branch}:") for fname, metric in val.items(): if not isinstance(metric, dict): lines.append("\t{}: {}".format(fname, str(metric))) continue lines.append(f"\t{fname}:") for key, value in flatten(format_dict(metric)).items(): lines.append(f"\t\t{key}: {value}") if missing: raise BadMetricError(missing) return "\n".join(lines)
def _update_names(names, items): for name, item in items: if isinstance(item, dict): item = flatten(item) names.update(item.keys()) else: names[name] = None
def metrics_table( metrics, all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, precision: int = None, round_digits: bool = False, ): from dvc.utils.diff import format_dict from dvc.utils.flatten import flatten td = TabularData(["Revision", "Path"], fill_value="-") for branch, val in metrics.items(): for fname, metric in val.get("data", {}).items(): row_data: Dict[str, str] = {"Revision": branch, "Path": fname} metric = metric.get("data", {}) flattened = (flatten(format_dict(metric)) if isinstance( metric, dict) else { "": metric }) row_data.update({ k: _format_field(v, precision, round_digits) for k, v in flattened.items() }) td.row_from_dict(row_data) rev, path, *metrics_headers = td.keys() td.project(rev, path, *sorted(metrics_headers)) if not any([all_branches, all_tags, all_commits]): td.drop("Revision") return td
def _extend_row(row, names, items, precision): from rich.text import Text if not items: row.extend(["-"] * len(names)) return for fname, item in items: if isinstance(item, dict): item = flatten(item) else: item = {fname: item} for name in names: if name in item: value = item[name] if value is None: text = "-" else: # wrap field data in rich.Text, otherwise rich may # interpret unescaped braces from list/dict types as rich # markup tags text = Text(str(_format_field(value, precision))) row.append(text) else: row.append("-")
def _sort(item): rev, exp = item tip = exp.get("checkpoint_tip") if tip and tip != rev: # Sort checkpoint experiments by tip commit return _sort((tip, experiments[tip])) data = exp.get(typ, {}).get(sort_path, {}) val = flatten(data).get(sort_name) return val is None, val
def _prepare_credentials(self, **config): from dvc.config import ConfigError from dvc.utils.flatten import flatten, unflatten login_info = defaultdict(dict) # credentials login_info["key"] = config.get("access_key_id") login_info["secret"] = config.get("secret_access_key") login_info["token"] = config.get("session_token") # session configuration login_info["profile"] = config.get("profile") login_info["use_ssl"] = config.get("use_ssl", True) # extra client configuration client = login_info["client_kwargs"] client["region_name"] = config.get("region") client["endpoint_url"] = config.get("endpointurl") client["verify"] = config.get("ssl_verify") # encryptions additional = login_info["s3_additional_kwargs"] additional["ServerSideEncryption"] = config.get("sse") additional["SSEKMSKeyId"] = config.get("sse_kms_key_id") additional["ACL"] = config.get("acl") for grant_option, grant_key in self._GRANTS.items(): if config.get(grant_option): if additional["ACL"]: raise ConfigError( "`acl` and `grant_*` AWS S3 config options " "are mutually exclusive") additional[grant_key] = config[grant_option] # config kwargs session_config = login_info["config_kwargs"] session_config["s3"] = self._load_aws_config_file( login_info["profile"]) shared_creds = config.get("credentialpath") if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) if (client["region_name"] is None and session_config["s3"].get("region_name") is None and os.getenv("AWS_REGION") is None): # Enable bucket region caching login_info["cache_regions"] = config.get("cache_regions", True) config_path = config.get("configpath") if config_path: os.environ.setdefault("AWS_CONFIG_FILE", config_path) return unflatten({ key: value for key, value in flatten(login_info).items() if value is not None })
def _sort(item): _, exp = item for fname, item in exp.get(typ, {}).items(): if isinstance(item, dict): item = flatten(item) else: item = {fname: item} if sort_by in item: val = item[sort_by] return (val is None, val) return (True, None)
def _sort(item): rev, exp = item tip = exp.get("checkpoint_tip") if tip and tip != rev: # Sort checkpoint experiments by tip commit return _sort((tip, experiments[tip])) for fname, item in exp.get(typ, {}).items(): if isinstance(item, dict): item = flatten(item) else: item = {fname: item} if sort_by in item: val = item[sort_by] return (val is None, val) return (True, None)
def _extend_row(row, names, items, precision): from rich.text import Text from dvc.compare import _format_field, with_value if not items: row.extend(FILL_VALUE for keys in names.values() for _ in keys) return for fname, item in items: item = flatten(item) if isinstance(item, dict) else {fname: item} for name in names[fname]: value = with_value(item.get(name), FILL_VALUE) # wrap field data in rich.Text, otherwise rich may # interpret unescaped braces from list/dict types as rich # markup tags row.append(Text(str(_format_field(value, precision))))
def _extend_row(row, names, items, precision, fill_value=FILL_VALUE): from dvc.compare import _format_field, with_value if not items: row.extend(fill_value for keys in names.values() for _ in keys) return for fname, data in items: item = data.get("data", {}) item = flatten(item) if isinstance(item, dict) else {fname: item} for name in names[fname]: value = with_value( item.get(name), FILL_VALUE_ERRORED if data.get("error", None) else fill_value, ) # wrap field data in ui.rich_text, otherwise rich may # interpret unescaped braces from list/dict types as rich # markup tags row.append(ui.rich_text(str(_format_field(value, precision))))
def _extend_row(row, names, headers, items, precision, fill_value=FILL_VALUE): from dvc.compare import _format_field, with_value for fname, data in items: item = data.get("data", {}) item = flatten(item) if isinstance(item, dict) else {fname: item} for name in names[fname]: value = with_value( item.get(name), FILL_VALUE_ERRORED if data.get("error", None) else fill_value, ) # wrap field data in ui.rich_text, otherwise rich may # interpret unescaped braces from list/dict types as rich # markup tags value = ui.rich_text(str(_format_field(value, precision))) if name in headers: row[name] = value else: row[f"{fname}:{name}"] = value
def _extend_row(row, names, items, precision): if not items: row.extend(["-"] * len(names)) return for fname, item in items: if isinstance(item, dict): item = flatten(item) else: item = {fname: item} for name in names: if name in item: value = item[name] if value is None: text = "-" else: text = _format_float(value, precision) row.append(text) else: row.append("-")
def _extend_row(row, names, items, precision): def _round(val): if isinstance(val, float): return round(val, precision) return val if not items: row.extend(["-"] * len(names)) return for fname, item in items: if isinstance(item, dict): item = flatten(item) else: item = {fname: item} for name in names: if name in item: value = item[name] text = str(_round(value)) if value is not None else "-" row.append(text) else: row.append("-")
def _format_config(config, prefix=""): from dvc.utils.flatten import flatten for key, value in flatten(config).items(): yield f"{prefix}{key}={value}"
def _format_config(config): for key, value in flatten(config).items(): yield f"{key}={value}"
def _update_names(names, items): for name, item in items: if isinstance(item, dict): item = flatten(item) names[name].update({key: None for key in item})
def _format_config(config, prefix=""): for key, value in flatten(config).items(): yield f"{prefix}{key}={value}"