def _clone_default_branch(url, rev, for_write=False):
"""Get or create a clean clone of the url.
The cloned is reactualized with git pull unless rev is a known sha.
"""
from scmrepo.git import Git
clone_path, shallow = CLONES.get(url, (None, False))
git = None
try:
if clone_path:
git = Git(clone_path)
# Do not pull for known shas, branches and tags might move
if not Git.is_sha(rev) or not git.has_rev(rev):
if shallow:
# If we are missing a rev in a shallow clone, fallback to
# a full (unshallowed) clone. Since fetching specific rev
# SHAs is only available in certain git versions, if we
# have need to reference multiple specific revs for a
# given repo URL it is easier/safer for us to work with
# full clones in this case.
logger.debug("erepo: unshallowing clone for '%s'", url)
_unshallow(git)
shallow = False
CLONES[url] = (clone_path, shallow)
else:
logger.debug("erepo: git pull '%s'", url)
git.pull()
else:
from dvc.scm import clone
logger.debug("erepo: git clone '%s' to a temporary dir", url)
clone_path = tempfile.mkdtemp("dvc-clone")
if not for_write and rev and not Git.is_sha(rev):
# If rev is a tag or branch name try shallow clone first
try:
git = clone(url, clone_path, shallow_branch=rev)
shallow = True
logger.debug("erepo: using shallow clone for branch '%s'",
rev)
except CloneError:
pass
if not git:
git = clone(url, clone_path)
shallow = False
CLONES[url] = (clone_path, shallow)
finally:
if git:
git.close()
return clone_path, shallow
def ls(repo, *args, rev=None, git_remote=None, all_=False, **kwargs):
from scmrepo.git import Git
from dvc.scm import RevError, resolve_rev
if rev:
try:
rev = resolve_rev(repo.scm, rev)
except RevError:
if not (git_remote and Git.is_sha(rev)):
# This could be a remote rev that has not been fetched yet
raise
elif not all_:
rev = repo.scm.get_rev()
results = defaultdict(list)
if rev:
if git_remote:
gen = remote_exp_refs_by_baseline(repo.scm, git_remote, rev)
else:
gen = exp_refs_by_baseline(repo.scm, rev)
for info in gen:
results[rev].append(info.name)
elif all_:
if git_remote:
gen = remote_exp_refs(repo.scm, git_remote)
else:
gen = exp_refs(repo.scm)
for info in gen:
results[info.baseline_sha].append(info.name)
return results
def _collect_rows(
base_rev,
experiments,
all_headers,
metric_headers,
param_headers,
metric_names,
param_names,
deps_names,
precision=DEFAULT_PRECISION,
sort_by=None,
sort_order=None,
fill_value=FILL_VALUE,
iso=False,
):
from scmrepo.git import Git
if sort_by:
sort_path, sort_name, sort_type = _sort_column(
sort_by, metric_names, param_names
)
reverse = sort_order == "desc"
experiments = _sort_exp(
experiments, sort_path, sort_name, sort_type, reverse
)
new_checkpoint = True
for i, (rev, results) in enumerate(experiments.items()):
fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
row_dict = {k: fill_value for k in all_headers}
exp = results.get("data", {})
if exp.get("running"):
state = "Running"
elif exp.get("queued"):
state = "Queued"
else:
state = fill_value
is_baseline = rev == "baseline"
if is_baseline:
name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
else:
name_rev = rev[:7]
tip = exp.get("checkpoint_tip")
parent_rev = exp.get("checkpoint_parent", "")
parent_exp = experiments.get(parent_rev, {}).get("data", {})
parent_tip = parent_exp.get("checkpoint_tip")
parent = ""
if is_baseline:
typ = "baseline"
elif tip:
if tip == parent_tip:
typ = (
"checkpoint_tip" if new_checkpoint else "checkpoint_commit"
)
elif parent_rev == base_rev:
typ = "checkpoint_base"
else:
typ = "checkpoint_commit"
parent = parent_rev[:7]
elif i < len(experiments) - 1:
typ = "branch_commit"
else:
typ = "branch_base"
if not is_baseline:
new_checkpoint = not (tip and tip == parent_tip)
row_dict["Experiment"] = exp.get("name", "")
row_dict["rev"] = name_rev
row_dict["typ"] = typ
row_dict["Created"] = _format_time(
exp.get("timestamp"), fill_value, iso
)
row_dict["parent"] = parent
row_dict["State"] = state
row_dict["Executor"] = exp.get("executor", fill_value)
_extend_row(
row_dict,
metric_names,
metric_headers,
exp.get("metrics", {}).items(),
precision,
fill_value=fill_value,
)
_extend_row(
row_dict,
param_names,
param_headers,
exp.get("params", {}).items(),
precision,
fill_value=fill_value,
)
for dep in deps_names:
hash_info = exp.get("deps", {}).get(dep, {}).get("hash")
if hash_info is not None:
hash_info = hash_info[:7]
row_dict[dep] = hash_info
yield list(row_dict.values())
def _collect_rows(
base_rev,
experiments,
metric_names,
param_names,
precision=DEFAULT_PRECISION,
sort_by=None,
sort_order=None,
fill_value=FILL_VALUE,
iso=False,
):
from scmrepo.git import Git
if sort_by:
sort_path, sort_name, sort_type = _sort_column(
sort_by, metric_names, param_names
)
reverse = sort_order == "desc"
experiments = _sort_exp(
experiments, sort_path, sort_name, sort_type, reverse
)
new_checkpoint = True
for i, (rev, results) in enumerate(experiments.items()):
exp = results.get("data", {})
if exp.get("running"):
state = "Running"
elif exp.get("queued"):
state = "Queued"
else:
state = fill_value
executor = exp.get("executor", fill_value)
is_baseline = rev == "baseline"
if is_baseline:
name_rev = base_rev[:7] if Git.is_sha(base_rev) else base_rev
else:
name_rev = rev[:7]
exp_name = exp.get("name", "")
tip = exp.get("checkpoint_tip")
parent_rev = exp.get("checkpoint_parent", "")
parent_exp = experiments.get(parent_rev, {}).get("data", {})
parent_tip = parent_exp.get("checkpoint_tip")
parent = ""
if is_baseline:
typ = "baseline"
elif tip:
if tip == parent_tip:
typ = (
"checkpoint_tip" if new_checkpoint else "checkpoint_commit"
)
elif parent_rev == base_rev:
typ = "checkpoint_base"
else:
typ = "checkpoint_commit"
parent = parent_rev[:7]
elif i < len(experiments) - 1:
typ = "branch_commit"
else:
typ = "branch_base"
if not is_baseline:
new_checkpoint = not (tip and tip == parent_tip)
row = [
exp_name,
name_rev,
typ,
_format_time(exp.get("timestamp"), fill_value, iso),
parent,
state,
executor,
]
fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
_extend_row(
row,
metric_names,
exp.get("metrics", {}).items(),
precision,
fill_value=fill_value,
)
_extend_row(
row,
param_names,
exp.get("params", {}).items(),
precision,
fill_value=fill_value,
)
yield row