def load(project, fname):
Stage._check_file_exists(fname)
Stage._check_dvc_filename(fname)
if not Stage.is_stage_file(fname):
raise StageFileIsNotDvcFileError(fname)
with open(fname, "r") as fd:
d = yaml.safe_load(fd) or {}
Stage.validate(d, fname=os.path.relpath(fname))
stage = Stage(
project=project,
path=os.path.abspath(fname),
cwd=os.path.dirname(os.path.abspath(fname)),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def load(repo, fname):
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a dvc filename
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
Stage._check_isfile(repo, fname)
d = load_stage_file_fobj(repo.tree.open(fname), fname)
Stage.validate(d, fname=os.path.relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(
os.path.dirname(path), d.get(Stage.PARAM_WDIR, ".")
)
),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def load(repo, fname):
Stage._check_file_exists(fname)
Stage._check_dvc_filename(fname)
if not Stage.is_stage_file(fname):
raise StageFileIsNotDvcFileError(fname)
d = load_stage_file(fname)
Stage.validate(d, fname=os.path.relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(os.path.dirname(path),
d.get(Stage.PARAM_WDIR, "."))),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def load_stage(cls, dvcfile, d, stage_text):
path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
stage._stage_text = stage_text # noqa, pylint:disable=protected-access
stage.deps = dependency.loadd_from(stage,
d.get(Stage.PARAM_DEPS) or [])
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS) or [])
return stage
def load_stage(cls, dvcfile, d, stage_text):
from dvc.stage import Stage, loads_from
path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
stage._stage_text = stage_text
stage.deps = dependency.loadd_from(stage,
d.get(Stage.PARAM_DEPS) or [])
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS) or [])
return stage
def _load_outs(cls, stage, data):
d = []
for key in data:
extra_kwargs = {}
if isinstance(key, str):
path = key
elif isinstance(key, dict):
path = first(key)
extra_kwargs = key[path]
else:
continue
d.append({"path": path, **extra_kwargs})
stage.outs.extend(output.loadd_from(stage, d))
def loadd(project, d, path):
Stage.validate(d)
path = os.path.abspath(path)
cwd = os.path.dirname(path)
cmd = d.get(Stage.PARAM_CMD, None)
md5 = d.get(Stage.PARAM_MD5, None)
stage = Stage(project=project, path=path, cmd=cmd, cwd=cwd, md5=md5)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def _loadd(project, d, path):
Stage.validate(d, fname=os.path.relpath(path))
path = os.path.abspath(path)
cwd = os.path.dirname(path)
cmd = d.get(Stage.PARAM_CMD, None)
md5 = d.get(Stage.PARAM_MD5, None)
locked = d.get(Stage.PARAM_LOCKED, False)
stage = Stage(project=project,
path=path,
cmd=cmd,
cwd=cwd,
md5=md5,
locked=locked)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def load(repo, fname):
fname, tag = Stage._get_path_tag(fname)
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a DVC-file
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
Stage._check_isfile(repo, fname)
with repo.tree.open(fname) as fd:
stage_text = fd.read()
d = parse_stage(stage_text, fname)
Stage.validate(d, fname=relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(
os.path.dirname(path), d.get(Stage.PARAM_WDIR, ".")
)
),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
tag=tag,
always_changed=d.get(Stage.PARAM_ALWAYS_CHANGED, False),
# We store stage text to apply updates to the same structure
stage_text=stage_text,
)
stage.deps = dependency.loadd_from(
stage, d.get(Stage.PARAM_DEPS) or []
)
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS) or [])
return stage
def load(repo, fname):
fname, tag = Stage._get_path_tag(fname)
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a DVC-file
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
Stage._check_isfile(repo, fname)
with repo.tree.open(fname) as fd:
d = load_stage_fd(fd, fname)
# Making a deepcopy since the original structure
# looses keys in deps and outs load
state = copy.deepcopy(d)
Stage.validate(d, fname=relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(
os.path.dirname(path), d.get(Stage.PARAM_WDIR, ".")
)
),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
tag=tag,
always_changed=d.get(Stage.PARAM_ALWAYS_CHANGED, False),
state=state,
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def _load_outs(cls, stage, data, typ=None):
from dvc.output.base import BaseOutput
d = []
for key in data:
if isinstance(key, str):
entry = {BaseOutput.PARAM_PATH: key}
if typ:
entry[typ] = True
d.append(entry)
continue
assert isinstance(key, dict)
assert len(key) == 1
path = first(key)
extra = key[path]
if not typ:
d.append({BaseOutput.PARAM_PATH: path, **extra})
continue
entry = {BaseOutput.PARAM_PATH: path}
persist = extra.pop(BaseOutput.PARAM_PERSIST, False)
if persist:
entry[BaseOutput.PARAM_PERSIST] = persist
cache = extra.pop(BaseOutput.PARAM_CACHE, True)
if not cache:
entry[BaseOutput.PARAM_CACHE] = cache
entry[typ] = extra or True
d.append(entry)
stage.outs.extend(output.loadd_from(stage, d))
