def _dump_pipeline_file(self, stage):
data = {}
if self.exists():
with open(self.path) as fd:
data = parse_yaml_for_update(fd.read(), self.path)
else:
logger.info("Creating '%s'", self.relpath)
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
stage_data = serialize.to_pipeline_file(stage)
existing_entry = stage.name in data["stages"]
action = "Modifying" if existing_entry else "Adding"
logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
if existing_entry:
orig_stage_data = data["stages"][stage.name]
if "meta" in orig_stage_data:
stage_data[stage.name]["meta"] = orig_stage_data["meta"]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
dump_yaml(self.path, data)
self.repo.scm.track_file(self.relpath)
def test_run_overwrite_order(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
tmp_dir.gen({"foo": "foo", "foo1": "foo1"})
run_copy("foo", "bar", name="copy-foo-bar")
run_copy("bar", "foobar", name="copy-bar-foobar")
run_copy("foo1", "bar1", name="copy-foo-bar", force=True)
data = parse_yaml_for_update((tmp_dir / PIPELINE_FILE).read_text(),
PIPELINE_FILE)
assert list(data["stages"].keys()) == ["copy-foo-bar", "copy-bar-foobar"]
def remove_stage(self, stage):
if not self.exists():
return
with open(self.path) as f:
d = parse_yaml_for_update(f.read(), self.path)
self.validate(d, self.path)
if stage.name not in d:
return
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
del d[stage.name]
dump_yaml(self.path, d)
def remove_stage(self, stage):
self._lockfile.remove_stage(stage)
if not self.exists():
return
with open(self.path, "r") as f:
d = parse_yaml_for_update(f.read(), self.path)
self.validate(d, self.path)
if stage.name not in d.get("stages", {}):
return
logger.debug("Removing '%s' from '%s'", stage.name, self.path)
del d["stages"][stage.name]
dump_yaml(self.path, d)
def dump(self, stage, **kwargs):
stage_data = serialize.to_lockfile(stage)
if not self.exists():
modified = True
logger.info("Generating lock file '%s'", self.relpath)
data = stage_data
open(self.path, "w+").close()
else:
with self.repo.tree.open(self.path, "r") as fd:
data = parse_yaml_for_update(fd.read(), self.path)
modified = data.get(stage.name, {}) != stage_data.get(
stage.name, {})
if modified:
logger.info("Updating lock file '%s'", self.relpath)
data.update(stage_data)
dump_yaml(self.path, data)
if modified:
self.repo.scm.track_file(self.relpath)
def to_single_stage_file(stage: "Stage"):
state = stage.dumpd()
# When we load a stage we parse yaml with a fast parser, which strips
# off all the comments and formatting. To retain those on update we do
# a trick here:
# - reparse the same yaml text with a slow but smart ruamel yaml parser
# - apply changes to a returned structure
# - serialize it
if stage._stage_text is not None:
saved_state = parse_yaml_for_update(stage._stage_text, stage.path)
# Stage doesn't work with meta in any way, so .dumpd() doesn't
# have it. We simply copy it over.
if "meta" in saved_state:
state["meta"] = saved_state["meta"]
apply_diff(state, saved_state)
state = saved_state
return state
def _dump_pipeline_file(self, stage):
data = {}
if self.exists():
with open(self.path) as fd:
data = parse_yaml_for_update(fd.read(), self.path)
else:
logger.info("Creating '%s'", self.relpath)
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
stage_data = serialize.to_pipeline_file(stage)
if data["stages"].get(stage.name):
orig_stage_data = data["stages"][stage.name]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
logger.info(
"Adding stage '%s' to '%s'",
stage.name,
self.relpath,
)
dump_yaml(self.path, data)
self.repo.scm.track_file(self.relpath)
def read_lock_file(file=PIPELINE_LOCK):
with open(file) as f:
data = parse_yaml_for_update(f.read(), file)
assert isinstance(data, OrderedDict)
return data
