def _dump_pipeline_file(self, stage):
data = {}
if self.exists():
with open(self.path) as fd:
data = parse_yaml_for_update(fd.read(), self.path)
else:
logger.info("Creating '%s'", self.relpath)
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
stage_data = serialize.to_pipeline_file(stage)
existing_entry = stage.name in data["stages"]
action = "Modifying" if existing_entry else "Adding"
logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath)
if existing_entry:
orig_stage_data = data["stages"][stage.name]
if "meta" in orig_stage_data:
stage_data[stage.name]["meta"] = orig_stage_data["meta"]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
dump_yaml(self.path, data)
self.repo.scm.track_file(self.relpath)
def test_apply_diff_is_inplace():
dest = MyDict()
dest.attr = 42 # pylint: disable=attribute-defined-outside-init
apply_diff({}, dest)
assert type(dest) is MyDict, "Preserves class"
assert dest.attr == 42, "Preserves custom attrs"
def dump(self):
fname = self.path
self._check_dvc_filename(fname)
logger.debug(
"Saving information to '{file}'.".format(file=relpath(fname)))
state = self.dumpd()
# When we load a stage we parse yaml with a fast parser, which strips
# off all the comments and formatting. To retain those on update we do
# a trick here:
# - reparse the same yaml text with a slow but smart ruamel yaml parser
# - apply changes to a returned structure
# - serialize it
if self._stage_text is not None:
saved_state = parse_stage_for_update(self._stage_text, fname)
# Stage doesn't work with meta in any way, so .dumpd() doesn't
# have it. We simply copy it over.
if "meta" in saved_state:
state["meta"] = saved_state["meta"]
apply_diff(state, saved_state)
state = saved_state
dump_stage_file(fname, state)
self.repo.scm.track_file(relpath(fname))
def test_apply_diff_is_inplace():
dest = MyDict()
dest.attr = 42
apply_diff({}, dest)
assert type(dest) is MyDict, "Preserves class"
assert dest.attr == 42, "Preserves custom attrs"
def test_apply_diff_seq():
src = [1]
dest = [MyInt(1)]
apply_diff(src, dest)
assert type(dest[0]) is MyInt, "Does not replace equals"
src = {"l": [1]}
inner = []
dest = {"l": inner}
apply_diff(src, dest)
assert dest["l"] is inner, "Updates inner lists"
def dump(self):
fname = self.path
self._check_dvc_filename(fname)
logger.info(
"Saving information to '{file}'.".format(file=relpath(fname)))
d = self.dumpd()
apply_diff(d, self._state)
dump_stage_file(fname, self._state)
self.repo.scm.track_file(relpath(fname))
def to_single_stage_file(stage: "Stage"):
state = stage.dumpd()
# When we load a stage we parse yaml with a fast parser, which strips
# off all the comments and formatting. To retain those on update we do
# a trick here:
# - reparse the same yaml text with a slow but smart ruamel yaml parser
# - apply changes to a returned structure
# - serialize it
text = stage._stage_text # noqa, pylint: disable=protected-access
if text is not None:
saved_state = parse_yaml_for_update(text, stage.path)
apply_diff(state, saved_state)
state = saved_state
return state
def to_single_stage_file(stage: "Stage"):
state = stage.dumpd()
# When we load a stage we parse yaml with a fast parser, which strips
# off all the comments and formatting. To retain those on update we do
# a trick here:
# - reparse the same yaml text with a slow but smart ruamel yaml parser
# - apply changes to a returned structure
# - serialize it
if stage._stage_text is not None:
saved_state = parse_stage_for_update(stage._stage_text, stage.path)
# Stage doesn't work with meta in any way, so .dumpd() doesn't
# have it. We simply copy it over.
if "meta" in saved_state:
state["meta"] = saved_state["meta"]
apply_diff(state, saved_state)
state = saved_state
return state
def _dump_pipeline_file(self, stage):
data = {}
if self.exists():
with open(self.path) as fd:
data = parse_stage_for_update(fd.read(), self.path)
else:
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
stage_data = serialize.to_pipeline_file(stage)
if data["stages"].get(stage.name):
orig_stage_data = data["stages"][stage.name]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
dump_stage_file(self.path, data)
self.repo.scm.track_file(relpath(self.path))
def _dump_pipeline_file(self, stage):
self._check_if_parametrized(stage)
stage_data = serialize.to_pipeline_file(stage)
with modify_yaml(self.path, fs=self.repo.fs) as data:
if not data:
logger.info("Creating '%s'", self.relpath)
data["stages"] = data.get("stages", {})
existing_entry = stage.name in data["stages"]
action = "Modifying" if existing_entry else "Adding"
logger.info("%s stage '%s' in '%s'", action, stage.name,
self.relpath)
if existing_entry:
orig_stage_data = data["stages"][stage.name]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
self.repo.scm.track_file(self.relpath)
def _dump_pipeline_file(self, stage):
data = {}
if self.exists():
with open(self.path) as fd:
data = parse_yaml_for_update(fd.read(), self.path)
else:
logger.info("Creating '%s'", self.relpath)
open(self.path, "w+").close()
data["stages"] = data.get("stages", {})
stage_data = serialize.to_pipeline_file(stage)
if data["stages"].get(stage.name):
orig_stage_data = data["stages"][stage.name]
apply_diff(stage_data[stage.name], orig_stage_data)
else:
data["stages"].update(stage_data)
logger.info(
"Adding stage '%s' to '%s'",
stage.name,
self.relpath,
)
dump_yaml(self.path, data)
self.repo.scm.track_file(self.relpath)
def test_apply_diff_mapping():
src = {"a": 1}
dest = {"b": 2}
apply_diff(src, dest)
assert dest == src, "Adds and removes keys"
src = {"a": 1}
dest = {"a": MyInt(1)}
apply_diff(src, dest)
assert type(dest["a"]) is MyInt, "Does not replace equals"
src = {"d": {"a": 1}}
inner = {}
dest = {"d": inner}
apply_diff(src, dest)
assert dest["d"] is inner, "Updates inner dicts"
