def _dump_pipeline_file(self, stage): data = {} if self.exists(): with open(self.path) as fd: data = parse_yaml_for_update(fd.read(), self.path) else: logger.info("Creating '%s'", self.relpath) open(self.path, "w+").close() data["stages"] = data.get("stages", {}) stage_data = serialize.to_pipeline_file(stage) existing_entry = stage.name in data["stages"] action = "Modifying" if existing_entry else "Adding" logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath) if existing_entry: orig_stage_data = data["stages"][stage.name] if "meta" in orig_stage_data: stage_data[stage.name]["meta"] = orig_stage_data["meta"] apply_diff(stage_data[stage.name], orig_stage_data) else: data["stages"].update(stage_data) dump_yaml(self.path, data) self.repo.scm.track_file(self.relpath)
def test_apply_diff_is_inplace(): dest = MyDict() dest.attr = 42 # pylint: disable=attribute-defined-outside-init apply_diff({}, dest) assert type(dest) is MyDict, "Preserves class" assert dest.attr == 42, "Preserves custom attrs"
def dump(self): fname = self.path self._check_dvc_filename(fname) logger.debug( "Saving information to '{file}'.".format(file=relpath(fname))) state = self.dumpd() # When we load a stage we parse yaml with a fast parser, which strips # off all the comments and formatting. To retain those on update we do # a trick here: # - reparse the same yaml text with a slow but smart ruamel yaml parser # - apply changes to a returned structure # - serialize it if self._stage_text is not None: saved_state = parse_stage_for_update(self._stage_text, fname) # Stage doesn't work with meta in any way, so .dumpd() doesn't # have it. We simply copy it over. if "meta" in saved_state: state["meta"] = saved_state["meta"] apply_diff(state, saved_state) state = saved_state dump_stage_file(fname, state) self.repo.scm.track_file(relpath(fname))
def test_apply_diff_is_inplace(): dest = MyDict() dest.attr = 42 apply_diff({}, dest) assert type(dest) is MyDict, "Preserves class" assert dest.attr == 42, "Preserves custom attrs"
def test_apply_diff_seq(): src = [1] dest = [MyInt(1)] apply_diff(src, dest) assert type(dest[0]) is MyInt, "Does not replace equals" src = {"l": [1]} inner = [] dest = {"l": inner} apply_diff(src, dest) assert dest["l"] is inner, "Updates inner lists"
def dump(self): fname = self.path self._check_dvc_filename(fname) logger.info( "Saving information to '{file}'.".format(file=relpath(fname))) d = self.dumpd() apply_diff(d, self._state) dump_stage_file(fname, self._state) self.repo.scm.track_file(relpath(fname))
def to_single_stage_file(stage: "Stage"): state = stage.dumpd() # When we load a stage we parse yaml with a fast parser, which strips # off all the comments and formatting. To retain those on update we do # a trick here: # - reparse the same yaml text with a slow but smart ruamel yaml parser # - apply changes to a returned structure # - serialize it text = stage._stage_text # noqa, pylint: disable=protected-access if text is not None: saved_state = parse_yaml_for_update(text, stage.path) apply_diff(state, saved_state) state = saved_state return state
def to_single_stage_file(stage: "Stage"): state = stage.dumpd() # When we load a stage we parse yaml with a fast parser, which strips # off all the comments and formatting. To retain those on update we do # a trick here: # - reparse the same yaml text with a slow but smart ruamel yaml parser # - apply changes to a returned structure # - serialize it if stage._stage_text is not None: saved_state = parse_stage_for_update(stage._stage_text, stage.path) # Stage doesn't work with meta in any way, so .dumpd() doesn't # have it. We simply copy it over. if "meta" in saved_state: state["meta"] = saved_state["meta"] apply_diff(state, saved_state) state = saved_state return state
def _dump_pipeline_file(self, stage): data = {} if self.exists(): with open(self.path) as fd: data = parse_stage_for_update(fd.read(), self.path) else: open(self.path, "w+").close() data["stages"] = data.get("stages", {}) stage_data = serialize.to_pipeline_file(stage) if data["stages"].get(stage.name): orig_stage_data = data["stages"][stage.name] apply_diff(stage_data[stage.name], orig_stage_data) else: data["stages"].update(stage_data) dump_stage_file(self.path, data) self.repo.scm.track_file(relpath(self.path))
def _dump_pipeline_file(self, stage): self._check_if_parametrized(stage) stage_data = serialize.to_pipeline_file(stage) with modify_yaml(self.path, fs=self.repo.fs) as data: if not data: logger.info("Creating '%s'", self.relpath) data["stages"] = data.get("stages", {}) existing_entry = stage.name in data["stages"] action = "Modifying" if existing_entry else "Adding" logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath) if existing_entry: orig_stage_data = data["stages"][stage.name] apply_diff(stage_data[stage.name], orig_stage_data) else: data["stages"].update(stage_data) self.repo.scm.track_file(self.relpath)
def _dump_pipeline_file(self, stage): data = {} if self.exists(): with open(self.path) as fd: data = parse_yaml_for_update(fd.read(), self.path) else: logger.info("Creating '%s'", self.relpath) open(self.path, "w+").close() data["stages"] = data.get("stages", {}) stage_data = serialize.to_pipeline_file(stage) if data["stages"].get(stage.name): orig_stage_data = data["stages"][stage.name] apply_diff(stage_data[stage.name], orig_stage_data) else: data["stages"].update(stage_data) logger.info( "Adding stage '%s' to '%s'", stage.name, self.relpath, ) dump_yaml(self.path, data) self.repo.scm.track_file(self.relpath)
def test_apply_diff_mapping(): src = {"a": 1} dest = {"b": 2} apply_diff(src, dest) assert dest == src, "Adds and removes keys" src = {"a": 1} dest = {"a": MyInt(1)} apply_diff(src, dest) assert type(dest["a"]) is MyInt, "Does not replace equals" src = {"d": {"a": 1}} inner = {} dest = {"d": inner} apply_diff(src, dest) assert dest["d"] is inner, "Updates inner dicts"