def test(self): """ Making sure that 'remote' syntax is handled properly for local outs. """ cwd = os.getcwd() remote = "myremote" ret = main(["remote", "add", remote, cwd]) self.assertEqual(ret, 0) self.dvc = DvcRepo() foo = "remote://{}/{}".format(remote, self.FOO) ret = main(["add", foo]) self.assertEqual(ret, 0) d = load_stage_file("foo.dvc") self.assertEqual(d["outs"][0]["path"], foo) bar = os.path.join(cwd, self.BAR) ret = main(["add", bar]) self.assertEqual(ret, 0) d = load_stage_file("bar.dvc") self.assertEqual(d["outs"][0]["path"], bar)
def test(self): # Use copy to test for changes in the inodes ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) ret = main(["add", self.DATA_DIR]) self.assertEqual(0, ret) stage_path = self.DATA_DIR + Stage.STAGE_FILE_SUFFIX stage = load_stage_file(stage_path) staged_files = self.outs_info(stage) # move instead of remove, to lock inode assigned to stage_files[0].path # if we were to use remove, we might end up with same inode assigned to # newly checked out file shutil.move(staged_files[0].path, "random_name") ret = main(["checkout", "--force", stage_path]) self.assertEqual(ret, 0) checkedout_files = self.outs_info(stage) self.assertEqual(len(staged_files), len(checkedout_files)) self.assertEqual(staged_files[0].path, checkedout_files[0].path) self.assertNotEqual(staged_files[0].inode, checkedout_files[0].inode) self.assertEqual(staged_files[1].inode, checkedout_files[1].inode)
def test(self): data_stage_file = self.DATA + Stage.STAGE_FILE_SUFFIX ret = main(["add", self.DATA]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(data_stage_file)) new_data_dir = "data_dir2" os.makedirs(new_data_dir) ret = main(["move", self.DATA, new_data_dir]) self.assertEqual(ret, 0) new_data_path = os.path.join(new_data_dir, os.path.basename(self.DATA)) new_data_stage_file = new_data_path + Stage.STAGE_FILE_SUFFIX self.assertFalse(os.path.exists(self.DATA)) self.assertFalse(os.path.exists(data_stage_file)) self.assertTrue(os.path.exists(new_data_path)) self.assertTrue(os.path.exists(new_data_stage_file)) new_stage_file = load_stage_file(new_data_stage_file) self.assertEqual( os.path.basename(self.DATA), new_stage_file["outs"][0]["path"] )
def load(repo, fname): Stage._check_file_exists(fname) Stage._check_dvc_filename(fname) if not Stage.is_stage_file(fname): raise StageFileIsNotDvcFileError(fname) d = load_stage_file(fname) Stage.validate(d, fname=os.path.relpath(fname)) path = os.path.abspath(fname) stage = Stage( repo=repo, path=path, wdir=os.path.abspath( os.path.join(os.path.dirname(path), d.get(Stage.PARAM_WDIR, "."))), cmd=d.get(Stage.PARAM_CMD), md5=d.get(Stage.PARAM_MD5), locked=d.get(Stage.PARAM_LOCKED, False), ) stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, [])) stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, [])) return stage
def _test(self): self._prepare_external_data() ret = main(["add", os.path.join(self.link_name, self.data_file_name)]) self.assertEqual(0, ret) stage_file = self.data_file_name + Stage.STAGE_FILE_SUFFIX self.assertTrue(os.path.exists(stage_file)) d = load_stage_file(stage_file) relative_data_path = posixpath.join(self.link_name, self.data_file_name) self.assertEqual(relative_data_path, d["outs"][0]["path"])
def test(self): import yaml stages = self.dvc.add(self.FOO) self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) d = load_stage_file(stage.relpath) # NOTE: checking that reloaded stage didn't change its checksum md5 = "11111111111111111111111111111111" d[stage.PARAM_MD5] = md5 with open(stage.relpath, "w") as fobj: yaml.safe_dump(d, fobj, default_flow_style=False) stage = Stage.load(self.dvc, stage.relpath) self.assertTrue(stage is not None) stage.dump() d = load_stage_file(stage.relpath) self.assertEqual(d[stage.PARAM_MD5], md5)
def test(self): stages = self.dvc.add(self.FOO, no_commit=True) self.assertEqual(len(stages), 1) stage = stages[0] st = load_stage_file(stage.path) st["md5"] = "1111111111" with open(stage.path, "w") as fobj: yaml.safe_dump(st, fobj, default_flow_style=False) with self.assertRaises(StageCommitError): self.dvc.commit(stage.path) self.dvc.commit(stage.path, force=True)
def test(self): foo_stage_file_path = self.FOO + Stage.STAGE_FILE_SUFFIX ret = main(["add", self.FOO]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(foo_stage_file_path)) target_foo_path = os.path.join(self.DATA_DIR, self.FOO) target_foo_stage_file_path = target_foo_path + Stage.STAGE_FILE_SUFFIX ret = main(["move", self.FOO, self.DATA_DIR]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.FOO)) self.assertFalse(os.path.exists(foo_stage_file_path)) self.assertTrue(os.path.exists(target_foo_path)) self.assertTrue(os.path.exists(target_foo_stage_file_path)) new_stage = load_stage_file(target_foo_stage_file_path) self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
def _test(self): url = get_local_url() self.main(["remote", "add", "-d", TEST_REMOTE, url]) stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar") self.main(["push"]) stage_file_path = stage.relpath content = load_stage_file(stage_file_path) del content["outs"][0]["md5"] with open(stage_file_path, "w") as stage_file: yaml.dump(content, stage_file) with self._caplog.at_level(logging.WARNING, logger="dvc"): self._caplog.clear() self.main(["status", "-c"]) expected_warning = ( "Output 'bar'(Stage: 'bar.dvc') is missing version info." " Cache for it will not be collected." " Use dvc repro to get your pipeline up to date.") assert expected_warning in self._caplog.text
def _test(self): url = get_local_url() self.main(["remote", "add", "-d", TEST_REMOTE, url]) stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar") self.main(["push"]) stage_file_path = stage.relpath content = load_stage_file(stage_file_path) del content["outs"][0]["md5"] with open(stage_file_path, "w") as stage_file: yaml.dump(content, stage_file) with MockLoggerHandlers(logger): reset_logger_standard_output() self.main(["status", "-c"]) self.assertIn( "Warning: Output 'bar'(Stage: 'bar.dvc') is " "missing version info. Cache for it will not be " "collected. Use dvc repro to get your pipeline up to " "date.", logger.handlers[0].stream.getvalue(), )
def stage_should_contain_persist_flag(self, stage_file): stage_file_content = load_stage_file(stage_file) self.assertEqual( True, stage_file_content["outs"][0][OutputBase.PARAM_PERSIST])