def test_outside_git_dir(self):
git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
config = ConfigI('data', 'cache', 'state')
path_factory = PathFactory(git, config)
with self.assertRaises(NotInDataDirError):
path_factory.data_item('file.txt')
pass
def setUp(self):
BasicEnvironment.init_environment(
self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
self.data_dir = 'da'
config = ConfigI(self.data_dir, 'ca', 'st')
self.path_factory = PathFactory(git, config)
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
self.data_path = self.path_factory.data_item(deep_path)
pass
def setUp(self):
BasicEnvironment.init_environment(self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
self.data_dir = 'da'
config = ConfigI()
self.path_factory = PathFactory(git, config)
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'dir1', 'd2', 'file.txt_eeeff8f')
self.data_path = self.path_factory.data_item(deep_path, cache_file=self.cache_file)
pass
def setUp(self):
curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj',
'mydir', 'dd3')
BasicEnvironment.init_environment(
self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), curr_dir)
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
self.data_path = self.path_factory.data_item(
os.path.join('..', '..', 'data', 'file1.txt'))
pass
def setUp(self):
self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test')
curr_dir = None
commit = 'abc1234'
BasicEnvironment.init_environment(self, self._test_dir, curr_dir)
self._commit = commit
self._git = GitWrapperI(git_dir=self._test_git_dir,
commit=self._commit)
self._config = ConfigI('data', 'cache', 'state', '.target')
self.path_factory = PathFactory(self._git, self._config)
self.settings = Settings(['target'], self._git, self._config)
os.chdir(self._test_git_dir)
os.mkdir(os.path.join('data', 'dir1'))
os.mkdir(os.path.join('cache', 'dir1'))
self.file1_cache = os.path.join('cache', 'dir1', 'file1')
self.file1_data = os.path.join('data', 'dir1', 'file1')
open(self.file1_cache, 'w').write('ddfdff')
System.symlink(self.file1_cache, self.file1_data)
self.file2_cache = os.path.join('cache', 'file2')
self.file2_data = os.path.join('data', 'file2')
open(self.file2_cache, 'w').write('ddfdff')
System.symlink(self.file2_cache, self.file2_data)
class TestDataFileObjLongPath(BasicEnvironment):
def setUp(self):
curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj', 'mydir', 'dd3')
BasicEnvironment.init_environment(self,
os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'),
curr_dir)
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
self._config = ConfigI('data')
self.path_factory = PathFactory(self._git, self._config)
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, 'file1.txt_123ed8')
self.data_path = self.path_factory.data_item(os.path.join('..', '..', 'data', 'file1.txt'), cache_file=self.cache_file)
pass
def test_data_file(self):
target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
self.assertEqual(self.data_path.data.abs, target)
def test_cache_file(self):
self.assertEqual(self.data_path.cache.abs, self.cache_file)
def test_state_file(self):
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file1.txt.state')
self.assertEqual(self.data_path.state.abs, target)
class TestDataPathInDataDir(BasicEnvironment):
def setUp(self):
BasicEnvironment.init_environment(
self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='eeeff8f')
self.data_dir = 'da'
config = ConfigI(self.data_dir)
self.path_factory = PathFactory(git, config)
deep_path = os.path.join(self.data_dir, 'dir1', 'd2', 'file.txt')
self.cache_file = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR,
ConfigI.CACHE_DIR, 'dir1', 'd2',
'file.txt_eeeff8f')
self.data_path = self.path_factory.data_item(
deep_path, cache_file=self.cache_file)
pass
def test_data_file(self):
target = os.path.join(self._test_git_dir, 'da', 'dir1', 'd2',
'file.txt')
self.assertEqual(self.data_path.data.abs, target)
def test_cache_file(self):
self.assertEqual(self.data_path.cache.abs, self.cache_file)
def test_state_file(self):
target = os.path.join(self._test_git_dir, ConfigI.CONFIG_DIR,
ConfigI.STATE_DIR, 'dir1', 'd2',
'file.txt.state')
self.assertEqual(self.data_path.state.abs, target)
def test_symlink(self):
expected = os.path.join('..', '..', '..', ConfigI.CONFIG_DIR,
ConfigI.CACHE_DIR, 'dir1', 'd2',
'file.txt_eeeff8f')
self.assertEqual(self.data_path.symlink_file, expected)
def test_data_dir(self):
data_path = self.path_factory.data_item(self.data_dir)
self.assertEqual(data_path.data.dvc, self.data_dir)
self.assertEqual(data_path.data_dvc_short, '')
def __init__(self, args, git, config):
self._args = args
self._git = git
self._config = config
self._path_factory = PathFactory(git, config)
self._dvc_home = os.environ.get('DVC_HOME')
if not self._dvc_home:
raise SettingsError('DVC_HOME environment variable is not defined')
if not os.path.exists(self._dvc_home):
raise SettingsError("DVC_HOME directory doesn't exists")
pass
def setUp(self):
BasicEnvironment.init_environment(self,
test_dir=os.path.join(
os.path.sep, 'tmp',
'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(git, config)
self.data_file = os.path.join('data', 'file.txt')
self.cache_file = os.path.join('cache', 'fsymlinc.txt')
fd = open(self.cache_file, 'w+')
fd.write('some text')
fd.close()
os.chdir('data')
System.symlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
def setUp(self):
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI()
self.path_factory = PathFactory(git, config)
self.data_file = os.path.join('data', 'file.txt')
dummy_md5 = 'fsymlinc.txt'
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
with open(self.cache_file, 'w+') as fd:
fd.write('some text')
with open(self.state_file, 'w+') as fd:
fd.write('{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
os.chdir('data')
System.hardlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
class TestPathFactory(BasicEnvironment):
def setUp(self):
BasicEnvironment.init_environment(self, test_dir=os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI()
self.path_factory = PathFactory(git, config)
self.data_file = os.path.join('data', 'file.txt')
dummy_md5 = 'fsymlinc.txt'
self.cache_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME, dummy_md5)
self.state_file = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, 'data', 'file.txt' + DataItem.STATE_FILE_SUFFIX)
with open(self.cache_file, 'w+') as fd:
fd.write('some text')
with open(self.state_file, 'w+') as fd:
fd.write('{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
os.chdir('data')
System.hardlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
def test_data_file_factory(self):
data_path = self.path_factory.data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
indirect_file_path = os.path.join('..', self._proj_dir, self.data_file)
data_path_indirect = self.path_factory.data_item(indirect_file_path)
self.assertEqual(data_path_indirect.data.dvc, self.data_file)
pass
def test_data_hardlink_factory(self):
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.cache.dvc, self.cache_file)
pass
def test_data_hardlink_factory_cache(self):
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
pass
def test_data_hardlink_factory_exception(self):
with self.assertRaises(DataItemError):
self.path_factory.existing_data_item(self.cache_file)
pass
def test_data_path(self):
file = os.path.join('data', 'file1')
path = self.path_factory.path(file)
self.assertEqual(path.dvc, file)
self.assertEqual(path.relative, file)
self.assertTrue(path.abs.endswith(file))
def init_environment(self,
test_dir=os.path.join(os.path.sep, 'tmp',
'ntx_unit_test'),
curr_dir=None,
commit='abc12345'):
''' Creates data environment with data, cache and state dirs.
data/
file1.txt --> ../cache/file1.txt_abc123
dir1/
file2.txt --> ../../cache/dir1/file2.txt_abc123
file3.txt --> ../../cache/dir1/file3.txt_abc123
dir11/
file4.txt --> ../../../cache/dir1/dir11/file4.txt_abc123
dir2/
file5.txt --> ../../cache/dir2/file5.txt_abc123
file6.txt --> an actual file
'''
BasicEnvironment.init_environment(self, test_dir, curr_dir)
self._commit = commit
self._git = GitWrapperI(git_dir=self._test_git_dir,
commit=self._commit)
self._config = ConfigI('data')
self.path_factory = PathFactory(self._git, self._config)
self.settings = Settings([], self._git, self._config)
self.dir1 = 'dir1'
self.dir11 = os.path.join('dir1', 'dir11')
self.dir2 = 'dir2'
self.create_dirs(self.dir1)
self.create_dirs(self.dir11)
self.create_dirs(self.dir2)
self.file1, self.cache1, self.state1 = self.crate_data_item(
'file1.txt')
self.file2, self.cache2, self.state2 = self.crate_data_item(
os.path.join(self.dir1, 'file2.txt'))
self.file3, self.cache3, self.state3 = self.crate_data_item(
os.path.join(self.dir1, 'file3.txt'))
self.file4, self.cache4, self.state4 = self.crate_data_item(
os.path.join(self.dir11, 'file4.txt'))
self.file5, self.cache5, self.state5 = self.crate_data_item(
os.path.join(self.dir2, 'file5.txt'))
self.file6, self.cache6, self.state6 = self.crate_data_item(
os.path.join(self.dir2, 'file6.txt'), cache_file=False)
pass
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self._old_curr_dir_abs = os.path.realpath(os.curdir)
self.tearDown()
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
os.mkdir('data')
self._devnull = open(os.devnull, 'w')
subprocess.Popen(['git', 'init'], stdout=self._devnull, stderr=None).wait()
self.git = GitWrapperI(self.test_dir)
self.config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self.git, self.config)
self.settings = Settings([], self.git, self.config)
def setUp(self):
self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test')
curr_dir = None
commit = 'abc1234'
BasicEnvironment.init_environment(self, self._test_dir, curr_dir)
self._commit = commit
self._git = GitWrapperI(git_dir=self._test_git_dir,
commit=self._commit)
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
self.settings = Settings('gc', self._git, self._config)
self.dir1 = 'dir1'
self.create_dirs(self.dir1)
def setUp(self):
self.test_dir = System.get_long_path(tempfile.mkdtemp())
self._old_curr_dir_abs = System.realpath(os.curdir)
self.tearDown()
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
os.mkdir('data')
os.mkdir('cache')
os.mkdir('state')
self.init_git_repo()
self.git = GitWrapper()
self.config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self.git, self.config)
self.settings = Settings([], self.git, self.config)
pass
def setUp(self):
self.test_dir = System.get_long_path(tempfile.mkdtemp())
self._old_curr_dir_abs = System.realpath(os.curdir)
self.tearDown()
os.mkdir(self.test_dir)
os.chdir(self.test_dir)
os.mkdir('data')
os.mkdir(ConfigI.CONFIG_DIR)
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME))
os.mkdir(os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME))
self.init_git_repo()
self.git = GitWrapper()
self.config = ConfigI('data')
self.path_factory = PathFactory(self.git, self.config)
self.settings = Settings('run cmd', self.git, self.config)
pass
def __init__(self, argv=None, git=None, config=None):
self._args = None
args = None
if argv != None and len(argv) != 0:
args = parse_args(argv)
self._args = argv[2:]
if git == None:
git = GitWrapper()
if config == None:
if args != None and args.cmd != 'init':
config = Config()
else:
config = ConfigI()
self._git = git
self._config = config
self._path_factory = PathFactory(git, config)
self._parsed_args = args
class TestDataFileObjLongPath(BasicEnvironment):
def setUp(self):
curr_dir = os.path.join(os.path.sep, 'tmp', 'ntx_unit_test', 'proj',
'mydir', 'dd3')
BasicEnvironment.init_environment(
self, os.path.join(os.path.sep, 'tmp', 'ntx_unit_test'), curr_dir)
self._git = GitWrapperI(git_dir=self._test_git_dir, commit='123ed8')
self._config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(self._git, self._config)
self.data_path = self.path_factory.data_item(
os.path.join('..', '..', 'data', 'file1.txt'))
pass
def test_data_file(self):
target = os.path.join(self._test_git_dir, 'data', 'file1.txt')
self.assertEqual(self.data_path.data.abs, target)
def test_cache_file(self):
target = os.path.join(self._test_git_dir, 'cache', 'file1.txt_123ed8')
self.assertEqual(self.data_path.cache.abs, target)
def test_state_file(self):
target = os.path.join(self._test_git_dir, 'state', 'file1.txt.state')
self.assertEqual(self.data_path.state.abs, target)
def test_file_name_only(self):
with self.assertRaises(NotInDataDirError):
self.path_factory.data_item('file.txt')
pass
def test_relative_git_dir(self):
with self.assertRaises(NotInDataDirError):
self.path_factory.data_item('data/file.txt')
pass
def test_relative_path_error(self):
with self.assertRaises(NotInDataDirError):
self.path_factory.data_item('../data/file.txt')
pass
def __init__(self, argv=None, git=None, config=None):
self._args = None
args = None
if argv is not None and len(argv) != 0:
args = parse_args(argv)
self._args = argv[2:]
if git is None:
git = GitWrapper()
if config is None:
if args is not None and args.cmd != 'init':
config = Config()
else:
config = ConfigI()
self._git = git
self._config = config
self._path_factory = PathFactory(self)
self._parsed_args = args
self._cloud = DataCloud(self)
class TestPathFactory(BasicEnvironment):
def setUp(self):
BasicEnvironment.init_environment(self,
test_dir=os.path.join(
os.path.sep, 'tmp',
'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(git, config)
self.data_file = os.path.join('data', 'file.txt')
self.cache_file = os.path.join('cache', 'fsymlinc.txt')
fd = open(self.cache_file, 'w+')
fd.write('some text')
fd.close()
os.chdir('data')
System.symlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
def test_data_file_factory(self):
data_path = self.path_factory.data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
indirect_file_path = os.path.join('..', self._proj_dir, self.data_file)
data_path_indirect = self.path_factory.data_item(indirect_file_path)
self.assertEqual(data_path_indirect.data.dvc, self.data_file)
pass
def test_data_symlink_factory(self):
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.cache.dvc, self.cache_file)
pass
def test_data_symlink_factory_cache(self):
data_path = self.path_factory.existing_data_item(self.data_file)
self.assertEqual(data_path.data.dvc, self.data_file)
pass
def test_data_symlink_factory_exception(self):
with self.assertRaises(DataItemError):
self.path_factory.existing_data_item(self.cache_file)
pass
def test_data_path(self):
file = os.path.join('data', 'file1')
path = self.path_factory.path(file)
self.assertEqual(path.dvc, file)
self.assertEqual(path.relative, file)
self.assertTrue(path.abs.endswith(file))
def test_to_data_path(self):
exclude_file = os.path.join('cache', 'file2')
data_path_file1 = os.path.join('data', 'file1')
data_path_file2 = os.path.join('data', 'file2')
files = [data_path_file1, exclude_file, data_path_file2]
data_path_list, exclude_file_list = self.path_factory.to_data_items(
files)
self.assertEqual(len(data_path_list), 2)
self.assertEqual(len(exclude_file_list), 1)
self.assertEqual(exclude_file_list[0], exclude_file)
data_path_set = set(x.data.dvc for x in data_path_list)
self.assertEqual(data_path_set, {data_path_file1, data_path_file2})
