def create_state_files(self, targets, lock):
"""
Create state files for all targets.
"""
for t in targets:
orig_target, processed_data_item = t
input, data_item = orig_target
output = data_item.data.relative
if processed_data_item == None:
Logger.debug(
'Skipping creating state file for failed import {}'.format(
data_item.state.relative))
continue
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
def move_data_to_cache(self):
cache_dir = os.path.dirname(self.cache.relative)
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
shutil.move(self.data.relative, self.cache.relative)
System.symlink(self.symlink_file, self.data.relative)
def test_get_gitignore_symlink(self, git, repo_dir):
link = os.path.join(repo_dir.root_dir, "link")
target = os.path.join(repo_dir.root_dir, repo_dir.DATA_SUB)
System.symlink(target, link)
entry, gitignore = Git(repo_dir._root_dir)._get_gitignore(link)
assert entry == "/link"
assert gitignore == os.path.join(repo_dir.root_dir, Git.GITIGNORE)
def setUp(self):
self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test')
curr_dir = None
commit = 'abc1234'
BasicEnvironment.init_environment(self, self._test_dir, curr_dir)
self._commit = commit
self._git = GitWrapperI(git_dir=self._test_git_dir,
commit=self._commit)
self._config = ConfigI('data', 'cache', 'state', '.target')
self.path_factory = PathFactory(self._git, self._config)
self.settings = Settings(['target'], self._git, self._config)
os.chdir(self._test_git_dir)
os.mkdir(os.path.join('data', 'dir1'))
os.mkdir(os.path.join('cache', 'dir1'))
self.file1_cache = os.path.join('cache', 'dir1', 'file1')
self.file1_data = os.path.join('data', 'dir1', 'file1')
open(self.file1_cache, 'w').write('ddfdff')
System.symlink(self.file1_cache, self.file1_data)
self.file2_cache = os.path.join('cache', 'file2')
self.file2_data = os.path.join('data', 'file2')
open(self.file2_cache, 'w').write('ddfdff')
System.symlink(self.file2_cache, self.file2_data)
def crate_data_item(self,
data_file,
cache_file=True,
content='random text'):
file_result = os.path.join('data', data_file)
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR,
data_file + DataItem.STATE_FILE_SUFFIX)
dummy_md5 = os.path.basename(
data_file) + DataItem.CACHE_FILE_SEP + self._commit
self.create_content_file(
state_result,
'{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}')
if cache_file:
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR,
dummy_md5)
self.create_content_file(cache_result, content)
relevant_dir = self.relevant_dir(data_file)
System.symlink(os.path.join(relevant_dir, cache_result),
file_result)
else:
cache_result = None
self.create_content_file(file_result, content)
return file_result, cache_result, state_result
def test_get_gitignore_symlink(tmp_dir, scm):
tmp_dir.gen({"dir": {"subdir": {"data": "contents"}}})
link = fspath(tmp_dir / "link")
target = fspath(tmp_dir / "dir" / "subdir" / "data")
System.symlink(target, link)
entry, gitignore = scm._get_gitignore(link)
assert entry == "/link"
assert gitignore == fspath(tmp_dir / ".gitignore")
def _prepare_external_data(self):
data_dir = self._get_data_dir()
self.data_file_name = "data_file"
external_data_path = os.path.join(data_dir, self.data_file_name)
with open(external_data_path, "w+") as f:
f.write("data")
self.link_name = "data_link"
System.symlink(data_dir, self.link_name)
def import_file(self, input, output, lock=False):
if not CmdImportFile.is_url(input):
if not os.path.exists(input):
raise ImportFileError(
'Input file "{}" does not exist'.format(input))
if not os.path.isfile(input):
raise ImportFileError(
'Input file "{}" has to be a regular file'.format(input))
if os.path.isdir(output):
output = os.path.join(output, os.path.basename(input))
data_item = self.settings.path_factory.data_item(output)
if os.path.exists(data_item.data.relative):
raise ImportFileError('Output file "{}" already exists'.format(
data_item.data.relative))
if not os.path.isdir(os.path.dirname(data_item.data.relative)):
raise ImportFileError(
'Output file directory "{}" does not exists'.format(
os.path.dirname(data_item.data.relative)))
cache_dir = os.path.dirname(data_item.cache.relative)
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
if CmdImportFile.is_url(input):
Logger.debug('Downloading file {} ...'.format(input))
self.download_file(input, data_item.cache.relative)
Logger.debug('Input file "{}" was downloaded to cache "{}"'.format(
input, data_item.cache.relative))
else:
copyfile(input, data_item.cache.relative)
Logger.debug('Input file "{}" was copied to cache "{}"'.format(
input, data_item.cache.relative))
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
pass
def create_empty_file(self):
empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME)
cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM
empty_cache_path = os.path.join(Config.CONFIG_DIR, Config.CACHE_DIR, cache_file_suffix)
open(empty_cache_path, 'w').close()
System.symlink(os.path.join('..', empty_cache_path), empty_data_path)
StateFile(StateFile.COMMAND_EMPTY_FILE,
self.settings.path_factory.data_item(empty_data_path),
self.settings,
input_files=[],
output_files=[],
lock=False).save(is_update_target_metrics=False)
pass
def create_empty_file(self):
empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME)
cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM
empty_cache_path = os.path.join(self.parsed_args.cache_dir, cache_file_suffix)
empty_state_path = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME + '.state')
open(empty_cache_path, 'w').close()
System.symlink(os.path.join('..', empty_cache_path), empty_data_path)
StateFile(StateFile.COMMAND_EMPTY_FILE,
empty_state_path,
self.settings,
input_files=[],
output_files=[],
lock=False).save()
pass
def crate_data_item_with_five_caches(self,
dir,
data_file,
content='random text'):
os.chdir(self._test_git_dir)
file_result = os.path.join('data', dir, data_file)
state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, dir,
data_file + DataItem.STATE_FILE_SUFFIX)
self.create_content_file(state_result, 'state content')
file_prefix = data_file + DataItem.CACHE_FILE_SEP
cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir,
file_prefix + self._commit)
self.create_content_file(cache_result, content)
relevant_dir = self.relevant_dir(os.path.join(dir, data_file))
cache_file = os.path.join(relevant_dir, cache_result)
d = os.path.join(self._test_git_dir, 'data', dir)
os.chdir(d)
print('---- CREATE SL {}: {} --> {}'.format(d, data_file, cache_file))
System.symlink(cache_file, data_file)
os.chdir(self._test_git_dir)
data_item = self.settings.path_factory.existing_data_item(file_result)
print('*DATA ITEM {}: {} {}'.format(file_result,
data_item.data.relative,
data_item.cache.relative))
print('----> {} {} {} {}'.format(dir, data_file, cache_file,
file_result))
# Additional cache files
self.create_content_file(
os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir,
file_prefix + 'aaaaaaa'))
self.create_content_file(
os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir,
file_prefix + '1111111'))
self.create_content_file(
os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir,
file_prefix + '5555555'))
self.create_content_file(
os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir,
file_prefix + '123abff'))
return
def crate_data_item(self, data_file, cache_file=True, content='random text'):
file_result = os.path.join('data', data_file)
state_result = os.path.join('state', data_file + DataItem.STATE_FILE_SUFFIX)
self.create_content_file(state_result, 'state content')
if cache_file:
cache_result = os.path.join('cache', data_file + DataItem.CACHE_FILE_SEP + self._commit)
self.create_content_file(cache_result, content)
relevant_dir = self.relevant_dir(data_file)
System.symlink(os.path.join(relevant_dir, cache_result), file_result)
else:
cache_result = None
self.create_content_file(file_result, content)
return file_result, cache_result, state_result
def test_broken_symlink(tmp_dir, dvc):
from dvc.system import System
tmp_dir.gen("file", "content")
System.symlink("file", "link")
os.remove("file")
entries = Repo.ls(os.fspath(tmp_dir))
assert entries == [
{
"isout": False,
"isdir": False,
"isexec": False,
"path": ".dvcignore",
},
]
def setUp(self):
BasicEnvironment.init_environment(self,
test_dir=os.path.join(
os.path.sep, 'tmp',
'ntx_unit_test'))
git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8')
config = ConfigI('data', 'cache', 'state')
self.path_factory = PathFactory(git, config)
self.data_file = os.path.join('data', 'file.txt')
self.cache_file = os.path.join('cache', 'fsymlinc.txt')
fd = open(self.cache_file, 'w+')
fd.write('some text')
fd.close()
os.chdir('data')
System.symlink(os.path.join('..', self.cache_file), 'file.txt')
os.chdir('..')
pass
def create_state_files(self, targets, lock):
"""
Create state files for all targets.
"""
for t in targets:
input = t[0]
output = t[1]
data_item = t[2]
Logger.debug('Creating symlink {} --> {}'.format(
data_item.symlink_file, data_item.data.relative))
System.symlink(data_item.symlink_file, data_item.data.relative)
state_file = StateFile(StateFile.COMMAND_IMPORT_FILE,
data_item.state.relative,
self.settings,
argv=[input, output],
input_files=[],
output_files=[output],
lock=lock)
state_file.save()
Logger.debug('State file "{}" was created'.format(
data_item.state.relative))
def symlink(from_info, to_info):
System.symlink(from_info, to_info)
def symlink(self, path1, path2):
return System.symlink(path1, path2)
def move_data_to_cache(self):
self.import_cache(self.data.relative)
System.symlink(self.symlink_file, self.data.relative)
