def create_state_files(self, targets, lock): """ Create state files for all targets. """ for t in targets: orig_target, processed_data_item = t input, data_item = orig_target output = data_item.data.relative if processed_data_item == None: Logger.debug( 'Skipping creating state file for failed import {}'.format( data_item.state.relative)) continue Logger.debug('Creating symlink {} --> {}'.format( data_item.symlink_file, data_item.data.relative)) System.symlink(data_item.symlink_file, data_item.data.relative) state_file = StateFile(StateFile.COMMAND_IMPORT_FILE, data_item, self.settings, argv=[input, output], input_files=[], output_files=[output], lock=lock) state_file.save() Logger.debug('State file "{}" was created'.format( data_item.state.relative))
def move_data_to_cache(self): cache_dir = os.path.dirname(self.cache.relative) if not os.path.isdir(cache_dir): os.makedirs(cache_dir) shutil.move(self.data.relative, self.cache.relative) System.symlink(self.symlink_file, self.data.relative)
def test_get_gitignore_symlink(self, git, repo_dir): link = os.path.join(repo_dir.root_dir, "link") target = os.path.join(repo_dir.root_dir, repo_dir.DATA_SUB) System.symlink(target, link) entry, gitignore = Git(repo_dir._root_dir)._get_gitignore(link) assert entry == "/link" assert gitignore == os.path.join(repo_dir.root_dir, Git.GITIGNORE)
def setUp(self): self._test_dir = os.path.join(os.path.sep, 'tmp', 'dvc_unit_test') curr_dir = None commit = 'abc1234' BasicEnvironment.init_environment(self, self._test_dir, curr_dir) self._commit = commit self._git = GitWrapperI(git_dir=self._test_git_dir, commit=self._commit) self._config = ConfigI('data', 'cache', 'state', '.target') self.path_factory = PathFactory(self._git, self._config) self.settings = Settings(['target'], self._git, self._config) os.chdir(self._test_git_dir) os.mkdir(os.path.join('data', 'dir1')) os.mkdir(os.path.join('cache', 'dir1')) self.file1_cache = os.path.join('cache', 'dir1', 'file1') self.file1_data = os.path.join('data', 'dir1', 'file1') open(self.file1_cache, 'w').write('ddfdff') System.symlink(self.file1_cache, self.file1_data) self.file2_cache = os.path.join('cache', 'file2') self.file2_data = os.path.join('data', 'file2') open(self.file2_cache, 'w').write('ddfdff') System.symlink(self.file2_cache, self.file2_data)
def crate_data_item(self, data_file, cache_file=True, content='random text'): file_result = os.path.join('data', data_file) state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, data_file + DataItem.STATE_FILE_SUFFIX) dummy_md5 = os.path.basename( data_file) + DataItem.CACHE_FILE_SEP + self._commit self.create_content_file( state_result, '{"Md5" : "' + dummy_md5 + '", "Command" : "run", "Cwd" : "dir"}') if cache_file: cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dummy_md5) self.create_content_file(cache_result, content) relevant_dir = self.relevant_dir(data_file) System.symlink(os.path.join(relevant_dir, cache_result), file_result) else: cache_result = None self.create_content_file(file_result, content) return file_result, cache_result, state_result
def test_get_gitignore_symlink(tmp_dir, scm): tmp_dir.gen({"dir": {"subdir": {"data": "contents"}}}) link = fspath(tmp_dir / "link") target = fspath(tmp_dir / "dir" / "subdir" / "data") System.symlink(target, link) entry, gitignore = scm._get_gitignore(link) assert entry == "/link" assert gitignore == fspath(tmp_dir / ".gitignore")
def _prepare_external_data(self): data_dir = self._get_data_dir() self.data_file_name = "data_file" external_data_path = os.path.join(data_dir, self.data_file_name) with open(external_data_path, "w+") as f: f.write("data") self.link_name = "data_link" System.symlink(data_dir, self.link_name)
def import_file(self, input, output, lock=False): if not CmdImportFile.is_url(input): if not os.path.exists(input): raise ImportFileError( 'Input file "{}" does not exist'.format(input)) if not os.path.isfile(input): raise ImportFileError( 'Input file "{}" has to be a regular file'.format(input)) if os.path.isdir(output): output = os.path.join(output, os.path.basename(input)) data_item = self.settings.path_factory.data_item(output) if os.path.exists(data_item.data.relative): raise ImportFileError('Output file "{}" already exists'.format( data_item.data.relative)) if not os.path.isdir(os.path.dirname(data_item.data.relative)): raise ImportFileError( 'Output file directory "{}" does not exists'.format( os.path.dirname(data_item.data.relative))) cache_dir = os.path.dirname(data_item.cache.relative) if not os.path.exists(cache_dir): os.makedirs(cache_dir) if CmdImportFile.is_url(input): Logger.debug('Downloading file {} ...'.format(input)) self.download_file(input, data_item.cache.relative) Logger.debug('Input file "{}" was downloaded to cache "{}"'.format( input, data_item.cache.relative)) else: copyfile(input, data_item.cache.relative) Logger.debug('Input file "{}" was copied to cache "{}"'.format( input, data_item.cache.relative)) Logger.debug('Creating symlink {} --> {}'.format( data_item.symlink_file, data_item.data.relative)) System.symlink(data_item.symlink_file, data_item.data.relative) state_file = StateFile(StateFile.COMMAND_IMPORT_FILE, data_item.state.relative, self.settings, argv=[input, output], input_files=[], output_files=[output], lock=lock) state_file.save() Logger.debug('State file "{}" was created'.format( data_item.state.relative)) pass
def create_empty_file(self): empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM empty_cache_path = os.path.join(Config.CONFIG_DIR, Config.CACHE_DIR, cache_file_suffix) open(empty_cache_path, 'w').close() System.symlink(os.path.join('..', empty_cache_path), empty_data_path) StateFile(StateFile.COMMAND_EMPTY_FILE, self.settings.path_factory.data_item(empty_data_path), self.settings, input_files=[], output_files=[], lock=False).save(is_update_target_metrics=False) pass
def create_empty_file(self): empty_data_path = os.path.join(self.parsed_args.data_dir, self.EMPTY_FILE_NAME) cache_file_suffix = self.EMPTY_FILE_NAME + '_' + self.EMPTY_FILE_CHECKSUM empty_cache_path = os.path.join(self.parsed_args.cache_dir, cache_file_suffix) empty_state_path = os.path.join(self.parsed_args.state_dir, self.EMPTY_FILE_NAME + '.state') open(empty_cache_path, 'w').close() System.symlink(os.path.join('..', empty_cache_path), empty_data_path) StateFile(StateFile.COMMAND_EMPTY_FILE, empty_state_path, self.settings, input_files=[], output_files=[], lock=False).save() pass
def crate_data_item_with_five_caches(self, dir, data_file, content='random text'): os.chdir(self._test_git_dir) file_result = os.path.join('data', dir, data_file) state_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR, dir, data_file + DataItem.STATE_FILE_SUFFIX) self.create_content_file(state_result, 'state content') file_prefix = data_file + DataItem.CACHE_FILE_SEP cache_result = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir, file_prefix + self._commit) self.create_content_file(cache_result, content) relevant_dir = self.relevant_dir(os.path.join(dir, data_file)) cache_file = os.path.join(relevant_dir, cache_result) d = os.path.join(self._test_git_dir, 'data', dir) os.chdir(d) print('---- CREATE SL {}: {} --> {}'.format(d, data_file, cache_file)) System.symlink(cache_file, data_file) os.chdir(self._test_git_dir) data_item = self.settings.path_factory.existing_data_item(file_result) print('*DATA ITEM {}: {} {}'.format(file_result, data_item.data.relative, data_item.cache.relative)) print('----> {} {} {} {}'.format(dir, data_file, cache_file, file_result)) # Additional cache files self.create_content_file( os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir, file_prefix + 'aaaaaaa')) self.create_content_file( os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir, file_prefix + '1111111')) self.create_content_file( os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir, file_prefix + '5555555')) self.create_content_file( os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR, dir, file_prefix + '123abff')) return
def crate_data_item(self, data_file, cache_file=True, content='random text'): file_result = os.path.join('data', data_file) state_result = os.path.join('state', data_file + DataItem.STATE_FILE_SUFFIX) self.create_content_file(state_result, 'state content') if cache_file: cache_result = os.path.join('cache', data_file + DataItem.CACHE_FILE_SEP + self._commit) self.create_content_file(cache_result, content) relevant_dir = self.relevant_dir(data_file) System.symlink(os.path.join(relevant_dir, cache_result), file_result) else: cache_result = None self.create_content_file(file_result, content) return file_result, cache_result, state_result
def test_broken_symlink(tmp_dir, dvc): from dvc.system import System tmp_dir.gen("file", "content") System.symlink("file", "link") os.remove("file") entries = Repo.ls(os.fspath(tmp_dir)) assert entries == [ { "isout": False, "isdir": False, "isexec": False, "path": ".dvcignore", }, ]
def setUp(self): BasicEnvironment.init_environment(self, test_dir=os.path.join( os.path.sep, 'tmp', 'ntx_unit_test')) git = GitWrapperI(git_dir=self._test_git_dir, commit='ad45ba8') config = ConfigI('data', 'cache', 'state') self.path_factory = PathFactory(git, config) self.data_file = os.path.join('data', 'file.txt') self.cache_file = os.path.join('cache', 'fsymlinc.txt') fd = open(self.cache_file, 'w+') fd.write('some text') fd.close() os.chdir('data') System.symlink(os.path.join('..', self.cache_file), 'file.txt') os.chdir('..') pass
def create_state_files(self, targets, lock): """ Create state files for all targets. """ for t in targets: input = t[0] output = t[1] data_item = t[2] Logger.debug('Creating symlink {} --> {}'.format( data_item.symlink_file, data_item.data.relative)) System.symlink(data_item.symlink_file, data_item.data.relative) state_file = StateFile(StateFile.COMMAND_IMPORT_FILE, data_item.state.relative, self.settings, argv=[input, output], input_files=[], output_files=[output], lock=lock) state_file.save() Logger.debug('State file "{}" was created'.format( data_item.state.relative))
def symlink(from_info, to_info): System.symlink(from_info, to_info)
def symlink(self, path1, path2): return System.symlink(path1, path2)
def move_data_to_cache(self): self.import_cache(self.data.relative) System.symlink(self.symlink_file, self.data.relative)