def test_submodule_init(tmpdir_factory, runner, run, project): """Test initializing submodules.""" src_project = Path(str(tmpdir_factory.mktemp('src_project'))) assert 0 == run(args=('init', str(src_project))) woop = src_project / 'woop' with woop.open('w') as fp: fp.write('woop') repo = git.Repo(str(src_project)) repo.git.add('--all') repo.index.commit('Added woop file') assert 0 == run(args=('dataset', 'create', 'foo')) assert 0 == run(args=('dataset', 'add', 'foo', str(woop))) imported_woop = Path(project) / 'data' / 'foo' / woop.name assert imported_woop.exists() dst_project = Path(str(tmpdir_factory.mktemp('dst_project'))) subprocess.call(['git', 'clone', project, str(dst_project)]) subprocess.call(['git', 'lfs', 'install', '--local'], cwd=str(dst_project)) dst_woop = Path(dst_project) / 'data' / 'foo' / 'woop' assert not dst_woop.exists() result = runner.invoke(cli.cli, [ '--path', str(dst_project), 'run', '--no-output', 'wc', str(dst_woop.absolute()) ], catch_exceptions=False) assert 0 == result.exit_code
def test_init(isolated_runner): """Test project initialization.""" runner = isolated_runner # 1. the directory should be automatically created new_project = Path('test-new-project') assert not new_project.exists() result = runner.invoke(cli.cli, ['init', 'test-new-project']) assert 0 == result.exit_code assert new_project.exists() # 2. test project directory creation os.mkdir('test-project') result = runner.invoke(cli.cli, ['init', 'test-project']) assert 0 == result.exit_code assert os.stat(os.path.join('test-project', '.git')) assert os.stat(os.path.join('test-project', '.renku')) # 3. test project init from already existing renku repository os.chdir('test-project') result = runner.invoke(cli.cli, ['init']) assert 0 != result.exit_code # 4. in case of init failure because of existing .git folder # .renku directory should not exist assert not os.path.exists(os.path.join('test-project', '.renku')) result = runner.invoke(cli.cli, ['init', '--force']) assert 0 == result.exit_code assert os.stat(os.path.join('.git')) assert os.stat(os.path.join('.renku')) # 5. check git lfs init options os.chdir('../') shutil.rmtree('test-project') os.mkdir('test-project') os.chdir('test-project') result = runner.invoke(cli.cli, ['init', '--no-external-storage']) with open('.git/config') as f: config = f.read() assert 'filter "lfs"' not in config result = runner.invoke(cli.cli, ['init', '-S']) with open('.git/config') as f: config = f.read() assert 'filter "lfs"' not in config result = runner.invoke(cli.cli, ['init', '--force']) with open('.git/config') as f: config = f.read() assert 'filter "lfs"' in config
def test_show_inputs(tmpdir_factory, project, runner, run): """Test show inputs with submodules.""" second_project = Path(str(tmpdir_factory.mktemp('second_project'))) assert 0 == run(args=('init', str(second_project))) woop = second_project / 'woop' with woop.open('w') as fp: fp.write('woop') second_repo = git.Repo(str(second_project)) second_repo.git.add('--all') second_repo.index.commit('Added woop file') assert 0 == run(args=('dataset', 'create', 'foo')) assert 0 == run(args=('dataset', 'add', 'foo', str(woop))) imported_woop = Path(project) / 'data' / 'foo' / woop.name assert imported_woop.exists() woop_wc = Path(project) / 'woop.wc' assert 0 == run(args=('run', 'wc'), stdin=imported_woop, stdout=woop_wc) result = runner.invoke(cli.cli, ['show', 'inputs'], catch_exceptions=False) assert {str(imported_woop.resolve().relative_to(Path(project).resolve())) } == set(result.output.strip().split('\n'))
def test_streams_cleanup(runner, project, run): """Test cleanup of standard streams.""" source = Path(project) / 'source.txt' stdout = Path(project) / 'result.txt' with source.open('w') as fp: fp.write('first,second,third') # File outside the Git index should be deleted. with source.open('r') as fp: assert fp.read() == 'first,second,third' assert not stdout.exists() result = runner.invoke(cli.cli, ['status']) # Dirty repository check. assert 1 == result.exit_code # File from the Git index should be restored. repo = git.Repo(project) with stdout.open('w') as fp: fp.write('1') repo.index.add(['result.txt']) with stdout.open('r') as fp: assert fp.read() == '1'
def test_init_force_in_empty_dir(isolated_runner): """Run init --force in empty directory.""" runner = isolated_runner new_project = Path('test-new-project') assert not new_project.exists() result = runner.invoke(cli.cli, ['init', '--force', 'test-new-project']) assert 0 == result.exit_code
def uninstall(client): """Uninstall Git hooks.""" from git.index.fun import hook_path as get_hook_path for hook in HOOKS: hook_path = Path(get_hook_path(hook, client.repo.git_dir)) if hook_path.exists(): hook_path.unlink()
def file_candidate(self, candidate): """Return a path instance if it exists in current directory.""" candidate = Path(candidate) if not candidate.is_absolute(): candidate = self.directory / candidate if candidate.exists(): return candidate
def test_file_modification_during_run(tmpdir, runner, project, client, run): """Test run in isolation.""" script = client.path / 'script.py' output = client.path / 'output' lock = Path(str(tmpdir.join('lock'))) with client.commit(): with script.open('w') as fp: fp.write('import os, time, sys\n' 'open("{lock}", "a")\n' 'while os.path.exists("{lock}"):\n' ' time.sleep(1)\n' 'sys.stdout.write(sys.stdin.read())\n' 'sys.stdout.flush()\n'.format(lock=str(lock))) prefix = [ sys.executable, '-m', 'renku', 'run', '--isolation', ] cmd = [ 'python', script.name, ] previous = client.repo.head.commit with output.open('wb') as stdout: process = subprocess.Popen(prefix + cmd, stdin=subprocess.PIPE, stdout=stdout) while not lock.exists(): time.sleep(1) with script.open('w') as fp: fp.write('print("edited")') lock.unlink() process.communicate(input=b'test') assert 0 == process.wait() with output.open('r') as fp: assert 'test' == fp.read().strip() diff = previous.diff(client.repo.head.commit) modifications = [ modification for modification in diff if modification.change_type == 'M' ] assert 0 == len(modifications)
def file_candidate(self, candidate, ignore=None): """Return a path instance if it exists in current directory.""" if ignore and candidate in ignore: return candidate = Path(candidate) if not candidate.is_absolute(): candidate = self.directory / candidate if candidate.exists(): return candidate.resolve()
def install(client, force): """Install Git hooks.""" import pkg_resources from git.index.fun import hook_path as get_hook_path for hook in HOOKS: hook_path = Path(get_hook_path(hook, client.repo.git_dir)) if hook_path.exists(): if not force: click.echo("Hook already exists. Skipping {0}".format( str(hook_path)), err=True) continue else: hook_path.unlink() # Make sure the hooks directory exists. hook_path.parent.mkdir(parents=True, exist_ok=True) Path(hook_path).write_bytes( pkg_resources.resource_string('renku.data', '{hook}.sh'.format(hook=hook))) hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC)
def test_streams_cleanup(runner, project, run): """Test cleanup of standard streams.""" source = Path(project) / 'source.txt' stdout = Path(project) / 'result.txt' with source.open('w') as fp: fp.write('first,second,third') # File outside the Git index should be deleted. assert 1 == run( args=('run', 'cat', source.name), stdout=stdout, ), 'The repo must be dirty.' with source.open('r') as fp: assert fp.read() == 'first,second,third' assert not stdout.exists() result = runner.invoke(cli.cli, ['status']) assert result.exit_code == 1 # File from the Git index should be restored. repo = git.Repo(project) with stdout.open('w') as fp: fp.write('1') repo.index.add(['result.txt']) assert 1 == run( args=('run', 'cat', 'source.txt'), stdout=stdout, ), 'The repo must be dirty.' with stdout.open('r') as fp: assert fp.read() == '1'
def worktree( self, path=None, branch_name=None, commit=None, merge_args=('--ff-only', ), ): """Create new worktree.""" from git import GitCommandError, NULL_TREE from renku._contexts import Isolation delete = path is None path = path or tempfile.mkdtemp() branch_name = branch_name or 'renku/run/isolation/' + uuid.uuid4().hex # TODO sys.argv if commit is NULL_TREE: args = ['add', '--detach', path] self.repo.git.worktree(*args) client = attr.evolve(self, path=path) client.repo.git.checkout('--orphan', branch_name) client.repo.git.rm('-rf', '*') else: args = ['add', '-b', branch_name, path] if commit: args.append(commit) self.repo.git.worktree(*args) client = attr.evolve(self, path=path) client.repo.config_reader = self.repo.config_reader # Keep current directory relative to repository root. relative = Path('.').resolve().relative_to(self.path) # Reroute standard streams original_mapped_std = _mapped_std_streams(self.candidate_paths) mapped_std = {} for name, stream in original_mapped_std.items(): stream_path = Path(path) / (Path(stream).relative_to(self.path)) stream_path = stream_path.absolute() if not stream_path.exists(): stream_path.parent.mkdir(parents=True, exist_ok=True) stream_path.touch() mapped_std[name] = stream_path _clean_streams(self.repo, original_mapped_std) new_cwd = Path(path) / relative new_cwd.mkdir(parents=True, exist_ok=True) with Isolation(cwd=str(new_cwd), **mapped_std): yield client try: self.repo.git.merge(branch_name, *merge_args) except GitCommandError: raise errors.FailedMerge(self.repo) if delete: shutil.rmtree(path) self.repo.git.worktree('prune') self.checkout_paths_from_storage()
def get_project_config_path(path=None): """Return project configuration folder if exist.""" project_path = Path(path or '.').absolute().joinpath(RENKU_HOME) if project_path.exists() and project_path.is_dir(): return str(project_path)
def commit(self, author_date=None, commit_only=None): """Automatic commit.""" from git import Actor from renku.version import __version__, version_url diff_before = set() author_date = author_date or formatdate(localtime=True) if commit_only == COMMIT_DIFF_STRATEGY: staged = {item.a_path for item in self.repo.index.diff(None)} modified = {item.a_path for item in self.repo.index.diff('HEAD')} if staged or modified: self.repo.git.reset() # Exclude files created by pipes. diff_before = { file_ for file_ in self.repo.untracked_files if STARTED_AT - int(Path(file_).stat().st_ctime * 1e3) >= 1e3 } if isinstance(commit_only, list): for path_ in commit_only: self.ensure_untracked(str(path_)) self.ensure_unstaged(str(path_)) yield committer = Actor('renku {0}'.format(__version__), version_url) if commit_only == COMMIT_DIFF_STRATEGY: # Get diff generated in command. staged_after = {item.a_path for item in self.repo.index.diff(None)} modified_after = { item.a_path for item in self.repo.index.diff('HEAD') } diff_after = set(self.repo.untracked_files)\ .union(staged_after)\ .union(modified_after) # Remove files not touched in command. commit_only = list(diff_after - diff_before) if isinstance(commit_only, list): for path_ in commit_only: p = Path(path_) if p.exists(): self.repo.git.add(path_) if not commit_only: self.repo.git.add('--all') argv = [os.path.basename(sys.argv[0])] + sys.argv[1:] # Ignore pre-commit hooks since we have already done everything. self.repo.index.commit( ' '.join(argv), author_date=author_date, committer=committer, skip_hooks=True, )