def test_submodule_init(tmpdir_factory, runner, run, project):
"""Test initializing submodules."""
src_project = Path(str(tmpdir_factory.mktemp('src_project')))
assert 0 == run(args=('init', str(src_project)))
woop = src_project / 'woop'
with woop.open('w') as fp:
fp.write('woop')
repo = git.Repo(str(src_project))
repo.git.add('--all')
repo.index.commit('Added woop file')
assert 0 == run(args=('dataset', 'create', 'foo'))
assert 0 == run(args=('dataset', 'add', 'foo', str(woop)))
imported_woop = Path(project) / 'data' / 'foo' / woop.name
assert imported_woop.exists()
dst_project = Path(str(tmpdir_factory.mktemp('dst_project')))
subprocess.call(['git', 'clone', project, str(dst_project)])
subprocess.call(['git', 'lfs', 'install', '--local'], cwd=str(dst_project))
dst_woop = Path(dst_project) / 'data' / 'foo' / 'woop'
assert not dst_woop.exists()
result = runner.invoke(cli.cli, [
'--path',
str(dst_project), 'run', '--no-output', 'wc',
str(dst_woop.absolute())
],
catch_exceptions=False)
assert 0 == result.exit_code
def test_init(isolated_runner):
"""Test project initialization."""
runner = isolated_runner
# 1. the directory should be automatically created
new_project = Path('test-new-project')
assert not new_project.exists()
result = runner.invoke(cli.cli, ['init', 'test-new-project'])
assert 0 == result.exit_code
assert new_project.exists()
# 2. test project directory creation
os.mkdir('test-project')
result = runner.invoke(cli.cli, ['init', 'test-project'])
assert 0 == result.exit_code
assert os.stat(os.path.join('test-project', '.git'))
assert os.stat(os.path.join('test-project', '.renku'))
# 3. test project init from already existing renku repository
os.chdir('test-project')
result = runner.invoke(cli.cli, ['init'])
assert 0 != result.exit_code
# 4. in case of init failure because of existing .git folder
# .renku directory should not exist
assert not os.path.exists(os.path.join('test-project', '.renku'))
result = runner.invoke(cli.cli, ['init', '--force'])
assert 0 == result.exit_code
assert os.stat(os.path.join('.git'))
assert os.stat(os.path.join('.renku'))
# 5. check git lfs init options
os.chdir('../')
shutil.rmtree('test-project')
os.mkdir('test-project')
os.chdir('test-project')
result = runner.invoke(cli.cli, ['init', '--no-external-storage'])
with open('.git/config') as f:
config = f.read()
assert 'filter "lfs"' not in config
result = runner.invoke(cli.cli, ['init', '-S'])
with open('.git/config') as f:
config = f.read()
assert 'filter "lfs"' not in config
result = runner.invoke(cli.cli, ['init', '--force'])
with open('.git/config') as f:
config = f.read()
assert 'filter "lfs"' in config
def test_show_inputs(tmpdir_factory, project, runner, run):
"""Test show inputs with submodules."""
second_project = Path(str(tmpdir_factory.mktemp('second_project')))
assert 0 == run(args=('init', str(second_project)))
woop = second_project / 'woop'
with woop.open('w') as fp:
fp.write('woop')
second_repo = git.Repo(str(second_project))
second_repo.git.add('--all')
second_repo.index.commit('Added woop file')
assert 0 == run(args=('dataset', 'create', 'foo'))
assert 0 == run(args=('dataset', 'add', 'foo', str(woop)))
imported_woop = Path(project) / 'data' / 'foo' / woop.name
assert imported_woop.exists()
woop_wc = Path(project) / 'woop.wc'
assert 0 == run(args=('run', 'wc'), stdin=imported_woop, stdout=woop_wc)
result = runner.invoke(cli.cli, ['show', 'inputs'], catch_exceptions=False)
assert {str(imported_woop.resolve().relative_to(Path(project).resolve()))
} == set(result.output.strip().split('\n'))
def test_streams_cleanup(runner, project, run):
"""Test cleanup of standard streams."""
source = Path(project) / 'source.txt'
stdout = Path(project) / 'result.txt'
with source.open('w') as fp:
fp.write('first,second,third')
# File outside the Git index should be deleted.
with source.open('r') as fp:
assert fp.read() == 'first,second,third'
assert not stdout.exists()
result = runner.invoke(cli.cli, ['status'])
# Dirty repository check.
assert 1 == result.exit_code
# File from the Git index should be restored.
repo = git.Repo(project)
with stdout.open('w') as fp:
fp.write('1')
repo.index.add(['result.txt'])
with stdout.open('r') as fp:
assert fp.read() == '1'
def test_init_force_in_empty_dir(isolated_runner):
"""Run init --force in empty directory."""
runner = isolated_runner
new_project = Path('test-new-project')
assert not new_project.exists()
result = runner.invoke(cli.cli, ['init', '--force', 'test-new-project'])
assert 0 == result.exit_code
def uninstall(client):
"""Uninstall Git hooks."""
from git.index.fun import hook_path as get_hook_path
for hook in HOOKS:
hook_path = Path(get_hook_path(hook, client.repo.git_dir))
if hook_path.exists():
hook_path.unlink()
def file_candidate(self, candidate):
"""Return a path instance if it exists in current directory."""
candidate = Path(candidate)
if not candidate.is_absolute():
candidate = self.directory / candidate
if candidate.exists():
return candidate
def test_file_modification_during_run(tmpdir, runner, project, client, run):
"""Test run in isolation."""
script = client.path / 'script.py'
output = client.path / 'output'
lock = Path(str(tmpdir.join('lock')))
with client.commit():
with script.open('w') as fp:
fp.write('import os, time, sys\n'
'open("{lock}", "a")\n'
'while os.path.exists("{lock}"):\n'
' time.sleep(1)\n'
'sys.stdout.write(sys.stdin.read())\n'
'sys.stdout.flush()\n'.format(lock=str(lock)))
prefix = [
sys.executable,
'-m',
'renku',
'run',
'--isolation',
]
cmd = [
'python',
script.name,
]
previous = client.repo.head.commit
with output.open('wb') as stdout:
process = subprocess.Popen(prefix + cmd,
stdin=subprocess.PIPE,
stdout=stdout)
while not lock.exists():
time.sleep(1)
with script.open('w') as fp:
fp.write('print("edited")')
lock.unlink()
process.communicate(input=b'test')
assert 0 == process.wait()
with output.open('r') as fp:
assert 'test' == fp.read().strip()
diff = previous.diff(client.repo.head.commit)
modifications = [
modification for modification in diff
if modification.change_type == 'M'
]
assert 0 == len(modifications)
def file_candidate(self, candidate, ignore=None):
"""Return a path instance if it exists in current directory."""
if ignore and candidate in ignore:
return
candidate = Path(candidate)
if not candidate.is_absolute():
candidate = self.directory / candidate
if candidate.exists():
return candidate.resolve()
def install(client, force):
"""Install Git hooks."""
import pkg_resources
from git.index.fun import hook_path as get_hook_path
for hook in HOOKS:
hook_path = Path(get_hook_path(hook, client.repo.git_dir))
if hook_path.exists():
if not force:
click.echo("Hook already exists. Skipping {0}".format(
str(hook_path)),
err=True)
continue
else:
hook_path.unlink()
# Make sure the hooks directory exists.
hook_path.parent.mkdir(parents=True, exist_ok=True)
Path(hook_path).write_bytes(
pkg_resources.resource_string('renku.data',
'{hook}.sh'.format(hook=hook)))
hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC)
def test_streams_cleanup(runner, project, run):
"""Test cleanup of standard streams."""
source = Path(project) / 'source.txt'
stdout = Path(project) / 'result.txt'
with source.open('w') as fp:
fp.write('first,second,third')
# File outside the Git index should be deleted.
assert 1 == run(
args=('run', 'cat', source.name),
stdout=stdout,
), 'The repo must be dirty.'
with source.open('r') as fp:
assert fp.read() == 'first,second,third'
assert not stdout.exists()
result = runner.invoke(cli.cli, ['status'])
assert result.exit_code == 1
# File from the Git index should be restored.
repo = git.Repo(project)
with stdout.open('w') as fp:
fp.write('1')
repo.index.add(['result.txt'])
assert 1 == run(
args=('run', 'cat', 'source.txt'),
stdout=stdout,
), 'The repo must be dirty.'
with stdout.open('r') as fp:
assert fp.read() == '1'
def worktree(
self,
path=None,
branch_name=None,
commit=None,
merge_args=('--ff-only', ),
):
"""Create new worktree."""
from git import GitCommandError, NULL_TREE
from renku._contexts import Isolation
delete = path is None
path = path or tempfile.mkdtemp()
branch_name = branch_name or 'renku/run/isolation/' + uuid.uuid4().hex
# TODO sys.argv
if commit is NULL_TREE:
args = ['add', '--detach', path]
self.repo.git.worktree(*args)
client = attr.evolve(self, path=path)
client.repo.git.checkout('--orphan', branch_name)
client.repo.git.rm('-rf', '*')
else:
args = ['add', '-b', branch_name, path]
if commit:
args.append(commit)
self.repo.git.worktree(*args)
client = attr.evolve(self, path=path)
client.repo.config_reader = self.repo.config_reader
# Keep current directory relative to repository root.
relative = Path('.').resolve().relative_to(self.path)
# Reroute standard streams
original_mapped_std = _mapped_std_streams(self.candidate_paths)
mapped_std = {}
for name, stream in original_mapped_std.items():
stream_path = Path(path) / (Path(stream).relative_to(self.path))
stream_path = stream_path.absolute()
if not stream_path.exists():
stream_path.parent.mkdir(parents=True, exist_ok=True)
stream_path.touch()
mapped_std[name] = stream_path
_clean_streams(self.repo, original_mapped_std)
new_cwd = Path(path) / relative
new_cwd.mkdir(parents=True, exist_ok=True)
with Isolation(cwd=str(new_cwd), **mapped_std):
yield client
try:
self.repo.git.merge(branch_name, *merge_args)
except GitCommandError:
raise errors.FailedMerge(self.repo)
if delete:
shutil.rmtree(path)
self.repo.git.worktree('prune')
self.checkout_paths_from_storage()
def get_project_config_path(path=None):
"""Return project configuration folder if exist."""
project_path = Path(path or '.').absolute().joinpath(RENKU_HOME)
if project_path.exists() and project_path.is_dir():
return str(project_path)
def commit(self, author_date=None, commit_only=None):
"""Automatic commit."""
from git import Actor
from renku.version import __version__, version_url
diff_before = set()
author_date = author_date or formatdate(localtime=True)
if commit_only == COMMIT_DIFF_STRATEGY:
staged = {item.a_path for item in self.repo.index.diff(None)}
modified = {item.a_path for item in self.repo.index.diff('HEAD')}
if staged or modified:
self.repo.git.reset()
# Exclude files created by pipes.
diff_before = {
file_
for file_ in self.repo.untracked_files
if STARTED_AT - int(Path(file_).stat().st_ctime * 1e3) >= 1e3
}
if isinstance(commit_only, list):
for path_ in commit_only:
self.ensure_untracked(str(path_))
self.ensure_unstaged(str(path_))
yield
committer = Actor('renku {0}'.format(__version__), version_url)
if commit_only == COMMIT_DIFF_STRATEGY:
# Get diff generated in command.
staged_after = {item.a_path for item in self.repo.index.diff(None)}
modified_after = {
item.a_path
for item in self.repo.index.diff('HEAD')
}
diff_after = set(self.repo.untracked_files)\
.union(staged_after)\
.union(modified_after)
# Remove files not touched in command.
commit_only = list(diff_after - diff_before)
if isinstance(commit_only, list):
for path_ in commit_only:
p = Path(path_)
if p.exists():
self.repo.git.add(path_)
if not commit_only:
self.repo.git.add('--all')
argv = [os.path.basename(sys.argv[0])] + sys.argv[1:]
# Ignore pre-commit hooks since we have already done everything.
self.repo.index.commit(
' '.join(argv),
author_date=author_date,
committer=committer,
skip_hooks=True,
)
