def test_package_hard_size_fail(tmpdir, monkeypatch, threshold):
monkeypatch.setattr(pkg, threshold, 200)
write_temp_files(tmpdir,
with_yaml=True,
with_gitignore=False,
large_file_size=10000)
with pytest.raises(PackageTooLarge):
pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_file_soft_size_warn(tmpdir, capsys, monkeypatch):
monkeypatch.setattr(termui, 'visible_prompt_func', lambda x: 'y\n')
write_temp_files(tmpdir,
with_yaml=True,
large_file_size=int(pkg.FILE_SIZE_WARN_THRESHOLD + 50))
pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
out, err = capsys.readouterr()
assert 'Large file large_file.dat' in err
def test_package_file_count_hard_fail(tmpdir, monkeypatch):
# Should not fail here
monkeypatch.setattr(pkg, 'FILE_COUNT_HARD_THRESHOLD', 3)
write_temp_files(tmpdir, with_yaml=True, with_gitignore=False)
pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
# With threshold of 2, should fail for 3 files
monkeypatch.setattr(pkg, 'FILE_COUNT_HARD_THRESHOLD', 2)
with pytest.raises(PackageTooLarge):
pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_no_files_in_rootdir(tmpdir):
tmpdir.mkdir('subway').join('asdf.bat').write_text('this file is required',
'utf8')
tarball = pkg.package_directory(directory=str(tmpdir),
yaml_path='valohai.yaml',
validate=False)
assert get_tar_files(tarball) == {'subway/asdf.bat'}
def package_adhoc_commit(project: Project, validate: bool = True) -> Dict[str, Any]:
"""
Create an ad-hoc tarball and commit of the project directory.
:return: Commit response object from API
"""
directory = project.directory
tarball = None
try:
description = ''
try:
description = describe_current_commit(directory)
except (NoGitRepo, NoCommit):
pass
except Exception as exc:
warn(f'Unable to derive Git description: {exc}')
if description:
click.echo(f'Packaging {directory} ({description})...')
else:
click.echo(f'Packaging {directory}...')
tarball = package_directory(directory, progress=True, validate=validate)
return create_adhoc_commit_from_tarball(project, tarball, description)
finally:
if tarball:
try:
os.unlink(tarball)
except OSError as err: # pragma: no cover
warn(f'Unable to remove temporary file: {err}')
def test_vhignore_entire_directory(tmp_path):
(tmp_path / ".vhignore").write_text('data/\n')
(tmp_path / "data").mkdir()
(tmp_path / "data" / "file1").write_text('this file is ignored')
(tmp_path / "hello.py").write_text('print("hello")')
(tmp_path / "valohai.yaml").write_text('')
tarball = pkg.package_directory(directory=str(tmp_path),
yaml_path='valohai.yaml')
assert get_tar_files(tarball) == {'hello.py', 'valohai.yaml'}
def test_package_no_git(tmpdir, with_gitignore, with_vhignore):
write_temp_files(tmpdir,
with_gitignore=with_gitignore,
with_vhignore=with_vhignore)
tarball = pkg.package_directory(directory=str(tmpdir),
yaml_path='valohai.yaml')
expected_filenames = get_expected_filenames(
{'asbestos', 'kahvikuppi', 'valohai.yaml'},
with_vhignore=with_vhignore,
with_gitignore=with_gitignore,
)
assert get_tar_files(tarball) == expected_filenames
def test_package_git(tmpdir, with_commit):
check_call('git init', cwd=str(tmpdir), shell=True)
write_temp_files(tmpdir)
if with_commit:
check_call('git add .', cwd=str(tmpdir), shell=True)
check_call('git commit -m test', cwd=str(tmpdir), shell=True)
assert describe_current_commit(str(tmpdir))
else:
with pytest.raises(NoCommit):
describe_current_commit(str(tmpdir))
tarball = pkg.package_directory(str(tmpdir))
# the dotfile and asbestos do not appear
assert get_tar_files(tarball) == {'kahvikuppi', 'valohai.yaml'}
def create_adhoc_commit(project):
"""
Create an ad-hoc tarball and commit of the project directory.
:param project: Project
:type project: valohai_cli.models.project.Project
:return: Commit response object from API
:rtype: dict[str, object]
"""
tarball = None
try:
click.echo('Packaging {dir}...'.format(dir=project.directory))
tarball = package_directory(project.directory, progress=True)
# TODO: We could check whether the commit is known already
size = os.stat(tarball).st_size
click.echo('Uploading {size:.2f} KiB...'.format(size=size / 1024.))
upload = MultipartEncoder(
{'data': ('data.tgz', open(tarball, 'rb'), 'application/gzip')})
prog = click.progressbar(length=upload.len, width=0)
prog.is_hidden = (size < 524288
) # Don't bother with the bar if the upload is small
with prog:
def callback(upload):
prog.pos = upload.bytes_read
prog.update(0) # Step is 0 because we set pos above
monitor = MultipartEncoderMonitor(upload, callback)
resp = request(
'post',
'/api/v0/projects/{id}/import-package/'.format(id=project.id),
data=monitor,
headers={
'Content-Type': monitor.content_type
},
).json()
success('Uploaded ad-hoc code {identifier}'.format(
identifier=resp['identifier']))
finally:
if tarball:
os.unlink(tarball)
return resp
def test_package_requires_yaml(tmpdir):
write_temp_files(tmpdir, with_yaml=False)
with pytest.raises(ConfigurationError):
pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_single_file_packaged_correctly(tmpdir):
tmpdir.join('valohai.yaml').write_text('this file is required', 'utf8')
tarball = pkg.package_directory(directory=str(tmpdir),
yaml_path='valohai.yaml')
assert get_tar_files(tarball) == {'valohai.yaml'}
def test_package_requires_yaml(tmpdir):
write_temp_files(tmpdir, with_yaml=False)
with pytest.raises(ConfigurationError):
package_directory(str(tmpdir))
def test_package_no_git(tmpdir):
write_temp_files(tmpdir)
tarball = package_directory(str(tmpdir))
# the dotfile is gone, but there's nothing to stop the asbestos
assert get_tar_files(tarball) == {'asbestos', 'kahvikuppi', 'valohai.yaml'}
def test_package_git(tmpdir):
check_call('git init', cwd=str(tmpdir), shell=True)
write_temp_files(tmpdir)
tarball = package_directory(str(tmpdir))
# the dotfile and asbestos do not appear
assert get_tar_files(tarball) == {'kahvikuppi', 'valohai.yaml'}
def test_package_hard_size_fail(tmpdir, monkeypatch, threshold):
monkeypatch.setattr(pkg, threshold, 200)
write_temp_files(tmpdir, with_yaml=True, large_file_size=10000)
with pytest.raises(PackageTooLarge):
pkg.package_directory(str(tmpdir))
