def test_package_hard_size_fail(tmpdir, monkeypatch, threshold): monkeypatch.setattr(pkg, threshold, 200) write_temp_files(tmpdir, with_yaml=True, with_gitignore=False, large_file_size=10000) with pytest.raises(PackageTooLarge): pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_file_soft_size_warn(tmpdir, capsys, monkeypatch): monkeypatch.setattr(termui, 'visible_prompt_func', lambda x: 'y\n') write_temp_files(tmpdir, with_yaml=True, large_file_size=int(pkg.FILE_SIZE_WARN_THRESHOLD + 50)) pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml') out, err = capsys.readouterr() assert 'Large file large_file.dat' in err
def test_package_file_count_hard_fail(tmpdir, monkeypatch): # Should not fail here monkeypatch.setattr(pkg, 'FILE_COUNT_HARD_THRESHOLD', 3) write_temp_files(tmpdir, with_yaml=True, with_gitignore=False) pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml') # With threshold of 2, should fail for 3 files monkeypatch.setattr(pkg, 'FILE_COUNT_HARD_THRESHOLD', 2) with pytest.raises(PackageTooLarge): pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_no_files_in_rootdir(tmpdir): tmpdir.mkdir('subway').join('asdf.bat').write_text('this file is required', 'utf8') tarball = pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml', validate=False) assert get_tar_files(tarball) == {'subway/asdf.bat'}
def package_adhoc_commit(project: Project, validate: bool = True) -> Dict[str, Any]: """ Create an ad-hoc tarball and commit of the project directory. :return: Commit response object from API """ directory = project.directory tarball = None try: description = '' try: description = describe_current_commit(directory) except (NoGitRepo, NoCommit): pass except Exception as exc: warn(f'Unable to derive Git description: {exc}') if description: click.echo(f'Packaging {directory} ({description})...') else: click.echo(f'Packaging {directory}...') tarball = package_directory(directory, progress=True, validate=validate) return create_adhoc_commit_from_tarball(project, tarball, description) finally: if tarball: try: os.unlink(tarball) except OSError as err: # pragma: no cover warn(f'Unable to remove temporary file: {err}')
def test_vhignore_entire_directory(tmp_path): (tmp_path / ".vhignore").write_text('data/\n') (tmp_path / "data").mkdir() (tmp_path / "data" / "file1").write_text('this file is ignored') (tmp_path / "hello.py").write_text('print("hello")') (tmp_path / "valohai.yaml").write_text('') tarball = pkg.package_directory(directory=str(tmp_path), yaml_path='valohai.yaml') assert get_tar_files(tarball) == {'hello.py', 'valohai.yaml'}
def test_package_no_git(tmpdir, with_gitignore, with_vhignore): write_temp_files(tmpdir, with_gitignore=with_gitignore, with_vhignore=with_vhignore) tarball = pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml') expected_filenames = get_expected_filenames( {'asbestos', 'kahvikuppi', 'valohai.yaml'}, with_vhignore=with_vhignore, with_gitignore=with_gitignore, ) assert get_tar_files(tarball) == expected_filenames
def test_package_git(tmpdir, with_commit): check_call('git init', cwd=str(tmpdir), shell=True) write_temp_files(tmpdir) if with_commit: check_call('git add .', cwd=str(tmpdir), shell=True) check_call('git commit -m test', cwd=str(tmpdir), shell=True) assert describe_current_commit(str(tmpdir)) else: with pytest.raises(NoCommit): describe_current_commit(str(tmpdir)) tarball = pkg.package_directory(str(tmpdir)) # the dotfile and asbestos do not appear assert get_tar_files(tarball) == {'kahvikuppi', 'valohai.yaml'}
def create_adhoc_commit(project): """ Create an ad-hoc tarball and commit of the project directory. :param project: Project :type project: valohai_cli.models.project.Project :return: Commit response object from API :rtype: dict[str, object] """ tarball = None try: click.echo('Packaging {dir}...'.format(dir=project.directory)) tarball = package_directory(project.directory, progress=True) # TODO: We could check whether the commit is known already size = os.stat(tarball).st_size click.echo('Uploading {size:.2f} KiB...'.format(size=size / 1024.)) upload = MultipartEncoder( {'data': ('data.tgz', open(tarball, 'rb'), 'application/gzip')}) prog = click.progressbar(length=upload.len, width=0) prog.is_hidden = (size < 524288 ) # Don't bother with the bar if the upload is small with prog: def callback(upload): prog.pos = upload.bytes_read prog.update(0) # Step is 0 because we set pos above monitor = MultipartEncoderMonitor(upload, callback) resp = request( 'post', '/api/v0/projects/{id}/import-package/'.format(id=project.id), data=monitor, headers={ 'Content-Type': monitor.content_type }, ).json() success('Uploaded ad-hoc code {identifier}'.format( identifier=resp['identifier'])) finally: if tarball: os.unlink(tarball) return resp
def test_package_requires_yaml(tmpdir): write_temp_files(tmpdir, with_yaml=False) with pytest.raises(ConfigurationError): pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml')
def test_single_file_packaged_correctly(tmpdir): tmpdir.join('valohai.yaml').write_text('this file is required', 'utf8') tarball = pkg.package_directory(directory=str(tmpdir), yaml_path='valohai.yaml') assert get_tar_files(tarball) == {'valohai.yaml'}
def test_package_requires_yaml(tmpdir): write_temp_files(tmpdir, with_yaml=False) with pytest.raises(ConfigurationError): package_directory(str(tmpdir))
def test_package_no_git(tmpdir): write_temp_files(tmpdir) tarball = package_directory(str(tmpdir)) # the dotfile is gone, but there's nothing to stop the asbestos assert get_tar_files(tarball) == {'asbestos', 'kahvikuppi', 'valohai.yaml'}
def test_package_git(tmpdir): check_call('git init', cwd=str(tmpdir), shell=True) write_temp_files(tmpdir) tarball = package_directory(str(tmpdir)) # the dotfile and asbestos do not appear assert get_tar_files(tarball) == {'kahvikuppi', 'valohai.yaml'}
def test_package_hard_size_fail(tmpdir, monkeypatch, threshold): monkeypatch.setattr(pkg, threshold, 200) write_temp_files(tmpdir, with_yaml=True, large_file_size=10000) with pytest.raises(PackageTooLarge): pkg.package_directory(str(tmpdir))