def test_error_if_exclude_and_include_overlap(cmdr, tmp_empty):
with pytest.raises(ClickException) as excinfo:
source.copy(cmdr, '.', 'dist', exclude=['file'], include=['file'])
expected = ("include and exclude must "
"not have overlapping elements: {'file'}")
assert expected == str(excinfo.value)
def test_no_git_but_exclude(cmdr, tmp_empty):
Path('file').touch()
Path('secrets.txt').touch()
source.copy(cmdr, '.', 'dist', exclude=['secrets.txt'])
expected = set(Path(p) for p in ('dist/file', ))
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_copy_with_gitignore(cmdr, tmp_empty):
Path('file').touch()
Path('ignoreme').touch()
Path('.gitignore').write_text('ignoreme')
git_init()
source.copy(cmdr, '.', 'dist')
expected = set({Path('dist/file')})
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_warns_if_fails_to_get_git_tracked_files(tmp_empty, capsys):
Path('file').touch()
Path('secrets.txt').touch()
with Commander() as cmdr:
source.copy(cmdr, '.', 'dist')
captured = capsys.readouterr()
assert 'Unable to get git tracked files' in captured.out
def test_no_git_but_exclude_entire_folder(cmdr, tmp_empty):
Path('file').touch()
Path('dir').mkdir()
Path('dir', 'secrets.txt').touch()
Path('dir', 'more-secrets.txt').touch()
source.copy(cmdr, '.', 'dist', exclude=['dir'])
expected = set(Path(p) for p in ('dist/file', ))
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_copy(cmdr, tmp_empty):
Path('file').touch()
Path('dir').mkdir()
Path('dir', 'another').touch()
git_init()
source.copy(cmdr, '.', 'dist')
expected = set(Path(p) for p in (
'dist/file',
'dist/dir/another',
))
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_override_git_with_exclude(cmdr, tmp_empty):
Path('file').touch()
Path('secrets.txt').touch()
# let git track everything
Path('.gitignore').touch()
git_init()
# exclude some file
source.copy(cmdr, '.', 'dist', exclude=['file'])
expected = set({Path('dist/secrets.txt')})
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_warns_on_dirty_git(tmp_empty, capsys):
Path('file').touch()
Path('secrets.txt').touch()
Path('.gitignore').write_text('secrets.txt')
git_init()
Path('new-file').touch()
with Commander() as cmdr:
source.copy(cmdr, '.', 'dist')
captured = capsys.readouterr()
assert 'Your git repository contains untracked' in captured.out
def test_ignores_pycache(cmdr, tmp_empty):
Path('file').touch()
dir_ = Path('__pycache__')
dir_.mkdir()
(dir_ / 'file').touch()
(dir_ / 'another').touch()
dir_another = Path('subdir', '__pycache__')
dir_another.mkdir(parents=True)
(dir_another / 'file').touch()
(dir_another / 'another').touch()
source.copy(cmdr, '.', 'dist')
expected = set(Path(p) for p in ('dist/file', ))
assert set(Path(p) for p in source.glob_all('dist')) == expected
def test_copy_override_gitignore_with_include(cmdr, tmp_empty):
Path('file').touch()
Path('secrets.txt').touch()
Path('.gitignore').write_text('secrets.txt')
git_init()
source.copy(cmdr, '.', 'dist', include=['secrets.txt'])
expected = set(Path(p) for p in (
'dist/file',
'dist/secrets.txt',
))
assert set(Path(p) for p in source.glob_all('dist')) == expected
def build(e, cfg, name, until, skip_tests=False):
"""Build a docker image
Parameters
----------
e : Commander
Commander instance
cfg
Configuration
name : str
Target environment name
until : str
Stop after certain starge
skip_tests : bool, default=False
Skip image testing (check dag loading and File.client configuration)
"""
# if this is a pkg, get the name
try:
pkg_name = default.find_package_name()
# if not a package, use the parent folder's name
except ValueError:
pkg_name = Path('.').resolve().name
version = 'latest'
else:
# if using versioneer, the version may contain "+"
version = importlib.import_module(pkg_name).__version__.replace(
'+', '-plus-')
if Path('requirements.lock.txt').exists():
e.cp('requirements.lock.txt')
elif Path('environment.lock.yml').exists():
e.cp('environment.lock.yml')
else:
raise ClickException('Expected environment.lock.yml or '
'requirements.txt.lock at the root '
'directory. Add one and try again')
# generate source distribution
if Path('setup.py').exists():
# .egg-info may cause issues if MANIFEST.in was recently updated
e.rm('dist', 'build', Path('src', pkg_name, f'{pkg_name}.egg-info'))
e.run('python', '-m', 'build', '--sdist', description='Packaging code')
# raise error if include is not None? and suggest to use MANIFEST.in
# instead
else:
e.rm('dist')
target = Path('dist', pkg_name)
e.info('Packaging code')
source.copy(cmdr=e,
src='.',
dst=target,
include=cfg.include,
exclude=cfg.exclude)
source.compress_dir(target, Path('dist', f'{pkg_name}.tar.gz'))
e.cp('dist')
e.cd(name)
image_local = f'{pkg_name}:{version}'
# how to allow passing --no-cache?
e.run('docker',
'build',
'.',
'--tag',
image_local,
description='Building image')
if not skip_tests:
# test "ploomber status" in docker image
e.run('docker',
'run',
image_local,
'ploomber',
'status',
description='Testing image',
error_message='Error while testing your docker image with',
hint=f'Use "docker run -it {image_local} /bin/bash" to '
'start an interactive session to debug your image')
# check that the pipeline in the image has a configured File.client
test_cmd = ('from ploomber.spec import DAGSpec; '
'print("File" in DAGSpec.find().to_dag().clients)')
e.run('docker',
'run',
image_local,
'python',
'-c',
test_cmd,
description='Testing File client',
error_message='Missing File client',
hint=f'Run "docker run -it {image_local} /bin/bash" to '
'to debug your image. Ensure a File client is configured',
capture_output=True,
expected_output='True\n',
show_cmd=False)
if until == 'build':
raise CommanderStop('Done. Run "docker images" to see your image.')
# TODO: validate format of cfg.repository
if cfg.repository:
image_target = f'{cfg.repository}:{version}'
e.run('docker',
'tag',
image_local,
image_target,
description='Tagging')
e.run('docker', 'push', image_target, description='Pushing image')
else:
image_target = image_local
if until == 'push':
raise CommanderStop('Done. Image pushed to repository.')
return pkg_name, image_target