def test_editable_by_host():
"""Test whether a new file created by the host environment, is
detected in the container"""
app = Repo2Docker()
app.initialize(['--editable', DIR])
app.run = False
app.start() # This just build the image and does not run it.
container = app.start_container()
# give the container a chance to start
time.sleep(1)
try:
with tempfile.NamedTemporaryFile(dir=DIR, prefix='testfile', suffix='.txt'):
status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt'])
assert status == 0
assert re.match(br'^testfile\w{8}\.txt\n$', output) is not None
# File should be removed in the container as well
status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt'])
assert status != 1
assert re.match(br'^testfile\w{8}\.txt\n$', output) is None
finally:
# stop the container
container.stop()
app.wait_for_container(container)
def test_clone_depth_full2():
"""Test a remote repository, with a refspec of the master commit hash"""
with TemporaryDirectory() as d:
app = Repo2Docker()
argv = ['--ref', '703322e', URL]
app.initialize(argv)
app.run = False
app.build = False
# turn of automatic clean up of the checkout so we can inspect it
# we also set the work directory explicitly so we know where to look
app.cleanup_checkout = False
app.git_workdir = d
app.start()
# Building the image has already put us in the cloned repository directory
cmd = ['git', 'rev-parse', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b'703322e9c6635ba1835d3b92eafbabeca0042c3e'
cmd = ['git', 'rev-list', '--count', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b'100'
with open(os.path.join(d, 'COMMIT')) as fp:
assert fp.read() == '100\n'
def __init__(self,
name,
path,
helm_substitution_path='jupyterhub.singleuser.image'):
"""
Create an Image from a local path
name: Fully qualified name of image
path: Absolute path to local directory with image contents
helm_substitution_path: Dot separated path in a helm file that should be populated with this image spec
Expects cwd to be inside the git repo we are operating in
"""
self.name = name
self.tag = gitutils.last_modified_commit(path)
self.path = path
self.helm_substitution_path = helm_substitution_path
self.image_spec = f'{self.name}:{self.tag}'
# Make r2d object here so we can use it to build & push
self.r2d = Repo2Docker()
self.r2d.subdir = self.path
self.r2d.output_image_spec = self.image_spec
self.r2d.user_id = 1000
self.r2d.user_name = 'jovyan'
self.r2d.target_repo_dir = '/srv/repo'
self.r2d.initialize()
def test_clone_depth_full2():
"""Test a remote repository, with a refspec of the master commit hash"""
with TemporaryDirectory() as d:
app = Repo2Docker(
repo=URL,
ref="703322e",
dry_run=True,
run=False,
# turn of automatic clean up of the checkout so we can inspect it
# we also set the work directory explicitly so we know where to look
cleanup_checkout=False,
git_workdir=d,
)
app.initialize()
app.start()
# Building the image has already put us in the cloned repository directory
cmd = ["git", "rev-parse", "HEAD"]
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b"703322e9c6635ba1835d3b92eafbabeca0042c3e"
cmd = ["git", "rev-list", "--count", "HEAD"]
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b"100"
with open(os.path.join(d, "COMMIT")) as fp:
assert fp.read() == "100\n"
def test_clone_depth_mid():
"""Test a remote repository, with a refspec of a commit hash halfway"""
with TemporaryDirectory() as d:
app = Repo2Docker()
argv = ['--ref', '8bc4f21', URL]
app.initialize(argv)
app.run = False
app.build = False
# turn of automatic clean up of the checkout so we can inspect it
# we also set the work directory explicitly so we know where to look
app.cleanup_checkout = False
app.git_workdir = d
app.start()
# Building the image has already put us in the cloned repository directory
cmd = ['git', 'rev-parse', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b'8bc4f216856f86f6fc25a788b744b93b87e9ba48'
cmd = ['git', 'rev-list', '--count', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b'50'
with open(os.path.join(d, 'COMMIT')) as fp:
assert fp.read() == '50\n'
def test_Repo2Docker_labels(ref, repo, expected_repo_label, tmpdir):
if repo is None:
repo = str(tmpdir)
if ref is not None:
argv = ['--ref', ref, repo]
else:
argv = [repo]
app = Repo2Docker()
# Add mock BuildPack to app
mock_buildpack = Mock()
mock_buildpack.return_value.labels = {}
app.buildpacks = [mock_buildpack]
app.initialize(argv)
app.build = False
app.run = False
app.start()
expected_labels = {
'repo2docker.ref': ref,
'repo2docker.repo': expected_repo_label,
'repo2docker.version': __version__,
}
assert mock_buildpack().labels == expected_labels
def test_clone_depth_mid():
"""Test a remote repository, with a refspec of a commit hash halfway"""
with TemporaryDirectory() as d:
app = Repo2Docker(
repo=URL,
ref="8bc4f21",
dry_run=True,
run=False,
# turn of automatic clean up of the checkout so we can inspect it
# we also set the work directory explicitly so we know where to look
cleanup_checkout=False,
git_workdir=d,
)
app.initialize()
app.start()
# Building the image has already put us in the cloned repository directory
cmd = ["git", "rev-parse", "HEAD"]
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b"8bc4f216856f86f6fc25a788b744b93b87e9ba48"
cmd = ["git", "rev-list", "--count", "HEAD"]
p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d)
assert p.stdout.strip() == b"50"
with open(os.path.join(d, "COMMIT")) as fp:
assert fp.read() == "50\n"
def __init__(self,
name,
path,
helm_substitution_path='jupyterhub.singleuser.image'):
"""
Create an Image from a local path
name: Fully qualified name of image
path: Absolute path to local directory with image contents
helm_substitution_path: Dot separated path in a helm file that should be populated with this image spec
Expects cwd to be inside the git repo we are operating in
"""
# name must not be empty
# FIXME: Validate name to conform to docker image name guidelines
if not name or name.strip() == '':
raise ValueError(
"Name of image to be built is not specified. Check hubploy.yaml of your deployment"
)
self.name = name
self.tag = utils.last_modified_commit(path)
self.path = path
self.helm_substitution_path = helm_substitution_path
self.image_spec = f'{self.name}:{self.tag}'
# Make r2d object here so we can use it to build & push
self.r2d = Repo2Docker()
self.r2d.subdir = self.path
self.r2d.output_image_spec = self.image_spec
self.r2d.user_id = 1000
self.r2d.user_name = 'jovyan'
self.r2d.target_repo_dir = '/srv/repo'
self.r2d.initialize()
async def start(self):
if self.repo is None:
raise ValueError("Repo2DockerSpawner.repo must be set")
resolved_ref = await resolve_ref(self.repo, self.ref)
repo_escaped = escape(self.repo, escape_char='-').lower()
image_spec = f'r2dspawner-{repo_escaped}:{resolved_ref}'
image_info = await self.inspect_image(image_spec)
if not image_info:
self.log.info(f'Image {image_spec} not present, building...')
r2d = Repo2Docker()
r2d.repo = self.repo
r2d.ref = resolved_ref
r2d.user_id = 1000
r2d.user_name = 'jovyan'
r2d.output_image_spec = image_spec
r2d.initialize()
await self.run_in_executor(r2d.build)
# HACK: DockerSpawner (and traitlets) don't seem to realize we're setting 'cmd',
# and refuse to use our custom command. Explicitly set this variable for
# now.
self._user_set_cmd = True
self.log.info(
f'Launching with image {image_spec} for {self.user.name}')
self.image = image_spec
return await super().start()
def test_subdir_in_image_name():
app = Repo2Docker(repo=TEST_REPO, subdir="a directory")
app.initialize()
app.build()
escaped_dirname = escapism.escape("a directory", escape_char="-").lower()
assert escaped_dirname in app.output_image_spec
def build_repo2docker(client, path, image_spec):
ENTRYPOINT = '''\
#!/bin/bash
cp --archive --recursive --no-clobber /srv/home/ -T ${HOME}
if [ -x binder/start ]; then
exec binder/start "$@"
else
exec "$@"
fi
'''
NEWLINE = '\n'
ESCAPED_NEWLINE = '\\n'
APPENDIX = f'''
USER root
RUN rm -rf .npm .cache && cp -ra ${{HOME}} /srv/home
RUN printf '{ENTRYPOINT.replace(NEWLINE, ESCAPED_NEWLINE)}' > /entrypoint.sh && chmod +x /entrypoint.sh
USER ${{NB_USER}}
ENTRYPOINT ["/entrypoint.sh"]
'''
from repo2docker.app import Repo2Docker
builder = Repo2Docker()
builder.initialize(['--subdir', path, '--image-name', image_spec,
'--no-run',
'--user-name', 'jovyan',
'--user-id', '1000',
'--appendix', APPENDIX,
'.',])
builder.start()
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument(
'image', help='Image to build. Subdirectory with this name must exist')
argparser.add_argument(
'--image-prefix',
help=
'Prefix for image to be built. Usually contains registry url and name',
default='pangeo/')
argparser.add_argument('--push',
help='Push the built image to the docker registry',
action='store_true',
default=False)
args = argparser.parse_args()
image_name = f'{args.image_prefix}{args.image}'
print(f'Building {image_name}')
client = docker.from_env()
cache_from = []
# Pull the most recent built image available for this docker image
# We can re-use the cache from that, significantly speeding up
# our image builds
for i in range(1, 100):
date = modified_date(i, '.')
# Stick to UTC for calver
existing_calver = date.astimezone(pytz.utc).strftime('%Y.%m.%d')
existing_image_spec = f'{image_name}:{existing_calver}'
if image_exists_in_registry(client, existing_image_spec):
print(f'Re-using cache from {existing_image_spec}')
cache_from = [existing_image_spec]
subprocess.check_call(['docker', 'pull', existing_image_spec])
break
calver = datetime.utcnow().strftime('%Y.%m.%d')
r2d = Repo2Docker()
r2d.subdir = args.image
r2d.output_image_spec = f'{image_name}:{calver}'
r2d.user_id = 1000
r2d.user_name = 'jovyan'
r2d.cache_from = cache_from
with open('appendix.txt') as f:
r2d.appendix = f.read()
r2d.initialize()
r2d.build()
if os.path.exists(os.path.join(r2d.subdir, 'binder/verify')):
print(f'Validating {image_name}')
# Validate the built image
subprocess.check_call([
'docker', 'run', '-i', '-t', f'{r2d.output_image_spec}',
'binder/verify'
])
else:
print(f'No verify script found for {image_name}')
def test_dryrun_works_without_docker(tmpdir, capsys):
with chdir(tmpdir):
with patch.object(DockerCLI, "__init__") as client:
client.side_effect = docker.errors.DockerException("Error: no Docker")
app = Repo2Docker(dry_run=True)
app.build()
captured = capsys.readouterr()
assert "Error: no Docker" not in captured.err
def test_root_not_allowed():
with TemporaryDirectory() as src, patch("os.geteuid") as geteuid:
geteuid.return_value = 0
argv = [src]
with pytest.raises(SystemExit) as exc:
app = make_r2d(argv)
assert exc.code == 1
with pytest.raises(ValueError):
app = Repo2Docker(repo=src, run=False)
app.build()
app = Repo2Docker(repo=src, user_id=1000, user_name="jovyan", run=False)
app.initialize()
with patch.object(DockerCLI, "build") as builds:
builds.return_value = []
app.build()
builds.assert_called_once()
def test_subdir_invalid(caplog):
# test an error is raised when requesting a non existent subdir
app = Repo2Docker(
repo=TEST_REPO,
subdir='invalid-sub-dir',
)
app.initialize()
with pytest.raises(FileNotFoundError):
app.build() # Just build the image and do not run it.
def test_error_log_without_docker(tmpdir, capsys):
with chdir(tmpdir):
with patch.object(DockerCLI, "__init__") as client:
client.side_effect = docker.errors.DockerException("Error: no Docker")
app = Repo2Docker()
with pytest.raises(SystemExit):
app.build()
captured = capsys.readouterr()
assert "Error: no Docker" in captured.err
def test_local_dir_image_name(repo_with_content):
upstream, sha1 = repo_with_content
app = Repo2Docker()
argv = ["--no-build", upstream]
app = make_r2d(argv)
app.start()
assert app.output_image_spec.startswith(
"r2d" + escapism.escape(upstream, escape_char="-").lower())
def test_image_name_remains_unchanged():
# if we specify an image name, it should remain unmodified
with TemporaryDirectory() as src:
app = Repo2Docker()
argv = ["--image-name", "a-special-name", "--no-build", src]
app = make_r2d(argv)
app.start()
assert app.output_image_spec == "a-special-name"
def r2d_build(image, image_spec, cache_from=None):
r2d = Repo2Docker()
r2d.subdir = image
r2d.output_image_spec = image_spec
r2d.user_id = 1000
r2d.user_name = 'jovyan'
if cache_from:
r2d.cache_from = [cache_from]
r2d.initialize()
r2d.build()
def test_image_name_contains_sha1(repo_with_content):
upstream, sha1 = repo_with_content
app = Repo2Docker()
# force selection of the git content provider by prefixing path with
# file://. This is important as the Local content provider does not
# store the SHA1 in the repo spec
argv = ["--no-build", "file://" + upstream]
app = make_r2d(argv)
app.start()
assert app.output_image_spec.endswith(sha1[:7])
def test_dont_find_image():
images = [{"RepoTags": ["some-org/some-image-name:latest"]}]
with patch("repo2docker.app.docker.APIClient") as FakeDockerClient:
instance = FakeDockerClient.return_value
instance.images.return_value = images
r2d = Repo2Docker()
r2d.output_image_spec = "some-org/some-other-image-name"
assert not r2d.find_image()
instance.images.assert_called_with()
def test_find_image():
images = [{'RepoTags': ['some-org/some-repo:latest']}]
with patch('repo2docker.app.docker.APIClient') as FakeDockerClient:
instance = FakeDockerClient.return_value
instance.images.return_value = images
r2d = Repo2Docker()
r2d.output_image_spec = 'some-org/some-repo'
assert r2d.find_image()
instance.images.assert_called_with()
def build_image(client, path, image_spec, cache_from=None, push=False):
r2d = Repo2Docker()
r2d.subdir = path
r2d.output_image_spec = image_spec
r2d.user_id = 1000
r2d.user_name = 'jovyan'
if cache_from:
r2d.cache_from = cache_from
r2d.initialize()
r2d.build()
if push:
r2d.push_image()
def test_subdir_invalid(caplog):
caplog.set_level(logging.INFO, logger='Repo2Docker')
app = Repo2Docker()
argv = ['--subdir', 'tests/conda/invalid', repo_path]
app.initialize(argv)
app.debug = True
app.run = False
with pytest.raises(SystemExit) as excinfo:
app.start() # Just build the image and do not run it.
# The build should fail
assert excinfo.value.code == 1
def __post_init__(self) -> None:
self.cache_dir = pathlib.Path(
_config.get('cache_dir')
or pathlib.Path.home() / '.singularity/cache').resolve()
self.cache_dir.mkdir(parents=True, exist_ok=True)
self.r2d = Repo2Docker()
self.r2d.repo = self.repo
self.r2d.ref = self.ref
self.r2d.output_image_spec = generate_image_name(
self.repo, self.r2d.ref)
self.sif_image = self.cache_dir / f'{self.r2d.output_image_spec}.sif'
self.apptainer_image = (
f"{_config.get('apptainer_in_docker.image')}:{_config.get('apptainer_in_docker.tag')}"
)
def test_subdir_invalid(caplog):
# test an error is raised when requesting a non existent subdir
#caplog.set_level(logging.INFO, logger='Repo2Docker')
app = Repo2Docker()
argv = ['--subdir', 'invalid-sub-dir', TEST_REPO]
app.initialize(argv)
app.debug = True
# no build does not imply no run
app.build = False
app.run = False
with pytest.raises(SystemExit) as excinfo:
app.start() # Just build the image and do not run it.
# The build should fail
assert excinfo.value.code == 1
def test_connect_url(tmpdir):
tmpdir.chdir()
#q = tmpdir.join("environment.yml")
#q.write("dependencies:\n"
# " - notebook==5.6.0")
p = tmpdir.join("requirements.txt")
p.write("notebook==5.6.0")
app = Repo2Docker()
argv = [
str(tmpdir),
]
app.initialize(argv)
app.debug = True
app.run = False
app.start() # This just build the image and does not run it.
container = app.start_container()
container_url = 'http://{}:{}/api'.format(app.hostname, app.port)
expected_url = 'http://{}:{}'.format(app.hostname, app.port)
# wait a bit for the container to be ready
# give the container a chance to start
time.sleep(1)
try:
# try a few times to connect
success = False
for i in range(1, 4):
container.reload()
assert container.status == 'running'
if expected_url not in container.logs().decode("utf8"):
time.sleep(i * 3)
continue
try:
info = requests.get(container_url).json()
except Exception as e:
print("Error: %s" % e)
time.sleep(i * 3)
else:
print(info)
success = True
break
assert success, "Notebook never started in %s" % container
finally:
# stop the container
container.stop()
app.wait_for_container(container)
def create_docker_image(gitlab_repo, commit_hash, image_name):
# Create Docker image from git repository using jupyter-repo2docker
r2d = Repo2Docker()
r2d.repo = f"https://*****:*****@{settings.GIT_HOST}/{gitlab_repo}.git"
r2d.ref = commit_hash
r2d.output_image_spec = image_name = '/'.join(
(settings.DOCKER_REGISTRY_HOST, image_name))
r2d.user_id = 2000
r2d.user_name = 'jovyan'
r2d.buildpacks = buildpacks
r2d.initialize()
r2d.build()
run_command = r2d.picked_buildpack.get_command()
return image_name, run_command
def test_connect_url(tmpdir):
tmpdir.chdir()
p = tmpdir.join("Dockerfile")
p.write(DOCKER_FILE)
# we set run=False so that we can start the container ourselves and
# get a handle to the container, used to inspect the logs
app = Repo2Docker(repo=str(tmpdir), run=False)
app.initialize()
app.start()
container = app.start_container()
container_url = "http://{}:{}/api".format(app.hostname, app.port)
expected_url = "http://{}:{}".format(app.hostname, app.port)
# wait a bit for the container to be ready
# give the container a chance to start
time.sleep(1)
try:
# try a few times to connect
success = False
for i in range(1, 4):
container.reload()
assert container.status == "running"
if expected_url not in container.logs().decode("utf8"):
time.sleep(i * 3)
continue
try:
info = requests.get(container_url).json()
except Exception as e:
print("Error: %s" % e)
time.sleep(i * 3)
else:
print(info)
success = True
break
assert success, "Notebook never started in %s" % container
finally:
# stop the container
container.stop()
app.wait_for_container(container)
def test_clone_depth_full():
"""Test a remote repository, with a refspec of 'master'"""
app = Repo2Docker()
argv = ['--ref', 'master', URL]
app.initialize(argv)
app.debug = True
app.run = False
app.cleanup_checkout = False
app.start() # This just build the image and does not run it.
# Building the image has already put us in the cloned repository directory
cmd = ['git', 'rev-parse', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE)
assert p.stdout.strip() == b'703322e9c6635ba1835d3b92eafbabeca0042c3e'
cmd = ['git', 'rev-list', '--count', 'HEAD']
p = subprocess.run(cmd, stdout=subprocess.PIPE)
assert p.stdout.strip() == b'100'
with open('COMMIT') as fp:
assert fp.read() == '100\n'
