def test_editable_by_host(): """Test whether a new file created by the host environment, is detected in the container""" app = Repo2Docker() app.initialize(['--editable', DIR]) app.run = False app.start() # This just build the image and does not run it. container = app.start_container() # give the container a chance to start time.sleep(1) try: with tempfile.NamedTemporaryFile(dir=DIR, prefix='testfile', suffix='.txt'): status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt']) assert status == 0 assert re.match(br'^testfile\w{8}\.txt\n$', output) is not None # File should be removed in the container as well status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt']) assert status != 1 assert re.match(br'^testfile\w{8}\.txt\n$', output) is None finally: # stop the container container.stop() app.wait_for_container(container)
def test_clone_depth_full2(): """Test a remote repository, with a refspec of the master commit hash""" with TemporaryDirectory() as d: app = Repo2Docker() argv = ['--ref', '703322e', URL] app.initialize(argv) app.run = False app.build = False # turn of automatic clean up of the checkout so we can inspect it # we also set the work directory explicitly so we know where to look app.cleanup_checkout = False app.git_workdir = d app.start() # Building the image has already put us in the cloned repository directory cmd = ['git', 'rev-parse', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b'703322e9c6635ba1835d3b92eafbabeca0042c3e' cmd = ['git', 'rev-list', '--count', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b'100' with open(os.path.join(d, 'COMMIT')) as fp: assert fp.read() == '100\n'
def __init__(self, name, path, helm_substitution_path='jupyterhub.singleuser.image'): """ Create an Image from a local path name: Fully qualified name of image path: Absolute path to local directory with image contents helm_substitution_path: Dot separated path in a helm file that should be populated with this image spec Expects cwd to be inside the git repo we are operating in """ self.name = name self.tag = gitutils.last_modified_commit(path) self.path = path self.helm_substitution_path = helm_substitution_path self.image_spec = f'{self.name}:{self.tag}' # Make r2d object here so we can use it to build & push self.r2d = Repo2Docker() self.r2d.subdir = self.path self.r2d.output_image_spec = self.image_spec self.r2d.user_id = 1000 self.r2d.user_name = 'jovyan' self.r2d.target_repo_dir = '/srv/repo' self.r2d.initialize()
def test_clone_depth_full2(): """Test a remote repository, with a refspec of the master commit hash""" with TemporaryDirectory() as d: app = Repo2Docker( repo=URL, ref="703322e", dry_run=True, run=False, # turn of automatic clean up of the checkout so we can inspect it # we also set the work directory explicitly so we know where to look cleanup_checkout=False, git_workdir=d, ) app.initialize() app.start() # Building the image has already put us in the cloned repository directory cmd = ["git", "rev-parse", "HEAD"] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b"703322e9c6635ba1835d3b92eafbabeca0042c3e" cmd = ["git", "rev-list", "--count", "HEAD"] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b"100" with open(os.path.join(d, "COMMIT")) as fp: assert fp.read() == "100\n"
def test_clone_depth_mid(): """Test a remote repository, with a refspec of a commit hash halfway""" with TemporaryDirectory() as d: app = Repo2Docker() argv = ['--ref', '8bc4f21', URL] app.initialize(argv) app.run = False app.build = False # turn of automatic clean up of the checkout so we can inspect it # we also set the work directory explicitly so we know where to look app.cleanup_checkout = False app.git_workdir = d app.start() # Building the image has already put us in the cloned repository directory cmd = ['git', 'rev-parse', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b'8bc4f216856f86f6fc25a788b744b93b87e9ba48' cmd = ['git', 'rev-list', '--count', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b'50' with open(os.path.join(d, 'COMMIT')) as fp: assert fp.read() == '50\n'
def test_Repo2Docker_labels(ref, repo, expected_repo_label, tmpdir): if repo is None: repo = str(tmpdir) if ref is not None: argv = ['--ref', ref, repo] else: argv = [repo] app = Repo2Docker() # Add mock BuildPack to app mock_buildpack = Mock() mock_buildpack.return_value.labels = {} app.buildpacks = [mock_buildpack] app.initialize(argv) app.build = False app.run = False app.start() expected_labels = { 'repo2docker.ref': ref, 'repo2docker.repo': expected_repo_label, 'repo2docker.version': __version__, } assert mock_buildpack().labels == expected_labels
def test_clone_depth_mid(): """Test a remote repository, with a refspec of a commit hash halfway""" with TemporaryDirectory() as d: app = Repo2Docker( repo=URL, ref="8bc4f21", dry_run=True, run=False, # turn of automatic clean up of the checkout so we can inspect it # we also set the work directory explicitly so we know where to look cleanup_checkout=False, git_workdir=d, ) app.initialize() app.start() # Building the image has already put us in the cloned repository directory cmd = ["git", "rev-parse", "HEAD"] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b"8bc4f216856f86f6fc25a788b744b93b87e9ba48" cmd = ["git", "rev-list", "--count", "HEAD"] p = subprocess.run(cmd, stdout=subprocess.PIPE, cwd=d) assert p.stdout.strip() == b"50" with open(os.path.join(d, "COMMIT")) as fp: assert fp.read() == "50\n"
def __init__(self, name, path, helm_substitution_path='jupyterhub.singleuser.image'): """ Create an Image from a local path name: Fully qualified name of image path: Absolute path to local directory with image contents helm_substitution_path: Dot separated path in a helm file that should be populated with this image spec Expects cwd to be inside the git repo we are operating in """ # name must not be empty # FIXME: Validate name to conform to docker image name guidelines if not name or name.strip() == '': raise ValueError( "Name of image to be built is not specified. Check hubploy.yaml of your deployment" ) self.name = name self.tag = utils.last_modified_commit(path) self.path = path self.helm_substitution_path = helm_substitution_path self.image_spec = f'{self.name}:{self.tag}' # Make r2d object here so we can use it to build & push self.r2d = Repo2Docker() self.r2d.subdir = self.path self.r2d.output_image_spec = self.image_spec self.r2d.user_id = 1000 self.r2d.user_name = 'jovyan' self.r2d.target_repo_dir = '/srv/repo' self.r2d.initialize()
async def start(self): if self.repo is None: raise ValueError("Repo2DockerSpawner.repo must be set") resolved_ref = await resolve_ref(self.repo, self.ref) repo_escaped = escape(self.repo, escape_char='-').lower() image_spec = f'r2dspawner-{repo_escaped}:{resolved_ref}' image_info = await self.inspect_image(image_spec) if not image_info: self.log.info(f'Image {image_spec} not present, building...') r2d = Repo2Docker() r2d.repo = self.repo r2d.ref = resolved_ref r2d.user_id = 1000 r2d.user_name = 'jovyan' r2d.output_image_spec = image_spec r2d.initialize() await self.run_in_executor(r2d.build) # HACK: DockerSpawner (and traitlets) don't seem to realize we're setting 'cmd', # and refuse to use our custom command. Explicitly set this variable for # now. self._user_set_cmd = True self.log.info( f'Launching with image {image_spec} for {self.user.name}') self.image = image_spec return await super().start()
def test_subdir_in_image_name(): app = Repo2Docker(repo=TEST_REPO, subdir="a directory") app.initialize() app.build() escaped_dirname = escapism.escape("a directory", escape_char="-").lower() assert escaped_dirname in app.output_image_spec
def build_repo2docker(client, path, image_spec): ENTRYPOINT = '''\ #!/bin/bash cp --archive --recursive --no-clobber /srv/home/ -T ${HOME} if [ -x binder/start ]; then exec binder/start "$@" else exec "$@" fi ''' NEWLINE = '\n' ESCAPED_NEWLINE = '\\n' APPENDIX = f''' USER root RUN rm -rf .npm .cache && cp -ra ${{HOME}} /srv/home RUN printf '{ENTRYPOINT.replace(NEWLINE, ESCAPED_NEWLINE)}' > /entrypoint.sh && chmod +x /entrypoint.sh USER ${{NB_USER}} ENTRYPOINT ["/entrypoint.sh"] ''' from repo2docker.app import Repo2Docker builder = Repo2Docker() builder.initialize(['--subdir', path, '--image-name', image_spec, '--no-run', '--user-name', 'jovyan', '--user-id', '1000', '--appendix', APPENDIX, '.',]) builder.start()
def main(): argparser = argparse.ArgumentParser() argparser.add_argument( 'image', help='Image to build. Subdirectory with this name must exist') argparser.add_argument( '--image-prefix', help= 'Prefix for image to be built. Usually contains registry url and name', default='pangeo/') argparser.add_argument('--push', help='Push the built image to the docker registry', action='store_true', default=False) args = argparser.parse_args() image_name = f'{args.image_prefix}{args.image}' print(f'Building {image_name}') client = docker.from_env() cache_from = [] # Pull the most recent built image available for this docker image # We can re-use the cache from that, significantly speeding up # our image builds for i in range(1, 100): date = modified_date(i, '.') # Stick to UTC for calver existing_calver = date.astimezone(pytz.utc).strftime('%Y.%m.%d') existing_image_spec = f'{image_name}:{existing_calver}' if image_exists_in_registry(client, existing_image_spec): print(f'Re-using cache from {existing_image_spec}') cache_from = [existing_image_spec] subprocess.check_call(['docker', 'pull', existing_image_spec]) break calver = datetime.utcnow().strftime('%Y.%m.%d') r2d = Repo2Docker() r2d.subdir = args.image r2d.output_image_spec = f'{image_name}:{calver}' r2d.user_id = 1000 r2d.user_name = 'jovyan' r2d.cache_from = cache_from with open('appendix.txt') as f: r2d.appendix = f.read() r2d.initialize() r2d.build() if os.path.exists(os.path.join(r2d.subdir, 'binder/verify')): print(f'Validating {image_name}') # Validate the built image subprocess.check_call([ 'docker', 'run', '-i', '-t', f'{r2d.output_image_spec}', 'binder/verify' ]) else: print(f'No verify script found for {image_name}')
def test_dryrun_works_without_docker(tmpdir, capsys): with chdir(tmpdir): with patch.object(DockerCLI, "__init__") as client: client.side_effect = docker.errors.DockerException("Error: no Docker") app = Repo2Docker(dry_run=True) app.build() captured = capsys.readouterr() assert "Error: no Docker" not in captured.err
def test_root_not_allowed(): with TemporaryDirectory() as src, patch("os.geteuid") as geteuid: geteuid.return_value = 0 argv = [src] with pytest.raises(SystemExit) as exc: app = make_r2d(argv) assert exc.code == 1 with pytest.raises(ValueError): app = Repo2Docker(repo=src, run=False) app.build() app = Repo2Docker(repo=src, user_id=1000, user_name="jovyan", run=False) app.initialize() with patch.object(DockerCLI, "build") as builds: builds.return_value = [] app.build() builds.assert_called_once()
def test_subdir_invalid(caplog): # test an error is raised when requesting a non existent subdir app = Repo2Docker( repo=TEST_REPO, subdir='invalid-sub-dir', ) app.initialize() with pytest.raises(FileNotFoundError): app.build() # Just build the image and do not run it.
def test_error_log_without_docker(tmpdir, capsys): with chdir(tmpdir): with patch.object(DockerCLI, "__init__") as client: client.side_effect = docker.errors.DockerException("Error: no Docker") app = Repo2Docker() with pytest.raises(SystemExit): app.build() captured = capsys.readouterr() assert "Error: no Docker" in captured.err
def test_local_dir_image_name(repo_with_content): upstream, sha1 = repo_with_content app = Repo2Docker() argv = ["--no-build", upstream] app = make_r2d(argv) app.start() assert app.output_image_spec.startswith( "r2d" + escapism.escape(upstream, escape_char="-").lower())
def test_image_name_remains_unchanged(): # if we specify an image name, it should remain unmodified with TemporaryDirectory() as src: app = Repo2Docker() argv = ["--image-name", "a-special-name", "--no-build", src] app = make_r2d(argv) app.start() assert app.output_image_spec == "a-special-name"
def r2d_build(image, image_spec, cache_from=None): r2d = Repo2Docker() r2d.subdir = image r2d.output_image_spec = image_spec r2d.user_id = 1000 r2d.user_name = 'jovyan' if cache_from: r2d.cache_from = [cache_from] r2d.initialize() r2d.build()
def test_image_name_contains_sha1(repo_with_content): upstream, sha1 = repo_with_content app = Repo2Docker() # force selection of the git content provider by prefixing path with # file://. This is important as the Local content provider does not # store the SHA1 in the repo spec argv = ["--no-build", "file://" + upstream] app = make_r2d(argv) app.start() assert app.output_image_spec.endswith(sha1[:7])
def test_dont_find_image(): images = [{"RepoTags": ["some-org/some-image-name:latest"]}] with patch("repo2docker.app.docker.APIClient") as FakeDockerClient: instance = FakeDockerClient.return_value instance.images.return_value = images r2d = Repo2Docker() r2d.output_image_spec = "some-org/some-other-image-name" assert not r2d.find_image() instance.images.assert_called_with()
def test_find_image(): images = [{'RepoTags': ['some-org/some-repo:latest']}] with patch('repo2docker.app.docker.APIClient') as FakeDockerClient: instance = FakeDockerClient.return_value instance.images.return_value = images r2d = Repo2Docker() r2d.output_image_spec = 'some-org/some-repo' assert r2d.find_image() instance.images.assert_called_with()
def build_image(client, path, image_spec, cache_from=None, push=False): r2d = Repo2Docker() r2d.subdir = path r2d.output_image_spec = image_spec r2d.user_id = 1000 r2d.user_name = 'jovyan' if cache_from: r2d.cache_from = cache_from r2d.initialize() r2d.build() if push: r2d.push_image()
def test_subdir_invalid(caplog): caplog.set_level(logging.INFO, logger='Repo2Docker') app = Repo2Docker() argv = ['--subdir', 'tests/conda/invalid', repo_path] app.initialize(argv) app.debug = True app.run = False with pytest.raises(SystemExit) as excinfo: app.start() # Just build the image and do not run it. # The build should fail assert excinfo.value.code == 1
def __post_init__(self) -> None: self.cache_dir = pathlib.Path( _config.get('cache_dir') or pathlib.Path.home() / '.singularity/cache').resolve() self.cache_dir.mkdir(parents=True, exist_ok=True) self.r2d = Repo2Docker() self.r2d.repo = self.repo self.r2d.ref = self.ref self.r2d.output_image_spec = generate_image_name( self.repo, self.r2d.ref) self.sif_image = self.cache_dir / f'{self.r2d.output_image_spec}.sif' self.apptainer_image = ( f"{_config.get('apptainer_in_docker.image')}:{_config.get('apptainer_in_docker.tag')}" )
def test_subdir_invalid(caplog): # test an error is raised when requesting a non existent subdir #caplog.set_level(logging.INFO, logger='Repo2Docker') app = Repo2Docker() argv = ['--subdir', 'invalid-sub-dir', TEST_REPO] app.initialize(argv) app.debug = True # no build does not imply no run app.build = False app.run = False with pytest.raises(SystemExit) as excinfo: app.start() # Just build the image and do not run it. # The build should fail assert excinfo.value.code == 1
def test_connect_url(tmpdir): tmpdir.chdir() #q = tmpdir.join("environment.yml") #q.write("dependencies:\n" # " - notebook==5.6.0") p = tmpdir.join("requirements.txt") p.write("notebook==5.6.0") app = Repo2Docker() argv = [ str(tmpdir), ] app.initialize(argv) app.debug = True app.run = False app.start() # This just build the image and does not run it. container = app.start_container() container_url = 'http://{}:{}/api'.format(app.hostname, app.port) expected_url = 'http://{}:{}'.format(app.hostname, app.port) # wait a bit for the container to be ready # give the container a chance to start time.sleep(1) try: # try a few times to connect success = False for i in range(1, 4): container.reload() assert container.status == 'running' if expected_url not in container.logs().decode("utf8"): time.sleep(i * 3) continue try: info = requests.get(container_url).json() except Exception as e: print("Error: %s" % e) time.sleep(i * 3) else: print(info) success = True break assert success, "Notebook never started in %s" % container finally: # stop the container container.stop() app.wait_for_container(container)
def create_docker_image(gitlab_repo, commit_hash, image_name): # Create Docker image from git repository using jupyter-repo2docker r2d = Repo2Docker() r2d.repo = f"https://*****:*****@{settings.GIT_HOST}/{gitlab_repo}.git" r2d.ref = commit_hash r2d.output_image_spec = image_name = '/'.join( (settings.DOCKER_REGISTRY_HOST, image_name)) r2d.user_id = 2000 r2d.user_name = 'jovyan' r2d.buildpacks = buildpacks r2d.initialize() r2d.build() run_command = r2d.picked_buildpack.get_command() return image_name, run_command
def test_connect_url(tmpdir): tmpdir.chdir() p = tmpdir.join("Dockerfile") p.write(DOCKER_FILE) # we set run=False so that we can start the container ourselves and # get a handle to the container, used to inspect the logs app = Repo2Docker(repo=str(tmpdir), run=False) app.initialize() app.start() container = app.start_container() container_url = "http://{}:{}/api".format(app.hostname, app.port) expected_url = "http://{}:{}".format(app.hostname, app.port) # wait a bit for the container to be ready # give the container a chance to start time.sleep(1) try: # try a few times to connect success = False for i in range(1, 4): container.reload() assert container.status == "running" if expected_url not in container.logs().decode("utf8"): time.sleep(i * 3) continue try: info = requests.get(container_url).json() except Exception as e: print("Error: %s" % e) time.sleep(i * 3) else: print(info) success = True break assert success, "Notebook never started in %s" % container finally: # stop the container container.stop() app.wait_for_container(container)
def test_clone_depth_full(): """Test a remote repository, with a refspec of 'master'""" app = Repo2Docker() argv = ['--ref', 'master', URL] app.initialize(argv) app.debug = True app.run = False app.cleanup_checkout = False app.start() # This just build the image and does not run it. # Building the image has already put us in the cloned repository directory cmd = ['git', 'rev-parse', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE) assert p.stdout.strip() == b'703322e9c6635ba1835d3b92eafbabeca0042c3e' cmd = ['git', 'rev-list', '--count', 'HEAD'] p = subprocess.run(cmd, stdout=subprocess.PIPE) assert p.stdout.strip() == b'100' with open('COMMIT') as fp: assert fp.read() == '100\n'