def build_lb_image_for_env(mock_config_with_repo):
# Create a labook
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenv",
description="Testing environment functions.")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def build_lb_image_for_env_conda(mock_config_with_repo):
"""A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda",
description="Testing environment functions.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}])
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def mutate_and_get_payload(cls, root, info, owner, labbook_name, no_cache=False, client_mutation_id=None):
username = get_logged_in_username()
if BuildImage.get_container_status(labbook_name, owner, username):
raise ValueError(f'Cannot build image for running container {owner}/{labbook_name}')
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Generate Dockerfile
# TODO BVB - Move to build_image ??
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
# Kick off building in a background thread
d = Dispatcher()
build_kwargs = {
'path': lb.root_dir,
'username': username,
'nocache': no_cache
}
metadata = {'labbook': lb.key,
'method': 'build_image'}
res = d.dispatch_task(jobs.build_labbook_image, kwargs=build_kwargs, metadata=metadata)
return BuildImage(environment=Environment(owner=owner, name=labbook_name),
background_job_key=res.key_str)
def test_build_and_start_and_stop_labbook_container(
self, mock_config_file):
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
image_id = jobs.build_labbook_image(**docker_kwargs)
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = jobs.start_labbook_container(**startc_kwargs)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
jobs.stop_labbook_container(container_id)
with pytest.raises(Exception):
# Should not be found because the stop job cleans up
get_docker_client().containers.get(container_id)
def test_list_versions_from_fallback(self, mock_config_with_repo):
"""Test list_versions command"""
username = "******"
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook(
'unittest',
'unittest',
'labbook-unittest-01',
description="From mock_config_from_repo fixture")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username=username)
# Test lookup
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
# Delete image
client.images.remove(docker_image_id, force=True, noprune=False)
# Test lookup still works
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
def test_bundled_app_lines(self, mock_labbook):
"""Test if the Dockerfile builds with bundled app ports"""
lb = mock_labbook[2]
bam = BundledAppManager(lb)
bam.add_bundled_app(8050, 'dash 1', 'a demo dash app 1',
'python app1.py')
bam.add_bundled_app(9000, 'dash 2', 'a demo dash app 2',
'python app2.py')
bam.add_bundled_app(9001, 'dash 3', 'a demo dash app 3',
'python app3.py')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{
"manager": "pip",
"package": "requests",
"version": "2.18.4"
}])
ib = ImageBuilder(lb)
dockerfile_text = ib.assemble_dockerfile(write=False)
test_lines = [
'# Bundled Application Ports', 'EXPOSE 8050', 'EXPOSE 9000',
'EXPOSE 9001'
]
docker_lines = dockerfile_text.split(os.linesep)
for line in test_lines:
assert line in docker_lines
def test_validate_dockerfile(self, mock_labbook):
"""Test if the Dockerfile builds and can launch the image. """
lb = mock_labbook[2]
populate_with_pkgs(lb)
package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env',
'package_manager')
with open(os.path.join(package_manager_dir, 'pip3_docker.yaml'),
'w') as apt_dep:
content = os.linesep.join([
'manager: pip3', 'package: docker', 'version: 2.0.1',
'from_base: false'
])
apt_dep.write(content)
with open(os.path.join(package_manager_dir, 'apt_docker.yaml'),
'w') as apt_dep:
content = os.linesep.join([
'manager: apt', 'package: docker', 'version: 1.2.3',
'from_base: false'
])
apt_dep.write(content)
with open(os.path.join(package_manager_dir, 'pip3_requests.yaml'),
'w') as apt_dep:
content = os.linesep.join([
'manager: pip3', 'package: requests', 'version: "2.18.4"',
'from_base: false'
])
apt_dep.write(content)
ib = ImageBuilder(lb)
n = ib.assemble_dockerfile(write=False)
with open(os.path.join(lb.root_dir, ".gigantum", "env", "Dockerfile"),
"w") as dockerfile:
dockerfile_text = ib.assemble_dockerfile(write=False)
dockerfile.write(dockerfile_text)
test_lines = [
'## Adding individual packages',
'RUN apt-get -y --no-install-recommends install docker',
'RUN pip install docker==2.0.1', 'RUN pip install requests==2.18.4'
]
for line in test_lines:
assert line in dockerfile_text.split(os.linesep)
def test_build_docker_image(self, temporary_worker, mock_config_file):
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
im = InventoryManager(mock_config_file[0])
lb = im.create_labbook('unittester',
'unittester',
'unittest-dispatcher-build-image',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, 'ut-busybox', 0)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(lb.root_dir, '.gigantum', 'env', 'Dockerfile'))
docker_kwargs = {'path': lb.root_dir, 'nocache': True}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs)
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
r = d.query_task(job_ref)
print(r.meta)
if elapsed_time > 2:
assert r.meta.get('feedback')
if status in ['success', 'failed', 'finished']:
print(r.exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
w.terminate()
res = d.query_task(job_ref)
assert res
assert res.status == 'finished'
def build_lb_image_for_jupyterlab(mock_config_with_repo):
with patch.object(Configuration, 'find_default_config', lambda self: mock_config_with_repo[0]):
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbook")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
ib = ImageBuilder(lb)
docker_lines = ib.assemble_dockerfile(write=True)
assert 'RUN pip install requests==2.18.4' in docker_lines
assert all(['==None' not in l for l in docker_lines.split()])
assert all(['=None' not in l for l in docker_lines.split()])
client = get_docker_client()
client.containers.prune()
assert os.path.exists(os.path.join(lb.root_dir, '.gigantum', 'env', 'entrypoint.sh'))
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
lb, container_id = ContainerOperations.start_container(lb, username="******")
assert isinstance(container_id, str)
yield lb, ib, client, docker_image_id, container_id, None, 'unittester'
try:
_, s = ContainerOperations.stop_container(labbook=lb, username="******")
except docker.errors.APIError:
client.containers.get(container_id=container_id).stop(timeout=2)
s = False
finally:
shutil.rmtree(lb.root_dir)
# Stop and remove container if it's still there
try:
client.containers.get(container_id=container_id).stop(timeout=2)
client.containers.get(container_id=container_id).remove()
except:
pass
# Remove image if it's still there
try:
ContainerOperations.delete_image(labbook=lb, username='******')
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def test_docker_snippet(self, mock_labbook):
lb = mock_labbook[2]
package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env', 'custom')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
custom = ['RUN true', 'RUN touch /tmp/cat', 'RUN rm /tmp/cat']
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
cm.add_docker_snippet('test-docker', custom, description="Apostrophe's and wėįrd çhårāčtêrś")
ib = ImageBuilder(lb)
l = ib.assemble_dockerfile()
assert all([any([i in l for i in custom]) for n in custom])
def test_success_import_export_zip(self, mock_config_with_repo):
# Create new LabBook to be exported
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester',
'unittester',
"unittest-lb-for-export-import-test",
description="Testing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile()
# Make sure the destination user exists locally
working_dir = lb.client_config.config['git']['working_directory']
os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2',
'labbooks'),
exist_ok=True)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
# Delete the labbook
shutil.rmtree(lb.root_dir)
assert not os.path.exists(
lb_root), f"LabBook at {lb_root} should not exist."
assert os.path.exists(tmp_archive_path)
# Now import the labbook as a new user, validating that the change of namespace works properly.
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=tmp_archive_path,
username='******',
owner='unittester2',
config_file=mock_config_with_repo[0])
assert not os.path.exists(tmp_archive_path)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
assert os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert imported_lb_path == lb_root.replace(
'/unittester/unittester/', '/unittester2/unittester2/')
import_lb = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
imported_lb_path)
ib = ImageBuilder(import_lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(imported_lb_path, '.gigantum', 'env',
'Dockerfile'))
assert not import_lb.has_remote
# Repeat the above, except with the original user (e.g., re-importing their own labbook)
user_import_lb = jobs.import_labboook_from_zip(
archive_path=tmp_archive_path,
username="******",
owner="unittester",
config_file=mock_config_with_repo[0])
assert not os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert user_import_lb
import_lb2 = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
user_import_lb)
# After importing, the new user (in this case "cat") should be the current, active workspace.
# And be created, if necessary.
assert not import_lb2.has_remote
build_kwargs = {
'path': lb.root_dir,
'username': '******',
'nocache': True
}
docker_image_id = jobs.build_labbook_image(**build_kwargs)
try:
client = get_docker_client()
client.images.remove(docker_image_id)
except Exception as e:
pprint.pprint(e)
raise
def test_success_import_export_lbk(self, mock_config_with_repo):
"""Test legacy .lbk extension still works"""
# Create new LabBook to be exported
lb = InventoryManager(mock_config_with_repo[0]).create_labbook(
'unittester',
'unittester',
"unittest-lb-for-export-import-test-lbk",
description="Testing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile()
# Make sure the destination user exists locally
working_dir = lb.client_config.config['git']['working_directory']
os.makedirs(os.path.join(working_dir, 'unittester2', 'unittester2',
'labbooks'),
exist_ok=True)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
lbk_archive_path = tmp_archive_path.replace(".zip", ".lbk")
lbk_archive_path = shutil.copy(tmp_archive_path, lbk_archive_path)
print(lbk_archive_path)
# Delete the labbook
shutil.rmtree(lb.root_dir)
assert not os.path.exists(
lb_root), f"LabBook at {lb_root} should not exist."
assert os.path.exists(tmp_archive_path)
# Now import the labbook as a new user, validating that the change of namespace works properly.
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=lbk_archive_path,
username='******',
owner='unittester2',
config_file=mock_config_with_repo[0])
assert not os.path.exists(lbk_archive_path)
tmp_archive_path = shutil.copy(exported_archive_path, '/tmp')
assert os.path.exists(tmp_archive_path)
# New path should reflect username of new owner and user.
assert imported_lb_path == lb_root.replace(
'/unittester/unittester/', '/unittester2/unittester2/')
import_lb = InventoryManager(
mock_config_with_repo[0]).load_labbook_from_directory(
imported_lb_path)
ib = ImageBuilder(import_lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(imported_lb_path, '.gigantum', 'env',
'Dockerfile'))
assert not import_lb.has_remote
def test_start_and_stop_docker_container(self, temporary_worker,
mock_config_file):
# start_docker_container(docker_image_id, exposed_ports, volumes_dict) -> str:
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
m = {'method': 'build_image', 'labbook': "unittest-start-stop-job"}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs,
metadata=m)
j = d.query_task(job_ref)
assert hasattr(j, 'meta')
assert j.meta.get('labbook') == "unittest-start-stop-job"
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
if status in ['success', 'failed', 'finished']:
print(d.query_task(job_ref).exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
res = d.query_task(job_ref)
assert res
print(res.status)
assert res.status == 'finished'
# Finish building image
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = bg_jobs.start_labbook_container(**startc_kwargs)
time.sleep(5)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
print(container_id)
bg_jobs.stop_labbook_container(container_id)
w.terminate()
def build_image_for_jupyterlab():
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# get environment data and index
erm = RepositoryManager(config_file)
erm.update_repositories()
erm.index_repositories()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
# Create a labook
im = InventoryManager(config_file)
lb = im.create_labbook('default',
'unittester',
"containerunittestbook",
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE,
ENV_UNIT_TEST_REV)
cm.add_packages("pip3", [{
"manager": "pip3",
"package": "requests",
"version": "2.18.4"
}])
bam = BundledAppManager(lb)
bam.add_bundled_app(9999, 'share', 'A bundled app for testing',
"cd /mnt; python3 -m http.server 9999")
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username="******")
# Note: The final field is the owner
yield lb, ib, docker_client, docker_image_id, client, "unittester"
finally:
try:
docker_client.containers.get(docker_image_id).stop()
docker_client.containers.get(docker_image_id).remove()
except:
pass
try:
docker_client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
shutil.rmtree(lb.root_dir)