def test_clone_repositories_branch(self):
"""Test cloning a branch"""
conf_file, working_dir = _create_temp_work_dir(
override_dict={
'environment': {
'repo_url': [
"https://github.com/gigantum/base-images-testing.git@test-branch-DONOTDELETE"
]
}
})
# Run clone and index operation
erm = RepositoryManager(conf_file)
erm.update_repositories()
assert os.path.exists(os.path.join(working_dir, ".labmanager")) is True
assert os.path.exists(
os.path.join(working_dir, ".labmanager",
"environment_repositories")) is True
assert os.path.exists(
os.path.join(working_dir, ".labmanager",
"environment_repositories",
ENV_UNIT_TEST_REPO)) is True
assert os.path.exists(
os.path.join(working_dir, ".labmanager",
"environment_repositories", ENV_UNIT_TEST_REPO,
"README.md")) is True
r = Repo(
os.path.join(working_dir, ".labmanager",
"environment_repositories", ENV_UNIT_TEST_REPO))
assert r.active_branch.name == "test-branch-DONOTDELETE"
def test_bundled_app_lines(self, mock_labbook):
"""Test if the Dockerfile builds with bundled app ports"""
lb = mock_labbook[2]
bam = BundledAppManager(lb)
bam.add_bundled_app(8050, 'dash 1', 'a demo dash app 1',
'python app1.py')
bam.add_bundled_app(9000, 'dash 2', 'a demo dash app 2',
'python app2.py')
bam.add_bundled_app(9001, 'dash 3', 'a demo dash app 3',
'python app3.py')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{
"manager": "pip",
"package": "requests",
"version": "2.18.4"
}])
ib = ImageBuilder(lb)
dockerfile_text = ib.assemble_dockerfile(write=False)
test_lines = [
'# Bundled Application Ports', 'EXPOSE 8050', 'EXPOSE 9000',
'EXPOSE 9001'
]
docker_lines = dockerfile_text.split(os.linesep)
for line in test_lines:
assert line in docker_lines
def mock_config_with_repo():
"""A pytest fixture that creates a temporary directory and a config file to match. Deletes directory after test"""
conf_file, working_dir = _create_temp_work_dir()
erm = RepositoryManager(conf_file)
erm.update_repositories()
erm.index_repositories()
yield conf_file, working_dir
shutil.rmtree(working_dir)
def test_build_and_start_and_stop_labbook_container(
self, mock_config_file):
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
image_id = jobs.build_labbook_image(**docker_kwargs)
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = jobs.start_labbook_container(**startc_kwargs)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
jobs.stop_labbook_container(container_id)
with pytest.raises(Exception):
# Should not be found because the stop job cleans up
get_docker_client().containers.get(container_id)
def test_docker_snippet(self, mock_labbook):
lb = mock_labbook[2]
package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env', 'custom')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
custom = ['RUN true', 'RUN touch /tmp/cat', 'RUN rm /tmp/cat']
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
cm.add_docker_snippet('test-docker', custom, description="Apostrophe's and wėįrd çhårāčtêrś")
ib = ImageBuilder(lb)
l = ib.assemble_dockerfile()
assert all([any([i in l for i in custom]) for n in custom])
def setup_index():
"""A pytest fixture that creates a temporary directory and a config file to match. Deletes directory after test"""
# Create a temporary working directory
conf_file, working_dir = _create_temp_work_dir()
# Run clone and index operation
erm = RepositoryManager(conf_file)
erm.update_repositories()
erm.index_repositories()
yield erm, working_dir, conf_file # provide the fixture value
# Remove the temp_dir
shutil.rmtree(working_dir)
def mock_labbook():
"""A pytest fixture that creates a temporary directory and a config file to match.
Deletes directory after test"""
conf_file, working_dir = _create_temp_work_dir()
erm = RepositoryManager(conf_file)
erm.update_repositories()
erm.index_repositories()
im = InventoryManager(conf_file)
# description was "my first labbook1"
lb = im.create_labbook('test',
'test',
'labbook1',
description="my test description")
yield conf_file, lb.root_dir, lb
shutil.rmtree(working_dir)
def test_build_docker_image(self, temporary_worker, mock_config_file):
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
im = InventoryManager(mock_config_file[0])
lb = im.create_labbook('unittester',
'unittester',
'unittest-dispatcher-build-image',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, 'ut-busybox', 0)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(lb.root_dir, '.gigantum', 'env', 'Dockerfile'))
docker_kwargs = {'path': lb.root_dir, 'nocache': True}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs)
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
r = d.query_task(job_ref)
print(r.meta)
if elapsed_time > 2:
assert r.meta.get('feedback')
if status in ['success', 'failed', 'finished']:
print(r.exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
w.terminate()
res = d.query_task(job_ref)
assert res
assert res.status == 'finished'
def fixture_working_dir_env_repo_scoped():
"""A pytest fixture that creates a temporary working directory, a config file to match, creates the schema,
and populates the environment component repository.
Class scope modifier attached
"""
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# get environment data and index
erm = RepositoryManager(config_file)
erm.update_repositories()
erm.index_repositories()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
# name of the config file, temporary working directory, the schema
yield config_file, temp_dir, client, schema
# Remove the temp_dir
shutil.rmtree(temp_dir)
def test_start_and_stop_docker_container(self, temporary_worker,
mock_config_file):
# start_docker_container(docker_image_id, exposed_ports, volumes_dict) -> str:
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
m = {'method': 'build_image', 'labbook': "unittest-start-stop-job"}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs,
metadata=m)
j = d.query_task(job_ref)
assert hasattr(j, 'meta')
assert j.meta.get('labbook') == "unittest-start-stop-job"
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
if status in ['success', 'failed', 'finished']:
print(d.query_task(job_ref).exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
res = d.query_task(job_ref)
assert res
print(res.status)
assert res.status == 'finished'
# Finish building image
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = bg_jobs.start_labbook_container(**startc_kwargs)
time.sleep(5)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
print(container_id)
bg_jobs.stop_labbook_container(container_id)
w.terminate()
loader = getattr(flask.request, 'labbook_loader', None)
if loader:
for key in loader.__dict__["_promise_cache"]:
try:
lb = loader.__dict__["_promise_cache"][key].value
lb.git.repo.__del__()
except AttributeError:
continue
return response
# TEMPORARY KLUDGE
logger.info("Cloning/Updating environment repositories.")
try:
erm = RepositoryManager()
update_successful = erm.update_repositories()
if update_successful:
logger.info("Indexing environment repositories.")
erm.index_repositories()
logger.info("Environment repositories updated and ready.")
else:
logger.info("Unable to update environment repositories at startup, most likely due to lack of internet access.")
except Exception as e:
logger.error(e)
raise
# Empty container-container share dir as it is ephemeral
share_dir = os.path.join(os.path.sep, 'mnt', 'share')
logger.info("Emptying container-container share folder: {}.".format(share_dir))
def build_image_for_jupyterlab():
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# get environment data and index
erm = RepositoryManager(config_file)
erm.update_repositories()
erm.index_repositories()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
# Create a labook
im = InventoryManager(config_file)
lb = im.create_labbook('default',
'unittester',
"containerunittestbook",
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE,
ENV_UNIT_TEST_REV)
cm.add_packages("pip3", [{
"manager": "pip3",
"package": "requests",
"version": "2.18.4"
}])
bam = BundledAppManager(lb)
bam.add_bundled_app(9999, 'share', 'A bundled app for testing',
"cd /mnt; python3 -m http.server 9999")
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username="******")
# Note: The final field is the owner
yield lb, ib, docker_client, docker_image_id, client, "unittester"
finally:
try:
docker_client.containers.get(docker_image_id).stop()
docker_client.containers.get(docker_image_id).remove()
except:
pass
try:
docker_client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
shutil.rmtree(lb.root_dir)