def build_lb_image_for_env(mock_config_with_repo):
# Create a labook
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenv",
description="Testing environment functions.")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def test_cuda_version_property(self, mock_config_with_repo):
"""Test getting the cuda version"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('test',
'test',
'labbook1',
description="my first labbook")
assert lb.cuda_version is None
# Add base without GPU support
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
base_yaml_file = glob.glob(
os.path.join(lb.root_dir, '.gigantum', 'env', 'base', '*.yaml'))[0]
assert lb.cuda_version is None
# Fake a version
with open(base_yaml_file, 'rt') as bf:
base_data = yaml.safe_load(bf)
base_data['cuda_version'] = '10.0'
with open(base_yaml_file, 'wt') as bf:
yaml.safe_dump(base_data, bf)
assert lb.cuda_version == '10.0'
def test_get_component_list_packages(self, mock_config_with_repo):
"""Test listing packages added a to labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}, {
"manager": "pip3",
"package": "gigantum",
"version": "0.5"
}]
cm.add_packages('pip3', pkgs)
packages = cm.get_component_list('package_manager')
assert len(packages) == 2
assert packages[1]['manager'] == 'pip3'
assert packages[1]['package'] == 'requests'
assert packages[1]['version'] == '2.18.2'
assert packages[0]['manager'] == 'pip3'
assert packages[0]['package'] == 'gigantum'
assert packages[0]['version'] == '0.5'
def mutate_and_get_payload(cls,
root,
info,
name,
description,
repository,
base_id,
revision,
is_untracked=False,
client_mutation_id=None):
username = get_logged_in_username()
inv_manager = InventoryManager()
if is_untracked:
lb = inv_manager.create_labbook_disabled_lfs(
username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
else:
lb = inv_manager.create_labbook(username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
if is_untracked:
FileOperations.set_untracked(lb, 'input')
FileOperations.set_untracked(lb, 'output')
input_set = FileOperations.is_set_untracked(lb, 'input')
output_set = FileOperations.is_set_untracked(lb, 'output')
if not (input_set and output_set):
raise ValueError(
f'{str(lb)} untracking for input/output in malformed state'
)
if not lb.is_repo_clean:
raise ValueError(
f'{str(lb)} should have clean Git state after setting for untracked'
)
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK,
show=False,
importance=0)
adr.add_value('text/plain', f"Created new LabBook: {username}/{name}")
# Create activity record
ar = ActivityRecord(ActivityType.LABBOOK,
message=f"Created new LabBook: {username}/{name}",
show=True,
importance=255,
linked_commit=lb.git.commit_hash)
ar.add_detail_object(adr)
store = ActivityStore(lb)
store.create_activity_record(ar)
cm = ComponentManager(lb)
cm.add_base(repository, base_id, revision)
return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def _add_package_components(cls, lb, packages):
manager = list(set([x['manager'] for x in packages]))
if len(manager) != 1:
raise ValueError(
"Only batch add packages via 1 package manager at a time.")
manager = manager[0]
# Set the cursor to the end of the collection of packages
glob_path = os.path.join(lb.root_dir, '.gigantum', 'env',
'package_manager', f"{manager}*")
cursor = len(glob.glob(glob_path))
# Create Component Manager
cm = ComponentManager(lb)
cm.add_packages(package_manager=manager,
packages=packages,
from_base=False,
force=True)
new_edges = list()
for cnt, pkg in enumerate(packages):
new_edges.append(
PackageComponentConnection.Edge(
node=PackageComponent(manager=manager,
package=pkg["package"],
version=pkg["version"],
schema=CURRENT_SCHEMA),
cursor=base64.b64encode(str(cursor +
cnt).encode()).decode()))
return new_edges
def test_build_image_no_cache(self, fixture_working_dir_env_repo_scoped, reset_images):
"""Test building a labbook's image"""
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default", "default", "labbook-build2",
description="building an env")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, "ut-busybox", 0)
query = """
{
labbook(name: "labbook-build2", owner: "default") {
environment {
imageStatus
containerStatus
}
}
}
"""
r = fixture_working_dir_env_repo_scoped[2].execute(query)
assert 'errors' not in r
assert r['data']['labbook']['environment']['imageStatus'] == 'DOES_NOT_EXIST'
assert r['data']['labbook']['environment']['containerStatus'] == 'NOT_RUNNING'
# Build the image
build_query = """
mutation myBuildImage {
buildImage(input: {labbookName: "labbook-build2", owner: "default", noCache: true}) {
environment {
imageStatus
containerStatus
}
}
}
"""
r = fixture_working_dir_env_repo_scoped[2].execute(build_query)
assert 'errors' not in r
assert r['data']['buildImage']['environment']['imageStatus'] in ['BUILD_QUEUED', 'BUILD_IN_PROGRESS']
assert r['data']['buildImage']['environment']['containerStatus'] == 'NOT_RUNNING'
# Wait for build to succeed for up to TIMEOUT_MAX seconds
success = False
for _ in range(TIMEOUT_MAX):
result = fixture_working_dir_env_repo_scoped[2].execute(query)
if result['data']['labbook']['environment']['imageStatus'] == 'EXISTS':
success = True
break
assert result['data']['labbook']['environment']['imageStatus'] == 'BUILD_IN_PROGRESS'
time.sleep(1)
assert success is True, f"Failed to build within {TIMEOUT_MAX} second timeout."
r = fixture_working_dir_env_repo_scoped[2].execute(query)
assert 'errors' not in r
assert r['data']['labbook']['environment']['imageStatus'] == 'EXISTS'
assert r['data']['labbook']['environment']['containerStatus'] == 'NOT_RUNNING'
def test_misconfigured_base_two_bases(self, mock_config_with_repo):
lb = create_tmp_labbook(mock_config_with_repo[0])
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-1", 0)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-2", 0)
with pytest.raises(ValueError):
a = cm.base_fields
def test_build_and_start_and_stop_labbook_container(
self, mock_config_file):
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
image_id = jobs.build_labbook_image(**docker_kwargs)
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = jobs.start_labbook_container(**startc_kwargs)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
jobs.stop_labbook_container(container_id)
with pytest.raises(Exception):
# Should not be found because the stop job cleans up
get_docker_client().containers.get(container_id)
def test_list_versions_from_fallback(self, mock_config_with_repo):
"""Test list_versions command"""
username = "******"
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook(
'unittest',
'unittest',
'labbook-unittest-01',
description="From mock_config_from_repo fixture")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username=username)
# Test lookup
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
# Delete image
client.images.remove(docker_image_id, force=True, noprune=False)
# Test lookup still works
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
def test_misconfigured_base_two_bases(self, mock_config_with_repo):
conf_file = mock_config_with_repo[0]
lb = create_tmp_labbook(conf_file)
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-1", 0)
# Updated logic to enable changing bases won't allow `add_base` again. Need to manually create a file
bad_base_config = Path(cm.env_dir, 'base',
'evil_repo_quantum-deathray.yaml')
bad_base_config.write_text("I'm gonna break you!")
with pytest.raises(ValueError):
a = cm.base_fields
def test_get_base(self, mock_config_with_repo):
lb = create_tmp_labbook(mock_config_with_repo[0])
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-1", 0)
base_data = cm.base_fields
assert type(base_data) == dict
assert base_data['name'] == 'Unit Test1'
assert base_data['os_class'] == 'ubuntu'
assert base_data['schema'] == 1
def test_add_duplicate_package(self, mock_config_with_repo):
"""Test adding a duplicate package to a labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# Add a component;
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}]
cm.add_packages('pip3', pkgs)
# Verify file
package_file = os.path.join(labbook_dir, '.gigantum', 'env',
'package_manager', 'pip3_requests.yaml')
assert os.path.exists(package_file) is True
# Add a component
with pytest.raises(ValueError):
cm.add_packages('pip3', pkgs)
# Force add a component
cm.add_packages('pip3', pkgs, force=True)
assert os.path.exists(package_file) is True
with open(package_file, 'rt') as pf:
data = yaml.safe_load(pf)
assert data['version'] == '2.18.2'
def test_get_base_empty_error(self, mock_config_with_repo):
lb = create_tmp_labbook(mock_config_with_repo[0])
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-1", 0)
base_filename = f"gigantum_base-images-testing_ut-jupyterlab-1.yaml"
base_final_path = os.path.join(cm.env_dir, 'base', base_filename)
with open(base_final_path, 'wt') as cf:
cf.write(yaml.safe_dump({}, default_flow_style=False))
with pytest.raises(ValueError):
cm.base_fields
def test_get_component_list_base(self, mock_config_with_repo):
"""Test listing base images added a to labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
bases = cm.get_component_list('base')
assert len(bases) == 1
assert bases[0]['id'] == gtmcore.fixtures.ENV_UNIT_TEST_BASE
assert bases[0]['revision'] == gtmcore.fixtures.ENV_UNIT_TEST_REV
def stop_labbook_monitor(labbook: LabBook,
username: str,
database: int = 1) -> None:
"""Method to stop a Development Environment Monitors for a given Lab Book
Args:
labbook(LabBook): A populated LabBook instance to start monitoring
username(str): Username of the logged in user
database(int): The redis database ID to use for key storage. Default should be 1
Returns:
None
"""
logger.info(f"Stopping labbook monitors for {labbook.name}")
# Connect to redis
redis_conn = redis.Redis(db=database)
# Get Dev envs in the lab book
cm = ComponentManager(labbook)
base_data = cm.base_fields
for dt in base_data['development_tools']:
owner = InventoryManager().query_owner(labbook)
dev_env_monitor_key = "dev_env_monitor:{}:{}:{}:{}".format(
username, owner, labbook.name, dt)
stop_dev_env_monitors(dev_env_monitor_key, redis_conn, labbook.name)
def test_initalize_labbook(self, mock_config_with_repo):
"""Test preparing an empty labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
pprint.pprint([n[0] for n in os.walk(labbook_dir)])
# Verify missing dir structure
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env', 'base')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env',
'package_manager')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env', 'custom')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env',
'entrypoint.sh')) is False
cm = ComponentManager(lb)
# Verify dir structure
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env', 'base')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env',
'package_manager')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env', 'custom')) is True
assert os.path.exists(
os.path.join(labbook_dir, '.gigantum', 'env',
'entrypoint.sh')) is True
def test_fail_import_export_zip(self, mock_config_with_repo):
# Create new LabBook to be exported
lb = InventoryManager(mock_config_with_repo[0]).create_labbook(
'test',
'test',
"lb-fail-export-import-test",
description="Failing import-export.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
lb_root = lb.root_dir
with tempfile.TemporaryDirectory() as temp_dir_path:
# Export the labbook
export_dir = os.path.join(mock_config_with_repo[1], "export")
try:
exported_archive_path = jobs.export_labbook_as_zip(
"/tmp", export_dir)
assert False, "Exporting /tmp should fail"
except Exception as e:
pass
# Export the labbook, then remove before re-importing
exported_archive_path = jobs.export_labbook_as_zip(
lb.root_dir, export_dir)
try:
imported_lb_path = jobs.import_labboook_from_zip(
archive_path=exported_archive_path,
username="******",
owner="test",
config_file=mock_config_with_repo[0])
assert False, f"Should not be able to import LabBook because it already exited at {lb_root}"
except Exception as e:
pass
try:
imported_lb_path = jobs.import_labboook_from_zip(
archive_path="/t",
username="******",
owner="test",
config_file=mock_config_with_repo[0])
assert False, f"Should not be able to import LabBook from strange directory /t"
except Exception as e:
pass
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# TODO - Should we cehck if a custom docker component already exists?
with lb.lock():
cm = ComponentManager(lb)
cm.remove_docker_snippet(cm.DEFAULT_CUSTOM_DOCKER_NAME)
return RemoveCustomDocker(
updated_environment=Environment(owner=owner, name=labbook_name))
def test_add_base(self, mock_config_with_repo):
"""Test adding a base to a labbook"""
# Create a labook
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
# Verify file
component_file = os.path.join(
labbook_dir, '.gigantum', 'env', 'base',
f"{gtmcore.fixtures.ENV_UNIT_TEST_REPO}_"
f"{gtmcore.fixtures.ENV_UNIT_TEST_BASE}.yaml")
assert os.path.exists(component_file) is True
with open(component_file, 'rt') as cf:
data = yaml.safe_load(cf)
preinstalled_pkgs = os.listdir(
os.path.join(labbook_dir, ".gigantum/env/package_manager"))
pkg_yaml_files = [n for n in preinstalled_pkgs if '.yaml' in n]
assert len(pkg_yaml_files) == 7
for p in pkg_yaml_files:
with open(
os.path.join(labbook_dir, ".gigantum/env/package_manager",
p)) as f:
assert 'from_base: true' in f.read()
assert data['id'] == gtmcore.fixtures.ENV_UNIT_TEST_BASE
assert data['revision'] == gtmcore.fixtures.ENV_UNIT_TEST_REV
# Verify git/activity
log = lb.git.log()
assert len(log) >= 4
assert "_GTM_ACTIVITY_START_" in log[0]["message"]
assert 'Added base:' in log[0]["message"]
assert "_GTM_ACTIVITY_START_" in log[2]["message"]
assert 'Added 6 pip3 package(s)' in log[2]["message"]
assert "_GTM_ACTIVITY_START_" in log[4]["message"]
assert 'Added 1 apt package(s)' in log[4]["message"]
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
docker_content,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
docker_lines = [n for n in docker_content.strip().split('\n') if n]
cm = ComponentManager(lb)
cm.add_docker_snippet(cm.DEFAULT_CUSTOM_DOCKER_NAME, docker_lines)
return AddCustomDocker(
updated_environment=Environment(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
manager,
packages,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
cm = ComponentManager(lb)
cm.remove_packages(package_manager=manager, package_names=packages)
return RemovePackageComponents(success=True)
def test_build_docker_image(self, temporary_worker, mock_config_file):
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
im = InventoryManager(mock_config_file[0])
lb = im.create_labbook('unittester',
'unittester',
'unittest-dispatcher-build-image',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, 'ut-busybox', 0)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
assert os.path.exists(
os.path.join(lb.root_dir, '.gigantum', 'env', 'Dockerfile'))
docker_kwargs = {'path': lb.root_dir, 'nocache': True}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs)
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
r = d.query_task(job_ref)
print(r.meta)
if elapsed_time > 2:
assert r.meta.get('feedback')
if status in ['success', 'failed', 'finished']:
print(r.exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
w.terminate()
res = d.query_task(job_ref)
assert res
assert res.status == 'finished'
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
repository,
base_id,
revision,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
cm = ComponentManager(lb)
cm.change_base(repository, base_id, revision)
return ChangeLabbookBase(labbook=Labbook(owner=owner, name=lb.name))
def test_docker_snippet(self, mock_labbook):
lb = mock_labbook[2]
package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env', 'custom')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
custom = ['RUN true', 'RUN touch /tmp/cat', 'RUN rm /tmp/cat']
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
cm.add_docker_snippet('test-docker', custom, description="Apostrophe's and wėįrd çhårāčtêrś")
ib = ImageBuilder(lb)
l = ib.assemble_dockerfile()
assert all([any([i in l for i in custom]) for n in custom])
def test_bundled_app_lines(self, mock_labbook):
"""Test if the Dockerfile builds with bundled app ports"""
lb = mock_labbook[2]
bam = BundledAppManager(lb)
bam.add_bundled_app(8050, 'dash 1', 'a demo dash app 1',
'python app1.py')
bam.add_bundled_app(9000, 'dash 2', 'a demo dash app 2',
'python app2.py')
bam.add_bundled_app(9001, 'dash 3', 'a demo dash app 3',
'python app3.py')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{
"manager": "pip",
"package": "requests",
"version": "2.18.4"
}])
ib = ImageBuilder(lb)
dockerfile_text = ib.assemble_dockerfile(write=False)
test_lines = [
'# Bundled Application Ports', 'EXPOSE 8050', 'EXPOSE 9000',
'EXPOSE 9001'
]
docker_lines = dockerfile_text.split(os.linesep)
for line in test_lines:
assert line in docker_lines
def build_lb_image_for_env_conda(mock_config_with_repo):
"""A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda",
description="Testing environment functions.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}])
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def test_add_duplicate_base(self, mock_config_with_repo):
"""Test adding a duplicate base to a labbook"""
# Create a labook
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
# Create Component Manager
cm = ComponentManager(lb)
# Add a component;
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
c = f"{gtmcore.fixtures.ENV_UNIT_TEST_REPO}_{gtmcore.fixtures.ENV_UNIT_TEST_BASE}.yaml"
# Verify file
component_file = os.path.join(labbook_dir, '.gigantum', 'env', 'base',
c)
assert os.path.exists(component_file) is True
# Add a component
with pytest.raises(ValueError):
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
gtmcore.fixtures.ENV_UNIT_TEST_BASE,
gtmcore.fixtures.ENV_UNIT_TEST_REV)
def test_add_package(self, mock_config_with_repo):
"""Test adding a package such as one from apt-get or pip3. """
# Create a labook
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
# Create Component Manager
cm = ComponentManager(lb)
# Add some sample components
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}, {
"manager": "pip3",
"package": "gigantum",
"version": "0.5"
}]
cm.add_packages('pip3', pkgs)
pkgs = [{
"manager": "apt",
"package": "ack",
"version": "1.0"
}, {
"manager": "apt",
"package": "docker",
"version": "3.5"
}]
cm.add_packages('apt', pkgs)
package_path = os.path.join(lb._root_dir, '.gigantum', 'env',
'package_manager')
assert os.path.exists(package_path)
# Ensure all four packages exist.
package_files = [f for f in os.listdir(package_path)]
package_files = [p for p in package_files if p != '.gitkeep']
assert len(package_files) == 4
# Ensure the fields in each of the 4 packages exist.
for file in package_files:
full_path = os.path.join(package_path, file)
with open(full_path) as package_yaml:
fields_dict = yaml.safe_load(package_yaml.read())
for required_field in [
'manager', 'package', 'from_base', 'version'
]:
assert required_field in fields_dict.keys()
# Verify git/activity
log = lb.git.log()
print(log)
assert len(log) == 7
assert "_GTM_ACTIVITY_START_" in log[0]["message"]
assert 'Added 2 apt package(s)' in log[0]["message"]
assert "_GTM_ACTIVITY_START_" in log[2]["message"]
assert 'Added 2 pip3 package(s)' in log[2]["message"]
def mutate_and_get_payload(cls,
root,
info,
name,
description,
repository,
base_id,
revision,
is_untracked=False,
client_mutation_id=None):
username = get_logged_in_username()
inv_manager = InventoryManager()
lb = inv_manager.create_labbook(username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK,
show=False,
importance=0)
adr.add_value('text/plain', f"Created new LabBook: {username}/{name}")
# Create activity record
ar = ActivityRecord(ActivityType.LABBOOK,
message=f"Created new LabBook: {username}/{name}",
show=True,
importance=255,
linked_commit=lb.git.commit_hash)
ar.add_detail_object(adr)
store = ActivityStore(lb)
store.create_activity_record(ar)
cm = ComponentManager(lb)
cm.add_base(repository, base_id, revision)
return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def test_try_configuring_two_bases(self, mock_config_with_repo):
conf_file = mock_config_with_repo[0]
lb = create_tmp_labbook(conf_file)
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-1", 0)
with pytest.raises(ValueError):
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO, "ut-jupyterlab-2",
0)
