def test_bundled_app_lines(self, mock_labbook):
"""Test if the Dockerfile builds with bundled app ports"""
lb = mock_labbook[2]
bam = BundledAppManager(lb)
bam.add_bundled_app(8050, 'dash 1', 'a demo dash app 1',
'python app1.py')
bam.add_bundled_app(9000, 'dash 2', 'a demo dash app 2',
'python app2.py')
bam.add_bundled_app(9001, 'dash 3', 'a demo dash app 3',
'python app3.py')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{
"manager": "pip",
"package": "requests",
"version": "2.18.4"
}])
ib = ImageBuilder(lb)
dockerfile_text = ib.assemble_dockerfile(write=False)
test_lines = [
'# Bundled Application Ports', 'EXPOSE 8050', 'EXPOSE 9000',
'EXPOSE 9001'
]
docker_lines = dockerfile_text.split(os.linesep)
for line in test_lines:
assert line in docker_lines
def test_get_component_list_packages(self, mock_config_with_repo):
"""Test listing packages added a to labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# mock_config_with_repo is a ComponentManager Instance
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}, {
"manager": "pip3",
"package": "gigantum",
"version": "0.5"
}]
cm.add_packages('pip3', pkgs)
packages = cm.get_component_list('package_manager')
assert len(packages) == 2
assert packages[1]['manager'] == 'pip3'
assert packages[1]['package'] == 'requests'
assert packages[1]['version'] == '2.18.2'
assert packages[0]['manager'] == 'pip3'
assert packages[0]['package'] == 'gigantum'
assert packages[0]['version'] == '0.5'
def build_lb_image_for_env_conda(mock_config_with_repo):
"""A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda",
description="Testing environment functions.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}])
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def _add_package_components(cls, lb, packages):
manager = list(set([x['manager'] for x in packages]))
if len(manager) != 1:
raise ValueError(
"Only batch add packages via 1 package manager at a time.")
manager = manager[0]
# Set the cursor to the end of the collection of packages
glob_path = os.path.join(lb.root_dir, '.gigantum', 'env',
'package_manager', f"{manager}*")
cursor = len(glob.glob(glob_path))
# Create Component Manager
cm = ComponentManager(lb)
cm.add_packages(package_manager=manager,
packages=packages,
from_base=False,
force=True)
new_edges = list()
for cnt, pkg in enumerate(packages):
new_edges.append(
PackageComponentConnection.Edge(
node=PackageComponent(manager=manager,
package=pkg["package"],
version=pkg["version"],
schema=CURRENT_SCHEMA),
cursor=base64.b64encode(str(cursor +
cnt).encode()).decode()))
return new_edges
def test_add_package(self, mock_config_with_repo):
"""Test adding a package such as one from apt-get or pip3. """
# Create a labook
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
# Create Component Manager
cm = ComponentManager(lb)
# Add some sample components
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}, {
"manager": "pip3",
"package": "gigantum",
"version": "0.5"
}]
cm.add_packages('pip3', pkgs)
pkgs = [{
"manager": "apt",
"package": "ack",
"version": "1.0"
}, {
"manager": "apt",
"package": "docker",
"version": "3.5"
}]
cm.add_packages('apt', pkgs)
package_path = os.path.join(lb._root_dir, '.gigantum', 'env',
'package_manager')
assert os.path.exists(package_path)
# Ensure all four packages exist.
package_files = [f for f in os.listdir(package_path)]
package_files = [p for p in package_files if p != '.gitkeep']
assert len(package_files) == 4
# Ensure the fields in each of the 4 packages exist.
for file in package_files:
full_path = os.path.join(package_path, file)
with open(full_path) as package_yaml:
fields_dict = yaml.safe_load(package_yaml.read())
for required_field in [
'manager', 'package', 'from_base', 'version'
]:
assert required_field in fields_dict.keys()
# Verify git/activity
log = lb.git.log()
print(log)
assert len(log) == 7
assert "_GTM_ACTIVITY_START_" in log[0]["message"]
assert 'Added 2 apt package(s)' in log[0]["message"]
assert "_GTM_ACTIVITY_START_" in log[2]["message"]
assert 'Added 2 pip3 package(s)' in log[2]["message"]
def build_lb_image_for_jupyterlab(mock_config_with_repo):
with patch.object(Configuration, 'find_default_config', lambda self: mock_config_with_repo[0]):
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbook")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
ib = ImageBuilder(lb)
docker_lines = ib.assemble_dockerfile(write=True)
assert 'RUN pip install requests==2.18.4' in docker_lines
assert all(['==None' not in l for l in docker_lines.split()])
assert all(['=None' not in l for l in docker_lines.split()])
client = get_docker_client()
client.containers.prune()
assert os.path.exists(os.path.join(lb.root_dir, '.gigantum', 'env', 'entrypoint.sh'))
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
lb, container_id = ContainerOperations.start_container(lb, username="******")
assert isinstance(container_id, str)
yield lb, ib, client, docker_image_id, container_id, None, 'unittester'
try:
_, s = ContainerOperations.stop_container(labbook=lb, username="******")
except docker.errors.APIError:
client.containers.get(container_id=container_id).stop(timeout=2)
s = False
finally:
shutil.rmtree(lb.root_dir)
# Stop and remove container if it's still there
try:
client.containers.get(container_id=container_id).stop(timeout=2)
client.containers.get(container_id=container_id).remove()
except:
pass
# Remove image if it's still there
try:
ContainerOperations.delete_image(labbook=lb, username='******')
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def test_docker_snippet(self, mock_labbook):
lb = mock_labbook[2]
package_manager_dir = os.path.join(lb.root_dir, '.gigantum', 'env', 'custom')
erm = RepositoryManager(mock_labbook[0])
erm.update_repositories()
erm.index_repositories()
cm = ComponentManager(lb)
custom = ['RUN true', 'RUN touch /tmp/cat', 'RUN rm /tmp/cat']
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
cm.add_docker_snippet('test-docker', custom, description="Apostrophe's and wėįrd çhårāčtêrś")
ib = ImageBuilder(lb)
l = ib.assemble_dockerfile()
assert all([any([i in l for i in custom]) for n in custom])
def test_remove_package_errors(self, mock_config_with_repo):
"""Test removing a package with expected errors"""
lb = create_tmp_labbook(mock_config_with_repo[0])
cm = ComponentManager(lb)
# Try removing package that doesn't exist
with pytest.raises(ValueError):
cm.remove_packages('apt', ['ack'])
# Add a package as if it's from the base
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}]
cm.add_packages('pip3', pkgs, from_base=True)
# Try removing package that you can't because it comes from a base
with pytest.raises(ValueError):
cm.remove_packages('pip3', ['requests'])
def test_add_duplicate_package(self, mock_config_with_repo):
"""Test adding a duplicate package to a labbook"""
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# Add a component;
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}]
cm.add_packages('pip3', pkgs)
# Verify file
package_file = os.path.join(labbook_dir, '.gigantum', 'env',
'package_manager', 'pip3_requests.yaml')
assert os.path.exists(package_file) is True
# Add a component
with pytest.raises(ValueError):
cm.add_packages('pip3', pkgs)
# Force add a component
cm.add_packages('pip3', pkgs, force=True)
assert os.path.exists(package_file) is True
with open(package_file, 'rt') as pf:
data = yaml.safe_load(pf)
assert data['version'] == '2.18.2'
def test_package_counts(self, fixture_working_dir_env_repo_scoped,
snapshot):
"""Test getting the a LabBook's package manager dependencies"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook5",
description="my first labbook10000")
cm = ComponentManager(lb)
# Add packages
cm.add_packages("apt", [{
"manager": "apt",
"package": "docker",
"version": ""
}])
pkgs = [{
"manager": "pip",
"package": "requests",
"version": "1.3"
}, {
"manager": "pip",
"package": "numpy",
"version": "1.12"
}]
cm.add_packages('pip', pkgs)
pkgs = [{
"manager": "conda2",
"package": "requests",
"version": "1.3"
}, {
"manager": "conda2",
"package": "numpy",
"version": "1.12"
}, {
"manager": "conda2",
"package": "matplotlib",
"version": "1.12"
}, {
"manager": "conda2",
"package": "plotly",
"version": "1.12"
}]
cm.add_packages('conda2', pkgs)
pkgs = [{
"manager": "conda3",
"package": "networkx",
"version": "1.3"
}, {
"manager": "conda3",
"package": "nibabel",
"version": "1.3"
}, {
"manager": "conda3",
"package": "scipy",
"version": "1.12"
}]
cm.add_packages('conda3', pkgs)
query = """
{
labbook(owner: "default", name: "labbook5") {
overview {
numAptPackages
numConda2Packages
numConda3Packages
numPipPackages
}
}
}
"""
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
def test_remove_package(self, mock_config_with_repo):
"""Test removing a package such as one from apt-get or pip3. """
lb = create_tmp_labbook(mock_config_with_repo[0])
labbook_dir = lb.root_dir
cm = ComponentManager(lb)
# Add some sample components
pkgs = [{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
}, {
"manager": "pip3",
"package": "docker",
"version": "0.5"
}]
cm.add_packages('pip3', pkgs)
pkgs = [{
"manager": "apt",
"package": "ack",
"version": "1.5"
}, {
"manager": "apt",
"package": "docker",
"version": "1.3"
}]
cm.add_packages('apt', pkgs)
pkgs = [{
"manager": "pip3",
"package": "matplotlib",
"version": "2.0.0"
}]
cm.add_packages('pip3', pkgs, from_base=True)
package_path = os.path.join(lb._root_dir, '.gigantum', 'env',
'package_manager')
assert os.path.exists(package_path)
# Ensure all four packages exist
assert os.path.exists(os.path.join(package_path, "apt_ack.yaml"))
assert os.path.exists(os.path.join(package_path, "pip3_requests.yaml"))
assert os.path.exists(os.path.join(package_path, "apt_docker.yaml"))
assert os.path.exists(os.path.join(package_path, "pip3_docker.yaml"))
assert os.path.exists(
os.path.join(package_path, "pip3_matplotlib.yaml"))
# Remove packages
cm.remove_packages("apt", ["ack", "docker"])
cm.remove_packages("pip3", ["requests", "docker"])
with pytest.raises(ValueError):
cm.remove_packages("pip3", ["matplotlib"])
# Ensure files are gone
assert not os.path.exists(os.path.join(package_path, "apt_ack.yaml"))
assert not os.path.exists(
os.path.join(package_path, "pip3_requests.yaml"))
assert not os.path.exists(os.path.join(package_path,
"apt_docker.yaml"))
assert not os.path.exists(
os.path.join(package_path, "pip3_docker.yaml"))
assert os.path.exists(
os.path.join(package_path, "pip3_matplotlib.yaml"))
# Ensure git is clean
status = lb.git.status()
assert status['untracked'] == []
assert status['staged'] == []
assert status['unstaged'] == []
# Ensure activity is being written
log = lb.git.log()
assert "_GTM_ACTIVITY_START_" in log[0]["message"]
assert 'Removed 2 pip3 managed package(s)' in log[0]["message"]
def test_change_base(self, mock_labbook):
"""change_base is used both for updating versions and truly changing the base"""
conf_file, root_dir, lb = mock_labbook
# Initial configuration for the labbook - base config taken from `test_add_base` and package config from
# `test_add_package` - so we don't test assertions (again) on this part
cm = ComponentManager(lb)
# We "misconfigure" a package that is not part of the base as if it was from a base
# This shouldn't happen, but we address it just in case
pkgs = [
{
"manager": "pip3",
"package": "gigantum",
"version": "0.5"
},
# pandas *is* part of the quickstart-juypter base, but we specify a different version here
{
"package": "pandas",
"version": "0.21"
}
]
cm.add_packages('pip3', pkgs, force=True, from_base=True)
packages = [p for p in cm.get_component_list('package_manager')]
assert (len(packages) == 2)
assert (all(p['from_base'] for p in packages))
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 1)
# After installing the base, we should have one version of matplotlib installed
packages = [
p for p in cm.get_component_list('package_manager')
if p['package'] == 'matplotlib'
]
assert (len(packages) == 1)
assert (packages[0]['version'] == '2.1.1')
# add_base() should have converted these to user-installed
packages = [
p for p in cm.get_component_list('package_manager')
if p['package'] in ['gigantum', 'pandas']
]
# If we had redundancy from fake base-installed pandas plus real base-installed pandas, this would be 3
assert (len(packages) == 2)
for p in packages:
# Fake base-installed is converted to user ("not from_base") installed
assert (not p['from_base'])
if p['package'] == 'pandas':
# we should still have the fake base-installed version
assert (p['version'] == '0.21')
pkgs = [
{
"manager": "pip3",
"package": "requests",
"version": "2.18.2"
},
# This will override an already installed package
{
"manager": "pip3",
"package": "matplotlib",
"version": "2.2"
}
]
cm.add_packages('pip3', pkgs, force=True)
pkgs = [{
"manager": "apt",
"package": "ack",
"version": "1.0"
}, {
"manager": "apt",
"package": "docker",
"version": "3.5"
}]
cm.add_packages('apt', pkgs)
# Installing a customized version of matplotlib is a new package compared to other tests,
# and is a critical piece of testing cm.change_base
packages = [
p for p in cm.get_component_list('package_manager')
if p['package'] == 'matplotlib'
]
assert (len(packages) == 1)
assert (packages[0]['version'] == '2.2')
# We upgrade our base
cm.change_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
# matplotlib still set up per "user" update?
packages = [
p for p in cm.get_component_list('package_manager')
if p['package'] == 'matplotlib'
]
assert (len(packages) == 1)
assert (packages[0]['version'] == '2.2')
# Base revision now 2?
assert (cm.base_fields['revision'] == 2)
def test_get_package_manager(self, fixture_working_dir_env_repo_scoped,
snapshot):
"""Test getting the a LabBook's package manager dependencies"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook4",
description="my first labbook10000")
query = """
{
labbook(owner: "default", name: "labbook4") {
environment {
packageDependencies {
edges {
node {
id
manager
package
version
fromBase
}
cursor
}
pageInfo {
hasNextPage
}
}
}
}
}
"""
# should be null
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
# Add a base image
cm = ComponentManager(lb)
pkgs = [{
"manager": "pip",
"package": "requests",
"version": "1.3"
}, {
"manager": "pip",
"package": "numpy",
"version": "1.12"
}, {
"manager": "pip",
"package": "gtmunit1",
"version": "0.2.4"
}]
cm.add_packages('pip', pkgs)
pkgs = [{
"manager": "conda3",
"package": "cdutil",
"version": "8.1"
}, {
"manager": "conda3",
"package": "nltk",
"version": '3.2.5'
}]
cm.add_packages('conda3', pkgs)
# Add one package without a version, which should cause an error in the API since version is required
pkgs = [{"manager": "apt", "package": "lxml", "version": "3.4"}]
cm.add_packages('apt', pkgs)
query = """
{
labbook(owner: "default", name: "labbook4") {
environment {
packageDependencies {
edges {
node {
id
manager
package
version
fromBase
}
cursor
}
pageInfo {
hasNextPage
}
}
}
}
}
"""
r1 = fixture_working_dir_env_repo_scoped[2].execute(query)
assert 'errors' not in r1
snapshot.assert_match(r1)
query = """
{
labbook(owner: "default", name: "labbook4") {
environment {
packageDependencies(first: 2, after: "MA==") {
edges {
node {
id
manager
package
version
fromBase
}
cursor
}
pageInfo {
hasNextPage
}
}
}
}
}
"""
r1 = fixture_working_dir_env_repo_scoped[2].execute(query)
assert 'errors' not in r1
snapshot.assert_match(r1)
def test_get_package_manager_metadata(self,
fixture_working_dir_env_repo_scoped,
snapshot):
"""Test getting the a LabBook's package manager dependencies"""
# Create labbook
im = InventoryManager(fixture_working_dir_env_repo_scoped[0])
lb = im.create_labbook("default",
"default",
"labbook4meta",
description="my first asdf")
query = """
{
labbook(owner: "default", name: "labbook4meta") {
environment {
packageDependencies {
edges {
node {
id
manager
package
version
fromBase
description
docsUrl
latestVersion
}
cursor
}
pageInfo {
hasNextPage
}
}
}
}
}
"""
# should be null
snapshot.assert_match(
fixture_working_dir_env_repo_scoped[2].execute(query))
# Add a base image
cm = ComponentManager(lb)
pkgs = [{
"manager": "pip",
"package": "gtmunit3",
"version": "5.0"
}, {
"manager": "pip",
"package": "gtmunit2",
"version": "12.2"
}, {
"manager": "pip",
"package": "gtmunit1",
"version": '0.2.1'
}]
cm.add_packages('pip', pkgs)
pkgs = [{
"manager": "conda3",
"package": "cdutil",
"version": "8.1"
}, {
"manager": "conda3",
"package": "python-coveralls",
"version": "2.5.0"
}]
cm.add_packages('conda3', pkgs)
r1 = fixture_working_dir_env_repo_scoped[2].execute(query)
assert 'errors' not in r1
snapshot.assert_match(r1)
def build_image_for_jupyterlab():
# Create temp dir
config_file, temp_dir = _create_temp_work_dir()
# Create user identity
insert_cached_identity(temp_dir)
# Create test client
schema = graphene.Schema(query=LabbookQuery, mutation=LabbookMutations)
# get environment data and index
erm = RepositoryManager(config_file)
erm.update_repositories()
erm.index_repositories()
with patch.object(Configuration, 'find_default_config',
lambda self: config_file):
# Load User identity into app context
app = Flask("lmsrvlabbook")
app.config["LABMGR_CONFIG"] = Configuration()
app.config["LABMGR_ID_MGR"] = get_identity_manager(Configuration())
with app.app_context():
# within this block, current_app points to app. Set current user explicitly (this is done in the middleware)
flask.g.user_obj = app.config["LABMGR_ID_MGR"].get_user_profile()
# Create a test client
client = Client(
schema,
middleware=[DataloaderMiddleware(), error_middleware],
context_value=ContextMock())
# Create a labook
im = InventoryManager(config_file)
lb = im.create_labbook('default',
'unittester',
"containerunittestbook",
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE,
ENV_UNIT_TEST_REV)
cm.add_packages("pip3", [{
"manager": "pip3",
"package": "requests",
"version": "2.18.4"
}])
bam = BundledAppManager(lb)
bam.add_bundled_app(9999, 'share', 'A bundled app for testing',
"cd /mnt; python3 -m http.server 9999")
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username="******")
# Note: The final field is the owner
yield lb, ib, docker_client, docker_image_id, client, "unittester"
finally:
try:
docker_client.containers.get(docker_image_id).stop()
docker_client.containers.get(docker_image_id).remove()
except:
pass
try:
docker_client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
shutil.rmtree(lb.root_dir)
