def test_put_single_file(self, build_lb_image_for_jupyterlab):
# Note - we are combining multiple tests in one to speed things up
# We do not want to build, start, execute, stop, and delete at container for each
# of the following test points.
fixture = ContainerFixture(build_lb_image_for_jupyterlab)
container = docker.from_env().containers.get(
fixture.docker_container_id)
# Test insert of a single file.
dst_dir_1 = "/home/giguser/sample-creds"
with tempfile.TemporaryDirectory() as tempdir:
with open(os.path.join(tempdir, 'secretfile'),
'w') as sample_secret:
sample_secret.write("<<Secret File Content>>")
t0 = time.time()
ContainerOperations.copy_into_container(
fixture.labbook,
fixture.username,
src_path=sample_secret.name,
dst_dir=dst_dir_1)
tf = time.time()
container.exec_run(f'sh -c "cat {dst_dir_1}/secretfile"')
# The copy_into_container should NOT remove the original file on disk.
assert os.path.exists(sample_secret.name)
assert tf - t0 < 1.0, \
f"Time to insert small file must be less than 1 sec - took {tf-t0:.2f}s"
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name)
d = Dispatcher()
lb_jobs = d.get_jobs_for_labbook(lb.key)
jobs = [
j for j in d.get_jobs_for_labbook(lb.key)
if j.meta.get('method') == 'build_image' and j.status == 'started'
]
if len(jobs) == 1:
d.abort_task(jobs[0].job_key)
ContainerOperations.delete_image(lb, username=username)
return CancelBuild(build_stopped=True, message="Stopped build")
elif len(jobs) == 0:
logger.warning(f"No build_image tasks found for {str(lb)}")
return CancelBuild(build_stopped=False,
message="No build task found")
else:
logger.warning(f"Multiple build jobs found for {str(lb)}")
return CancelBuild(build_stopped=False,
message="Multiple builds found")
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
if confirm:
logger.info(f"Deleting {str(lb)}...")
try:
lb, stopped = ContainerOperations.stop_container(
labbook=lb, username=username)
except OSError as e:
logger.warning(e)
lb, docker_removed = ContainerOperations.delete_image(
labbook=lb, username=username)
if not docker_removed:
raise ValueError(
f'Cannot delete docker image for {str(lb)} - unable to delete Project from disk'
)
datasets_to_schedule = InventoryManager().delete_labbook(
username, owner, labbook_name)
# Schedule jobs to clean the file cache for any linked datasets (if no other references exist)
for cleanup_job in datasets_to_schedule:
# Schedule Job to clear file cache if dataset is no longer in use
job_metadata = {'method': 'clean_dataset_file_cache'}
job_kwargs = {
'logged_in_username': username,
'dataset_owner': cleanup_job.namespace,
'dataset_name': cleanup_job.name,
'cache_location': cleanup_job.cache_root
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(
jobs.clean_dataset_file_cache,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched clean_dataset_file_cache({ cleanup_job.namespace}/{cleanup_job.name})"
f" to Job {job_key}")
# Verify Delete worked
if os.path.exists(lb.root_dir):
logger.error(
f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!'
)
return DeleteLabbook(success=False)
else:
return DeleteLabbook(success=True)
else:
logger.info(f"Dry run in deleting {str(lb)} -- not deleted.")
return DeleteLabbook(success=False)
def test_run_command(self, build_lb_image_for_jupyterlab):
my_lb = build_lb_image_for_jupyterlab[0]
docker_image_id = build_lb_image_for_jupyterlab[3]
result = ContainerOperations.run_command("echo My sample message",
my_lb,
username="******")
assert result.decode().strip() == 'My sample message'
result = ContainerOperations.run_command("pip search gigantum",
my_lb,
username="******")
assert any([
'Gigantum Platform' in l
for l in result.decode().strip().split('\n')
])
result = ContainerOperations.run_command("/bin/true",
my_lb,
username="******")
assert result.decode().strip() == ""
result = ContainerOperations.run_command("/bin/false",
my_lb,
username="******")
assert result.decode().strip() == ""
def _stop_container(cls, lb, username):
"""Stop container and also do necessary cleanup of confhttpproxy, monitors, etc.
Currently, this supports two cases, applications monitored by MITMProxy,
and Jupyter. So, for now, if we can't find an mitmproxy endpoint, we assume
we're dealing with a jupyter container.
"""
pr = confhttpproxy.ProxyRouter.get_proxy(
lb.client_config.config['proxy'])
# Remove route from proxy
lb_name = ContainerOperations.labbook_image_name(lb, username)
if MITMProxyOperations.get_mitmendpoint(lb_name):
# there is an MITMProxy (currently only used for RStudio)
proxy_endpoint = MITMProxyOperations.stop_mitm_proxy(lb_name)
tool = 'rserver'
else:
lb_ip = ContainerOperations.get_labbook_ip(lb, username)
# The only alternative to mitmproxy (currently) is jupyter
# TODO in #453: Construction of this URL should be encapsulated in Jupyter Dev Tool logic
proxy_endpoint = f'http://{lb_ip}:8888'
tool = 'jupyter'
est_target = pr.get_matching_routes(proxy_endpoint, tool)
for i, target in enumerate(est_target):
if i == 1:
# We have > 1 entry in the router, which shouldn't happen
logger.warning(
f'Removing multiple routes for {tool} on {proxy_endpoint} during Project container stop.'
)
pr.remove(target[1:])
wf = LabbookWorkflow(lb)
wf.garbagecollect()
# Clean up empty bind mount dirs from datasets if needed
submodules = lb.git.list_submodules()
for submodule in submodules:
namespace, dataset_name = submodule['name'].split("&")
bind_location = os.path.join(lb.root_dir, 'input', dataset_name)
if os.path.isdir(bind_location):
os.rmdir(bind_location)
# stop labbook monitor
stop_labbook_monitor(lb, username)
lb, stopped = ContainerOperations.stop_container(labbook=lb,
username=username)
if not stopped:
# TODO DK: Why would stopped=False? Should this move up??
raise ValueError(f"Failed to stop labbook {lb.name}")
def latest_version(self, package_name: str, labbook: LabBook,
username: str) -> str:
"""Method to get the latest version string for a package
Args:
package_name: Name of the package to query
labbook: Subject LabBook
username: username of current user
Returns:
str: latest version string
"""
result = ContainerOperations.run_command(
f"conda install --dry-run --no-deps --json {package_name}",
labbook,
username,
override_image_tag=self.fallback_image(labbook))
data = json.loads(result.decode().strip())
if data.get('message') == 'All requested packages already installed.':
# We enter this block if the given package_name is already installed to the latest version.
# Then we have to retrieve the latest version using conda list
result = ContainerOperations.run_command(
"conda list --json",
labbook,
username,
override_image_tag=self.fallback_image(labbook))
data = json.loads(result.decode().strip())
for pkg in data:
if pkg.get('name') == package_name:
return pkg.get('version')
else:
if isinstance(data.get('actions'), dict) is True:
# New method - added when bases updated to conda 4.5.1
for p in data.get('actions').get('LINK'):
if p.get('name') == package_name:
return p.get("version")
else:
# legacy methods to handle older bases built on conda 4.3.31
try:
for p in [
x.get('LINK')[0] for x in data.get('actions') if x
]:
if p.get('name') == package_name:
return p.get("version")
except Exception:
for p in [x.get('LINK') for x in data.get('actions') if x]:
if p.get('name') == package_name:
return p.get("version")
# if you get here, failed to find the package in the result from conda
raise ValueError(
f"Could not retrieve version list for provided package name: {package_name}"
)
def build_lb_image_for_jupyterlab(mock_config_with_repo):
with patch.object(Configuration, 'find_default_config', lambda self: mock_config_with_repo[0]):
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbook")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages("pip", [{"manager": "pip", "package": "requests", "version": "2.18.4"}])
ib = ImageBuilder(lb)
docker_lines = ib.assemble_dockerfile(write=True)
assert 'RUN pip install requests==2.18.4' in docker_lines
assert all(['==None' not in l for l in docker_lines.split()])
assert all(['=None' not in l for l in docker_lines.split()])
client = get_docker_client()
client.containers.prune()
assert os.path.exists(os.path.join(lb.root_dir, '.gigantum', 'env', 'entrypoint.sh'))
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
lb, container_id = ContainerOperations.start_container(lb, username="******")
assert isinstance(container_id, str)
yield lb, ib, client, docker_image_id, container_id, None, 'unittester'
try:
_, s = ContainerOperations.stop_container(labbook=lb, username="******")
except docker.errors.APIError:
client.containers.get(container_id=container_id).stop(timeout=2)
s = False
finally:
shutil.rmtree(lb.root_dir)
# Stop and remove container if it's still there
try:
client.containers.get(container_id=container_id).stop(timeout=2)
client.containers.get(container_id=container_id).remove()
except:
pass
# Remove image if it's still there
try:
ContainerOperations.delete_image(labbook=lb, username='******')
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def build_lb_image_for_env_conda(mock_config_with_repo):
"""A fixture that installs an old version of matplotlib and latest version of requests to increase code coverage"""
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenvconda",
description="Testing environment functions.")
cm = ComponentManager(lb)
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
cm.add_packages('conda3', [{'package': 'python-coveralls', 'version': '2.7.0'}])
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def list_available_updates(self, labbook: LabBook,
username: str) -> List[Dict[str, str]]:
"""Method to get a list of all installed packages that could be updated and the new version string
Note, this will return results for the computer/container in which it is executed. To get the properties of
a LabBook container, a docker exec command would be needed from the Gigantum application container.
return format is a list of dicts with the format
{name: <package name>, version: <currently installed version string>, latest_version: <latest version string>}
Returns:
list
"""
result = ContainerOperations.run_command(
"apt list --upgradable",
labbook,
username,
fallback_image=self.fallback_image(labbook))
packages = []
if result:
lines = result.decode('utf-8').split('\n')
for line in lines:
if line is not None and line != "Listing..." and "/" in line:
package_name, version_info_t = line.split("/")
version_info = version_info_t.split(' ')
packages.append({
'name': package_name,
'latest_version': version_info[1],
'version': version_info[5][:-1]
})
return packages
def build_lb_image_for_env(mock_config_with_repo):
# Create a labook
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook('unittester', 'unittester', "containerunittestbookenv",
description="Testing environment functions.")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
client.containers.prune()
try:
lb, docker_image_id = ContainerOperations.build_image(labbook=lb, username="******")
yield lb, 'unittester'
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id, force=True, noprune=False)
except:
pass
def search(self, search_str: str, labbook: LabBook,
username: str) -> List[str]:
"""Method to search a package manager for packages based on a string. The string can be a partial string.
Args:
search_str: The string to search on
labbook: Subject LabBook
username: username of current user
Returns:
list(str): The list of package names that match the search string
"""
result = ContainerOperations.run_command(
f"apt-cache search {search_str}",
labbook,
username,
fallback_image=self.fallback_image(labbook))
packages = []
if result:
lines = result.decode('utf-8').split('\n')
for l in lines:
if l:
packages.append(l.split(" - ")[0])
return packages
def list_versions(self, package_name: str, labbook: LabBook,
username: str) -> List[str]:
"""Method to list all available versions of a package based on the package name
Args:
package_name: Name of the package to query
labbook: Subject LabBook
username: Username of current user
Returns:
list(str): Version strings
"""
result = ContainerOperations.run_command(
f"apt-cache madison {package_name}",
labbook,
username,
override_image_tag=self.fallback_image(labbook))
package_versions: List[str] = []
if result:
lines = result.decode('utf-8').split('\n')
for l in lines:
if l:
parts = l.split(" | ")
if parts[1] not in package_versions:
package_versions.append(parts[1].strip())
else:
raise ValueError(f"Package {package_name} not found in apt.")
return package_versions
def list_installed_packages(self, labbook: LabBook,
username: str) -> List[Dict[str, str]]:
"""Method to get a list of all packages that are currently installed
Note, this will return results for the computer/container in which it is executed. To get the properties of
a LabBook container, a docker exec command would be needed from the Gigantum application container.
return format is a list of dicts with the format (name: <package name>, version: <version string>)
Returns:
list
"""
result = ContainerOperations.run_command(
"apt list --installed",
labbook,
username,
fallback_image=self.fallback_image(labbook))
packages = []
if result:
lines = result.decode('utf-8').split('\n')
for line in lines:
if line is not None and line != "Listing..." and "/" in line:
parts = line.split(" ")
package_name, _ = parts[0].split("/")
version = parts[1].strip()
packages.append({'name': package_name, 'version': version})
return packages
def _start_bundled_app(cls, labbook: LabBook, router: ProxyRouter, username: str, bundled_app: dict,
container_override_id: str = None):
tool_port = bundled_app['port']
labbook_ip = ContainerOperations.get_labbook_ip(labbook, username)
endpoint = f'http://{labbook_ip}:{tool_port}'
route_prefix = quote_plus(bundled_app['name'])
matched_routes = router.get_matching_routes(endpoint, route_prefix)
run_command = True
suffix = None
if len(matched_routes) == 1:
logger.info(f"Found existing {bundled_app['name']} in route table for {str(labbook)}.")
suffix = matched_routes[0]
run_command = False
elif len(matched_routes) > 1:
raise ValueError(f"Multiple {bundled_app['name']} instances found in route table "
f"for {str(labbook)}! Restart container.")
if run_command:
logger.info(f"Adding {bundled_app['name']} to route table for {str(labbook)}.")
suffix, _ = router.add(endpoint, route_prefix)
suffix = "/" + suffix
# Start app
logger.info(f"Starting {bundled_app['name']} in {str(labbook)}.")
start_bundled_app(labbook, username, bundled_app['command'], tag=container_override_id)
return suffix
def _start_rstudio(cls,
lb: LabBook,
pr: ProxyRouter,
username: str,
container_override_id: str = None):
lb_ip = ContainerOperations.get_labbook_ip(lb, username)
lb_endpoint = f'http://{lb_ip}:8787'
mitm_endpoint = MITMProxyOperations.get_mitmendpoint(lb_endpoint)
# start mitm proxy if it doesn't exist
if mitm_endpoint is None:
# get a proxy prefix
unique_key = unique_id()
# start proxy
mitm_endpoint = MITMProxyOperations.start_mitm_proxy(
lb_endpoint, unique_key)
# Ensure we start monitor when starting MITM proxy
start_labbook_monitor(
lb,
username,
"rstudio",
# This is the endpoint for the proxy and not the rserver?
url=f'{lb_endpoint}/{unique_key}',
author=get_logged_in_author())
# All messages will come through MITM, so we don't need to monitor rserver directly
start_rserver(lb, username, tag=container_override_id)
# add route
rt_prefix, _ = pr.add(mitm_endpoint, f'rserver/{unique_key}/')
# Warning: RStudio will break if there is a trailing slash!
suffix = f'/{rt_prefix}'
else:
# existing route to MITM or not?
matched_routes = pr.get_matching_routes(mitm_endpoint, 'rserver')
if len(matched_routes) == 1:
suffix = matched_routes[0]
elif len(matched_routes) == 0:
logger.warning(
'Creating missing route for existing RStudio mitmproxy_proxy'
)
# TODO DC: This feels redundant with already getting the mitm_endpoint above
# Can we refactor this into a more coherent single operation? Maybe an MITMProxy instance?
unique_key = MITMProxyOperations.get_mitmkey(lb_endpoint)
# add route
rt_prefix, _ = pr.add(mitm_endpoint, f'rserver/{unique_key}/')
# Warning: RStudio will break if there is a trailing slash!
suffix = f'/{rt_prefix}'
else:
raise ValueError(
f"Multiple RStudio proxy instances for {str(lb)}. Please restart the Project "
"or manually delete stale containers.")
return suffix
def test_start_bundled_app(self, build_image_for_jupyterlab):
"""Test bundled app
Note: In the fixture `build_image_for_jupyterlab`, the environment has been augmented to add the bundled app
"share" with the command "cd /mnt; python3 -m http.server 9999" on port 9999. This test then verifies that
the command is executed and route opened through the CHP.
The reason the dev tool is called "share" is because this test simply uses http.server to run a simple web
server in the /mnt directory. There will be a folder there called "share". http.server doesn't support
route prefixes unless you write a bit of code, so simply calling the app "share" is a simple work around
to create a funtioning test. So if you go to the route provided you'll get a 200 back if everything started
and is wired up OK.
"""
# Start the container
_, container_id = ContainerOperations.start_container(
build_image_for_jupyterlab[0], username='******')
lb = build_image_for_jupyterlab[0]
docker_client = build_image_for_jupyterlab[2]
gql_client = build_image_for_jupyterlab[4]
owner = build_image_for_jupyterlab[-1]
try:
q = f"""
mutation x {{
startDevTool(input: {{
owner: "{owner}",
labbookName: "{lb.name}",
devTool: "share"
}}) {{
path
}}
}}
"""
r = gql_client.execute(q)
assert 'errors' not in r
time.sleep(5)
url = f"http://localhost{r['data']['startDevTool']['path']}"
response = requests.get(url)
assert response.status_code == 200
# Running again should work and just load route
r = gql_client.execute(q)
assert 'errors' not in r
url = f"http://localhost{r['data']['startDevTool']['path']}"
response = requests.get(url)
assert response.status_code == 200
finally:
# Remove the container you fired up
docker_client.containers.get(container_id=container_id).stop(
timeout=10)
docker_client.containers.get(container_id=container_id).remove()
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
lb, container_id = ContainerOperations.start_container(
labbook=lb, username=username)
logger.info(f'Started new {lb} container ({container_id})')
return StartContainer(environment=Environment(owner=owner, name=labbook_name))
def test_list_versions_from_fallback(self, mock_config_with_repo):
"""Test list_versions command"""
username = "******"
im = InventoryManager(mock_config_with_repo[0])
lb = im.create_labbook(
'unittest',
'unittest',
'labbook-unittest-01',
description="From mock_config_from_repo fixture")
# Create Component Manager
cm = ComponentManager(lb)
# Add a component
cm.add_base(ENV_UNIT_TEST_REPO, ENV_UNIT_TEST_BASE, ENV_UNIT_TEST_REV)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
try:
lb, docker_image_id = ContainerOperations.build_image(
labbook=lb, username=username)
# Test lookup
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
# Delete image
client.images.remove(docker_image_id, force=True, noprune=False)
# Test lookup still works
mrg = PipPackageManager()
result = mrg.search("peppercorn", lb, username)
assert len(result) == 2
result = mrg.search("gigantum", lb, username)
assert len(result) == 4
assert result[0] == "gigantum"
finally:
shutil.rmtree(lb.root_dir)
# Remove image if it's still there
try:
client.images.remove(docker_image_id,
force=True,
noprune=False)
except:
pass
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
if confirm:
logger.info(f"Deleting {str(lb)}...")
try:
lb, stopped = ContainerOperations.stop_container(
labbook=lb, username=username)
except OSError as e:
logger.warning(e)
lb, docker_removed = ContainerOperations.delete_image(
labbook=lb, username=username)
if not docker_removed:
raise ValueError(
f'Cannot delete docker image for {str(lb)} - unable to delete LB from disk'
)
# TODO - gtmcore should contain routine to properly delete a labbook
shutil.rmtree(lb.root_dir, ignore_errors=True)
if os.path.exists(lb.root_dir):
logger.error(
f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!'
)
return DeleteLabbook(success=False)
else:
return DeleteLabbook(success=True)
else:
logger.info(f"Dry run in deleting {str(lb)} -- not deleted.")
return DeleteLabbook(success=False)
def _start_jupyter_tool(cls,
lb: LabBook,
pr: ProxyRouter,
username: str,
container_override_id: str = None):
tool_port = 8888
lb_ip = ContainerOperations.get_labbook_ip(lb, username)
lb_endpoint = f'http://{lb_ip}:{tool_port}'
matched_routes = pr.get_matching_routes(lb_endpoint, 'jupyter')
run_start_jupyter = True
suffix = None
if len(matched_routes) == 1:
logger.info(
f'Found existing Jupyter instance in route table for {str(lb)}.'
)
suffix = matched_routes[0]
# wait for jupyter to be up
try:
check_jupyter_reachable(lb_ip, tool_port, suffix)
run_start_jupyter = False
except GigantumException:
logger.warning(
f'Detected stale route. Attempting to restart Jupyter and clean up route table.'
)
pr.remove(suffix[1:])
elif len(matched_routes) > 1:
raise ValueError(
f"Multiple Jupyter instances found in route table for {str(lb)}! Restart container."
)
if run_start_jupyter:
rt_prefix = unique_id()
rt_prefix, _ = pr.add(lb_endpoint, f'jupyter/{rt_prefix}')
# Start jupyterlab
suffix = start_jupyter(lb,
username,
tag=container_override_id,
proxy_prefix=rt_prefix)
# Ensure we start monitor IFF jupyter isn't already running.
start_labbook_monitor(lb,
username,
'jupyterlab',
url=f'{lb_endpoint}/{rt_prefix}',
author=get_logged_in_author())
return suffix
def test_old_dockerfile_removed_when_new_build_fails(
self, build_lb_image_for_jupyterlab):
# Test that when a new build fails, old images are removed so they cannot be launched.
my_lb = build_lb_image_for_jupyterlab[0]
docker_image_id = build_lb_image_for_jupyterlab[3]
my_lb, stopped = ContainerOperations.stop_container(
my_lb, username="******")
assert stopped
olines = open(os.path.join(my_lb.root_dir,
'.gigantum/env/Dockerfile')).readlines()[:6]
with open(os.path.join(my_lb.root_dir, '.gigantum/env/Dockerfile'),
'w') as dockerfile:
dockerfile.write('\n'.join(olines))
dockerfile.write('\nRUN /bin/false')
# We need to remove cache data otherwise the following tests won't work
remove_image_cache_data()
with pytest.raises(ContainerBuildException):
ContainerOperations.build_image(labbook=my_lb,
username="******")
with pytest.raises(docker.errors.ImageNotFound):
owner = InventoryManager().query_owner(my_lb)
get_docker_client().images.get(
infer_docker_image_name(labbook_name=my_lb.name,
owner=owner,
username="******"))
with pytest.raises(requests.exceptions.HTTPError):
# Image not found so container cannot be started
ContainerOperations.start_container(labbook=my_lb,
username="******")
def get_packages_metadata(self, package_list: List[str], labbook: LabBook,
username: str) -> List[PackageMetadata]:
"""Method to get package metadata
Args:
package_list: List of package names
labbook(str): The labbook instance
username(str): The username for the logged in user
Returns:
list
"""
results = list()
for package in package_list:
result = ContainerOperations.run_command(
f"apt-cache search {package}",
labbook,
username,
fallback_image=self.fallback_image(labbook))
description = None
latest_version = None
if result:
lines = result.decode('utf-8').split('\n')
for l in lines:
if l:
pkg_name, pkg_description = l.split(" - ", 1)
if pkg_name == package:
description = pkg_description.strip()
break
# Get the latest version of the package
try:
versions = self.list_versions(package, labbook, username)
if versions:
latest_version = versions[0]
except ValueError:
# If package isn't found, just set to None
pass
results.append(
PackageMetadata(package_manager="apt",
package=package,
latest_version=latest_version,
description=description,
docs_url=None))
return results
def list_installed_packages(self, labbook: LabBook,
username: str) -> List[Dict[str, str]]:
"""Method to get a list of all packages that are currently installed
Note, this will return results for the computer/container in which it is executed. To get the properties of
a LabBook container, a docker exec command would be needed from the Gigantum application container.
return format is a list of dicts with the format (name: <package name>, version: <version string>)
Returns:
list
"""
packages = ContainerOperations.run_command(
'pip list --format=json',
labbook,
username,
fallback_image=self.fallback_image(labbook))
return json.loads(packages.decode())
def list_installed_packages(self, labbook: LabBook,
username: str) -> List[Dict[str, str]]:
"""Method to get a list of all packages that are currently installed
Note, this will return results for the computer/container in which it is executed. To get the properties of
a LabBook container, a docker exec command would be needed from the Gigantum application container.
return format is a list of dicts with the format (name: <package name>, version: <version string>)
Returns:
list
"""
result = ContainerOperations.run_command(f"conda list --no-pip --json",
labbook, username)
data = json.loads(result.decode().strip())
if data:
return [{"name": x['name'], 'version': x['version']} for x in data]
else:
return []
def search(self, search_str: str, labbook: LabBook,
username: str) -> List[str]:
"""Method to search a package manager for packages based on a string. The string can be a partial string.
Args:
search_str: The string to search on
labbook: Subject LabBook
username: username of current user
Returns:
list(str): The list of package names that match the search string
"""
# Add wildcard for search
if search_str[-1] != '*':
search_str = search_str + '*'
try:
result = ContainerOperations.run_command(
f'conda search --json "{search_str}"',
labbook=labbook,
username=username,
fallback_image=self.fallback_image(labbook))
except ContainerException as e:
logger.error(e)
return list()
data = json.loads(result.decode())
if 'exception_name' in data:
if data.get('exception_name') in [
'PackagesNotFoundError', 'PackageNotFoundError'
]:
# This means you entered an invalid package name that didn't resolve to anything
return list()
else:
raise Exception(
f"An error occurred while searching for packages: {data.get('exception_name')}"
)
if data:
return list(data.keys())
else:
return list()
def search(self, search_str: str, labbook: LabBook,
username: str) -> List[str]:
"""Method to search a package manager for packages based on a string. The string can be a partial string.
Args:
search_str: The string to search on
labbook: Subject LabBook
username: username of current user
Returns:
list(str): The list of package names that match the search string
"""
search_result = ContainerOperations.run_command(
f'pip search {search_str}',
labbook,
username,
fallback_image=self.fallback_image(labbook))
lines = search_result.decode().splitlines()
packages = [x.split(' ')[0] for x in lines]
return sorted(packages)
def test_start_jupyterlab(self, build_image_for_jupyterlab):
"""Test listing labbooks"""
# Start the container
_, container_id = ContainerOperations.start_container(
build_image_for_jupyterlab[0], username='******')
lb = build_image_for_jupyterlab[0]
docker_client = build_image_for_jupyterlab[2]
gql_client = build_image_for_jupyterlab[4]
owner = build_image_for_jupyterlab[-1]
try:
q = f"""
mutation x {{
startDevTool(input: {{
owner: "{owner}",
labbookName: "{lb.name}",
devTool: "jupyterlab"
}}) {{
path
}}
}}
"""
r = gql_client.execute(q)
assert 'errors' not in r
assert ':10000/jupyter/' in r['data']['startDevTool']['path']
rc, t = docker_client.containers.get(
container_id=container_id).exec_run(
'sh -c "ps aux | grep jupyter-lab | grep -v \' grep \'"',
user='******')
l = [a for a in t.decode().split('\n') if a]
assert len(l) == 1
finally:
# Remove the container you fired up
docker_client.containers.get(container_id=container_id).stop(
timeout=10)
docker_client.containers.get(container_id=container_id).remove()
def latest_versions(self, package_names: List[str], labbook: LabBook,
username: str) -> List[str]:
"""Method to get the latest version string for a list of packages
Args:
package_names: list of names of the packages to query
labbook: Subject LabBook
username: username of current user
Returns:
list: latest version strings
"""
cmd = [
'conda', 'install', '--dry-run', '--no-deps', '--json',
*package_names
]
try:
result = ContainerOperations.run_command(
' '.join(cmd),
labbook,
username,
override_image_tag=self.fallback_image(
labbook)).decode().strip()
except Exception as e:
logger.error(e)
pkgs = ", ".join(package_names)
raise ValueError(
f"Could not retrieve latest versions due to invalid package name in list: {pkgs}"
)
versions = {pn: "" for pn in package_names}
if result:
data = json.loads(result)
if data.get('exception_name') == "PackagesNotFoundError":
# Conda failed because of invalid packages. indicate failure
err_pkgs = [x for x in data.get('packages')]
raise ValueError(
f"Could not retrieve latest versions due to invalid package name in list: {err_pkgs}"
)
if data.get('actions') is not None:
for package_name in package_names:
if isinstance(data.get('actions'), dict) is True:
# New method - added when bases updated to conda 4.5.1
for p in data.get('actions').get('LINK'):
if p.get('name') == package_name:
versions[package_name] = p.get("version")
else:
# legacy methods to handle older bases built on conda 4.3.31
try:
for p in [
x.get('LINK')[0]
for x in data.get('actions') if x
]:
if p.get('name') == package_name:
versions[package_name] = p.get("version")
except Exception as e:
for p in [
x.get('LINK') for x in data.get('actions')
if x
]:
if p.get('name') == package_name:
versions[package_name] = p.get("version")
# For any packages whose versions could not be found (because they are installed and latest)
# just look up the installed versions
missing_keys = [k for k in versions.keys() if versions[k] == ""]
if missing_keys:
cmd = ['conda', 'list', '--no-pip', '--json']
result = ContainerOperations.run_command(
' '.join(cmd),
labbook,
username,
override_image_tag=self.fallback_image(
labbook)).decode().strip()
installed_info = json.loads(result)
installed_versions = {
pkg['name']: pkg['version']
for pkg in installed_info
}
for pn in missing_keys:
versions[pn] = installed_versions[pn]
# Reformat into list and return
output_versions = [versions[p] for p in package_names]
return output_versions
def validate_packages(self, package_list: List[Dict[str, str]], labbook: LabBook, username: str) \
-> List[PackageResult]:
"""Method to validate a list of packages, and if needed fill in any missing versions
Should check both the provided package name and version. If the version is omitted, it should be generated
from the latest version.
Args:
package_list(list): A list of dictionaries of packages to validate
labbook(str): The labbook instance
username(str): The username for the logged in user
Returns:
namedtuple: namedtuple indicating if the package and version are valid
"""
# Build install string
pkgs = list()
for p in package_list:
if p['version']:
pkgs.append(f"{p['package']}={p['version']}")
else:
pkgs.append(p['package'])
cmd = ['conda', 'install', '--dry-run', '--no-deps', '--json', *pkgs]
try:
cmd_result = ContainerOperations.run_command(
' '.join(cmd),
labbook,
username,
override_image_tag=self.fallback_image(labbook))
container_result = cmd_result.decode().strip()
except Exception as e:
logger.error(e)
raise ValueError(f"An error occured while validating packages")
if not container_result:
raise ValueError(
f"Failed to get response from Docker while querying for package info"
)
# Good to process
data = json.loads(container_result)
if data.get('exception_name') == "PackagesNotFoundError":
# Conda failed because of invalid packages. indicate failures.
result = list()
for pkg_str, pkg_data in zip(pkgs, package_list):
if pkg_str in data.get('packages'):
result.append(
PackageResult(package=pkg_data['package'],
version=pkg_data['version'],
error=True))
else:
result.append(
PackageResult(package=pkg_data['package'],
version=pkg_data['version'],
error=False))
return result
# All packages are valid, collect data
conda_data = dict()
if isinstance(data.get('actions'), dict) is True:
# New method - added when bases updated to conda 4.5.1
for p in data.get('actions').get('LINK'):
conda_data[p.get('name')] = p.get('version')
else:
# legacy methods to handle older bases built on conda 4.3.31
try:
for p in [x.get('LINK')[0] for x in data.get('actions') if x]:
conda_data[p.get('name')] = p.get('version')
except Exception:
for p in [x.get('LINK') for x in data.get('actions') if x]:
conda_data[p.get('name')] = p.get('version')
# Return properly formatted data
return [
PackageResult(package=x['package'],
version=conda_data[x['package']],
error=False) for x in package_list
]
def list_versions(self, package_name: str, labbook: LabBook,
username: str) -> List[str]:
"""Method to list all available versions of a package based on the package name
Args:
package_name: Name of the package to query
labbook: Subject LabBook
username: username of current user
Returns:
list(str): Version strings
"""
try:
result = ContainerOperations.run_command(
f"conda info --json {package_name}",
labbook,
username,
fallback_image=self.fallback_image(labbook))
data = json.loads(result.decode())
except ContainerException as e:
logger.error(e)
data = {}
# TODO: Conda does not seem to throw this anymore. Remove once confirmed
if 'exception_name' in data:
raise ValueError(
f"An error occurred while getting package versions: {data.get('exception_name')}"
)
if len(data.keys()) == 0 or len(data.get(package_name)) == 0:
raise ValueError(f"Package {package_name} not found")
# Check to see if this is a python package. If so, filter based on the current version of python (set in child)
if any([
True for x in data.get(package_name)
if self.python_depends_str in x.get('depends')
]):
versions = [
x.get('version') for x in data.get(package_name)
if self.python_depends_str in x.get('depends')
]
else:
versions = [x.get('version') for x in data.get(package_name)]
versions = list(OrderedDict.fromkeys(versions))
try:
versions.sort(key=StrictVersion)
except ValueError as e:
if 'invalid version number' in str(e):
try:
versions.sort(key=LooseVersion)
except Exception:
versions = natsorted(
versions, key=lambda x: x.replace('.', '~') + 'z')
else:
raise e
versions.reverse()
return versions