def test_get_sessions(self, redis_client, monkeypatch):
"""Test getting the session information from jupyterlab"""
monkeypatch.setattr(requests, 'get', MockSessionsResponse)
monkeypatch.setattr(JupyterLabMonitor, 'get_container_ip', mock_ip)
monitor = JupyterLabMonitor()
# prep redis data
dev_env_key = "dev_env_monitor:{}:{}:{}:{}".format(
'default', 'default', 'test-labbook', 'jupyterlab')
lb_key = infer_docker_image_name('test-labbook', 'default', 'default')
redis_client.set(f"{lb_key}-jupyter-token", "afaketoken")
redis_client.hset(dev_env_key, "url",
"http://localhost:10000/jupyter/asdf/")
sessions = monitor.get_sessions(dev_env_key, redis_conn=redis_client)
assert sessions['6e529520-2a6d-4adb-a2a1-de10b85b86a6'][
'kernel_id'] == '6e529520-2a6d-4adb-a2a1-de10b85b86a6'
assert sessions['6e529520-2a6d-4adb-a2a1-de10b85b86a6'][
'kernel_name'] == 'python3'
assert sessions['6e529520-2a6d-4adb-a2a1-de10b85b86a6'][
'kernel_type'] == 'notebook'
assert sessions['6e529520-2a6d-4adb-a2a1-de10b85b86a6'][
'path'] == 'Untitled2.ipynb'
assert sessions['146444cd-b4ec-4c00-b29c-d8cef4a503b0'][
'kernel_id'] == '146444cd-b4ec-4c00-b29c-d8cef4a503b0'
assert sessions['146444cd-b4ec-4c00-b29c-d8cef4a503b0'][
'kernel_name'] == 'python3'
assert sessions['146444cd-b4ec-4c00-b29c-d8cef4a503b0'][
'kernel_type'] == 'notebook'
assert sessions['146444cd-b4ec-4c00-b29c-d8cef4a503b0'][
'path'] == 'code/Untitled.ipynb'
def copy_into_container(cls, labbook: LabBook, username: str, src_path: str, dst_dir: str):
"""Copy the given file in src_path into the project's container.
Args:
labbook: Project under consideration.
username: Active username
src_path: Source path ON THE HOST of the file - callers responsibility to sanitize
dst_dir: Destination directory INSIDE THE CONTAINER.
"""
if not labbook.owner:
raise ContainerException(f"{str(labbook)} has no owner")
if not os.path.isfile(src_path):
raise ContainerException(f"Source file {src_path} is not a file")
docker_key = infer_docker_image_name(labbook_name=labbook.name,
owner=labbook.owner,
username=username)
lb_container = docker.from_env().containers.get(docker_key)
r = lb_container.exec_run(f'sh -c "mkdir -p {dst_dir}"')
# Tar up the src file to copy into container
tarred_secret_file = tempfile.NamedTemporaryFile()
t = tarfile.open(mode='w', fileobj=tarred_secret_file)
abs_path = os.path.abspath(src_path)
t.add(abs_path, arcname=os.path.basename(src_path), recursive=True)
t.close()
tarred_secret_file.seek(0)
try:
logger.info(f"Copying file {src_path} into {dst_dir} in {str(labbook)}")
docker.from_env().api.put_archive(docker_key, dst_dir, tarred_secret_file)
finally:
# Make sure the temporary Tar archive gets deleted.
tarred_secret_file.close()
def start_rserver(labbook: LabBook,
username: str,
tag: Optional[str] = None,
check_reachable: bool = True) -> None:
""" Main entrypoint to launch rstudio-server. Note, the caller must
determine for themselves the host and port.
Raises an exception if there's a problem.
Returns:
Path to rstudio-server
"""
owner = InventoryManager().query_owner(labbook)
lb_key = tag or infer_docker_image_name(
labbook_name=labbook.name, owner=owner, username=username)
docker_client = get_docker_client()
lb_container = docker_client.containers.get(lb_key)
if lb_container.status != 'running':
raise GigantumException(f"{str(labbook)} container is not running")
rserver_ps = ps_search(lb_container, 'rserver')
if len(rserver_ps) == 1:
# we have an existing rstudio-server instance
return
elif len(rserver_ps) == 0:
_start_rserver_process(lb_container)
else:
# If "ps aux" for rserver returns multiple hits - this should never happen.
for n, l in enumerate(rserver_ps):
logger.error(
f'Multiple RStudio-Server instances - ({n+1} of {len(rserver_ps)}) - {l}'
)
raise ValueError(
f'Multiple ({len(rserver_ps)}) RStudio Server instances detected')
def delete_image(cls,
labbook: LabBook,
override_image_tag: Optional[str] = None,
username: Optional[str] = None) -> Tuple[LabBook, bool]:
""" Delete the Docker image for the given LabBook
Args:
labbook: Subject LabBook.
override_image_tag: Tag of docker image (optional)
username: The current logged in username
Returns:
A tuple containing the labbook, docker image id.
"""
owner = InventoryManager().query_owner(labbook)
image_name = override_image_tag or infer_docker_image_name(
labbook_name=labbook.name, owner=owner, username=username)
# We need to remove any images pertaining to this labbook before triggering a build.
try:
get_docker_client().images.get(name=image_name)
get_docker_client().images.remove(image_name)
except docker.errors.ImageNotFound:
pass
except Exception as e:
logger.error("Error deleting docker images for {str(lb)}: {e}")
return labbook, False
return labbook, True
def stop_container(cls,
labbook: LabBook,
username: Optional[str] = None) -> Tuple[LabBook, bool]:
""" Stop the given labbook. Returns True in the second field if stopped,
otherwise False (False can simply imply no container was running).
Args:
labbook: Subject labbook
username: Optional username of active user
Returns:
A tuple of (Labbook, boolean indicating whether a container was successfully stopped).
"""
owner = InventoryManager().query_owner(labbook)
n = infer_docker_image_name(labbook_name=labbook.name,
owner=owner,
username=username)
logger.info(f"Stopping {str(labbook)} ({n})")
try:
stopped = stop_labbook_container(n)
finally:
# Save state of LB when container turned off.
labbook.sweep_uncommitted_changes()
return labbook, stopped
def get_sessions(key: str, redis_conn: redis.Redis) -> Dict[str, Any]:
"""Method to get and reformat session info from JupyterLab
Args:
key(str): The unique string used as the key in redis to track this DevEnvMonitor instance
redis_conn(redis.Redis): A redis client
Returns:
dict
"""
_, username, owner, labbook_name, _ = key.split(':')
lb_key = infer_docker_image_name(labbook_name, owner, username)
token = redis_conn.get(f"{lb_key}-jupyter-token").decode()
url = redis_conn.hget(key, "url").decode()
# Get List of active sessions
path = f'{url}/api/sessions?token={token}'
r = requests.get(path)
if r.status_code != 200:
raise IOError(
f"Failed to get session listing from JupyterLab {path}")
sessions = r.json()
data = {}
for session in sessions:
data[session['kernel']['id']] = {
"kernel_id": session['kernel']['id'],
"kernel_name": session['kernel']['name'],
"kernel_type": session['type'],
"path": session['path']
}
return data
def test_build_and_start_and_stop_labbook_container(
self, mock_config_file):
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
image_id = jobs.build_labbook_image(**docker_kwargs)
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = jobs.start_labbook_container(**startc_kwargs)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
jobs.stop_labbook_container(container_id)
with pytest.raises(Exception):
# Should not be found because the stop job cleans up
get_docker_client().containers.get(container_id)
def labbook_image_name(cls, labbook: LabBook, username: str) -> str:
"""Return the image name of the LabBook container
Args:
labbook: Subject LabBook
username: Username of active user
Returns:
Externally facing IP
"""
owner = InventoryManager().query_owner(labbook)
return infer_docker_image_name(labbook_name=labbook.name,
owner=owner,
username=username)
def get_container_status(labbook_name: str, owner: str, username: str) -> bool:
labbook_key = infer_docker_image_name(labbook_name=labbook_name, owner=owner,
username=username)
try:
client = get_docker_client()
container = client.containers.get(labbook_key)
if container.status == "running":
return True
else:
return False
except:
pass
return False
def get_container_ip(self) -> Optional[str]:
"""Method to get the monitored lab book container's IP address on the Docker bridge network
Returns:
str
"""
client = get_docker_client()
lb_key = infer_docker_image_name(self.labbook_name, self.owner,
self.user)
container = client.containers.get(lb_key)
ip = container.attrs['NetworkSettings']['Networks']['bridge'][
'IPAddress']
logger.info("container {} IP: {}".format(container.name, ip))
return ip
def get_labbook_ip(cls, labbook: LabBook, username: str) -> str:
"""Return the IP on the docker network of the LabBook container
Args:
labbook: Subject LabBook
username: Username of active user
Returns:
Externally facing IP
"""
owner = InventoryManager().query_owner(labbook)
docker_key = infer_docker_image_name(labbook_name=labbook.name,
owner=owner,
username=username)
return get_container_ip(docker_key)
def resolve_container_status(self, info):
"""Resolve the image_status field"""
# Check if the container is running by looking up the container
labbook_key = infer_docker_image_name(
labbook_name=self.name,
owner=self.owner,
username=get_logged_in_username())
try:
client = get_docker_client()
container = client.containers.get(labbook_key)
if container.status == "running":
container_status = ContainerStatus.RUNNING
else:
container_status = ContainerStatus.NOT_RUNNING
except (NotFound, requests.exceptions.ConnectionError):
container_status = ContainerStatus.NOT_RUNNING
return container_status.value
def test_run(self, redis_client, monkeypatch):
"""Test running the monitor process"""
monkeypatch.setattr(requests, 'get', MockSessionsResponse)
monkeypatch.setattr(JupyterLabMonitor, 'get_container_ip', mock_ip)
# TODO: Mock dispatch methods once added
monitor = JupyterLabMonitor()
dev_env_key = "dev_env_monitor:{}:{}:{}:{}".format(
'default', 'default', 'test-labbook', 'jupyterlab-ubuntu1604')
lb_key = infer_docker_image_name('test-labbook', 'default', 'default')
redis_client.set(f"{lb_key}-jupyter-token", "afaketoken")
redis_client.hset(dev_env_key, "author_name", "default")
redis_client.hset(dev_env_key, "author_email", "*****@*****.**")
redis_client.hset(dev_env_key, "url",
"http://localhost:10000/jupyter/asdf/")
monitor.run(dev_env_key)
data = redis_client.hgetall(
'{}:activity_monitor:6e529520-2a6d-4adb-a2a1-de10b85b86a6'.format(
dev_env_key))
assert data[b'kernel_id'] == b'6e529520-2a6d-4adb-a2a1-de10b85b86a6'
assert data[b'kernel_name'] == b'python3'
assert data[b'kernel_type'] == b'notebook'
assert data[b'path'] == b'Untitled2.ipynb'
assert data[b'dev_env_monitor'].decode() == dev_env_key
assert 'rq:job' in data[b'process_id'].decode()
data = redis_client.hgetall(
'{}:activity_monitor:146444cd-b4ec-4c00-b29c-d8cef4a503b0'.format(
dev_env_key))
assert data[b'kernel_id'] == b'146444cd-b4ec-4c00-b29c-d8cef4a503b0'
assert data[b'kernel_name'] == b'python3'
assert data[b'kernel_type'] == b'notebook'
assert data[b'path'] == b'code/Untitled.ipynb'
assert data[b'dev_env_monitor'].decode() == dev_env_key
assert 'rq:job' in data[b'process_id'].decode()
def test_old_dockerfile_removed_when_new_build_fails(
self, build_lb_image_for_jupyterlab):
# Test that when a new build fails, old images are removed so they cannot be launched.
my_lb = build_lb_image_for_jupyterlab[0]
docker_image_id = build_lb_image_for_jupyterlab[3]
my_lb, stopped = ContainerOperations.stop_container(
my_lb, username="******")
assert stopped
olines = open(os.path.join(my_lb.root_dir,
'.gigantum/env/Dockerfile')).readlines()[:6]
with open(os.path.join(my_lb.root_dir, '.gigantum/env/Dockerfile'),
'w') as dockerfile:
dockerfile.write('\n'.join(olines))
dockerfile.write('\nRUN /bin/false')
# We need to remove cache data otherwise the following tests won't work
remove_image_cache_data()
with pytest.raises(ContainerBuildException):
ContainerOperations.build_image(labbook=my_lb,
username="******")
with pytest.raises(docker.errors.ImageNotFound):
owner = InventoryManager().query_owner(my_lb)
get_docker_client().images.get(
infer_docker_image_name(labbook_name=my_lb.name,
owner=owner,
username="******"))
with pytest.raises(requests.exceptions.HTTPError):
# Image not found so container cannot be started
ContainerOperations.start_container(labbook=my_lb,
username="******")
def helper_resolve_image_status(self, labbook):
"""Helper to resolve the image status of a labbook"""
labbook_image_key = infer_docker_image_name(
labbook_name=self.name,
owner=self.owner,
username=get_logged_in_username())
dispatcher = Dispatcher()
lb_jobs = [
dispatcher.query_task(j.job_key)
for j in dispatcher.get_jobs_for_labbook(labbook.key)
]
for j in lb_jobs:
logger.debug("Current job for labbook: status {}, meta {}".format(
j.status, j.meta))
# First, check if image exists or not -- The first step of building an image untags any existing ones.
# Therefore, we know that if one exists, there most likely is not one being built.
try:
client = get_docker_client()
client.images.get(labbook_image_key)
image_status = ImageStatus.EXISTS
except (ImageNotFound, requests.exceptions.ConnectionError):
image_status = ImageStatus.DOES_NOT_EXIST
if any([
j.status == 'failed' and j.meta.get('method') == 'build_image'
for j in lb_jobs
]):
logger.debug("Image status for {} is BUILD_FAILED".format(
labbook.key))
if image_status == ImageStatus.EXISTS:
# The indication that there's a failed job is probably lingering from a while back, so don't
# change the status to FAILED. Only do that if there is no Docker image.
logger.debug(
f'Got failed build_image for labbook {labbook.key}, but image exists.'
)
else:
image_status = ImageStatus.BUILD_FAILED
if any([
j.status in ['started']
and j.meta.get('method') == 'build_image' for j in lb_jobs
]):
logger.debug(
f"Image status for {labbook.key} is BUILD_IN_PROGRESS")
# build_image being in progress takes precedence over if image already exists (unlikely event).
if image_status == ImageStatus.EXISTS:
logger.warning(
f'Got build_image for labbook {labbook.key}, but image exists.'
)
image_status = ImageStatus.BUILD_IN_PROGRESS
if any([
j.status in ['queued']
and j.meta.get('method') == 'build_image' for j in lb_jobs
]):
logger.warning(
f"build_image for {labbook.key} stuck in queued state")
image_status = ImageStatus.BUILD_QUEUED
return image_status.value
def start_labbook_monitor(labbook: LabBook,
username: str,
dev_tool: str,
url: str,
database: int = 1,
author: Optional[GitAuthor] = None) -> None:
"""Method to start Development Environment Monitors for a given Lab Book if available
Args:
labbook(LabBook): A populated LabBook instance to start monitoring
username(str): The username of the logged in user
dev_tool(str): The name of the development tool to monitor
url(str): URL (from LabManager) at which this dev tool can be reached.
database(int): The redis database ID to use for key storage. Default should be 1
author(GitAuthor): A GitAuthor instance for the current logged in user starting the monitor
Returns:
None
"""
# Connect to redis
redis_conn = redis.Redis(db=database)
# Get all dev env monitors currently running
dev_env_monitors = redis_conn.keys("dev_env_monitor:*")
# Clean up after Lab Books that have "closed" by checking if the container is running
docker_client = get_docker_client()
for key in dev_env_monitors:
if "activity_monitor" in key.decode():
# Ignore all associated activity monitors, as they'll get cleaned up with the dev env monitor
continue
container_name = redis_conn.hget(key, 'container_name')
try:
docker_client.containers.get(container_name.decode())
except NotFound:
# Container isn't running, clean up
logger.warn(
"Shutting down zombie Activity Monitoring for {}.".format(
key.decode()))
stop_dev_env_monitors(key.decode(), redis_conn, labbook.name)
# Check if Dev Env is supported and then start Dev Env Monitor
dev_env_mgr = DevEnvMonitorManager(database=database)
if dev_env_mgr.is_available(dev_tool):
# Add record to redis for Dev Env Monitor
owner = InventoryManager().query_owner(labbook)
dev_env_monitor_key = "dev_env_monitor:{}:{}:{}:{}".format(
username, owner, labbook.name, dev_tool)
if redis_conn.exists(dev_env_monitor_key):
# Assume already set up properly (it wasn't cleaned up above)
logger.info(
f'Found existing entry for {dev_env_monitor_key}, skipping setup'
)
return
owner = InventoryManager().query_owner(labbook)
redis_conn.hset(dev_env_monitor_key, "container_name",
infer_docker_image_name(labbook.name, owner, username))
redis_conn.hset(dev_env_monitor_key, "labbook_root", labbook.root_dir)
redis_conn.hset(dev_env_monitor_key, "url", url)
# Set author information so activity records can be committed on behalf of the user
if author:
redis_conn.hset(dev_env_monitor_key, "author_name", author.name)
redis_conn.hset(dev_env_monitor_key, "author_email", author.email)
# Schedule dev env
d = Dispatcher()
kwargs = {'dev_env_name': dev_tool, 'key': dev_env_monitor_key}
job_key = d.schedule_task(run_dev_env_monitor,
kwargs=kwargs,
repeat=None,
interval=3)
redis_conn.hset(dev_env_monitor_key, "process_id", job_key.key_str)
logger.info("Started `{}` dev env monitor for lab book `{}`".format(
dev_tool, labbook.name))
else:
raise ValueError(
f"{dev_tool} Developer Tool does not support monitoring")
def start_labbook_container(labbook_root: str,
config_path: str,
username: str,
override_image_id: Optional[str] = None) -> str:
""" Start a Docker container from a given image_name.
Args:
labbook_root: Root dir of labbook
config_path: Path to LabBook configuration file.
override_image_id: Optional explicit docker image id (do not infer).
username: Username of active user. Do not use with override_image_id.
Returns:
Tuple containing docker container id, dict mapping of exposed ports.
Raises:
"""
if username and override_image_id:
raise ValueError(
'Argument username and override_image_id cannot both be set')
lb = InventoryManager(
config_file=config_path).load_labbook_from_directory(labbook_root)
if not override_image_id:
owner = InventoryManager().query_owner(lb)
tag = infer_docker_image_name(lb.name, owner, username)
else:
tag = override_image_id
mnt_point = labbook_root.replace('/mnt/gigantum',
os.environ['HOST_WORK_DIR'])
volumes_dict = {
mnt_point: {
'bind': '/mnt/labbook',
'mode': 'cached'
},
'labmanager_share_vol': {
'bind': '/mnt/share',
'mode': 'rw'
}
}
# Set up additional bind mounts for datasets if needed.
submodules = lb.git.list_submodules()
for submodule in submodules:
try:
namespace, dataset_name = submodule['name'].split("&")
submodule_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets',
namespace, dataset_name)
ds = InventoryManager().load_dataset_from_directory(submodule_dir)
ds.namespace = namespace
cm_class = get_cache_manager_class(ds.client_config)
cm = cm_class(ds, username)
ds_cache_dir = cm.current_revision_dir.replace(
'/mnt/gigantum', os.environ['HOST_WORK_DIR'])
volumes_dict[ds_cache_dir] = {
'bind': f'/mnt/labbook/input/{ds.name}',
'mode': 'ro'
}
except InventoryException:
continue
# If re-mapping permissions, be sure to configure the container
if 'LOCAL_USER_ID' in os.environ:
env_var = [f"LOCAL_USER_ID={os.environ['LOCAL_USER_ID']}"]
else:
env_var = ["WINDOWS_HOST=1"]
# Get resource limits
resource_args = dict()
memory_limit = lb.client_config.config['container']['memory']
cpu_limit = lb.client_config.config['container']['cpu']
gpu_shared_mem = lb.client_config.config['container']['gpu_shared_mem']
if memory_limit:
# If memory_limit not None, pass to Docker to limit memory allocation to container
resource_args["mem_limit"] = memory_limit
if cpu_limit:
# If cpu_limit not None, pass to Docker to limit CPU allocation to container
# "nano_cpus" is an integer in factional parts of a CPU
resource_args["nano_cpus"] = round(cpu_limit * 1e9)
docker_client = get_docker_client()
# run with nvidia-docker if we have GPU support on the Host compatible with the project
should_run_nvidia, reason = should_launch_with_cuda_support(
lb.cuda_version)
if should_run_nvidia:
logger.info(f"Launching container with GPU support:{reason}")
if gpu_shared_mem:
resource_args["shm_size"] = gpu_shared_mem
container_id = docker_client.containers.run(tag,
detach=True,
init=True,
name=tag,
environment=env_var,
volumes=volumes_dict,
runtime='nvidia',
**resource_args).id
else:
logger.info(f"Launching container without GPU support. {reason}")
container_id = docker_client.containers.run(tag,
detach=True,
init=True,
name=tag,
environment=env_var,
volumes=volumes_dict,
**resource_args).id
labmanager_ip = ""
try:
labmanager_ip = get_labmanager_ip() or ""
except IndexError:
logger.warning("Cannot find labmanager IP")
labmanager_ip = labmanager_ip.strip()
cmd = f"echo {labmanager_ip} > /home/giguser/labmanager_ip"
for timeout in range(20):
time.sleep(0.5)
if docker_client.containers.get(container_id).status == 'running':
r = docker_client.containers.get(container_id).exec_run(
f'sh -c "{cmd}"')
logger.info(f"Response to write labmanager_ip in {tag}: {r}")
break
else:
logger.error(
"After 10 seconds could not write IP to labmanager container."
f" Container status = {docker_client.containers.get(container_id).status}"
)
return container_id
def build_docker_image(root_dir: str,
username: str,
nocache: bool = False,
override_image_tag: Optional[str] = None,
feedback_callback: Optional[Callable] = None) -> str:
"""
Build a new docker image from the Dockerfile at the given directory, give this image
the name defined by the image_name argument.
Note! This method is static, it should **NOT** use any global variables or any other
reference to global state.
Also note - This will delete any existing image pertaining to the given labbook.
Thus if this call fails, there will be no docker images pertaining to that labbook.
Args:
root_dir: LabBook root directory (obtained by LabBook.root_dir)
override_image_tag: Tag of docker image; in general this should not be explicitly set.
username: Username of active user.
nocache: If True do not use docker cache.
feedback_callback: Optional method taking one argument (a string) to process each line of output
Returns:
A string container the short docker id of the newly built image.
Raises:
ContainerBuildException if container build fails.
"""
if not os.path.exists(root_dir):
raise ValueError(
f'Expected env directory `{root_dir}` does not exist.')
env_dir = os.path.join(root_dir, '.gigantum', 'env')
lb = InventoryManager().load_labbook_from_directory(root_dir)
# Build image
owner = InventoryManager().query_owner(lb)
image_name = override_image_tag or infer_docker_image_name(
labbook_name=lb.name, owner=owner, username=username)
reuse_image_id = _get_cached_image(env_dir, image_name)
if reuse_image_id:
logger.info(f"Reusing Docker image for {str(lb)}")
if feedback_callback:
feedback_callback(f"Using cached image {reuse_image_id}")
return reuse_image_id
try:
image_id = None
# From: https://docker-py.readthedocs.io/en/stable/api.html#docker.api.build.BuildApiMixin.build
# This builds the image and generates output status text.
for line in docker.from_env().api.build(path=env_dir,
tag=image_name,
pull=True,
nocache=nocache,
forcerm=True):
ldict = json.loads(line)
stream = (ldict.get("stream") or "").strip()
if feedback_callback:
feedback_callback(stream)
status = (ldict.get("status") or "").strip()
if feedback_callback:
feedback_callback(status)
if 'Successfully built'.lower() in stream.lower():
# When build, final line is in form of "Successfully build 02faas3"
# There is no other (simple) way to grab the image ID
image_id = stream.split(' ')[-1]
except docker.errors.BuildError as e:
_remove_docker_image(image_name)
raise ContainerBuildException(e)
if not image_id:
_remove_docker_image(image_name)
raise ContainerBuildException(
f"Cannot determine docker image on LabBook from {root_dir}")
return image_id
def run_command(cls,
cmd_text: str,
labbook: LabBook,
username: Optional[str] = None,
override_image_tag: Optional[str] = None,
fallback_image: str = None) -> bytes:
"""Run a command executed in the context of the LabBook's docker image.
Args:
cmd_text: Content of command to be executed.
labbook: Subject labbook
username: Optional active username
override_image_tag: If set, does not automatically infer container name.
fallback_image: If LabBook image can't be found, use this one instead.
Returns:
A tuple containing the labbook, Docker container id, and port mapping.
"""
image_name = override_image_tag
if not image_name:
owner = InventoryManager().query_owner(labbook)
image_name = infer_docker_image_name(labbook_name=labbook.name,
owner=owner,
username=username)
# Get a docker client instance
client = get_docker_client()
# Verify image name exists. If it doesn't, fallback and use the base image
try:
client.images.get(image_name)
except docker.errors.ImageNotFound:
# Image not found...assume build has failed and fallback to base
if not fallback_image:
raise
logger.warning(f"LabBook image not available for package query."
f"Falling back to base image `{fallback_image}`.")
image_name = fallback_image
t0 = time.time()
try:
# Note, for container docs see: http://docker-py.readthedocs.io/en/stable/containers.html
container = client.containers.run(image_name,
cmd_text,
entrypoint=[],
remove=False,
detach=True,
stdout=True)
while container.status != "exited":
time.sleep(.25)
container.reload()
result = container.logs(stdout=True, stderr=False)
container.remove(v=True)
except docker.errors.ContainerError as e:
tfail = time.time()
logger.error(
f'Command ({cmd_text}) failed after {tfail-t0:.2f}s - '
f'output: {e.exit_status}, {e.stderr}')
raise ContainerException(e)
ts = time.time()
if ts - t0 > 5.0:
logger.warning(
f'Command ({cmd_text}) in {str(labbook)} took {ts-t0:.2f} sec')
return result
def test_start_and_stop_docker_container(self, temporary_worker,
mock_config_file):
# start_docker_container(docker_image_id, exposed_ports, volumes_dict) -> str:
w, d = temporary_worker
erm = RepositoryManager(mock_config_file[0])
erm.update_repositories()
erm.index_repositories()
# Create a labbook
lb = InventoryManager(mock_config_file[0]).create_labbook(
'unittester',
'unittester',
'unittest-start-stop-job',
description="Testing docker building.")
cm = ComponentManager(lb)
cm.add_base(gtmcore.fixtures.ENV_UNIT_TEST_REPO,
'quickstart-jupyterlab', 2)
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
docker_kwargs = {
'path': lb.root_dir,
'nocache': True,
'username': '******'
}
client = get_docker_client()
img_list = client.images.list()
try:
from gtmcore.container.utils import infer_docker_image_name
owner = InventoryManager().query_owner(lb)
client.images.remove(
infer_docker_image_name(labbook_name=lb.name,
owner=owner,
username='******'))
except:
pass
m = {'method': 'build_image', 'labbook': "unittest-start-stop-job"}
job_ref = d.dispatch_task(bg_jobs.build_labbook_image,
kwargs=docker_kwargs,
metadata=m)
j = d.query_task(job_ref)
assert hasattr(j, 'meta')
assert j.meta.get('labbook') == "unittest-start-stop-job"
elapsed_time = 0
while True:
status = d.query_task(job_ref).status
print(status)
if status in ['success', 'failed', 'finished']:
print(d.query_task(job_ref).exc_info)
break
if elapsed_time > 60:
w.terminate()
assert False, "timed out {}".format(status)
elapsed_time = elapsed_time + 1
time.sleep(1)
res = d.query_task(job_ref)
assert res
print(res.status)
assert res.status == 'finished'
# Finish building image
startc_kwargs = {
'root': lb.root_dir,
'config_path': lb.client_config.config_file,
'username': '******'
}
# Start the docker container, and then wait till it's done.
container_id = bg_jobs.start_labbook_container(**startc_kwargs)
time.sleep(5)
assert get_docker_client().containers.get(
container_id).status == 'running'
# Stop the docker container, and wait until that is done.
print(container_id)
bg_jobs.stop_labbook_container(container_id)
w.terminate()
