def initiate_swarm(): client.swarm.init() print('Swarm initiated...') benchmark = client.services.create( 'nclcloudcomputing/javabenchmarkapp:latest', endpoint_spec=EndpointSpec(mode='vip', ports={4000: 8080}), mode=ServiceMode(mode='replicated', replicas=2), name='benchmarkapp', ) print('Benchmark service initialized with id ' + benchmark.id) visualizer = client.services.create( 'dockersamples/visualizer', endpoint_spec=EndpointSpec(mode='vip', ports={5000: 8080}), mounts=['/var/run/docker.sock:/var/run/docker.sock'], name='visualizer', ) print('Visualizer service initialized with id ' + visualizer.id) mongo = client.services.create( 'mongo', endpoint_spec=EndpointSpec(mode='vip', ports={27017: 27017}), mounts=['db:/data/db'], name='db', ) print('Mongo service initialized with id ' + mongo.id)
def deploy_component(workflow_id, component): endpoint_spec = None if "port" in component: port = component.pop("port") endpoint_spec = EndpointSpec(ports={port: port}) if "publishedPort" in component and "targetPort" in component: publishedPort = component.pop("publishedPort") targetPort = component.pop("targetPort") endpoint_spec = EndpointSpec(ports={publishedPort: targetPort}) log(workflow_id, f"Starting component {component['name']} with {replicas} replicas") client.services.create(**component, endpoint_spec=endpoint_spec).scale( replicas=replicas) log(workflow_id, f"Service {component['name']} is now available")
def createService(self, service_name, image, ports, restart, cpu_limit, cpu_reservation, mem_reservation, mem_limit, volumes, environment, labels, command, networks): try: self.is_deploying = True resources = DockerResources(mem_limit=mem_limit, mem_reservation=mem_reservation, cpu_limit=cpu_limit, cpu_reservation=cpu_reservation) restart_policy = RestartPolicy( condition=restart['name'], delay=10, max_attempts=restart['MaximumRetryCount']) endpoints = EndpointSpec(mode='vip', ports=ports) self.docker_client.services.create(image=image, name=service_name, hostname=service_name, restart_policy=restart_policy, endpoint_spec=endpoints, resources=resources, mounts=volumes, env=environment, labels=labels, command=command, networks=networks) logging.info("{0} component deployed".format(service_name)) self.is_deploying = False return resources except Exception as e: print(e) self.is_deploying = False return None
def run_service(service_name, image_name, command, restart_condition, networks, s_and_t, ports, mode='global', replicas=1): docker = MANAGER_CLIENR from dsp.image.valify import valify #if not valify(image_name): # return "找不到应用指定的镜像!" # Mount对象的集合 mounts = [] image_name = '192.168.123.251/'+image_name for s_t in s_and_t: type = False if s_t.get('read_only', '') == 'yes': type = True m = Mount(source=s_t.get('source', ''), target=s_t.get('target', ''), read_only=type) mounts.append(m) print s_t.get('source', '') #containSpec = ContainerSpec( image=image_name, mounts=mounts, command=command, tty=True, open_stdin=True) containSpec = ContainerSpec( image=image_name, command=command, tty=True, open_stdin=True) # 声明TaskTemplate--->task # restart_condition --->none, on - failure, or any restart_policy = RestartPolicy(condition=restart_condition) task = TaskTemplate(container_spec=containSpec, restart_policy=restart_policy) # 应用启动模式 if mode == "global": replicas = None service_mode = ServiceMode(mode=mode, replicas=replicas) # 接入点,包括负载均衡模式设置等 end_point = EndpointSpec(ports=ports) service_id = docker.create_service(task_template=task, name=service_name, mode=service_mode, networks=networks, endpoint_spec=end_point) return service_id
def create_elasticsearch_service(self): try: elasticsearch = self.client.services.create( image= 'docker.elastic.co/elasticsearch/elasticsearch-basic:6.1.1', env=[ "discovery.type=single-node", "ES_JAVA_OPTS=-Xms512m -Xmx512m", "bootstrap.memory_lock=true" ], name='elasticsearch', # command='jmeter.sh -n -t ./scripts/' + script_file + # ' -j ./results/jmeter.log', mounts=['esdata1:/usr/share/elasticsearch/data'], endpoint_spec=EndpointSpec(ports={ 9200: 9200, 9300: 9300 }), networks=['juggernaut'], labels={'app': 'juggernaut'}, # workdir='/opt/apache-jmeter-3.3/bin/', # mode=ServiceMode(mode='replicated', replicas=replicas), # update_config=UpdateConfig(parallelism=1, delay=60) ) except APIError as e: print(e) else: return elasticsearch
def create_service(username): """Created a new server service Args: username (string): The username for the OP user Returns: Service: The mc server service """ # global port_last # vol = {check_label+"_user_"+username: {'bind': '/server', 'mode': 'rw'}} if username != None else False env = [f"OP_USERNAME={username}" ] if username != None else [f"OP_USERNAME="******"tcp")}), labels={ stack_name: '', 'username': username }, env=env, networks=[stack_name + "_default"])
def launch_docker_kernel(kernel_id, response_addr, spark_context_init_mode): # Launches a containerized kernel. # Can't proceed if no image was specified. image_name = os.environ.get('KERNEL_IMAGE', None) if image_name is None: sys.exit("ERROR - KERNEL_IMAGE not found in environment - kernel launch terminating!") # Container name is composed of KERNEL_USERNAME and KERNEL_ID container_name = os.environ.get('KERNEL_USERNAME', '') + '-' + kernel_id # Determine network. If EG_DOCKER_NETWORK has not been propagated, fall back to 'bridge'... docker_network = os.environ.get('EG_DOCKER_NETWORK', 'bridge') # Build labels - these will be modelled similar to kubernetes: kernel_id, component, app, ... labels = dict() labels['kernel_id'] = kernel_id labels['component'] = 'kernel' labels['app'] = 'enterprise-gateway' # Capture env parameters... param_env = dict() param_env['EG_RESPONSE_ADDRESS'] = response_addr param_env['KERNEL_SPARK_CONTEXT_INIT_MODE'] = spark_context_init_mode # Since the environment is specific to the kernel (per env stanza of kernelspec, KERNEL_ and ENV_WHITELIST) # just add the env here. param_env.update(os.environ) param_env.pop('PATH') # Let the image PATH be used. Since this is relative to images, we're probably safe. client = DockerClient.from_env() if swarm_mode: networks = list() networks.append(docker_network) mounts = list() mounts.append("/usr/local/share/jupyter/kernels:/usr/local/share/jupyter/kernels:ro") endpoint_spec = EndpointSpec(mode='dnsrr') restart_policy = RestartPolicy(condition='none') kernel_service = client.services.create(image_name, name=container_name, endpoint_spec=endpoint_spec, restart_policy=restart_policy, env=param_env, container_labels=labels, labels=labels, #mounts=mounts, # Enable if necessary networks=networks) else: volumes = {'/usr/local/share/jupyter/kernels': {'bind': '/usr/local/share/jupyter/kernels', 'mode': 'ro'}} kernel_container = client.containers.run(image_name, name=container_name, hostname=container_name, environment=param_env, labels=labels, remove=remove_container, network=docker_network, #volumes=volumes, # Enable if necessary detach=True)
def create_object(self): """Start the single-user server in a docker service.""" container_kwargs = dict( image=self.image, env=self.get_env(), args=(yield self.get_command()), mounts=self.mounts, ) container_kwargs.update(self.extra_container_spec) container_spec = ContainerSpec(**container_kwargs) for mount in self.mounts: if mount['Target'] == '/home/jovyan/work': if not os.path.exists(mount['Source']): os.makedirs(mount['Source']) resources_kwargs = dict( mem_limit=self.mem_limit, mem_reservation=self.mem_guarantee, cpu_limit=int(self.cpu_limit * 1e9) if self.cpu_limit else None, cpu_reservation=int(self.cpu_guarantee * 1e9) if self.cpu_guarantee else None, ) resources_kwargs.update(self.extra_resources_spec) resources_spec = Resources(**resources_kwargs) placement_kwargs = dict( constraints=None, preferences=None, platforms=None, ) placement_kwargs.update(self.extra_placement_spec) placement_spec = Placement(**placement_kwargs) task_kwargs = dict( container_spec=container_spec, resources=resources_spec, networks=[self.network_name] if self.network_name else [], placement=placement_spec, ) task_kwargs.update(self.extra_task_spec) task_spec = TaskTemplate(**task_kwargs) endpoint_kwargs = {} if not self.use_internal_ip: endpoint_kwargs["ports"] = {None: (self.port, "tcp")} endpoint_kwargs.update(self.extra_endpoint_spec) endpoint_spec = EndpointSpec(**endpoint_kwargs) create_kwargs = dict(task_template=task_spec, endpoint_spec=endpoint_spec, name=self.service_name) create_kwargs.update(self.extra_create_kwargs) return (yield self.docker("create_service", **create_kwargs))
def create_object(self): """Start the single-user server in a docker service.""" container_kwargs = dict( image=self.image, env=self.get_env(), args=(yield self.get_command()), mounts=self.mounts, ) container_kwargs.update(self.extra_container_spec) container_spec = ContainerSpec(**container_kwargs) resources_kwargs = dict( mem_limit=self.mem_limit, mem_reservation=self.mem_guarantee, cpu_limit=int(self.cpu_limit * 1e9) if self.cpu_limit else None, cpu_reservation=int(self.cpu_guarantee * 1e9) if self.cpu_guarantee else None, ) resources_kwargs.update(self.extra_resources_spec) resources_spec = Resources(**resources_kwargs) placement_kwargs = dict( constraints=None, preferences=None, platforms=None, ) placement_kwargs.update(self.extra_placement_spec) placement_spec = Placement(**placement_kwargs) task_kwargs = dict( container_spec=container_spec, resources=resources_spec, networks=[self.network_name] if self.network_name else [], placement=placement_spec, ) task_kwargs.update(self.extra_task_spec) task_spec = TaskTemplate(**task_kwargs) endpoint_kwargs = {} if not self.use_internal_ip: endpoint_kwargs["ports"] = {None: (self.port, "tcp")} endpoint_kwargs.update(self.extra_endpoint_spec) endpoint_spec = EndpointSpec(**endpoint_kwargs) create_kwargs = dict(task_template=task_spec, endpoint_spec=endpoint_spec, name=self.service_name) create_kwargs.update(self.extra_create_kwargs) result = yield self.docker("create_service", **create_kwargs) # Chenglu added: inspect_service right after create_servce may raise # Service not found error yield gen.sleep(1) self.log.debug("Docker >>> create_service with %s", json.dumps(create_kwargs)) return result
def ensure_port(self, port, override=None, **kwargs): match = next((p for p in self.ports if ServiceUpdater._obj_match(p, **kwargs)), None) if match and not override: return match port = override or port if match and match['PublishedPort'] == port: return match if match: match['PublishedPort'] = port else: self.ports.append(dict(kwargs, PublishedPort=port)) self.updates['endpoint_spec'] = EndpointSpec(ports={ p['PublishedPort']: (p['TargetPort'], p.get('Protocol', 'tcp')) for p in self.ports })
def create_object(self): """Start the single-user server in a docker service.""" container_kwargs = dict( image=self.image, env=self.get_env(), args=(yield self.get_command()), mounts=self.mounts, ) container_kwargs.update(self.extra_container_spec) container_spec = ContainerSpec(**container_kwargs) resources_kwargs = dict( mem_limit=self.mem_limit, mem_reservation=self.mem_guarantee, cpu_limit=int(self.cpu_limit * 1e9) if self.cpu_limit else None, cpu_reservation=int(self.cpu_guarantee * 1e9) if self.cpu_guarantee else None, ) resources_kwargs.update(self.extra_resources_spec) resources_spec = Resources(**resources_kwargs) task_kwargs = dict( container_spec=container_spec, resources=resources_spec, networks=[self.network_name] if self.network_name else [], ) task_kwargs.update(self.extra_task_spec) task_spec = TaskTemplate(**task_kwargs) endpoint_kwargs = {} if not self.use_internal_ip: endpoint_kwargs["ports"] = {None: (self.port, "tcp")} endpoint_kwargs.update(self.extra_endpoint_spec) endpoint_spec = EndpointSpec(**endpoint_kwargs) create_kwargs = dict(task_template=task_spec, endpoint_spec=endpoint_spec, name=self.service_name) create_kwargs.update(self.extra_create_kwargs) return (yield self.docker("create_service", **create_kwargs))
# hub config hub_config = join(dirname(realpath(__file__)), 'configs', 'mount_jupyterhub_config.py') hub_sshfs_service = { 'image': HUB_IMAGE_TAG, 'name': HUB_SERVICE_NAME, 'mounts': [ ':'.join(['/var/run/docker.sock', '/var/run/docker.sock', 'rw']), ':'.join([hub_config, '/etc/jupyterhub/jupyterhub_config.py', 'ro']) ], 'networks': [NETWORK_NAME], 'endpoint_spec': EndpointSpec(ports={8000: 8000}), 'env': ['JUPYTERHUB_CRYPT_KEY=' + rand_key], 'command': ['jupyterhub', '-f', '/etc/jupyterhub/jupyterhub_config.py'] } mount_service = { 'image': MOUNT_IMAGE_TAG, 'name': MOUNT_SERVICE_NAME, 'endpoint_spec': EndpointSpec(ports={2222: 22}) } @pytest.mark.parametrize('image', [hub_image], indirect=['image']) @pytest.mark.parametrize('swarm', [swarm_config], indirect=['swarm']) @pytest.mark.parametrize('network', [network_config], indirect=['network']) def test_sshfs_mount_hub(image, swarm, network, make_service):
hub_path = dirname(dirname(__file__)) hub_image = {'path': hub_path, 'tag': HUB_IMAGE_TAG, 'rm': True, 'pull': False} swarm_config = {'advertise_addr': '192.168.99.100'} network_config = {'name': NETWORK_NAME, 'driver': 'overlay', 'options': {'subnet': '192.168.0.0/20'}, 'attachable': True} hub_config = join(dirname(realpath(__file__)), 'configs', 'jupyterhub_config.py') hub_service = {'image': HUB_IMAGE_TAG, 'name': HUB_SERVICE_NAME, 'mounts': [ ':'.join(['/var/run/docker.sock', '/var/run/docker.sock', 'rw']), ':'.join([hub_config, '/etc/jupyterhub/jupyterhub_config.py', 'ro']) ], 'networks': [NETWORK_NAME], 'endpoint_spec': EndpointSpec(ports={8000: 8000}), 'command': ['jupyterhub', '-f', '/etc/jupyterhub/jupyterhub_config.py']} @pytest.mark.parametrize('image', [hub_image], indirect=['image']) @pytest.mark.parametrize('swarm', [swarm_config], indirect=['swarm']) @pytest.mark.parametrize('network', [network_config], indirect=['network']) def test_creates_service(image, swarm, network, make_service): """Test that logging in as a new user creates a new docker service.""" make_service(hub_service) client = docker.from_env() # jupyterhub service should be running at this point services_before_spawn = client.services.list() with requests.Session() as s: ready = False
def launch_docker_kernel(kernel_id, port_range, response_addr, public_key, spark_context_init_mode): # Launches a containerized kernel. # Can't proceed if no image was specified. image_name = os.environ.get('KERNEL_IMAGE', None) if image_name is None: sys.exit( "ERROR - KERNEL_IMAGE not found in environment - kernel launch terminating!" ) # Container name is composed of KERNEL_USERNAME and KERNEL_ID container_name = os.environ.get('KERNEL_USERNAME', '') + '-' + kernel_id # Determine network. If EG_DOCKER_NETWORK has not been propagated, fall back to 'bridge'... docker_network = os.environ.get( 'DOCKER_NETWORK', os.environ.get('EG_DOCKER_NETWORK', 'bridge')) # Build labels - these will be modelled similar to kubernetes: kernel_id, component, app, ... labels = dict() labels['kernel_id'] = kernel_id labels['component'] = 'kernel' labels['app'] = 'enterprise-gateway' # Capture env parameters... param_env = dict() param_env['PORT_RANGE'] = port_range param_env['PUBLIC_KEY'] = public_key param_env['RESPONSE_ADDRESS'] = response_addr param_env['KERNEL_SPARK_CONTEXT_INIT_MODE'] = spark_context_init_mode # Since the environment is specific to the kernel (per env stanza of kernelspec, KERNEL_ and ENV_WHITELIST) # just add the env here. param_env.update(os.environ) param_env.pop( 'PATH' ) # Let the image PATH be used. Since this is relative to images, we're probably safe. user = param_env.get('KERNEL_UID') group = param_env.get('KERNEL_GID') # setup common args kwargs = dict() kwargs['name'] = container_name kwargs['hostname'] = container_name kwargs['user'] = user kwargs['labels'] = labels client = DockerClient.from_env() if swarm_mode: networks = list() networks.append(docker_network) mounts = list() mounts.append( "/usr/local/share/jupyter/kernels:/usr/local/share/jupyter/kernels:ro" ) endpoint_spec = EndpointSpec(mode='dnsrr') restart_policy = RestartPolicy(condition='none') # finish args setup kwargs['env'] = param_env kwargs['endpoint_spec'] = endpoint_spec kwargs['restart_policy'] = restart_policy kwargs['container_labels'] = labels kwargs['networks'] = networks kwargs['groups'] = [group, '100'] if param_env.get('KERNEL_WORKING_DIR'): kwargs['workdir'] = param_env.get('KERNEL_WORKING_DIR') # kwargs['mounts'] = mounts # Enable if necessary # print("service args: {}".format(kwargs)) # useful for debug kernel_service = client.services.create(image_name, **kwargs) else: volumes = { '/usr/local/share/jupyter/kernels': { 'bind': '/usr/local/share/jupyter/kernels', 'mode': 'ro' } } # finish args setup kwargs['environment'] = param_env kwargs['remove'] = remove_container kwargs['network'] = docker_network kwargs['group_add'] = [group, '100'] kwargs['detach'] = True if param_env.get('KERNEL_WORKING_DIR'): kwargs['working_dir'] = param_env.get('KERNEL_WORKING_DIR') # kwargs['volumes'] = volumes # Enable if necessary # print("container args: {}".format(kwargs)) # useful for debug kernel_container = client.containers.run(image_name, **kwargs)
def create_object(self): """Start the single-user server in a docker service.""" container_kwargs = dict( image=self.image, env=self.get_env(), args=(yield self.get_command()), mounts=self.mounts, ) container_kwargs.update(self.extra_container_spec) container_spec = ContainerSpec(**container_kwargs) resources_kwargs = dict( mem_limit=self.mem_limit, mem_reservation=self.mem_guarantee, cpu_limit=int(self.cpu_limit * 1e9) if self.cpu_limit else None, cpu_reservation=int( self.cpu_guarantee * 1e9 ) if self.cpu_guarantee else None, ) resources_kwargs.update(self.extra_resources_spec) resources_spec = Resources(**resources_kwargs) placement_kwargs = dict( constraints=None, preferences=None, platforms=None, ) placement_kwargs.update(self.extra_placement_spec) placement_spec = Placement(**placement_kwargs) task_kwargs = dict( container_spec=container_spec, resources=resources_spec, networks=[self.network_name] if self.network_name else [], placement=placement_spec, ) task_kwargs.update(self.extra_task_spec) task_spec = TaskTemplate(**task_kwargs) endpoint_kwargs = {} if not self.use_internal_ip: endpoint_kwargs["ports"] = {None: (self.port, "tcp")} endpoint_kwargs.update(self.extra_endpoint_spec) endpoint_spec = EndpointSpec(**endpoint_kwargs) create_kwargs = dict( task_template=task_spec, endpoint_spec=endpoint_spec, name=self.service_name ) create_kwargs.update(self.extra_create_kwargs) service = yield self.docker("create_service", **create_kwargs) while True: tasks = yield self.docker( "tasks", filters={"service": self.service_name}, ) if len(tasks) > 0: break yield gen.sleep(1.0) return service
'attachable': True } hub_config = join(dirname(realpath(__file__)), 'configs', 'jupyterhub_config.py') hub_service = { 'image': HUB_IMAGE_TAG, 'name': HUB_SERVICE_NAME, 'mounts': [ ':'.join(['/var/run/docker.sock', '/var/run/docker.sock', 'rw']), ':'.join([hub_config, '/etc/jupyterhub/jupyterhub_config.py', 'ro']) ], 'networks': [NETWORK_NAME], 'endpoint_spec': EndpointSpec(ports={8000: 8000}), 'command': ['jupyterhub', '-f', '/etc/jupyterhub/jupyterhub_config.py'] } remote_hub_config = join(dirname(realpath(__file__)), 'configs', 'remote_auth_jupyterhub_config.py') remote_hub_service = { 'image': HUB_IMAGE_TAG, 'name': HUB_SERVICE_NAME, 'mounts': [ ':'.join(['/var/run/docker.sock', '/var/run/docker.sock', 'rw']), ':'.join( [remote_hub_config, '/etc/jupyterhub/jupyterhub_config.py', 'ro']) ],
def start(self): """Start the single-user server in a docker service. You can specify the params for the service through jupyterhub_config.py or using the user_options """ self.log.debug("User: {}, start spawn".format(self.user.__dict__)) # https://github.com/jupyterhub/jupyterhub # /blob/master/jupyterhub/user.py#L202 # By default jupyterhub calls the spawner passing user_options if self.use_user_options: user_options = self.user_options else: user_options = {} service = yield self.get_service() if service is None: # Validate state if hasattr(self, "container_spec") and self.container_spec is not None: container_spec = dict(**self.container_spec) elif user_options == {}: self.log.error("User: {} is trying to create a service" " without a container_spec".format(self.user)) raise Exception("That notebook is missing a specification" "to launch it, contact the admin to resolve " "this issue") # Setup service container_spec.update(user_options.get("container_spec", {})) # Which image to spawn if self.use_user_options and "user_selected_image" in user_options: self.log.debug( "User options received: {}".format(user_options)) image_name = user_options["user_selected_name"] image_value = user_options["user_selected_image"] selected_image = None for di in self.images: if image_name == di["name"] and image_value == di["image"]: selected_image = copy.deepcopy(di) if selected_image is None: err_msg = "User selected image: {} couldn't be found".format( image_value) self.log.error(err_msg) raise Exception(err_msg) self.log.info( "Using the user selected image: {}".format(selected_image)) else: # Default image selected_image = self.images[0] self.log.info( "Using the default image: {}".format(selected_image)) self.log.debug("Image info: {}".format(selected_image)) # Does that image have restricted access if "access" in selected_image: # Check for static or db users allowed = False if self.service_owner in selected_image["access"]: allowed = True else: if os.path.exists(selected_image["access"]): db_path = selected_image["access"] try: self.log.info("Checking db: {} for " "User: {}".format( db_path, self.service_owner)) with open(db_path, "r") as db: users = [ user.rstrip("\n").rstrip("\r\n") for user in db ] if self.service_owner in users: allowed = True except IOError as err: self.log.error("User: {} tried to open db file {}," "Failed {}".format( self.service_owner, db_path, err)) if not allowed: self.log.error( "User: {} tried to launch {} without access".format( self.service_owner, selected_image["image"])) raise Exception( "You don't have permission to launch that image") self.log.debug("Container spec: {}".format(container_spec)) # Assign the image name as a label container_spec["labels"] = {"image_name": selected_image["name"]} # Setup mounts mounts = [] # Global mounts if "mounts" in container_spec: mounts.extend(container_spec["mounts"]) container_spec["mounts"] = [] # Image mounts if "mounts" in selected_image: mounts.extend(selected_image["mounts"]) for mount in mounts: if isinstance(mount, dict): m = VolumeMounter(mount) m = yield m.create(owner=self.service_owner) else: # Expects a mount_class that supports 'create' if hasattr(self.user, "data"): m = yield mount.create(self.user.data, owner=self.service_owner) else: m = yield mount.create(owner=self.service_owner) container_spec["mounts"].append(m) # Some envs are required by the single-user-image if "env" in container_spec: container_spec["env"].update(self.get_env()) else: container_spec["env"] = self.get_env() # Env of image if "env" in selected_image and isinstance(selected_image["env"], dict): container_spec["env"].update(selected_image["env"]) # Dynamic update of env values for env_key, env_value in container_spec["env"].items(): stripped_value = env_value.lstrip("{").rstrip("}") if hasattr(self, stripped_value) and isinstance( getattr(self, stripped_value), str): container_spec["env"][env_key] = getattr( self, stripped_value) if hasattr(self.user, stripped_value) and isinstance( getattr(self.user, stripped_value), str): container_spec["env"][env_key] = getattr( self.user, stripped_value) if (hasattr(self.user, "data") and hasattr(self.user.data, stripped_value) and isinstance(getattr(self.user.data, stripped_value), str)): container_spec["env"][env_key] = getattr( self.user.data, stripped_value) # Args of image if "args" in selected_image and isinstance(selected_image["args"], list): container_spec.update({"args": selected_image["args"]}) if ("command" in selected_image and isinstance(selected_image["command"], list) or "command" in selected_image and isinstance(selected_image["command"], str)): container_spec.update({"command": selected_image["command"]}) # Log mounts config self.log.debug("User: {} container_spec mounts: {}".format( self.user, container_spec["mounts"])) # Global resource_spec resource_spec = {} if hasattr(self, "resource_spec"): resource_spec = self.resource_spec resource_spec.update(user_options.get("resource_spec", {})) networks = None if hasattr(self, "networks"): networks = self.networks if user_options.get("networks") is not None: networks = user_options.get("networks") # Global Log driver log_driver = None if hasattr(self, "log_driver"): log_driver = self.log_driver if user_options.get("log_driver") is not None: log_driver = user_options.get("log_driver") accelerators = [] if hasattr(self, "accelerators"): accelerators = self.accelerators if user_options.get("accelerators") is not None: accelerators = user_options.get("accelerators") # Global placement placement = None if hasattr(self, "placement"): placement = self.placement if user_options.get("placement") is not None: placement = user_options.get("placement") # Image to spawn image = selected_image["image"] # Image resources if "resource_spec" in selected_image: resource_spec = selected_image["resource_spec"] # Accelerators attached to the image if "accelerators" in selected_image: accelerators = selected_image["accelerators"] # Placement of image if "placement" in selected_image: placement = selected_image["placement"] # Logdriver of image if "log_driver" in selected_image: log_driver = selected_image["log_driver"] # Configs attached to image if "configs" in selected_image and isinstance( selected_image["configs"], list): for c in selected_image["configs"]: if isinstance(c, dict): self.configs.append(c) endpoint_spec = {} if "endpoint_spec" in selected_image: endpoint_spec = selected_image["endpoint_spec"] if self.configs: # Check that the supplied configs already exists current_configs = yield self.docker("configs") config_error_msg = ( "The server has a misconfigured config, " "please contact an administrator to resolve this") for c in self.configs: if "config_name" not in c: self.log.error("Config: {} does not have a " "required config_name key".format(c)) raise Exception(config_error_msg) if "config_id" not in c: # Find the id from the supplied name config_ids = [ cc["ID"] for cc in current_configs if cc["Spec"]["Name"] == c["config_name"] ] if not config_ids: self.log.error( "A config with name {} could not be found") raise Exception(config_error_msg) c["config_id"] = config_ids[0] container_spec.update( {"configs": [ConfigReference(**c) for c in self.configs]}) # Prepare the accelerators and attach it to the environment if accelerators: for accelerator in accelerators: accelerator_id = accelerator.aquire(self.user.name) # NVIDIA_VISIBLE_DEVICES=0:0 container_spec["env"][ "NVIDIA_VISIBLE_DEVICES"] = "{}".format(accelerator_id) # Global container user uid_gid = None if "uid_gid" in container_spec: uid_gid = copy.deepcopy(container_spec["uid_gid"]) del container_spec["uid_gid"] # Image user if "uid_gid" in selected_image: uid_gid = selected_image["uid_gid"] self.log.info("gid info {}".format(uid_gid)) if isinstance(uid_gid, str): if ":" in uid_gid: uid, gid = uid_gid.split(":") else: uid, gid = uid_gid, None if (uid == "{uid}" and hasattr(self.user, "uid") and self.user.uid is not None): uid = self.user.uid if (gid is not None and gid == "{gid}" and hasattr(self.user, "gid") and self.user.gid is not None): gid = self.user.gid if uid: container_spec.update({"user": str(uid)}) if uid and gid: container_spec.update({"user": str(uid) + ":" + str(gid)}) # Global container user if "user" in container_spec: container_spec["user"] = str(container_spec["user"]) # Image user if "user" in selected_image: container_spec.update({"user": str(selected_image["user"])}) dynamic_holders = [Spawner, self, self.user] if hasattr(self.user, "data"): dynamic_holders.append(self.user.data) # Expand container_spec before start for construct in dynamic_holders: try: if not hasattr(construct, "__dict__"): continue recursive_format(container_spec, construct.__dict__) except TypeError: pass # Log driver log_driver_name, log_driver_options = None, None if log_driver and isinstance(log_driver, dict): if "name" in log_driver: log_driver_name = log_driver["name"] if "options" in log_driver: log_driver_options = log_driver["options"] # Create the service container_spec = ContainerSpec(image, **container_spec) resources = Resources(**resource_spec) placement = Placement(**placement) task_log_driver = None if log_driver_name: task_log_driver = DriverConfig(log_driver_name, options=log_driver_options) task_spec = { "container_spec": container_spec, "resources": resources, "placement": placement, } if task_log_driver: task_spec.update({"log_driver": task_log_driver}) task_tmpl = TaskTemplate(**task_spec) self.log.debug("task temp: {}".format(task_tmpl)) # Set endpoint spec endpoint_spec = EndpointSpec(**endpoint_spec) resp = yield self.docker( "create_service", task_tmpl, name=self.service_name, networks=networks, endpoint_spec=endpoint_spec, ) self.service_id = resp["ID"] self.log.info("Created Docker service {} (id: {}) from image {}" " for user {}".format(self.service_name, self.service_id[:7], image, self.user)) yield self.wait_for_running_tasks() else: self.log.info("Found existing Docker service '{}' (id: {})".format( self.service_name, self.service_id[:7])) # Handle re-using API token. # Get the API token from the environment variables # of the running service: envs = service["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"] for line in envs: if line.startswith("JPY_API_TOKEN="): self.api_token = line.split("=", 1)[1] break ip = self.service_name port = self.service_port self.log.debug("Active service: '{}' with user '{}'".format( self.service_name, self.user)) # we use service_name instead of ip # https://docs.docker.com/engine/swarm/networking/#use-swarm-mode-service-discovery # service_port is actually equal to 8888 return ip, port
network_config = { "name": NETWORK_NAME, "driver": "overlay", "options": {"subnet": "192.168.0.0/24"}, "attachable": True, } hub_config = join(dirname(realpath(__file__)), "configs", "jupyterhub_config.py") hub_service = { "image": HUB_IMAGE_TAG, "name": HUB_SERVICE_NAME, "mounts": [ ":".join(["/var/run/docker.sock", "/var/run/docker.sock", "rw"]), ":".join([hub_config, "/etc/jupyterhub/jupyterhub_config.py", "ro"]), ], "networks": [NETWORK_NAME], "endpoint_spec": EndpointSpec(ports={PORT: PORT}), "command": ["jupyterhub", "-f", "/etc/jupyterhub/jupyterhub_config.py"], } remote_hub_config = join( dirname(realpath(__file__)), "configs", "remote_auth_jupyterhub_config.py" ) remote_hub_service = { "image": HUB_IMAGE_TAG, "name": HUB_SERVICE_NAME, "mounts": [ ":".join(["/var/run/docker.sock", "/var/run/docker.sock", "rw"]), ":".join([remote_hub_config, "/etc/jupyterhub/jupyterhub_config.py", "ro"]), ], "networks": [NETWORK_NAME], "endpoint_spec": EndpointSpec(ports={PORT: PORT}),
def launch_docker_kernel(kernel_id, port_range, response_addr, public_key, spark_context_init_mode): # Launches a containerized kernel. # Can't proceed if no image was specified. image_name = os.environ.get("KERNEL_IMAGE", None) if image_name is None: sys.exit("ERROR - KERNEL_IMAGE not found in environment - kernel launch terminating!") # Container name is composed of KERNEL_USERNAME and KERNEL_ID container_name = os.environ.get("KERNEL_USERNAME", "") + "-" + kernel_id # Determine network. If EG_DOCKER_NETWORK has not been propagated, fall back to 'bridge'... docker_network = os.environ.get("DOCKER_NETWORK", os.environ.get("EG_DOCKER_NETWORK", "bridge")) # Build labels - these will be modelled similar to kubernetes: kernel_id, component, app, ... labels = dict() labels["kernel_id"] = kernel_id labels["component"] = "kernel" labels["app"] = "enterprise-gateway" # Capture env parameters... param_env = dict() param_env["PORT_RANGE"] = port_range param_env["PUBLIC_KEY"] = public_key param_env["RESPONSE_ADDRESS"] = response_addr param_env["KERNEL_SPARK_CONTEXT_INIT_MODE"] = spark_context_init_mode # Since the environment is specific to the kernel (per env stanza of kernelspec, KERNEL_ and EG_CLIENT_ENVS) # just add the env here. param_env.update(os.environ) param_env.pop( "PATH" ) # Let the image PATH be used. Since this is relative to images, we're probably safe. user = param_env.get("KERNEL_UID") group = param_env.get("KERNEL_GID") # setup common args kwargs = dict() kwargs["name"] = container_name kwargs["hostname"] = container_name kwargs["user"] = user kwargs["labels"] = labels client = DockerClient.from_env() if swarm_mode: networks = list() networks.append(docker_network) # mounts = list() # Enable if necessary # mounts.append("/usr/local/share/jupyter/kernels:/usr/local/share/jupyter/kernels:ro") endpoint_spec = EndpointSpec(mode="dnsrr") restart_policy = RestartPolicy(condition="none") # finish args setup kwargs["env"] = param_env kwargs["endpoint_spec"] = endpoint_spec kwargs["restart_policy"] = restart_policy kwargs["container_labels"] = labels kwargs["networks"] = networks kwargs["groups"] = [group, "100"] if param_env.get("KERNEL_WORKING_DIR"): kwargs["workdir"] = param_env.get("KERNEL_WORKING_DIR") # kwargs['mounts'] = mounts # Enable if necessary # print("service args: {}".format(kwargs)) # useful for debug client.services.create(image_name, **kwargs) # noqa else: # volumes = { # Enable if necessary # "/usr/local/share/jupyter/kernels": { # "bind": "/usr/local/share/jupyter/kernels", # "mode": "ro", # } # } # finish args setup kwargs["environment"] = param_env kwargs["remove"] = remove_container kwargs["network"] = docker_network kwargs["group_add"] = [group, "100"] kwargs["detach"] = True if param_env.get("KERNEL_WORKING_DIR"): kwargs["working_dir"] = param_env.get("KERNEL_WORKING_DIR") # kwargs['volumes'] = volumes # Enable if necessary # print("container args: {}".format(kwargs)) # useful for debug client.containers.run(image_name, **kwargs) # noqa
def test_restart_service_retains_settings(self): from docker.types import EndpointSpec, Resources, RestartPolicy, SecretReference, UpdateConfig join_command = self.init_swarm() with self.with_dind_container() as second_dind: self.prepare_images('alpine', client=self.dind_client(second_dind)) second_dind.exec_run(join_command) network = self.remote_client.networks.create('pygen-net', driver='overlay') secret = self.remote_client.secrets.create(name='pygen-secret', data='TopSecret') secret.reload() service = self.remote_client.services.create( 'alpine', name='target-svc', mode='global', command='sh -c "date +%s ; sleep 3600"', constraints=['node.hostname != non-existing-node'], container_labels={'container.label': 'testing'}, dns_config={'Nameservers': ['8.8.8.8']}, endpoint_spec=EndpointSpec(mode='vip', ports={14002: 1234}), env=['TEST_ENV_VAR=12345'], labels={'service.label': 'on-service'}, mounts=['/tmp:/data/hosttmp:ro'], networks=[network.name], resources=Resources(mem_limit=24000000), restart_policy=RestartPolicy(condition='any', delay=5, max_attempts=3), secrets=[ SecretReference(secret_id=secret.id, secret_name=secret.name) ], stop_grace_period=1, update_config=UpdateConfig(parallelism=1, delay=1, monitor=7200000000), user='******', workdir='/data/hosttmp', tty=True) self.wait_for_service_start(service, num_tasks=2) service.reload() initial_spec = service.attrs['Spec'] command = ['--template #ok', '--restart target-svc', '--one-shot'] self.remote_client.containers.run( 'pygen-build', command=' '.join(command), remove=True, volumes=['/var/run/docker.sock:/var/run/docker.sock:ro']) self.wait_for_service_start(service, num_tasks=4) service = self.remote_client.services.get(service.id) service.reload() newer_spec = service.attrs['Spec'] del initial_spec['TaskTemplate']['ForceUpdate'] del newer_spec['TaskTemplate']['ForceUpdate'] initial_networks = initial_spec.pop( 'Networks', initial_spec['TaskTemplate'].pop('Networks', [])) newer_networks = newer_spec.pop( 'Networks', newer_spec['TaskTemplate'].pop('Networks', [])) self.maxDiff = None self.assertGreater(len(newer_networks), 0) self.assertEqual(newer_networks, initial_networks) self.assertDictEqual(newer_spec, initial_spec)
def launch_docker_kernel(kernel_id, response_addr, spark_context_init_mode): # Launches a containerized kernel. # Can't proceed if no image was specified. image_name = os.environ.get("KERNEL_IMAGE", None) if image_name is None: sys.exit( "ERROR - KERNEL_IMAGE not found in environment - kernel launch terminating!" ) # Container name is composed of KERNEL_USERNAME and KERNEL_ID container_name = os.environ.get("KERNEL_USERNAME", "") + "-" + kernel_id # Determine network. If EG_DOCKER_NETWORK has not been propagated, fall back to 'bridge'... docker_network = os.environ.get("EG_DOCKER_NETWORK", "bridge") # Build labels - these will be modelled similar to kubernetes: kernel_id, component, app, ... labels = dict() labels["kernel_id"] = kernel_id labels["component"] = "kernel" labels["app"] = "enterprise-gateway" # Capture env parameters... param_env = dict() param_env["EG_RESPONSE_ADDRESS"] = response_addr param_env["KERNEL_SPARK_CONTEXT_INIT_MODE"] = spark_context_init_mode # Since the environment is specific to the kernel (per env stanza of kernelspec, KERNEL_ and ENV_WHITELIST) # just add the env here. param_env.update(os.environ) param_env.pop( "PATH" ) # Let the image PATH be used. Since this is relative to images, we're probably safe. user = param_env.get("KERNEL_UID") group = param_env.get("KERNEL_GID") # setup common args kwargs = dict() kwargs["name"] = container_name kwargs["user"] = user kwargs["labels"] = labels client = DockerClient.from_env() if swarm_mode: print("Started Jupyter kernel in swarm-mode") networks = list() networks.append(docker_network) mounts = list() mounts.append( "/usr/local/share/jupyter/kernels:/usr/local/share/jupyter/kernels:ro" ) endpoint_spec = EndpointSpec(mode="dnsrr") restart_policy = RestartPolicy(condition="none") # finish args setup kwargs["env"] = param_env kwargs["endpoint_spec"] = endpoint_spec kwargs["restart_policy"] = restart_policy kwargs["container_labels"] = labels kwargs["networks"] = networks kwargs["groups"] = [group, "100"] if param_env.get("KERNEL_WORKING_DIR"): kwargs["workdir"] = param_env.get("KERNEL_WORKING_DIR") # kwargs['mounts'] = mounts # Enable if necessary # print("service args: {}".format(kwargs)) # useful for debug kernel_service = client.services.create(image_name, **kwargs) else: print("Started Jupyter kernel in normal docker mode") # Note: seems to me that the kernels don't need to be mounted on a container that runs a single kernel # mount the kernel working directory from EG to kernel container # finish args setup kwargs["hostname"] = container_name kwargs["environment"] = param_env kwargs["remove"] = remove_container kwargs["network"] = docker_network kwargs["group_add"] = [ group, "100", ] # NOTE: "group_add" for newer versions of docker kwargs["detach"] = True if param_env.get("KERNEL_WORKING_DIR"): kwargs["working_dir"] = param_env.get("KERNEL_WORKING_DIR") # print("container args: {}".format(kwargs)) # useful for debug orchest_mounts = get_orchest_mounts( project_dir=param_env.get("KERNEL_WORKING_DIR"), host_project_dir=param_env.get("ORCHEST_HOST_PROJECT_DIR"), ) orchest_mounts += [ get_volume_mount( param_env.get("ORCHEST_PIPELINE_UUID"), param_env.get("ORCHEST_PROJECT_UUID"), ) ] # Extract environment_uuid from the image name (last 36 characters) extracted_environment_uuid = image_name[-36:] device_requests = get_device_requests( extracted_environment_uuid, param_env.get("ORCHEST_PROJECT_UUID") ) kernel_container = client.containers.run( image_name, mounts=orchest_mounts, device_requests=device_requests, **kwargs )
def gen_director_service_params( # pylint: disable=unused-argument client: DockerClient, site_id: int, site_data: Dict[str, Any]) -> Dict[str, Any]: extra_env = { "PORT": "80", "HOST": "0.0.0.0", } params = gen_director_shared_params(client, site_id, site_data) env = params.pop("env", []) env.extend("{}={}".format(name, val) for name, val in extra_env.items()) # We do the run.sh path detection in the shell so that it can adapt to the path changing without # updating the Docker service # The killing of the child process is based off of # https://unix.stackexchange.com/a/146770/306760 shell_command = """date +'DIRECTOR: Starting server at %Y-%m-%d %H:%M:%S %Z' # See docs/UMASK.md before touching this umask "$1" for path in /site/run.sh /site/private/run.sh /site/public/run.sh; do if [ -x "$path" ]; then term() { date +'DIRECTOR: Stopping server at %Y-%m-%d %H:%M:%S %Z' kill "$child" } trap term TERM "$path" & child="$!" while ! wait; do true; done exec date +'DIRECTOR: Stopped server at %Y-%m-%d %H:%M:%S %Z' fi done echo 'DIRECTOR: No run.sh file found -- if it exists, make sure it is set as executable' exec sleep 2147483647""" params.update({ "name": get_director_service_name(site_id), "read_only": True, # See docs/UMASK.md before touching this "command": ["sh", "-c", shell_command, "sh", oct(settings.SITE_UMASK)[2:]], "workdir": "/site/public", "networks": ["director-sites"], "resources": Resources( # 0.1 CPUs, 100M or so of memory cpu_limit=convert_cpu_limit(site_data["resource_limits"]["cpus"]), mem_limit=convert_memory_limit( site_data["resource_limits"]["mem_limit"]), ), "env": env, "log_driver": "json-file", "log_driver_options": { # Keep minimal logs "max-size": "500k", "max-file": "1", }, "hosts": params.pop("extra_hosts"), "stop_grace_period": 3, "endpoint_spec": EndpointSpec(mode="vip", ports={}), "mode": ServiceMode(mode="replicated", replicas=1 if site_data["is_being_served"] else 0), "restart_policy": RestartPolicy(condition="any", delay=5, max_attempts=5, window=0), "update_config": UpdateConfig( parallelism=1, order="stop-first", failure_action="rollback", max_failure_ratio=0, # delay and monitor are in nanoseconds (1e9 seconds) delay=int(5 * (10**9)), monitor=int(5 * (10**9)), ), }) return params
"attachable": True, } hub_config = join(dirname(realpath(__file__)), "configs", "jupyterhub_config.py") hub_service = { "image": HUB_IMAGE_TAG, "name": HUB_SERVICE_NAME, "mounts": [ ":".join(["/var/run/docker.sock", "/var/run/docker.sock", "rw"]), ":".join([hub_config, "/etc/jupyterhub/jupyterhub_config.py", "ro"]), ], "networks": [NETWORK_NAME], "endpoint_spec": EndpointSpec(ports={PORT: PORT}), "env": ["JUPYTERHUB_CRYPT_KEY=" + rand_key], "command": ["jupyterhub", "-f", "/etc/jupyterhub/jupyterhub_config.py"], } @pytest.mark.parametrize("image", [hub_image], indirect=["image"]) @pytest.mark.parametrize("swarm", [swarm_config], indirect=["swarm"]) @pytest.mark.parametrize("network", [network_config], indirect=["network"]) def test_creates_service(image, swarm, network, make_service): """Test that logging in as a new user creates a new docker service.""" test_logger.info("Start of service testing") make_service(hub_service) client = docker.from_env() # jupyterhub service should be running at this point services_before_spawn = client.services.list()
def launch_docker_kernel(kernel_id, response_addr, spark_context_init_mode): # Launches a containerized kernel. # Can't proceed if no image was specified. image_name = os.environ.get('KERNEL_IMAGE', None) if image_name is None: sys.exit( "ERROR - KERNEL_IMAGE not found in environment - kernel launch terminating!" ) # Container name is composed of KERNEL_USERNAME and KERNEL_ID container_name = os.environ.get('KERNEL_USERNAME', '') + '-' + kernel_id # Determine network. If EG_DOCKER_NETWORK has not been propagated, fall back to 'bridge'... docker_network = os.environ.get('EG_DOCKER_NETWORK', 'bridge') # Build labels - these will be modelled similar to kubernetes: kernel_id, component, app, ... labels = dict() labels['kernel_id'] = kernel_id labels['component'] = 'kernel' labels['app'] = 'enterprise-gateway' # Capture env parameters... param_env = dict() param_env['EG_RESPONSE_ADDRESS'] = response_addr param_env['KERNEL_SPARK_CONTEXT_INIT_MODE'] = spark_context_init_mode # Since the environment is specific to the kernel (per env stanza of kernelspec, KERNEL_ and ENV_WHITELIST) # just add the env here. param_env.update(os.environ) param_env.pop( 'PATH' ) # Let the image PATH be used. Since this is relative to images, we're probably safe. user = param_env.get('KERNEL_UID') group = param_env.get('KERNEL_GID') # setup common args kwargs = dict() kwargs['name'] = container_name kwargs['user'] = user kwargs['labels'] = labels client = DockerClient.from_env() if swarm_mode: print("Started Jupyter kernel in swarm-mode") networks = list() networks.append(docker_network) mounts = list() mounts.append( "/usr/local/share/jupyter/kernels:/usr/local/share/jupyter/kernels:ro" ) endpoint_spec = EndpointSpec(mode='dnsrr') restart_policy = RestartPolicy(condition='none') # finish args setup kwargs['env'] = param_env kwargs['endpoint_spec'] = endpoint_spec kwargs['restart_policy'] = restart_policy kwargs['container_labels'] = labels kwargs['networks'] = networks kwargs['groups'] = [group, '100'] if param_env.get('KERNEL_WORKING_DIR'): kwargs['workdir'] = param_env.get('KERNEL_WORKING_DIR') # kwargs['mounts'] = mounts # Enable if necessary # print("service args: {}".format(kwargs)) # useful for debug kernel_service = client.services.create(image_name, **kwargs) else: print("Started Jupyter kernel in normal docker mode") # Note: seems to me that the kernels don't need to be mounted on a container that runs a single kernel # mount the kernel working directory from EG to kernel container # TODO: mount pipeline directory # finish args setup kwargs['hostname'] = container_name kwargs['environment'] = param_env kwargs['remove'] = remove_container kwargs['network'] = docker_network kwargs['group_add'] = [ group, '100' ] # NOTE: "group_add" for newer versions of docker kwargs['detach'] = True if param_env.get('KERNEL_WORKING_DIR'): kwargs['working_dir'] = param_env.get('KERNEL_WORKING_DIR') pipeline_dir_mount = Mount(target=param_env.get('KERNEL_WORKING_DIR'), source=param_env.get('HOST_PIPELINE_DIR'), type='bind') mounts = [pipeline_dir_mount] # dynamically mount host-dir sources dynamic_mounts = get_dynamic_mounts(param_env) mounts = mounts + dynamic_mounts # print("container args: {}".format(kwargs)) # useful for debug kernel_container = client.containers.run(image_name, mounts=mounts, **kwargs)