def __init__(self, zkobj, this_ip, this_url, config): super(DockerManager, self).__init__() self.zkobj = zkobj and zkobj._cast_as(Docker) self.this_ip = this_ip self.this_url = this_url self.config = config self.uuid = str(uuid.uuid4()) self.used_slots = 0 self.scheduler_cache = {} self.image_cond = threading.Condition() self.images = {} self._containers = Cache(self.zkobj.containers(), update=self._delete) self._to_start = Cache(self.zkobj.to_start(), update=self._start) self._starting = self.zkobj.starting() self._scheduler_info = Cache(self.zkobj.scheduler_info(), update=self._notify_scheduler) self._load() self._update_scheduler_info()
class DockerManager(object): def __init__(self, zkobj, this_ip, this_url, config): super(DockerManager, self).__init__() self.zkobj = zkobj and zkobj._cast_as(Docker) self.this_ip = this_ip self.this_url = this_url self.config = config self.uuid = str(uuid.uuid4()) self.used_slots = 0 self.scheduler_cache = {} self.image_cond = threading.Condition() self.images = {} self._containers = Cache(self.zkobj.containers(), update=self._delete) self._to_start = Cache(self.zkobj.to_start(), update=self._start) self._starting = self.zkobj.starting() self._scheduler_info = Cache(self.zkobj.scheduler_info(), update=self._notify_scheduler) self._load() self._update_scheduler_info() def _extract_ip(self, instance_id): try: # Extract the private port. instance_info = self.config.client().inspect_container(instance_id) network_info = instance_info["NetworkSettings"] if network_info != None and network_info.get("Ports"): # This handles the case for Docker >= 0.6.5. tcp_ports = network_info.get("Ports").items() if tcp_ports and len(tcp_ports) > 0: private_port = tcp_ports[0][1][0].get("HostPort") return '%s:%s' % (self.this_ip, private_port) elif network_info != None and network_info.get("PortMapping"): # This handles the case for Docker < 0.6.5. tcp_ports = network_info["PortMapping"].get("Tcp") if tcp_ports and len(tcp_ports) > 0: private_port = tcp_ports.items()[0][1] return '%s:%s' % (self.this_ip, private_port) except Exception: traceback.print_exc() # No port available. # Unfortunately this means this isn't # a running instance and we will kill it. return None def _load(self): # Check that our instances match expectations. for instance_id in map(lambda x: x['Id'][:12], self.config.client().containers(all=True)): # Is it one of ours? try: instance_info = self.config.client().inspect_container(instance_id) except Exception: traceback.print_exc() continue env = instance_info['Config']['Env'] if env != None: env = dict(map(lambda x: x.split("=", 1), env)) if SLOTS_ENVIRONMENT_KEY in env: if instance_id in self._containers.list(): # NOTE: We update the IP here as well because it # may have changed if this was a reboot event or # something similar. ip_address = self._extract_ip(instance_id) self.used_slots += int(env[SLOTS_ENVIRONMENT_KEY]) if ip_address: # Reset the IP address in the store. self._containers.add( instance_id, ip_address, mustexist=True) else: # No need to do any accounting here. self.config.client().kill(instance_id) self.config.client().remove_container(instance_id) # Clean out non-running instances. # NOTE: This duplicates a lot of the functionality # above, but it's important to ensure that we get # the accounting correct. self._delete() def _notify_scheduler(self): # Update scheduler information. Either a new # instance has been started, or a new server # has come online, or some combination. if hasattr(self, '_scheduler_info'): self.scheduler_cache = self._scheduler_info.as_map() # We've changed scheduler info, so we'd best # rerun our schedule() function and ensure that # everything gets scheduled. self._start() def _update_scheduler_info(self): # Update our scheduler information. # NOTE: This will generate _notify_scheduler() # asynchronously, so there's no need to call it. self._scheduler_info.add(self.uuid, (self.used_slots, self.config.slots), ephemeral=True) def instances(self): # Clean up all the dead containers. # We don't care about failure here, as # this is just a best effort call. try: self._delete() except Exception: pass # List the aggregate of all servers. return self._containers.as_map() def start(self, config, timeout=10.0, params=None): # Submit a scheduling request. # We do so by writing the uuid to # a unique node in the to_start pool. # This will be picked up by the a # server that thinks it has the most # (relative) slots available. start = time.time() this_uuid = str(uuid.uuid4()) # Serialize the configuration. # This installs an ephermal node in Zookeeper # (basically as an RPC entry) which will fire # appropriately when it is changed. self._to_start.add(this_uuid, config._values(), ephemeral=True) ref = self._to_start._get_child(this_uuid, clazz=JSONObject) try: cond = threading.Condition() def _notify(value): cond.acquire() cond.notifyAll() cond.release() cond.acquire() instance_info = ref._get_data(watch=_notify) try: while True: # Check if we're set. # This is a pretty loose system for # RPC, where we know the result is either # a tuple (the result), an exception (str) # or the original dictionary configuration. if isinstance(instance_info, str): # Shit, an exception on the other end. raise Exception(instance_info) elif isinstance(instance_info, dict): # Still our configuration, waiting. pass else: # Successfully spawned. return instance_info # Check if we're out of time. now = time.time() if now >= start + timeout: break # Wait for a notification and refresh. cond.wait(start + timeout - now) instance_info = ref._get_data() finally: cond.release() finally: ref._delete() # No success. raise Exception("timed out") def _spawn(self, config): # Ensure that this image is available. self._pull(config.image) # Figure out the right port. # NOTE: This is derived from the endpoint. if config.port(): port_key = "%s/tcp" % (config.port(),) ports = {} ports[port_key] = {} port_bindings = {} port_bindings[port_key] = [{'HostIp': '', 'HostPort': ''}] else: ports = None port_bindings = None # Add the slots to the environment. env = config.get_environment() env[SLOTS_ENVIRONMENT_KEY] = str(config.slots) # Add the reactor URL to the environment. env[REACTOR_ENVIRONMENT_KEY] = str(self.this_url) # Create the container. instance_info = self.config.client().create_container( config.image, config.command, hostname=config.hostname or None, user=config.user or None, detach=True, stdin_open=False, tty=False, mem_limit=config.mem_limit or None, ports=ports, environment=env, dns=config.dns or None, privileged=False) # Grab the id to work with here. instance_id = instance_info['Id'][:12] # Start the container. try: # This will work well for Docker >= 0.6.5. self.config.client().start(instance_id, port_bindings=port_bindings) except TypeError: # If port_bindings is not a known key, then assume Docker < 0.6.5. self.config.client().start(instance_id) # Grab the local IP for mapping. ip_address = self._extract_ip(instance_id) # Off to the races. return (instance_id, config.slots, ip_address) def _pull(self, image): self.image_cond.acquire() try: # Wait while pending. while self.images.has_key(image) and \ not self.images[image]: self.image_cond.wait() # Start the request. self.images[image] = False self.config.client().pull(image) # Success. self.images[image] = True except Exception: del self.images[image] raise finally: self.image_cond.notifyAll() self.image_cond.release() def _start(self, clean_uuid=None): # Check for an early notification. if not hasattr(self, '_to_start'): return # Read all instances waiting to be scheduled. for (this_uuid, config) in self._to_start.as_map().items(): # This is a result or something else. # Will eventually be pruned (ephemeral) # or removed by the caller. if not isinstance(config, dict): continue # Get the config. # This is currently a hack that is required to # break the circular reference between the manager and # the connection. If there is a reference, then neither # the manager nor the connection will ever be cleaned up. config = DockerEndpointConfig(values=config, section='cloud:docker') # Figure out if we should schedule this one. # We do this by figuring out which host will # have the least slot used by *percentage* # after the process is completed. min_used = 1.0 best_hosts = [] for (host_uuid, (used_slots, total_slots)) in self.scheduler_cache.items(): this_used = float(used_slots + config.slots) / total_slots if this_used < min_used: min_used = this_used best_hosts = [host_uuid] elif this_used == min_used: best_hosts.append(host_uuid) # Are we a candidate? if self.uuid in best_hosts: if self._starting.lock([this_uuid]): try: # First, do the actual launch and update # our scheduling information so people can # make the right decisions. (instance_id, slots, ip_address) = self._spawn(config) self.used_slots += slots self._update_scheduler_info() # Finally add the container to the list of # containers and return to the caller. if self._to_start.add(this_uuid, (instance_id, ip_address), mustexist=True): self._containers.add(instance_id, ip_address) except Exception, e: # Write the result out as an exception. self._to_start.add(this_uuid, (str(e),), mustexist=True) finally: # Always clear the lock. self._starting.remove(this_uuid)