def watchTermination(self, container, component):
report('Monitor check started', level=ReportLevels.BACKGROUND)
client = getDockerClient()
# Send the termination signal(s) to the container
signals = []
for signal in component.config.termination_signals:
signals.append((signal, buildTerminationSignal(signal)))
report('Sending %s termination signals' % len(signals), component=component)
for (config, signal) in signals:
report('Sending termination signal: ' + config.getTitle(), component=component)
result = signal.run(container, report)
if not result:
report('Termination signal failed', component=component)
# Now wait until all of the termination conditions are met
checks = []
for check in component.config.termination_checks:
checks.append((check, buildHealthCheck(check)))
report('Waiting for %s termination checks' % len(checks), component=component)
for (config, check) in checks:
check_passed = False
while not check_passed:
report('Running termination check: ' + config.getTitle(), component=component)
result = check.run(container, report)
if not result:
report('Termination check failed', component=component)
report('Sleeping ' + str(config.timeout) + ' second(s)...', component=component)
time.sleep(config.timeout)
else:
check_passed = True
report('Monitor check finished', level=ReportLevels.BACKGROUND)
setContainerStatus(container, 'shutting-down')
report('Shutting down container: ' + container['Id'][0:12], level=ReportLevels.BACKGROUND)
client.stop(container)
removeContainerMetadata(container)
def checkProxy(self):
""" Checks to see if a draining container can be shutdown. """
counter = 0
client = getDockerClient()
while True:
# Wait until something of interest is avaliable to check.
self.watcher_event.wait()
self.watcher_event.clear()
while True:
# Load the containers to check (under the lock).
containers = None
with self.watcher_lock:
containers = list(self.containers_watched)
# If none, we're done.
if not containers:
break
# Find the containers that no longer need to be running. Any container with no
# valid connections coming in and a status of 'draining', can be shutdown.
report('Monitor check started', level = ReportLevels.BACKGROUND)
containers_to_shutdown = self.findConnectionLessContainers(containers)
if len(containers_to_shutdown) > 0:
with self.watcher_lock:
for container in containers_to_shutdown:
self.containers_watched.remove(container)
for container in containers_to_shutdown:
setContainerStatus(container, 'shutting-down')
report('Shutting down container: ' + container['Id'][0:12], level = ReportLevels.BACKGROUND)
client.stop(container)
removeContainerMetadata(container)
# Determine how many residual containers are left over.
difference = len(containers) - len(containers_to_shutdown)
if difference > 0:
report(str(difference) + ' additional containers to monitor. Sleeping for 10 seconds', level = ReportLevels.BACKGROUND)
time.sleep(10)
counter = counter + 1
report('Monitor check finished', level = ReportLevels.BACKGROUND)
if not self.daemon_mode:
# Quit now that we're done.
return
def stop(self, kill=False):
""" Stops all containers for this component. """
if not self.isRunning():
return
self.logger.debug('Stopping component %s', self.getName())
client = getDockerClient()
# Mark all the containers as draining.
report('Draining all containers...', component=self)
self.elbManager.deregisterAllContainers()
for container in self.getAllContainers(client):
setContainerStatus(container, 'draining')
self.manager.terminateContainer(container, self)
# Kill any associated containers if asked.
if kill:
for container in self.getAllContainers(client):
report('Killing container ' + container['Id'][:12], component=self)
client.kill(container)
removeContainerMetadata(container)
def stop(self, kill=False):
""" Stops all containers for this component. """
if not self.isRunning():
return
self.logger.debug("Stopping component %s", self.getName())
client = getDockerClient()
# Mark all the containers as draining.
report("Draining all containers...", component=self)
self.elb_manager.deregisterAllContainers()
for container in self.getAllContainers(client):
setContainerStatus(container, "draining")
self.manager.terminateContainer(container, self)
# Kill any associated containers if asked.
if kill:
for container in self.getAllContainers(client):
report("Killing container " + container["Id"][:12], component=self)
client.kill(container)
removeContainerMetadata(container)
def stop(self, kill = False):
""" Stops all containers for this component. """
if not self.isRunning():
return
self.logger.debug('Stopping component %s', self.getName())
client = getDockerClient()
# Mark all the containers as draining.
report('Draining all containers...', component = self)
for container in self.getAllContainers(client):
setContainerStatus(container, 'draining')
# Kill any associated containers if asked.
if kill:
for container in self.getAllContainers(client):
report('Killing container ' + container['Id'][0:12], component = self)
client.kill(container)
removeContainerMetadata(container)
# Clear the proxy and rebuild its routes for the running components.
self.manager.adjustForStoppingComponent(self)
