def balance_containers():
try:
# Clean up before interacting with the pool:
# - checks in containers that are checked out longer than the max lifetime
# - stops containers that aren't running if their image is out of date
# - stops containers from the pool not running selenium
for container in interactions.containers():
if container.checked_out:
if ((datetime.utcnow() -
container.checked_out).total_seconds() >
max_checkout_time):
logger.info(
'Container %s checked out longer than %d seconds, forcing stop',
container.name, max_checkout_time)
interactions.stop(container)
continue
else:
if container.image_id != interactions.last_pulled_image_id:
logger.info('Container %s running an old image',
container.name)
interactions.stop(container)
continue
pool_balanced = False
while not pool_balanced:
# Grabs/releases the lock each time through the loop so checkouts don't have to wait
# too long if a container's being destroyed
with lock:
# Make sure the number of running containers that aren't checked out
containers = interactions.containers()
running = interactions.running(*containers)
not_running = containers - running
checked_out = set(
filter(lambda c: bool(c.checked_out), running))
# Reset the global pool based on the current derived state
pool.clear()
pool.update(running - checked_out)
pool_stat_str = '%d/%d - %d checked out - %d to destroy' % (
len(pool), pool_size, len(checked_out), len(not_running))
containers_to_start = pool_size - len(pool)
containers_to_stop = len(pool) - pool_size
# Starting containers can happen at-will, and shouldn't be done under lock
# so that checkouts don't have to block unless the pool is exhausted
if containers_to_start > 0:
if containers_to_start > 4:
# limit the number of ocntainers to start so we
# don't spend *too* much time refilling the pool if
# there's more work to be done
containers_to_start = 4
logger.info('Pool %s, adding %d containers', pool_stat_str,
containers_to_start)
new_containers = []
for __ in range(containers_to_start):
new_containers.append(
interactions.create_container(image_name))
interactions.start(*new_containers)
# after starting, continue the loop to ensure that
# starting new containers happens before destruction
continue
# Stopping containers does need to happen under lock to ensure that
# simultaneous balance_containers don't attempt to stop the same container
# This should be rare, since containers are never returned to the pool,
# but can happen if, for example, the configured pool size changes
if containers_to_stop > 0:
logger.debug('%d containers to stop', containers_to_stop)
with lock:
oldest_container = sorted(pool)[0]
logger.info('Pool %s, removing oldest container %s',
pool_stat_str, oldest_container.name)
interactions.stop(oldest_container)
# again, continue the loop here to save destroys for last
continue
# after balancing the pool, destroy oldest stopped container
if not_running:
if len(not_running) % 4 == 0:
logger.info('Pool %s' % pool_stat_str)
interactions.destroy(sorted(not_running)[0])
continue
# If we've made it this far...
logger.info('Pool balanced, %s', pool_stat_str)
pool_balanced = True
except (APIError, RequestException) as exc:
logger.error('%s while balancing containers, retrying.' %
type(exc).__name__)
logger.exception(exc)
balance_containers.trigger()
def status():
containers = interactions.running()
return jsonify(**{
container.name: container_info(container)
for container in containers
})
def status():
containers = interactions.running()
return jsonify(**{container.name: container_info(container) for container in containers})
def balance_containers():
try:
# Clean up before interacting with the pool:
# - checks in containers that are checked out longer than the max lifetime
# - stops containers that aren't running if their image is out of date
# - stops containers from the pool not running selenium
for container in interactions.containers():
if container.checked_out:
if ((datetime.utcnow() - container.checked_out).total_seconds()
> max_checkout_time):
logger.info('Container %s checked out longer than %d seconds, forcing stop',
container.name, max_checkout_time)
interactions.stop(container)
continue
else:
if container.image_id != interactions.last_pulled_image_id:
logger.info('Container %s running an old image', container.name)
interactions.stop(container)
continue
pool_balanced = False
while not pool_balanced:
# Grabs/releases the lock each time through the loop so checkouts don't have to wait
# too long if a container's being destroyed
with lock:
# Make sure the number of running containers that aren't checked out
containers = interactions.containers()
running = interactions.running(*containers)
not_running = containers - running
checked_out = set(filter(lambda c: bool(c.checked_out), running))
# Reset the global pool based on the current derived state
pool.clear()
pool.update(running - checked_out)
pool_stat_str = '%d/%d - %d checked out - %d to destroy' % (
len(pool), pool_size, len(checked_out), len(not_running))
containers_to_start = pool_size - len(pool)
containers_to_stop = len(pool) - pool_size
# Starting containers can happen at-will, and shouldn't be done under lock
# so that checkouts don't have to block unless the pool is exhausted
if containers_to_start > 0:
if containers_to_start > 4:
# limit the number of ocntainers to start so we
# don't spend *too* much time refilling the pool if
# there's more work to be done
containers_to_start = 4
logger.info('Pool %s, adding %d containers', pool_stat_str, containers_to_start)
new_containers = []
for __ in range(containers_to_start):
new_containers.append(interactions.create_container(image_name))
interactions.start(*new_containers)
# after starting, continue the loop to ensure that
# starting new containers happens before destruction
continue
# Stopping containers does need to happen under lock to ensure that
# simultaneous balance_containers don't attempt to stop the same container
# This should be rare, since containers are never returned to the pool,
# but can happen if, for example, the configured pool size changes
if containers_to_stop > 0:
logger.debug('%d containers to stop', containers_to_stop)
with lock:
oldest_container = sorted(pool)[0]
logger.info('Pool %s, removing oldest container %s',
pool_stat_str, oldest_container.name)
interactions.stop(oldest_container)
# again, continue the loop here to save destroys for last
continue
# after balancing the pool, destroy oldest stopped container
if not_running:
if len(not_running) % 4 == 0:
logger.info('Pool %s' % pool_stat_str)
interactions.destroy(sorted(not_running)[0])
continue
# If we've made it this far...
logger.info('Pool balanced, %s', pool_stat_str)
pool_balanced = True
except (APIError, RequestException) as exc:
logger.error('%s while balancing containers, retrying.' % type(exc).__name__)
logger.exception(exc)
balance_containers.trigger()
def balance_containers():
try:
stop_outdated()
pool_balanced = False
while not pool_balanced:
# Grabs/releases the lock each time through the loop so checkouts don't have to wait
# too long if a container's being destroyed
with lock:
# Make sure the number of running containers that aren't checked out
containers = interactions.containers()
running = interactions.running(*containers)
not_running = containers - running
checked_out = set(
filter(lambda c: bool(c.checked_out), running))
# Reset the global pool based on the current derived state
pool.clear()
pool.update(running - checked_out)
pool_stat_str = "%d/%d - %d checked out - %d to destroy" % (
len(pool),
pool_size,
len(checked_out),
len(not_running),
)
containers_to_start = pool_size - len(pool)
containers_to_stop = len(pool) - pool_size
# Starting containers can happen at-will, and shouldn't be done under lock
# so that checkouts don't have to block unless the pool is exhausted
if containers_to_start > 0:
if containers_to_start > 4:
# limit the number of ocntainers to start so we
# don't spend *too* much time refilling the pool if
# there's more work to be done
containers_to_start = 4
logger.info("Pool %s, adding %d containers", pool_stat_str,
containers_to_start)
interactions.create_containers(image_name, containers_to_start)
# after starting, continue the loop to ensure that
# starting new containers happens before destruction
continue
# Stopping containers does need to happen under lock to ensure that
# simultaneous balance_containers don't attempt to stop the same container
# This should be rare, since containers are never returned to the pool,
# but can happen if, for example, the configured pool size changes
if containers_to_stop > 0:
logger.debug("%d containers to stop", containers_to_stop)
with lock:
oldest_container = min(pool, key=attrgetter("created"))
logger.info(
"Pool %s, removing oldest container %s",
pool_stat_str,
oldest_container.name,
)
interactions.stop(oldest_container)
# again, continue the loop here to save destroys for last
continue
# If we've made it this far...
logger.info("Pool balanced, %s", pool_stat_str)
pool_balanced = True
except (APIError, RequestException) as exc:
logger.error("%s while balancing containers, retrying." %
type(exc).__name__)
logger.exception(exc)
balance_containers.trigger()
