def stop_outdated(): # Clean up before interacting with the pool: # - checks in containers that are checked out longer than the max lifetime # - stops containers that aren't running if their image is out of date # - stops containers from the pool not running selenium for container in interactions.containers(): if container.checked_out: checked_out_time = (datetime.utcnow() - container.checked_out).total_seconds() if checked_out_time > max_checkout_time: logger.info( "Container %s checked out longer than %d seconds, forcing stop", container.name, max_checkout_time, ) interactions.stop(container) continue else: if container.image_id != interactions.last_pulled_image_id: logger.info("Container %s running an old image", container.name) interactions.stop(container) continue
def balance_containers(): try: # Clean up before interacting with the pool: # - checks in containers that are checked out longer than the max lifetime # - stops containers that aren't running if their image is out of date # - stops containers from the pool not running selenium for container in interactions.containers(): if container.checked_out: if ((datetime.utcnow() - container.checked_out).total_seconds() > max_checkout_time): logger.info( 'Container %s checked out longer than %d seconds, forcing stop', container.name, max_checkout_time) interactions.stop(container) continue else: if container.image_id != interactions.last_pulled_image_id: logger.info('Container %s running an old image', container.name) interactions.stop(container) continue pool_balanced = False while not pool_balanced: # Grabs/releases the lock each time through the loop so checkouts don't have to wait # too long if a container's being destroyed with lock: # Make sure the number of running containers that aren't checked out containers = interactions.containers() running = interactions.running(*containers) not_running = containers - running checked_out = set( filter(lambda c: bool(c.checked_out), running)) # Reset the global pool based on the current derived state pool.clear() pool.update(running - checked_out) pool_stat_str = '%d/%d - %d checked out - %d to destroy' % ( len(pool), pool_size, len(checked_out), len(not_running)) containers_to_start = pool_size - len(pool) containers_to_stop = len(pool) - pool_size # Starting containers can happen at-will, and shouldn't be done under lock # so that checkouts don't have to block unless the pool is exhausted if containers_to_start > 0: if containers_to_start > 4: # limit the number of ocntainers to start so we # don't spend *too* much time refilling the pool if # there's more work to be done containers_to_start = 4 logger.info('Pool %s, adding %d containers', pool_stat_str, containers_to_start) new_containers = [] for __ in range(containers_to_start): new_containers.append( interactions.create_container(image_name)) interactions.start(*new_containers) # after starting, continue the loop to ensure that # starting new containers happens before destruction continue # Stopping containers does need to happen under lock to ensure that # simultaneous balance_containers don't attempt to stop the same container # This should be rare, since containers are never returned to the pool, # but can happen if, for example, the configured pool size changes if containers_to_stop > 0: logger.debug('%d containers to stop', containers_to_stop) with lock: oldest_container = sorted(pool)[0] logger.info('Pool %s, removing oldest container %s', pool_stat_str, oldest_container.name) interactions.stop(oldest_container) # again, continue the loop here to save destroys for last continue # after balancing the pool, destroy oldest stopped container if not_running: if len(not_running) % 4 == 0: logger.info('Pool %s' % pool_stat_str) interactions.destroy(sorted(not_running)[0]) continue # If we've made it this far... logger.info('Pool balanced, %s', pool_stat_str) pool_balanced = True except (APIError, RequestException) as exc: logger.error('%s while balancing containers, retrying.' % type(exc).__name__) logger.exception(exc) balance_containers.trigger()
def _stop_async_worker(container): interactions.stop(container) balance_containers.trigger()
def balance_containers(): try: # Clean up before interacting with the pool: # - checks in containers that are checked out longer than the max lifetime # - stops containers that aren't running if their image is out of date # - stops containers from the pool not running selenium for container in interactions.containers(): if container.checked_out: if ((datetime.utcnow() - container.checked_out).total_seconds() > max_checkout_time): logger.info('Container %s checked out longer than %d seconds, forcing stop', container.name, max_checkout_time) interactions.stop(container) continue else: if container.image_id != interactions.last_pulled_image_id: logger.info('Container %s running an old image', container.name) interactions.stop(container) continue pool_balanced = False while not pool_balanced: # Grabs/releases the lock each time through the loop so checkouts don't have to wait # too long if a container's being destroyed with lock: # Make sure the number of running containers that aren't checked out containers = interactions.containers() running = interactions.running(*containers) not_running = containers - running checked_out = set(filter(lambda c: bool(c.checked_out), running)) # Reset the global pool based on the current derived state pool.clear() pool.update(running - checked_out) pool_stat_str = '%d/%d - %d checked out - %d to destroy' % ( len(pool), pool_size, len(checked_out), len(not_running)) containers_to_start = pool_size - len(pool) containers_to_stop = len(pool) - pool_size # Starting containers can happen at-will, and shouldn't be done under lock # so that checkouts don't have to block unless the pool is exhausted if containers_to_start > 0: if containers_to_start > 4: # limit the number of ocntainers to start so we # don't spend *too* much time refilling the pool if # there's more work to be done containers_to_start = 4 logger.info('Pool %s, adding %d containers', pool_stat_str, containers_to_start) new_containers = [] for __ in range(containers_to_start): new_containers.append(interactions.create_container(image_name)) interactions.start(*new_containers) # after starting, continue the loop to ensure that # starting new containers happens before destruction continue # Stopping containers does need to happen under lock to ensure that # simultaneous balance_containers don't attempt to stop the same container # This should be rare, since containers are never returned to the pool, # but can happen if, for example, the configured pool size changes if containers_to_stop > 0: logger.debug('%d containers to stop', containers_to_stop) with lock: oldest_container = sorted(pool)[0] logger.info('Pool %s, removing oldest container %s', pool_stat_str, oldest_container.name) interactions.stop(oldest_container) # again, continue the loop here to save destroys for last continue # after balancing the pool, destroy oldest stopped container if not_running: if len(not_running) % 4 == 0: logger.info('Pool %s' % pool_stat_str) interactions.destroy(sorted(not_running)[0]) continue # If we've made it this far... logger.info('Pool balanced, %s', pool_stat_str) pool_balanced = True except (APIError, RequestException) as exc: logger.error('%s while balancing containers, retrying.' % type(exc).__name__) logger.exception(exc) balance_containers.trigger()
def balance_containers(): try: stop_outdated() pool_balanced = False while not pool_balanced: # Grabs/releases the lock each time through the loop so checkouts don't have to wait # too long if a container's being destroyed with lock: # Make sure the number of running containers that aren't checked out containers = interactions.containers() running = interactions.running(*containers) not_running = containers - running checked_out = set( filter(lambda c: bool(c.checked_out), running)) # Reset the global pool based on the current derived state pool.clear() pool.update(running - checked_out) pool_stat_str = "%d/%d - %d checked out - %d to destroy" % ( len(pool), pool_size, len(checked_out), len(not_running), ) containers_to_start = pool_size - len(pool) containers_to_stop = len(pool) - pool_size # Starting containers can happen at-will, and shouldn't be done under lock # so that checkouts don't have to block unless the pool is exhausted if containers_to_start > 0: if containers_to_start > 4: # limit the number of ocntainers to start so we # don't spend *too* much time refilling the pool if # there's more work to be done containers_to_start = 4 logger.info("Pool %s, adding %d containers", pool_stat_str, containers_to_start) interactions.create_containers(image_name, containers_to_start) # after starting, continue the loop to ensure that # starting new containers happens before destruction continue # Stopping containers does need to happen under lock to ensure that # simultaneous balance_containers don't attempt to stop the same container # This should be rare, since containers are never returned to the pool, # but can happen if, for example, the configured pool size changes if containers_to_stop > 0: logger.debug("%d containers to stop", containers_to_stop) with lock: oldest_container = min(pool, key=attrgetter("created")) logger.info( "Pool %s, removing oldest container %s", pool_stat_str, oldest_container.name, ) interactions.stop(oldest_container) # again, continue the loop here to save destroys for last continue # If we've made it this far... logger.info("Pool balanced, %s", pool_stat_str) pool_balanced = True except (APIError, RequestException) as exc: logger.error("%s while balancing containers, retrying." % type(exc).__name__) logger.exception(exc) balance_containers.trigger()