def load_recent_jobs_at_startup(cls):
"""
Loads all of the un-finished jobs into the job cache. This is
performed when :doc:`../feederd` starts.
"""
# Use print here because logging isn't fully configured at this point?
print("Populating job cache from SimpleDB.")
jobs = JobStateBackend.get_unfinished_jobs()
for job in jobs:
cls.update_job(job)
print("Jobs loaded from SDB to cache:")
for job in jobs:
print('* %s (State: %s -- Finished: %s)' %
(job.unique_id, job.job_state, job.is_finished()))
def load_recent_jobs_at_startup(cls):
"""
Loads all of the un-finished jobs into the job cache. This is
performed when :doc:`../feederd` starts.
"""
# Use print here because logging isn't fully configured at this point?
print("Populating job cache from SimpleDB.")
jobs = JobStateBackend.get_unfinished_jobs()
for job in jobs:
cls.update_job(job)
print("Jobs loaded from SDB to cache:")
for job in jobs:
print('* %s (State: %s -- Finished: %s)' % (
job.unique_id, job.job_state,
job.is_finished())
)
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
if overage <= 0:
# Adding in the overflow thresh brought this under the
# overage level. No need for spawning instances.
return None
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances,
settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
