def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
if changed_jobs:
logger.info("Job state changes found: %s" % changed_jobs)
for job in changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
return changed_jobs
def task_check_for_new_jobs():
"""
Looks at the number of currently active threads and compares it against the
:py:data:`MAX_ENCODING_JOBS_PER_EC2_INSTANCE <media_nommer.conf.settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE>`
setting. If we are under the max, fire up another thread for encoding
additional job(s).
The interval at which :doc:`../ec2nommerd` checks for new jobs is
determined by the
:py:data:`NOMMERD_NEW_JOB_CHECK_INTERVAL <media_nommer.conf.settings.NOMMERD_NEW_JOB_CHECK_INTERVAL>`
setting.
Calls :py:func:`threaded_encode_job` for any jobs to encode.
"""
num_active_threads = NodeStateManager.get_num_active_threads()
max_threads = settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
num_jobs_to_pop = max(0, max_threads - num_active_threads)
if num_jobs_to_pop > 0:
# We have more room for encoding threads, determine how many.
logger.debug("task_check_for_new_jobs: " \
"Popping up to %d new jobs." % num_jobs_to_pop)
# This is an iterable of BaseEncodingJob sub-classed instances for
# each job returned from the queue.
jobs = JobStateBackend.pop_new_jobs_from_queue(num_jobs_to_pop)
if jobs:
logger.debug("* Popped %d jobs from the queue." % len(jobs))
for job in jobs:
# For each job returned, render in another thread.
logger.debug("* Starting encoder thread for job: %s" % job.unique_id)
reactor.callInThread(threaded_encode_job, job)
def load_recent_jobs_at_startup(cls):
"""
Loads all of the un-finished jobs into the job cache. This is
performed when :doc:`../feederd` starts.
"""
# Use print here because logging isn't fully configured at this point?
print("Populating job cache from SimpleDB.")
jobs = JobStateBackend.get_unfinished_jobs()
for job in jobs:
cls.update_job(job)
print("Jobs loaded from SDB to cache:")
for job in jobs:
print('* %s (State: %s -- Finished: %s)' %
(job.unique_id, job.job_state, job.is_finished()))
def load_recent_jobs_at_startup(cls):
"""
Loads all of the un-finished jobs into the job cache. This is
performed when :doc:`../feederd` starts.
"""
# Use print here because logging isn't fully configured at this point?
print("Populating job cache from SimpleDB.")
jobs = JobStateBackend.get_unfinished_jobs()
for job in jobs:
cls.update_job(job)
print("Jobs loaded from SDB to cache:")
for job in jobs:
print('* %s (State: %s -- Finished: %s)' % (
job.unique_id, job.job_state,
job.is_finished())
)
def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
# Pops up to 10 changed jobs that we think may have changed. There are
# some false alarms in here, whch brings us to...
popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
# A temporary list that stores the jobs that actually changed. This
# will be returned at the completion of this method's path.
changed_jobs = []
if popped_changed_jobs:
logger.debug("Potential job state changes found: %s" %
popped_changed_jobs)
for job in popped_changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
# This one actually changed, append this for returning.
changed_jobs.append(job)
if new_state == 'ERROR':
logger.error('Error trace from ec2nommerd:')
logger.error(job.job_state_details)
return changed_jobs
def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
# Pops up to 10 changed jobs that we think may have changed. There are
# some false alarms in here, whch brings us to...
popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
# A temporary list that stores the jobs that actually changed. This
# will be returned at the completion of this method's path.
changed_jobs = []
if popped_changed_jobs:
logger.debug("Potential job state changes found: %s" % popped_changed_jobs)
for job in popped_changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
# This one actually changed, append this for returning.
changed_jobs.append(job)
if new_state == 'ERROR':
logger.error('Error trace from ec2nommerd:')
logger.error(job.job_state_details)
return changed_jobs
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
if overage <= 0:
# Adding in the overflow thresh brought this under the
# overage level. No need for spawning instances.
return None
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances,
settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
