def threaded_encode_job(job):
"""
Given a job, run it through its encoding workflow in a non-blocking manner.
"""
# Update the timestamp for when the node last did something so it
# won't terminate itself.
NodeStateManager.i_did_something()
job.nommer.onomnom()
def wrapped_set_job_state(self, *args, **kwargs):
"""
Wraps set_job_state() to perform extra actions before and/or after
job state updates.
:param str new_state: The job state to set.
"""
# Tracks the fact that we did something, prevents the node from
# terminating itself.
NodeStateManager.i_did_something()
self.job.set_job_state(*args, **kwargs)
def wrapped_set_job_state(self, *args, **kwargs):
"""
Wraps set_job_state() to perform extra actions before and/or after
job state updates.
:param str new_state: The job state to set.
"""
# Tracks the fact that we did something, prevents the node from
# terminating itself.
NodeStateManager.i_did_something()
self.job.set_job_state(*args, **kwargs)
def start_tasks(self, options):
"""
Tasks are started by importing the interval_tasks module. Only do this
once the settings have been loaded by self.load_settings().
"""
from media_nommer.ec2nommerd.node_state import NodeStateManager
is_local = options.get('local', 0) == 1
# If we're developing local, don't try to get an instance ID from AWS.
# No need to store the result of this, we just want it to be cached
# for the heartbeat task to use after startup.
NodeStateManager.get_instance_id(is_local=is_local)
from media_nommer.ec2nommerd import interval_tasks
interval_tasks.register_tasks()
def start_tasks(self, options):
"""
Tasks are started by importing the interval_tasks module. Only do this
once the settings have been loaded by self.load_settings().
"""
from media_nommer.ec2nommerd.node_state import NodeStateManager
is_local = options.get('local', 0) == 1
# If we're developing local, don't try to get an instance ID from AWS.
# No need to store the result of this, we just want it to be cached
# for the heartbeat task to use after startup.
NodeStateManager.get_instance_id(is_local=is_local)
from media_nommer.ec2nommerd import interval_tasks
interval_tasks.register_tasks()
def task_check_for_new_jobs():
"""
Looks at the number of currently active threads and compares it against the
:py:data:`MAX_ENCODING_JOBS_PER_EC2_INSTANCE <media_nommer.conf.settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE>`
setting. If we are under the max, fire up another thread for encoding
additional job(s).
The interval at which :doc:`../ec2nommerd` checks for new jobs is
determined by the
:py:data:`NOMMERD_NEW_JOB_CHECK_INTERVAL <media_nommer.conf.settings.NOMMERD_NEW_JOB_CHECK_INTERVAL>`
setting.
Calls :py:func:`threaded_encode_job` for any jobs to encode.
"""
num_active_threads = NodeStateManager.get_num_active_threads()
max_threads = settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
num_jobs_to_pop = max(0, max_threads - num_active_threads)
if num_jobs_to_pop > 0:
# We have more room for encoding threads, determine how many.
logger.debug("task_check_for_new_jobs: " \
"Popping up to %d new jobs." % num_jobs_to_pop)
# This is an iterable of BaseEncodingJob sub-classed instances for
# each job returned from the queue.
jobs = JobStateBackend.pop_new_jobs_from_queue(num_jobs_to_pop)
if jobs:
logger.debug("* Popped %d jobs from the queue." % len(jobs))
for job in jobs:
# For each job returned, render in another thread.
logger.debug("* Starting encoder thread for job: %s" % job.unique_id)
reactor.callInThread(threaded_encode_job, job)
def threaded_heartbeat():
"""
Fires off a threaded task to check in with feederd via SimpleDB_. There
is a domain that contains all of the running EC2_ instances and their
unique IDs, along with some state data.
The interval at which heartbeats occur is determined by the
:py:data:`NOMMERD_HEARTBEAT_INTERVAL <media_nommer.conf.settings.NOMMERD_HEARTBEAT_INTERVAL`
setting.
"""
if settings.NOMMERD_TERMINATE_WHEN_IDLE:
# thread_count_mod factors out this thread when counting active threads.
is_terminated = NodeStateManager.contemplate_termination(thread_count_mod= -1)
else:
is_terminated = False
if not is_terminated:
NodeStateManager.send_instance_state_update()