Esempio n. 1
0
def threaded_encode_job(job):
    """
    Given a job, run it through its encoding workflow in a non-blocking manner.
    """
    # Update the timestamp for when the node last did something so it
    # won't terminate itself.
    NodeStateManager.i_did_something()
    job.nommer.onomnom()
Esempio n. 2
0
 def wrapped_set_job_state(self, *args, **kwargs):
     """
     Wraps set_job_state() to perform extra actions before and/or after
     job state updates.
     
     :param str new_state: The job state to set.
     """
     # Tracks the fact that we did something, prevents the node from
     # terminating itself.
     NodeStateManager.i_did_something()
     self.job.set_job_state(*args, **kwargs)
Esempio n. 3
0
 def wrapped_set_job_state(self, *args, **kwargs):
     """
     Wraps set_job_state() to perform extra actions before and/or after
     job state updates.
     
     :param str new_state: The job state to set.
     """
     # Tracks the fact that we did something, prevents the node from
     # terminating itself.
     NodeStateManager.i_did_something()
     self.job.set_job_state(*args, **kwargs)
Esempio n. 4
0
    def start_tasks(self, options):
        """
        Tasks are started by importing the interval_tasks module. Only do this
        once the settings have been loaded by self.load_settings().
        """
        from media_nommer.ec2nommerd.node_state import NodeStateManager

        is_local = options.get('local', 0) == 1
        # If we're developing local, don't try to get an instance ID from AWS.
        # No need to store the result of this, we just want it to be cached
        # for the heartbeat task to use after startup.
        NodeStateManager.get_instance_id(is_local=is_local)

        from media_nommer.ec2nommerd import interval_tasks
        interval_tasks.register_tasks()
Esempio n. 5
0
    def start_tasks(self, options):
        """
        Tasks are started by importing the interval_tasks module. Only do this
        once the settings have been loaded by self.load_settings().
        """
        from media_nommer.ec2nommerd.node_state import NodeStateManager

        is_local = options.get('local', 0) == 1
        # If we're developing local, don't try to get an instance ID from AWS.
        # No need to store the result of this, we just want it to be cached
        # for the heartbeat task to use after startup.
        NodeStateManager.get_instance_id(is_local=is_local)

        from media_nommer.ec2nommerd import interval_tasks
        interval_tasks.register_tasks()
Esempio n. 6
0
def task_check_for_new_jobs():
    """
    Looks at the number of currently active threads and compares it against the 
    :py:data:`MAX_ENCODING_JOBS_PER_EC2_INSTANCE <media_nommer.conf.settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE>` 
    setting. If we are under the max, fire up another thread for encoding 
    additional job(s). 
    
    The interval at which :doc:`../ec2nommerd` checks for new jobs is 
    determined by the 
    :py:data:`NOMMERD_NEW_JOB_CHECK_INTERVAL <media_nommer.conf.settings.NOMMERD_NEW_JOB_CHECK_INTERVAL>`
    setting.
    
    Calls :py:func:`threaded_encode_job` for any jobs to encode.
    """
    num_active_threads = NodeStateManager.get_num_active_threads()
    max_threads = settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
    num_jobs_to_pop = max(0, max_threads - num_active_threads)

    if num_jobs_to_pop > 0:
        # We have more room for encoding threads, determine how many.
        logger.debug("task_check_for_new_jobs: " \
                     "Popping up to %d new jobs." % num_jobs_to_pop)
        # This is an iterable of BaseEncodingJob sub-classed instances for
        # each job returned from the queue.
        jobs = JobStateBackend.pop_new_jobs_from_queue(num_jobs_to_pop)
        if jobs:
            logger.debug("* Popped %d jobs from the queue." % len(jobs))

        for job in jobs:
            # For each job returned, render in another thread.
            logger.debug("* Starting encoder thread for job: %s" % job.unique_id)
            reactor.callInThread(threaded_encode_job, job)
Esempio n. 7
0
def threaded_heartbeat():
    """
    Fires off a threaded task to check in with feederd via SimpleDB_. There
    is a domain that contains all of the running EC2_ instances and their
    unique IDs, along with some state data.
    
    The interval at which heartbeats occur is determined by the
    :py:data:`NOMMERD_HEARTBEAT_INTERVAL <media_nommer.conf.settings.NOMMERD_HEARTBEAT_INTERVAL` 
    setting.
    """
    if settings.NOMMERD_TERMINATE_WHEN_IDLE:
        # thread_count_mod factors out this thread when counting active threads.
        is_terminated = NodeStateManager.contemplate_termination(thread_count_mod= -1)
    else:
        is_terminated = False

    if not is_terminated:
        NodeStateManager.send_instance_state_update()