def refresh_jobs_with_state_changes(cls): """ Looks at the state SQS queue specified by the :py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>` setting and refreshes any jobs that have changed. This simply reloads the job's details from SimpleDB_. :rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>` :returns: A list of changed :py:class:`EncodingJob` objects. """ logger.debug("JobCache.refresh_jobs_with_state_changes(): " \ "Checking state change queue.") changed_jobs = JobStateBackend.pop_state_changes_from_queue(10) if changed_jobs: logger.info("Job state changes found: %s" % changed_jobs) for job in changed_jobs: if cls.is_job_cached(job): current_state = cls.get_job(job).job_state new_state = job.job_state if current_state != new_state: logger.info("* Job state changed %s: %s -> %s" % ( job.unique_id, # Current job state in cache current_state, # New incoming job state new_state, )) cls.update_job(job) return changed_jobs
def task_check_for_new_jobs(): """ Looks at the number of currently active threads and compares it against the :py:data:`MAX_ENCODING_JOBS_PER_EC2_INSTANCE <media_nommer.conf.settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE>` setting. If we are under the max, fire up another thread for encoding additional job(s). The interval at which :doc:`../ec2nommerd` checks for new jobs is determined by the :py:data:`NOMMERD_NEW_JOB_CHECK_INTERVAL <media_nommer.conf.settings.NOMMERD_NEW_JOB_CHECK_INTERVAL>` setting. Calls :py:func:`threaded_encode_job` for any jobs to encode. """ num_active_threads = NodeStateManager.get_num_active_threads() max_threads = settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE num_jobs_to_pop = max(0, max_threads - num_active_threads) if num_jobs_to_pop > 0: # We have more room for encoding threads, determine how many. logger.debug("task_check_for_new_jobs: " \ "Popping up to %d new jobs." % num_jobs_to_pop) # This is an iterable of BaseEncodingJob sub-classed instances for # each job returned from the queue. jobs = JobStateBackend.pop_new_jobs_from_queue(num_jobs_to_pop) if jobs: logger.debug("* Popped %d jobs from the queue." % len(jobs)) for job in jobs: # For each job returned, render in another thread. logger.debug("* Starting encoder thread for job: %s" % job.unique_id) reactor.callInThread(threaded_encode_job, job)
def load_recent_jobs_at_startup(cls): """ Loads all of the un-finished jobs into the job cache. This is performed when :doc:`../feederd` starts. """ # Use print here because logging isn't fully configured at this point? print("Populating job cache from SimpleDB.") jobs = JobStateBackend.get_unfinished_jobs() for job in jobs: cls.update_job(job) print("Jobs loaded from SDB to cache:") for job in jobs: print('* %s (State: %s -- Finished: %s)' % (job.unique_id, job.job_state, job.is_finished()))
def load_recent_jobs_at_startup(cls): """ Loads all of the un-finished jobs into the job cache. This is performed when :doc:`../feederd` starts. """ # Use print here because logging isn't fully configured at this point? print("Populating job cache from SimpleDB.") jobs = JobStateBackend.get_unfinished_jobs() for job in jobs: cls.update_job(job) print("Jobs loaded from SDB to cache:") for job in jobs: print('* %s (State: %s -- Finished: %s)' % ( job.unique_id, job.job_state, job.is_finished()) )
def refresh_jobs_with_state_changes(cls): """ Looks at the state SQS queue specified by the :py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>` setting and refreshes any jobs that have changed. This simply reloads the job's details from SimpleDB_. :rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>` :returns: A list of changed :py:class:`EncodingJob` objects. """ logger.debug("JobCache.refresh_jobs_with_state_changes(): " \ "Checking state change queue.") # Pops up to 10 changed jobs that we think may have changed. There are # some false alarms in here, whch brings us to... popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10) # A temporary list that stores the jobs that actually changed. This # will be returned at the completion of this method's path. changed_jobs = [] if popped_changed_jobs: logger.debug("Potential job state changes found: %s" % popped_changed_jobs) for job in popped_changed_jobs: if cls.is_job_cached(job): current_state = cls.get_job(job).job_state new_state = job.job_state if current_state != new_state: logger.info("* Job state changed %s: %s -> %s" % ( job.unique_id, # Current job state in cache current_state, # New incoming job state new_state, )) cls.update_job(job) # This one actually changed, append this for returning. changed_jobs.append(job) if new_state == 'ERROR': logger.error('Error trace from ec2nommerd:') logger.error(job.job_state_details) return changed_jobs
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH if overage <= 0: # Adding in the overflow thresh brought this under the # overage level. No need for spawning instances. return None logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None