def load_recent_jobs_at_startup(cls): """ Loads all of the un-finished jobs into the job cache. This is performed when :doc:`../feederd` starts. """ # Use print here because logging isn't fully configured at this point? print("Populating job cache from SimpleDB.") jobs = JobStateBackend.get_unfinished_jobs() for job in jobs: cls.update_job(job) print("Jobs loaded from SDB to cache:") for job in jobs: print('* %s (State: %s -- Finished: %s)' % (job.unique_id, job.job_state, job.is_finished()))
def load_recent_jobs_at_startup(cls): """ Loads all of the un-finished jobs into the job cache. This is performed when :doc:`../feederd` starts. """ # Use print here because logging isn't fully configured at this point? print("Populating job cache from SimpleDB.") jobs = JobStateBackend.get_unfinished_jobs() for job in jobs: cls.update_job(job) print("Jobs loaded from SDB to cache:") for job in jobs: print('* %s (State: %s -- Finished: %s)' % ( job.unique_id, job.job_state, job.is_finished()) )
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH if overage <= 0: # Adding in the overflow thresh brought this under the # overage level. No need for spawning instances. return None logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None