def _start_encoding(self): """ Best thought of as a ``main()`` method for the Nommer. This is the main bit of logic that directs the encoding process. """ logger.info("Starting to encode job %s" % self.job.unique_id) fobj = self.download_source_file() # Encode the file. The return value is a tempfile with the output. self.wrapped_set_job_state('ENCODING') out_fobj = self.__run_ffmpeg(fobj) if not out_fobj: # Failure! We're going nowhere. fobj.close() return False # Upload the encoding output file to its final destination. self.upload_to_destination(out_fobj) self.wrapped_set_job_state('FINISHED') logger.info("FFmpegNommer: Job %s has been successfully encoded." % self.job.unique_id) # Clean these up explicitly, just in case. fobj.close() out_fobj.close() return True
def spawn_instances(cls, num_instances): """ Spawns the number of instances specified. :param int num_instances: The number of instances to spawn. :rtype: :py:class:`boto.ec2.instance.Reservation` :returns: A boto Reservation for the started instance(s). """ logger.info("EC2InstanceManager.spawn_instances(): " \ "Spawning %d new instances" % num_instances) conn = cls._aws_ec2_connection() try: image = conn.get_all_images(image_ids=settings.EC2_AMI_ID) image = image[0] except EC2ResponseError: logger.error("EC2InstanceManager.spawn_instances(): " \ "No AMI with ID %s could be found." % settings.EC2_AMI_ID) logger.error() return except IndexError: logger.error("EC2InstanceManager.spawn_instances(): " \ "No AMI with ID %s could be found." % settings.EC2_AMI_ID) logger.error() return # The boto Reservation object. Its 'instances' attribute is the # important bit. return image.run(min_count=num_instances, max_count=num_instances, instance_type=settings.EC2_INSTANCE_TYPE, security_groups=settings.EC2_SECURITY_GROUPS, key_name=settings.EC2_KEY_NAME, user_data=cls._gen_ec2_user_data())
def _onomnom(self): """ Best thought of as a ``main()`` method for the Nommer. This is the main bit of logic that directs the encoding process. """ logger.info("Starting to encode job %s" % self.job.unique_id) fobj = self.download_source_file() # Encode the file. The return value is a tempfile with the output. self.wrapped_set_job_state('ENCODING') out_fobj = self.__run_ffmpeg(fobj) if not out_fobj: # Failure! We're going nowhere. fobj.close() return False # Upload the encoding output file to its final destination. self.upload_to_destination(out_fobj) self.wrapped_set_job_state('FINISHED') logger.info("FFmpegNommer: Job %s has been successfully encoded." % self.job.unique_id) # Clean these up explicitly, just in case. fobj.close() out_fobj.close() return True
def refresh_jobs_with_state_changes(cls): """ Looks at the state SQS queue specified by the :py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>` setting and refreshes any jobs that have changed. This simply reloads the job's details from SimpleDB_. :rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>` :returns: A list of changed :py:class:`EncodingJob` objects. """ logger.debug("JobCache.refresh_jobs_with_state_changes(): " \ "Checking state change queue.") changed_jobs = JobStateBackend.pop_state_changes_from_queue(10) if changed_jobs: logger.info("Job state changes found: %s" % changed_jobs) for job in changed_jobs: if cls.is_job_cached(job): current_state = cls.get_job(job).job_state new_state = job.job_state if current_state != new_state: logger.info("* Job state changed %s: %s -> %s" % ( job.unique_id, # Current job state in cache current_state, # New incoming job state new_state, )) cls.update_job(job) return changed_jobs
def upload_settings(nomconf_module): """ Given a user-defined nomconf module (already imported), push said file to the S3 conf bucket, as defined by settings.CONFIG_S3_BUCKET. This is used by the nommers that require access to the config, like FFmpegNommer. :param module nomconf_module: The user's ``nomconf`` module. This may be called something other than ``nomconf``, but the uploaded filename will always be ``nomconf.py``, so the EC2 nodes can find it in your settings.CONFIG_S3_BUCKET. """ logger.info("Uploading nomconf.py to S3.") nomconf_py_path = nomconf_module.__file__ if nomconf_py_path.endswith('.pyc'): # Don't want to upload the .pyc, looking for the .py. nomconf_py_path = nomconf_py_path[:-1] conn = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) bucket = conn.create_bucket(settings.CONFIG_S3_BUCKET) key = bucket.new_key('nomconf.py') key.set_contents_from_filename(nomconf_py_path) logger.info("nomconf.py uploaded successfully.")
def uncache_finished_jobs(cls): """ Clears jobs from the cache after they have been finished. TODO: We'll eventually want to clear jobs from the cache that haven't been accessed by the web API recently. """ for id, job in cls.CACHE.items(): if job.is_finished(): logger.info("Removing job %s from job cache." % id) cls.remove_job(id)
def download_settings(nomconf_uri): """ Given the URI to a S3 location with a valid nomconf.py, download it to the current user's home directory. .. tip:: This is used on the media-nommer EC2 AMIs. This won't run on local development machines. :param str nomconf_uri: The URI to your setup's ``nomconf.py`` file. Make Sure to specify AWS keys and IDs if using the S3 protocol. """ logger.info("Downloading nomconf.py from S3.") nomconf_path = os.path.expanduser('~/nomconf.py') fobj = open(nomconf_path, 'w') S3Backend.download_file(nomconf_uri, fobj) logger.info("nomconf.py downloaded from S3 successfully.")
def contemplate_termination(cls, thread_count_mod=0): """ Looks at how long it's been since this worker has done something, and decides whether to self-terminate. :param int thread_count_mod: Add this to the amount returned by the call to :py:meth:`get_num_active_threads`. This is useful when calling this method from a non-encoder thread. :rtype: bool :returns: ``True`` if this instance terminated itself, ``False`` if not. """ if not cls.is_ec2_instance(): # Developing locally, don't go here. return False # This is -1 since this is also a thread doing the contemplation. # This would always be 1, even if we had no jobs encoding, if we # didn't take into account this thread. num_active_threads = cls.get_num_active_threads() + thread_count_mod if num_active_threads > 0: # Encoding right now, don't terminate. return False tdelt = datetime.datetime.now() - cls.last_dtime_i_did_something # Total seconds of inactivity. inactive_secs = total_seconds(tdelt) # If we're over the inactivity threshold... if inactive_secs > settings.NOMMERD_MAX_INACTIVITY: instance_id = cls.get_instance_id() conn = cls._aws_ec2_connection() # Find this particular EC2 instance via boto. reservations = conn.get_all_instances(instance_ids=[instance_id]) # This should only be one match, but in the interest of # playing along... for reservation in reservations: for instance in reservation.instances: # Here's the instance, terminate it. logger.info("Goodbye, cruel world.") cls.send_instance_state_update(state='TERMINATED') instance.terminate() # Seeya later! return True # Continue existence, no termination. return False
def cb_response_received(response, unique_id, req_url): """ This is a callback function that is hit when a response comes back from the remote server given in EncodingJob.notify_url. We'll just log it here for troubleshooting purposes. :param str unique_id: The job's unique ID. :param str req_url: The URL that we notified. """ # Shouldn't ever happen in this case, but... http_code = getattr(response, 'code', 'N/A') logger.info( 'Job state change notification (HTTP %s) Response received for job: %s' % ( http_code, unique_id ) )
def refresh_jobs_with_state_changes(cls): """ Looks at the state SQS queue specified by the :py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>` setting and refreshes any jobs that have changed. This simply reloads the job's details from SimpleDB_. :rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>` :returns: A list of changed :py:class:`EncodingJob` objects. """ logger.debug("JobCache.refresh_jobs_with_state_changes(): " \ "Checking state change queue.") # Pops up to 10 changed jobs that we think may have changed. There are # some false alarms in here, whch brings us to... popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10) # A temporary list that stores the jobs that actually changed. This # will be returned at the completion of this method's path. changed_jobs = [] if popped_changed_jobs: logger.debug("Potential job state changes found: %s" % popped_changed_jobs) for job in popped_changed_jobs: if cls.is_job_cached(job): current_state = cls.get_job(job).job_state new_state = job.job_state if current_state != new_state: logger.info("* Job state changed %s: %s -> %s" % ( job.unique_id, # Current job state in cache current_state, # New incoming job state new_state, )) cls.update_job(job) # This one actually changed, append this for returning. changed_jobs.append(job) if new_state == 'ERROR': logger.error('Error trace from ec2nommerd:') logger.error(job.job_state_details) return changed_jobs
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None
def spawn_if_needed(cls): """ Spawns additional EC2 instances if needed. :rtype: :py:class:`boto.ec2.instance.Reservation` or ``None`` :returns: If instances are spawned, return a boto Reservation object. If no instances are spawned, ``None`` is returned. """ instances = cls.get_instances() num_instances = len(instances) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current active instances: %d" % num_instances) if num_instances >= settings.MAX_NUM_EC2_INSTANCES: # No more instances, no spawning allowed. return unfinished_jobs = JobStateBackend.get_unfinished_jobs() num_unfinished_jobs = len(unfinished_jobs) logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Current unfinished jobs: %d" % num_unfinished_jobs) if num_unfinished_jobs == 0: # No unfinished jobs, no need to go any further. return job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE if job_capacity == 0: # Don't factor in overflow thresh or anything if we have no # instances or capacity. cap_plus_thresh = 0 else: cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH logger.debug("EC2InstanceManager.spawn_if_needed(): " \ "Job capacity (%d w/ thresh): %d" % (job_capacity, cap_plus_thresh)) is_over_capacity = num_unfinished_jobs >= cap_plus_thresh # Disgregard the overflow thresh if there are jobs but no instances. if is_over_capacity or num_instances == 0: overage = num_unfinished_jobs - job_capacity if job_capacity > 0: # Only factor overhold threshold in when we have capacity # available in some form. overage -= settings.JOB_OVERFLOW_THRESH if overage <= 0: # Adding in the overflow thresh brought this under the # overage level. No need for spawning instances. return None logger.info("EC2InstanceManager.spawn_if_needed(): " \ "Observed labor shortage of: %d" % overage) # Raw # of instances needing to be spawned. num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE # At this point, we know there's an overage, even with the overflow # thresh factored in (if there is at least one EC2 instance # already running). num_new_instances = max(num_new_instances, 1) # Also don't spawn more than the max configured instances. num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES) # The boto Reservation object. Its 'instances' attribute is the # important bit. if num_new_instances > 0: return cls.spawn_instances(num_new_instances) # No new instances. return None