def _start_encoding(self):
"""
Best thought of as a ``main()`` method for the Nommer. This is the
main bit of logic that directs the encoding process.
"""
logger.info("Starting to encode job %s" % self.job.unique_id)
fobj = self.download_source_file()
# Encode the file. The return value is a tempfile with the output.
self.wrapped_set_job_state('ENCODING')
out_fobj = self.__run_ffmpeg(fobj)
if not out_fobj:
# Failure! We're going nowhere.
fobj.close()
return False
# Upload the encoding output file to its final destination.
self.upload_to_destination(out_fobj)
self.wrapped_set_job_state('FINISHED')
logger.info("FFmpegNommer: Job %s has been successfully encoded." %
self.job.unique_id)
# Clean these up explicitly, just in case.
fobj.close()
out_fobj.close()
return True
def spawn_instances(cls, num_instances):
"""
Spawns the number of instances specified.
:param int num_instances: The number of instances to spawn.
:rtype: :py:class:`boto.ec2.instance.Reservation`
:returns: A boto Reservation for the started instance(s).
"""
logger.info("EC2InstanceManager.spawn_instances(): " \
"Spawning %d new instances" % num_instances)
conn = cls._aws_ec2_connection()
try:
image = conn.get_all_images(image_ids=settings.EC2_AMI_ID)
image = image[0]
except EC2ResponseError:
logger.error("EC2InstanceManager.spawn_instances(): " \
"No AMI with ID %s could be found." % settings.EC2_AMI_ID)
logger.error()
return
except IndexError:
logger.error("EC2InstanceManager.spawn_instances(): " \
"No AMI with ID %s could be found." % settings.EC2_AMI_ID)
logger.error()
return
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
return image.run(min_count=num_instances, max_count=num_instances,
instance_type=settings.EC2_INSTANCE_TYPE,
security_groups=settings.EC2_SECURITY_GROUPS,
key_name=settings.EC2_KEY_NAME,
user_data=cls._gen_ec2_user_data())
def _onomnom(self):
"""
Best thought of as a ``main()`` method for the Nommer. This is the
main bit of logic that directs the encoding process.
"""
logger.info("Starting to encode job %s" % self.job.unique_id)
fobj = self.download_source_file()
# Encode the file. The return value is a tempfile with the output.
self.wrapped_set_job_state('ENCODING')
out_fobj = self.__run_ffmpeg(fobj)
if not out_fobj:
# Failure! We're going nowhere.
fobj.close()
return False
# Upload the encoding output file to its final destination.
self.upload_to_destination(out_fobj)
self.wrapped_set_job_state('FINISHED')
logger.info("FFmpegNommer: Job %s has been successfully encoded." % self.job.unique_id)
# Clean these up explicitly, just in case.
fobj.close()
out_fobj.close()
return True
def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
if changed_jobs:
logger.info("Job state changes found: %s" % changed_jobs)
for job in changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
return changed_jobs
def upload_settings(nomconf_module):
"""
Given a user-defined nomconf module (already imported), push said file
to the S3 conf bucket, as defined by settings.CONFIG_S3_BUCKET.
This is used by the nommers that require access to the config, like
FFmpegNommer.
:param module nomconf_module: The user's ``nomconf`` module. This may
be called something other than ``nomconf``, but the uploaded filename
will always be ``nomconf.py``, so the EC2 nodes can find it in your
settings.CONFIG_S3_BUCKET.
"""
logger.info("Uploading nomconf.py to S3.")
nomconf_py_path = nomconf_module.__file__
if nomconf_py_path.endswith('.pyc'):
# Don't want to upload the .pyc, looking for the .py.
nomconf_py_path = nomconf_py_path[:-1]
conn = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = conn.create_bucket(settings.CONFIG_S3_BUCKET)
key = bucket.new_key('nomconf.py')
key.set_contents_from_filename(nomconf_py_path)
logger.info("nomconf.py uploaded successfully.")
def spawn_instances(cls, num_instances):
"""
Spawns the number of instances specified.
:param int num_instances: The number of instances to spawn.
:rtype: :py:class:`boto.ec2.instance.Reservation`
:returns: A boto Reservation for the started instance(s).
"""
logger.info("EC2InstanceManager.spawn_instances(): " \
"Spawning %d new instances" % num_instances)
conn = cls._aws_ec2_connection()
try:
image = conn.get_all_images(image_ids=settings.EC2_AMI_ID)
image = image[0]
except EC2ResponseError:
logger.error("EC2InstanceManager.spawn_instances(): " \
"No AMI with ID %s could be found." % settings.EC2_AMI_ID)
logger.error()
return
except IndexError:
logger.error("EC2InstanceManager.spawn_instances(): " \
"No AMI with ID %s could be found." % settings.EC2_AMI_ID)
logger.error()
return
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
return image.run(min_count=num_instances,
max_count=num_instances,
instance_type=settings.EC2_INSTANCE_TYPE,
security_groups=settings.EC2_SECURITY_GROUPS,
key_name=settings.EC2_KEY_NAME,
user_data=cls._gen_ec2_user_data())
def uncache_finished_jobs(cls):
"""
Clears jobs from the cache after they have been finished.
TODO: We'll eventually want to clear jobs from the cache that haven't
been accessed by the web API recently.
"""
for id, job in cls.CACHE.items():
if job.is_finished():
logger.info("Removing job %s from job cache." % id)
cls.remove_job(id)
def uncache_finished_jobs(cls):
"""
Clears jobs from the cache after they have been finished.
TODO: We'll eventually want to clear jobs from the cache that haven't
been accessed by the web API recently.
"""
for id, job in cls.CACHE.items():
if job.is_finished():
logger.info("Removing job %s from job cache." % id)
cls.remove_job(id)
def download_settings(nomconf_uri):
"""
Given the URI to a S3 location with a valid nomconf.py, download it to
the current user's home directory.
.. tip:: This is used on the media-nommer EC2 AMIs. This won't run on
local development machines.
:param str nomconf_uri: The URI to your setup's ``nomconf.py`` file. Make
Sure to specify AWS keys and IDs if using the S3 protocol.
"""
logger.info("Downloading nomconf.py from S3.")
nomconf_path = os.path.expanduser('~/nomconf.py')
fobj = open(nomconf_path, 'w')
S3Backend.download_file(nomconf_uri, fobj)
logger.info("nomconf.py downloaded from S3 successfully.")
def contemplate_termination(cls, thread_count_mod=0):
"""
Looks at how long it's been since this worker has done something, and
decides whether to self-terminate.
:param int thread_count_mod: Add this to the amount returned by the call
to :py:meth:`get_num_active_threads`. This is useful when calling
this method from a non-encoder thread.
:rtype: bool
:returns: ``True`` if this instance terminated itself, ``False``
if not.
"""
if not cls.is_ec2_instance():
# Developing locally, don't go here.
return False
# This is -1 since this is also a thread doing the contemplation.
# This would always be 1, even if we had no jobs encoding, if we
# didn't take into account this thread.
num_active_threads = cls.get_num_active_threads() + thread_count_mod
if num_active_threads > 0:
# Encoding right now, don't terminate.
return False
tdelt = datetime.datetime.now() - cls.last_dtime_i_did_something
# Total seconds of inactivity.
inactive_secs = total_seconds(tdelt)
# If we're over the inactivity threshold...
if inactive_secs > settings.NOMMERD_MAX_INACTIVITY:
instance_id = cls.get_instance_id()
conn = cls._aws_ec2_connection()
# Find this particular EC2 instance via boto.
reservations = conn.get_all_instances(instance_ids=[instance_id])
# This should only be one match, but in the interest of
# playing along...
for reservation in reservations:
for instance in reservation.instances:
# Here's the instance, terminate it.
logger.info("Goodbye, cruel world.")
cls.send_instance_state_update(state='TERMINATED')
instance.terminate()
# Seeya later!
return True
# Continue existence, no termination.
return False
def contemplate_termination(cls, thread_count_mod=0):
"""
Looks at how long it's been since this worker has done something, and
decides whether to self-terminate.
:param int thread_count_mod: Add this to the amount returned by the call
to :py:meth:`get_num_active_threads`. This is useful when calling
this method from a non-encoder thread.
:rtype: bool
:returns: ``True`` if this instance terminated itself, ``False``
if not.
"""
if not cls.is_ec2_instance():
# Developing locally, don't go here.
return False
# This is -1 since this is also a thread doing the contemplation.
# This would always be 1, even if we had no jobs encoding, if we
# didn't take into account this thread.
num_active_threads = cls.get_num_active_threads() + thread_count_mod
if num_active_threads > 0:
# Encoding right now, don't terminate.
return False
tdelt = datetime.datetime.now() - cls.last_dtime_i_did_something
# Total seconds of inactivity.
inactive_secs = total_seconds(tdelt)
# If we're over the inactivity threshold...
if inactive_secs > settings.NOMMERD_MAX_INACTIVITY:
instance_id = cls.get_instance_id()
conn = cls._aws_ec2_connection()
# Find this particular EC2 instance via boto.
reservations = conn.get_all_instances(instance_ids=[instance_id])
# This should only be one match, but in the interest of
# playing along...
for reservation in reservations:
for instance in reservation.instances:
# Here's the instance, terminate it.
logger.info("Goodbye, cruel world.")
cls.send_instance_state_update(state='TERMINATED')
instance.terminate()
# Seeya later!
return True
# Continue existence, no termination.
return False
def cb_response_received(response, unique_id, req_url):
"""
This is a callback function that is hit when a response comes back from
the remote server given in EncodingJob.notify_url. We'll just log it here
for troubleshooting purposes.
:param str unique_id: The job's unique ID.
:param str req_url: The URL that we notified.
"""
# Shouldn't ever happen in this case, but...
http_code = getattr(response, 'code', 'N/A')
logger.info(
'Job state change notification (HTTP %s) Response received for job: %s' % (
http_code,
unique_id
)
)
def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
# Pops up to 10 changed jobs that we think may have changed. There are
# some false alarms in here, whch brings us to...
popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
# A temporary list that stores the jobs that actually changed. This
# will be returned at the completion of this method's path.
changed_jobs = []
if popped_changed_jobs:
logger.debug("Potential job state changes found: %s" %
popped_changed_jobs)
for job in popped_changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
# This one actually changed, append this for returning.
changed_jobs.append(job)
if new_state == 'ERROR':
logger.error('Error trace from ec2nommerd:')
logger.error(job.job_state_details)
return changed_jobs
def refresh_jobs_with_state_changes(cls):
"""
Looks at the state SQS queue specified by the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting and refreshes any jobs that have changed. This simply reloads
the job's details from SimpleDB_.
:rtype: ``list`` of :py:class:`EncodingJob <media_nommer.core.job_state_backend.EncodingJob>`
:returns: A list of changed :py:class:`EncodingJob` objects.
"""
logger.debug("JobCache.refresh_jobs_with_state_changes(): " \
"Checking state change queue.")
# Pops up to 10 changed jobs that we think may have changed. There are
# some false alarms in here, whch brings us to...
popped_changed_jobs = JobStateBackend.pop_state_changes_from_queue(10)
# A temporary list that stores the jobs that actually changed. This
# will be returned at the completion of this method's path.
changed_jobs = []
if popped_changed_jobs:
logger.debug("Potential job state changes found: %s" % popped_changed_jobs)
for job in popped_changed_jobs:
if cls.is_job_cached(job):
current_state = cls.get_job(job).job_state
new_state = job.job_state
if current_state != new_state:
logger.info("* Job state changed %s: %s -> %s" % (
job.unique_id,
# Current job state in cache
current_state,
# New incoming job state
new_state,
))
cls.update_job(job)
# This one actually changed, append this for returning.
changed_jobs.append(job)
if new_state == 'ERROR':
logger.error('Error trace from ec2nommerd:')
logger.error(job.job_state_details)
return changed_jobs
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances, settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
def spawn_if_needed(cls):
"""
Spawns additional EC2 instances if needed.
:rtype: :py:class:`boto.ec2.instance.Reservation` or ``None``
:returns: If instances are spawned, return a boto Reservation
object. If no instances are spawned, ``None`` is returned.
"""
instances = cls.get_instances()
num_instances = len(instances)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current active instances: %d" % num_instances)
if num_instances >= settings.MAX_NUM_EC2_INSTANCES:
# No more instances, no spawning allowed.
return
unfinished_jobs = JobStateBackend.get_unfinished_jobs()
num_unfinished_jobs = len(unfinished_jobs)
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Current unfinished jobs: %d" % num_unfinished_jobs)
if num_unfinished_jobs == 0:
# No unfinished jobs, no need to go any further.
return
job_capacity = num_instances * settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
if job_capacity == 0:
# Don't factor in overflow thresh or anything if we have no
# instances or capacity.
cap_plus_thresh = 0
else:
cap_plus_thresh = job_capacity + settings.JOB_OVERFLOW_THRESH
logger.debug("EC2InstanceManager.spawn_if_needed(): " \
"Job capacity (%d w/ thresh): %d" % (job_capacity,
cap_plus_thresh))
is_over_capacity = num_unfinished_jobs >= cap_plus_thresh
# Disgregard the overflow thresh if there are jobs but no instances.
if is_over_capacity or num_instances == 0:
overage = num_unfinished_jobs - job_capacity
if job_capacity > 0:
# Only factor overhold threshold in when we have capacity
# available in some form.
overage -= settings.JOB_OVERFLOW_THRESH
if overage <= 0:
# Adding in the overflow thresh brought this under the
# overage level. No need for spawning instances.
return None
logger.info("EC2InstanceManager.spawn_if_needed(): " \
"Observed labor shortage of: %d" % overage)
# Raw # of instances needing to be spawned.
num_new_instances = overage / settings.MAX_ENCODING_JOBS_PER_EC2_INSTANCE
# At this point, we know there's an overage, even with the overflow
# thresh factored in (if there is at least one EC2 instance
# already running).
num_new_instances = max(num_new_instances, 1)
# Also don't spawn more than the max configured instances.
num_new_instances = min(num_new_instances,
settings.MAX_NUM_EC2_INSTANCES)
# The boto Reservation object. Its 'instances' attribute is the
# important bit.
if num_new_instances > 0:
return cls.spawn_instances(num_new_instances)
# No new instances.
return None
