def threaded_check_for_job_state_changes():
"""
Checks the SQS queue specified in the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting for announcements of state changes from the EC2_ instances running
:doc:`../ec2nommerd`. This lets :doc:`../feederd` know it needs to get
updated job details from the SimpleDB_ domain defined in the
:py:data:`SIMPLEDB_JOB_STATE_DOMAIN <media_nommer.conf.settings.SIMPLEDB_JOB_STATE_DOMAIN>`
setting.
"""
JobCache.refresh_jobs_with_state_changes()
# If jobs have completed, remove them from the job cache.
JobCache.uncache_finished_jobs()
def threaded_prune_jobs():
"""
Sometimes failure happens, but a Nommer doesn't handle said failure
gracefully. Instead of state changing to ``ERROR``, it gets stuck in
some un-finished state in the SimpleDB_ domain defined in
:py:data:`SIMPLEDB_JOB_STATE_DOMAIN <media_nommer.conf.settings.SIMPLEDB_JOB_STATE_DOMAIN>`
setting.
This process finds jobs that haven't been updated in a very long time
(a day or so) that are probably dead. It marks them with an ``ABANDONED``
state, letting us know something went really wrong.
"""
JobCache.abandon_stale_jobs()
# Expire any newly abandoned jobs, too. Removes them from job cache.
JobCache.uncache_finished_jobs()
def threaded_prune_jobs():
"""
Sometimes failure happens, but a Nommer doesn't handle said failure
gracefully. Instead of state changing to ``ERROR``, it gets stuck in
some un-finished state in the SimpleDB_ domain defined in
:py:data:`SIMPLEDB_JOB_STATE_DOMAIN <media_nommer.conf.settings.SIMPLEDB_JOB_STATE_DOMAIN>`
setting.
This process finds jobs that haven't been updated in a very long time
(a day or so) that are probably dead. It marks them with an ``ABANDONED``
state, letting us know something went really wrong.
"""
JobCache.abandon_stale_jobs()
# Expire any newly abandoned jobs, too. Removes them from job cache.
JobCache.uncache_finished_jobs()
def threaded_check_for_job_state_changes():
"""
Checks the SQS queue specified in the
:py:data:`SQS_JOB_STATE_CHANGE_QUEUE_NAME <media_nommer.conf.settings.SQS_JOB_STATE_CHANGE_QUEUE_NAME>`
setting for announcements of state changes from the EC2_ instances running
:doc:`../ec2nommerd`. This lets :doc:`../feederd` know it needs to get
updated job details from the SimpleDB_ domain defined in the
:py:data:`SIMPLEDB_JOB_STATE_DOMAIN <media_nommer.conf.settings.SIMPLEDB_JOB_STATE_DOMAIN>`
setting.
"""
changed_jobs = JobCache.refresh_jobs_with_state_changes()
for job in changed_jobs:
job_state_notifier.send_notification(job)
# If jobs have completed, remove them from the job cache.
JobCache.uncache_finished_jobs()
def view(self):
print "REQ", self.request.args
print "KW", self.kwargs
print "CONT", self.context
source_path = cgi.escape(self.request.args["source_path"][0])
dest_path = cgi.escape(self.request.args["dest_path"][0])
notify_url = cgi.escape(self.request.args["notify_url"][0])
preset = cgi.escape(self.request.args["preset"][0])
user_job_options = cgi.escape(self.request.args["job_options"][0])
user_job_options = simplejson.loads(user_job_options)
print "SOURCE", source_path
print "DEST", dest_path
print "NOTIFY", notify_url
print "OPTIONS", user_job_options
# Retrieve the given preset from nomconf.
try:
preset_dict = settings.PRESETS[preset]
except KeyError:
self.set_error("No such preset.")
return
# Determine the nommer based on the preset.
nommer = preset_dict["nommer"]
# Get the preset's job options dict.
job_options = preset_dict["options"]
# Override preset's options with user-specified values.
# TODO: Fix this for multi-pass!
# job_options.update(user_job_options)
print "NEW OPTS", job_options
# Create a new job and save it to the DB/queue.
job = EncodingJob(source_path, dest_path, nommer, job_options, notify_url=notify_url)
unique_job_id = job.save()
# Add the job to the local job cache.
JobCache.update_job(job)
# This is serialized and returned to the user.
self.context.update({"job_id": unique_job_id})
def set_context(self, request):
payload = self.user_input
print(payload)
for key in self.required_keys:
if not payload.get(key):
msg = "Missing/invalid required key+val: ['%s']" % key
self.set_error(msg)
return
for key in self.required_job_options_keys:
if not payload['job_options'].get(key):
msg = "Missing/invalid required key+val: ['job_options'][%s]" % key
self.set_error(msg)
return
source_path = payload['source_path']
dest_path = payload['dest_path']
notify_url = payload.get('notify_url')
job_options = payload['job_options']['options']
nommer = payload['job_options']['nommer']
#print "SOURCE", source_path
#print "DEST", dest_path
#print "NOTIFY", notify_url
#print "OPTIONS", job_options
#print "NOMMER", nommer
# Create a new job and save it to the DB/queue.
job = EncodingJob(source_path,
dest_path,
nommer,
job_options,
notify_url=notify_url)
unique_job_id = job.save()
# Add the job to the local job cache.
JobCache.update_job(job)
# This is serialized and returned to the user.
self.context.update({'job_id': unique_job_id})
def set_context(self, request):
payload = self.user_input
print(payload)
for key in self.required_keys:
if not payload.get(key):
msg = "Missing/invalid required key+val: ['%s']" % key
self.set_error(msg)
return
for key in self.required_job_options_keys:
if not payload['job_options'].get(key):
msg = "Missing/invalid required key+val: ['job_options'][%s]" % key
self.set_error(msg)
return
source_path = payload['source_path']
dest_path = payload['dest_path']
notify_url = payload.get('notify_url')
job_options = payload['job_options']['options']
nommer = payload['job_options']['nommer']
#print "SOURCE", source_path
#print "DEST", dest_path
#print "NOTIFY", notify_url
#print "OPTIONS", job_options
#print "NOMMER", nommer
# Create a new job and save it to the DB/queue.
job = EncodingJob(source_path, dest_path, nommer, job_options,
notify_url=notify_url)
unique_job_id = job.save()
# Add the job to the local job cache.
JobCache.update_job(job)
# This is serialized and returned to the user.
self.context.update({'job_id': unique_job_id})
def load_job_cache(self):
"""
Loads a portion of recently modified jobs into the job cache, where
they may be quickly accessed.
"""
JobCache.load_recent_jobs_at_startup()
def load_job_cache(self):
"""
Loads a portion of recently modified jobs into the job cache, where
they may be quickly accessed.
"""
JobCache.load_recent_jobs_at_startup()
