def run(args, media_id=None):
if not media_id:
media_id = args.media_id
try:
media_entry, manager = get_entry_and_processing_manager(media_id)
# TODO: (maybe?) This could probably be handled entirely by the
# processor class...
try:
processor_class = manager.get_processor(
args.reprocess_command, media_entry)
except ProcessorDoesNotExist:
print('No such processor "{}" for media with id "{}"'.format(
args.reprocess_command, media_entry.id))
return
except ProcessorNotEligible:
print('Processor "{}" exists but media "{}" is not eligible'.format(
args.reprocess_command, media_entry.id))
return
reprocess_parser = processor_class.generate_parser()
reprocess_args = reprocess_parser.parse_args(args.reprocess_args)
reprocess_request = processor_class.args_to_request(reprocess_args)
run_process_media(
media_entry,
reprocess_action=args.reprocess_command,
reprocess_info=reprocess_request)
except ProcessingManagerDoesNotExist:
entry = MediaEntry.query.filter_by(id=media_id).first()
print('No such processing manager for {}'.format(entry.media_type))
def run(args, media_id=None):
if not media_id:
media_id = args.media_id
try:
media_entry, manager = get_entry_and_processing_manager(media_id)
# TODO: (maybe?) This could probably be handled entirely by the
# processor class...
try:
processor_class = manager.get_processor(
args.reprocess_command, media_entry)
except ProcessorDoesNotExist:
print 'No such processor "%s" for media with id "%s"' % (
args.reprocess_command, media_entry.id)
return
except ProcessorNotEligible:
print 'Processor "%s" exists but media "%s" is not eligible' % (
args.reprocess_command, media_entry.id)
return
reprocess_parser = processor_class.generate_parser()
reprocess_args = reprocess_parser.parse_args(args.reprocess_args)
reprocess_request = processor_class.args_to_request(reprocess_args)
run_process_media(
media_entry,
reprocess_action=args.reprocess_command,
reprocess_info=reprocess_request)
except ProcessingManagerDoesNotExist:
entry = MediaEntry.query.filter_by(id=media_id).first()
print 'No such processing manager for {0}'.format(entry.media_type)
def processing_cleanup(entry_id):
_log.debug('Entered processing_cleanup')
entry, manager = get_entry_and_processing_manager(entry_id)
with CommonVideoProcessor(manager, entry) as processor:
# no need to specify a resolution here
processor.common_setup()
processor.copy_original()
processor.keep_best()
processor.delete_queue_file()
_log.debug('Deleted queue_file')
def initial(args):
"""
Reprocess all failed media
"""
query = MediaEntry.query.filter_by(state='failed')
for entry in query:
try:
media_entry, manager = get_entry_and_processing_manager(entry.id)
run_process_media(media_entry, reprocess_action='initial')
except ProcessingManagerDoesNotExist:
print 'No such processing manager for {0}'.format(entry.media_type)
def complementary_task(entry_id, resolution, medium_size, **process_info):
"""
Side celery task to transcode the video to other resolutions
"""
entry, manager = get_entry_and_processing_manager(entry_id)
with CommonVideoProcessor(manager, entry) as processor:
processor.common_setup(resolution)
processor.transcode(medium_size=tuple(medium_size),
vp8_quality=process_info['vp8_quality'],
vp8_threads=process_info['vp8_threads'],
vorbis_quality=process_info['vorbis_quality'])
_log.info('MediaEntry ID {} is transcoded to {}'.format(
entry.id, medium_size))
def initial(args):
"""
Reprocess all failed media
"""
query = MediaEntry.query.filter_by(state='failed')
for entry in query:
try:
media_entry, manager = get_entry_and_processing_manager(entry.id)
run_process_media(
media_entry,
reprocess_action='initial')
except ProcessingManagerDoesNotExist:
print 'No such processing manager for {0}'.format(entry.media_type)
def thumbs(args):
"""
Regenerate thumbs for all processed media
"""
query = MediaEntry.query.filter_by(state='processed')
for entry in query:
try:
media_entry, manager = get_entry_and_processing_manager(entry.id)
# TODO: (maybe?) This could probably be handled entirely by the
# processor class...
try:
processor_class = manager.get_processor(
'resize', media_entry)
except ProcessorDoesNotExist:
print('No such processor "{}" for media with id "{}"'.format(
'resize', media_entry.id))
return
except ProcessorNotEligible:
print('Processor "{}" exists but media "{}" is not eligible'.format(
'resize', media_entry.id))
return
reprocess_parser = processor_class.generate_parser()
# prepare filetype and size to be passed into reprocess_parser
if args.size:
extra_args = 'thumb --{} {} {}'.format(
processor_class.thumb_size,
args.size[0],
args.size[1])
else:
extra_args = 'thumb'
reprocess_args = reprocess_parser.parse_args(extra_args.split())
reprocess_request = processor_class.args_to_request(reprocess_args)
run_process_media(
media_entry,
reprocess_action='resize',
reprocess_info=reprocess_request)
except ProcessingManagerDoesNotExist:
print('No such processing manager for {}'.format(entry.media_type))
def thumbs(args):
"""
Regenerate thumbs for all processed media
"""
query = MediaEntry.query.filter_by(state='processed')
for entry in query:
try:
media_entry, manager = get_entry_and_processing_manager(entry.id)
# TODO: (maybe?) This could probably be handled entirely by the
# processor class...
try:
processor_class = manager.get_processor(
'resize', media_entry)
except ProcessorDoesNotExist:
print 'No such processor "%s" for media with id "%s"' % (
'resize', media_entry.id)
return
except ProcessorNotEligible:
print 'Processor "%s" exists but media "%s" is not eligible' % (
'resize', media_entry.id)
return
reprocess_parser = processor_class.generate_parser()
# prepare filetype and size to be passed into reprocess_parser
if args.size:
extra_args = 'thumb --{0} {1} {2}'.format(
processor_class.thumb_size,
args.size[0],
args.size[1])
else:
extra_args = 'thumb'
reprocess_args = reprocess_parser.parse_args(extra_args.split())
reprocess_request = processor_class.args_to_request(reprocess_args)
run_process_media(
media_entry,
reprocess_action='resize',
reprocess_info=reprocess_request)
except ProcessingManagerDoesNotExist:
print 'No such processing manager for {0}'.format(entry.media_type)
def main_task(entry_id, resolution, medium_size, **process_info):
"""
Main celery task to transcode the video to the default resolution
and store original video metadata.
"""
_log.debug('MediaEntry processing')
entry, manager = get_entry_and_processing_manager(entry_id)
with CommonVideoProcessor(manager, entry) as processor:
processor.common_setup(resolution)
processor.transcode(medium_size=tuple(medium_size),
vp8_quality=process_info['vp8_quality'],
vp8_threads=process_info['vp8_threads'],
vorbis_quality=process_info['vorbis_quality'])
processor.generate_thumb(thumb_size=process_info['thumb_size'])
processor.store_orig_metadata()
# Make state of entry as processed
entry.state = 'processed'
entry.save()
_log.info('MediaEntry ID {} is processed (transcoded to default'
' resolution): {}'.format(entry.id, medium_size))
_log.debug('MediaEntry processed')
def available(args):
# Get the media type, either by looking up media id, or by specific type
try:
media_id = int(args.id_or_type)
media_entry, manager = get_entry_and_processing_manager(media_id)
media_type = media_entry.media_type
except ValueError:
media_type = args.id_or_type
media_entry = None
manager = get_processing_manager_for_type(media_type)
except ProcessingManagerDoesNotExist:
entry = MediaEntry.query.filter_by(id=args.id_or_type).first()
print('No such processing manager for {}'.format(entry.media_type))
if args.state:
processors = manager.list_all_processors_by_state(args.state)
elif media_entry is None:
processors = manager.list_all_processors()
else:
processors = manager.list_eligible_processors(media_entry)
print("Available processors:")
print("=====================")
print("")
if args.action_help:
for processor in processors:
print(processor.name)
print("-" * len(processor.name))
parser = processor.generate_parser()
parser.print_help()
print("")
else:
for processor in processors:
if processor.description:
print(" - {}: {}".format(processor.name, processor.description))
else:
print(" - %s" % processor.name)
def available(args):
# Get the media type, either by looking up media id, or by specific type
try:
media_id = int(args.id_or_type)
media_entry, manager = get_entry_and_processing_manager(media_id)
media_type = media_entry.media_type
except ValueError:
media_type = args.id_or_type
media_entry = None
manager = get_processing_manager_for_type(media_type)
except ProcessingManagerDoesNotExist:
entry = MediaEntry.query.filter_by(id=args.id_or_type).first()
print 'No such processing manager for {0}'.format(entry.media_type)
if args.state:
processors = manager.list_all_processors_by_state(args.state)
elif media_entry is None:
processors = manager.list_all_processors()
else:
processors = manager.list_eligible_processors(media_entry)
print "Available processors:"
print "====================="
print ""
if args.action_help:
for processor in processors:
print processor.name
print "-" * len(processor.name)
parser = processor.generate_parser()
parser.print_help()
print ""
else:
for processor in processors:
if processor.description:
print " - %s: %s" % (processor.name, processor.description)
else:
print " - %s" % processor.name
def run_process_media(entry,
feed_url=None,
reprocess_action="initial",
reprocess_info=None):
"""Process the media asynchronously
:param entry: MediaEntry() instance to be processed.
:param feed_url: A string indicating the feed_url that the PuSH servers
should be notified of. This will be sth like: `request.urlgen(
'mediagoblin.user_pages.atom_feed',qualified=True,
user=request.user.username)`
:param reprocess_action: What particular action should be run.
:param reprocess_info: A dict containing all of the necessary reprocessing
info for the given media_type"""
entry, manager = get_entry_and_processing_manager(entry.id)
try:
wf = manager.workflow(entry, feed_url, reprocess_action,
reprocess_info)
if wf is None:
ProcessMedia().apply_async(
[entry.id, feed_url, reprocess_action, reprocess_info], {},
task_id=entry.queued_task_id)
else:
chord(wf[0])(wf[1])
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry.id, exc)
# re-raise the exception
raise
def run(self, media_id, feed_url, reprocess_action, reprocess_info=None):
"""
Pass the media entry off to the appropriate processing function
(for now just process_image...)
:param media_id: MediaEntry().id
:param feed_url: The feed URL that the PuSH server needs to be
updated for.
:param reprocess_action: What particular action should be run. For
example, 'initial'.
:param reprocess: A dict containing all of the necessary reprocessing
info for the media_type.
"""
reprocess_info = reprocess_info or {}
entry, manager = get_entry_and_processing_manager(media_id)
# Try to process, and handle expected errors.
try:
processor_class = manager.get_processor(reprocess_action, entry)
with processor_class(manager, entry) as processor:
# Initial state change has to be here because
# the entry.state gets recorded on processor_class init
entry.state = u'processing'
entry.save()
_log.debug('Processing {0}'.format(entry))
try:
processor.process(**reprocess_info)
except Exception as exc:
if processor.entry_orig_state == 'processed':
_log.error(
'Entry {0} failed to process due to the following'
' error: {1}'.format(entry.id, exc))
_log.info(
'Setting entry.state back to "processed"')
pass
else:
raise
# We set the state to processed and save the entry here so there's
# no need to save at the end of the processing stage, probably ;)
entry.state = u'processed'
entry.save()
# Notify the PuSH servers as async task
if mgg.app_config["push_urls"] and feed_url:
handle_push_urls.subtask().delay(feed_url)
json_processing_callback(entry)
except BaseProcessingFail as exc:
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
return
except ImportError as exc:
_log.error(
'Entry {0} failed to process due to an import error: {1}'\
.format(
entry.title,
exc))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
except Exception as exc:
_log.error('An unhandled exception was raised while'
+ ' processing {0}'.format(
entry))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
raise
def run(self, media_id, feed_url, reprocess_action, reprocess_info=None):
"""
Pass the media entry off to the appropriate processing function
(for now just process_image...)
:param feed_url: The feed URL that the PuSH server needs to be
updated for.
:param reprocess: A dict containing all of the necessary reprocessing
info for the media_type.
"""
reprocess_info = reprocess_info or {}
entry, manager = get_entry_and_processing_manager(media_id)
# Try to process, and handle expected errors.
try:
processor_class = manager.get_processor(reprocess_action, entry)
with processor_class(manager, entry) as processor:
# Initial state change has to be here because
# the entry.state gets recorded on processor_class init
entry.state = u'processing'
entry.save()
_log.debug('Processing {0}'.format(entry))
try:
processor.process(**reprocess_info)
except Exception as exc:
if processor.entry_orig_state == 'processed':
_log.error(
'Entry {0} failed to process due to the following'
' error: {1}'.format(entry.id, exc))
_log.info(
'Setting entry.state back to "processed"')
pass
else:
raise
# We set the state to processed and save the entry here so there's
# no need to save at the end of the processing stage, probably ;)
entry.state = u'processed'
entry.save()
# Notify the PuSH servers as async task
if mgg.app_config["push_urls"] and feed_url:
handle_push_urls.subtask().delay(feed_url)
json_processing_callback(entry)
except BaseProcessingFail as exc:
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
return
except ImportError as exc:
_log.error(
'Entry {0} failed to process due to an import error: {1}'\
.format(
entry.title,
exc))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
except Exception as exc:
_log.error('An unhandled exception was raised while'
+ ' processing {0}'.format(
entry))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
raise
