def run_process_media(entry,
feed_url=None,
reprocess_action="initial",
reprocess_info=None):
"""Process the media asynchronously
:param entry: MediaEntry() instance to be processed.
:param feed_url: A string indicating the feed_url that the PuSH servers
should be notified of. This will be sth like: `request.urlgen(
'mediagoblin.user_pages.atom_feed',qualified=True,
user=request.user.username)`
:param reprocess_action: What particular action should be run.
:param reprocess_info: A dict containing all of the necessary reprocessing
info for the given media_type"""
try:
ProcessMedia().apply_async(
[entry.id, feed_url, reprocess_action, reprocess_info], {},
task_id=entry.queued_task_id)
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry.id, exc)
# re-raise the exception
raise
def run_process_media(entry, feed_url=None,
reprocess_action="initial", reprocess_info=None):
"""Process the media asynchronously
:param entry: MediaEntry() instance to be processed.
:param feed_url: A string indicating the feed_url that the PuSH servers
should be notified of. This will be sth like: `request.urlgen(
'mediagoblin.user_pages.atom_feed',qualified=True,
user=request.user.username)`
:param reprocess_action: What particular action should be run.
:param reprocess_info: A dict containing all of the necessary reprocessing
info for the given media_type"""
try:
ProcessMedia().apply_async(
[entry.id, feed_url, reprocess_action, reprocess_info], {},
task_id=entry.queued_task_id)
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry.id, exc)
# re-raise the exception
raise
def import_file(self, media):
try:
media_type, media_manager = sniff_media(media)
except (InvalidFileType, FileTypeNotSupported) as e:
print u"File error {0}: {1}".format(media.filename, repr(e)).encode("utf-8")
return
entry = self.db.MediaEntry()
entry.media_type = unicode(media_type)
entry.title = unicode(os.path.splitext(media.filename)[0])
entry.uploader = 1
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts("")
# Generate a slug from the title
entry.generate_slug()
task_id = unicode(uuid.uuid4())
entry.queued_media_file = media.filename.split("/")
entry.queued_task_id = task_id
entry.save()
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async( [unicode(entry.id)], {}, task_id=task_id)
except BaseException as exc:
mark_entry_failed(entry.id, exc)
raise
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
If the processing failed we should mark that in the database.
Assuming that the exception raised is a subclass of
BaseProcessingFail, we can use that to get more information
about the failure and store that for conveying information to
users about the failure, etc.
"""
entry_id = args[0]
mark_entry_failed(entry_id, exc)
entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first()
json_processing_callback(entry)
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
If the processing failed we should mark that in the database.
Assuming that the exception raised is a subclass of
BaseProcessingFail, we can use that to get more information
about the failure and store that for conveying information to
users about the failure, etc.
"""
entry_id = args[0]
mark_entry_failed(entry_id, exc)
entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first()
json_processing_callback(entry)
def run_process_media(entry):
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async(
[unicode(entry.id)], {},
task_id=entry.queued_task_id)
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry.id, exc)
# re-raise the exception
raise
def run(self, media_id, feed_url):
"""
Pass the media entry off to the appropriate processing function
(for now just process_image...)
:param feed_url: The feed URL that the PuSH server needs to be
updated for.
"""
entry = MediaEntry.query.get(media_id)
# Try to process, and handle expected errors.
try:
entry.state = u'processing'
entry.save()
_log.debug('Processing {0}'.format(entry))
proc_state = ProcessingState(entry)
with mgg.workbench_manager.create() as workbench:
proc_state.set_workbench(workbench)
# run the processing code
entry.media_manager.processor(proc_state)
# We set the state to processed and save the entry here so there's
# no need to save at the end of the processing stage, probably ;)
entry.state = u'processed'
entry.save()
# Notify the PuSH servers as async task
if mgg.app_config["push_urls"] and feed_url:
handle_push_urls.subtask().delay(feed_url)
json_processing_callback(entry)
except BaseProcessingFail as exc:
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
return
except ImportError as exc:
_log.error(
'Entry {0} failed to process due to an import error: {1}'\
.format(
entry.title,
exc))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
except Exception as exc:
_log.error('An unhandled exception was raised while'
+ ' processing {0}'.format(
entry))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
raise
def run(self, media_id):
"""
Pass the media entry off to the appropriate processing function
(for now just process_image...)
"""
entry = mgg.database.MediaEntry.one(
{'_id': ObjectId(media_id)})
# Try to process, and handle expected errors.
try:
manager = get_media_manager(entry.media_type)
entry.state = u'processing'
entry.save()
_log.debug('Processing {0}'.format(entry))
manager['processor'](entry)
entry.state = u'processed'
entry.save()
json_processing_callback(entry)
except BaseProcessingFail as exc:
mark_entry_failed(entry._id, exc)
json_processing_callback(entry)
return
except ImportError as exc:
_log.error(
'Entry {0} failed to process due to an import error: {1}'\
.format(
entry.title,
exc))
mark_entry_failed(entry._id, exc)
json_processing_callback(entry)
except Exception as exc:
_log.error('An unhandled exception was raised while'
+ ' processing {0}'.format(
entry))
mark_entry_failed(entry._id, exc)
json_processing_callback(entry)
raise
def run(self, media_id):
"""
Pass the media entry off to the appropriate processing function
(for now just process_image...)
"""
entry = MediaEntry.query.get(media_id)
# Try to process, and handle expected errors.
try:
entry.state = u'processing'
entry.save()
_log.debug('Processing {0}'.format(entry))
# run the processing code
entry.media_manager['processor'](entry)
# We set the state to processed and save the entry here so there's
# no need to save at the end of the processing stage, probably ;)
entry.state = u'processed'
entry.save()
json_processing_callback(entry)
except BaseProcessingFail as exc:
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
return
except ImportError as exc:
_log.error(
'Entry {0} failed to process due to an import error: {1}'\
.format(
entry.title,
exc))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
except Exception as exc:
_log.error('An unhandled exception was raised while'
+ ' processing {0}'.format(
entry))
mark_entry_failed(entry.id, exc)
json_processing_callback(entry)
raise
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form)
if request.method == 'POST' and submit_form.validate():
if not ('file' in request.files
and isinstance(request.files['file'], FileStorage)
and request.files['file'].stream):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = request.db.MediaEntry()
entry.id = ObjectId()
entry.media_type = unicode(media_type)
entry.title = (
unicode(request.form['title'])
or unicode(splitext(filename)[0]))
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', "")) or None
entry.uploader = request.user._id
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
request.form.get('tags'))
# Generate a slug from the title
entry.generate_slug()
# We generate this ourselves so we know what the taks id is for
# retrieval later.
# (If we got it off the task's auto-generation, there'd be
# a risk of a race condition when we'd save after sending
# off the task)
task_id = unicode(uuid.uuid4())
# Now store generate the queueing related filename
queue_filepath = request.app.queue_store.get_unique_filepath(
['media_entries',
task_id,
secure_filename(filename)])
# queue appropriately
queue_file = request.app.queue_store.get_file(
queue_filepath, 'wb')
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Add queued filename to the entry
entry.queued_media_file = queue_filepath
entry.queued_task_id = task_id
# Save now so we have this data before kicking off processing
entry.save(validate=True)
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async(
[unicode(entry._id)], {},
task_id=task_id)
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry._id, exc)
# re-raise the exception
raise
if mg_globals.app_config["push_urls"]:
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
hubparameters = {
'hub.mode': 'publish',
'hub.url': feed_url}
hubdata = urllib.urlencode(hubparameters)
hubheaders = {
"Content-type": "application/x-www-form-urlencoded",
"Connection": "close"}
for huburl in mg_globals.app_config["push_urls"]:
hubrequest = urllib2.Request(huburl, hubdata, hubheaders)
try:
hubresponse = urllib2.urlopen(hubrequest)
except urllib2.HTTPError as exc:
# This is not a big issue, the item will be fetched
# by the PuSH server next time we hit it
_log.warning(
"push url %r gave error %r", huburl, exc.code)
except urllib2.URLError as exc:
_log.warning(
"push url %r is unreachable %r", huburl, exc.reason)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
return exc.HTTPBadRequest()
if not 'file' in request.files \
or not isinstance(request.files['file'], FileStorage) \
or not request.files['file'].stream:
_log.debug('File field not found')
return exc.HTTPBadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = request.db.MediaEntry()
entry.id = ObjectId()
entry.media_type = unicode(media_type)
entry.title = unicode(request.form.get('title')
or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.uploader = request.user.id
entry.generate_slug()
task_id = unicode(uuid.uuid4())
# Now store generate the queueing related filename
queue_filepath = request.app.queue_store.get_unique_filepath(
['media_entries',
task_id,
secure_filename(media_file.filename)])
# queue appropriately
queue_file = request.app.queue_store.get_file(
queue_filepath, 'wb')
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Add queued filename to the entry
entry.queued_media_file = queue_filepath
entry.queued_task_id = task_id
# Save now so we have this data before kicking off processing
entry.save(validate=True)
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async(
[unicode(entry._id)], {},
task_id=task_id)
except BaseException as e:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry._id, e)
# re-raise the exception
raise
return json_response(get_entry_serializable(entry, request.urlgen))
