def run_process_media(entry, feed_url=None, reprocess_action="initial", reprocess_info=None): """Process the media asynchronously :param entry: MediaEntry() instance to be processed. :param feed_url: A string indicating the feed_url that the PuSH servers should be notified of. This will be sth like: `request.urlgen( 'mediagoblin.user_pages.atom_feed',qualified=True, user=request.user.username)` :param reprocess_action: What particular action should be run. :param reprocess_info: A dict containing all of the necessary reprocessing info for the given media_type""" try: ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that # the failure handling won't happen on Celery end. Since we # expect a lot of users to run things in this way we have to # capture stuff here. # # ... not completely the diaper pattern because the # exception is re-raised :) mark_entry_failed(entry.id, exc) # re-raise the exception raise
def import_file(self, media): try: media_type, media_manager = sniff_media(media) except (InvalidFileType, FileTypeNotSupported) as e: print u"File error {0}: {1}".format(media.filename, repr(e)).encode("utf-8") return entry = self.db.MediaEntry() entry.media_type = unicode(media_type) entry.title = unicode(os.path.splitext(media.filename)[0]) entry.uploader = 1 # Process the user's folksonomy "tags" entry.tags = convert_to_tag_list_of_dicts("") # Generate a slug from the title entry.generate_slug() task_id = unicode(uuid.uuid4()) entry.queued_media_file = media.filename.split("/") entry.queued_task_id = task_id entry.save() process_media = registry.tasks[ProcessMedia.name] try: process_media.apply_async( [unicode(entry.id)], {}, task_id=task_id) except BaseException as exc: mark_entry_failed(entry.id, exc) raise
def on_failure(self, exc, task_id, args, kwargs, einfo): """ If the processing failed we should mark that in the database. Assuming that the exception raised is a subclass of BaseProcessingFail, we can use that to get more information about the failure and store that for conveying information to users about the failure, etc. """ entry_id = args[0] mark_entry_failed(entry_id, exc) entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first() json_processing_callback(entry)
def run_process_media(entry): process_media = registry.tasks[ProcessMedia.name] try: process_media.apply_async( [unicode(entry.id)], {}, task_id=entry.queued_task_id) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that # the failure handling won't happen on Celery end. Since we # expect a lot of users to run things in this way we have to # capture stuff here. # # ... not completely the diaper pattern because the # exception is re-raised :) mark_entry_failed(entry.id, exc) # re-raise the exception raise
def run(self, media_id, feed_url): """ Pass the media entry off to the appropriate processing function (for now just process_image...) :param feed_url: The feed URL that the PuSH server needs to be updated for. """ entry = MediaEntry.query.get(media_id) # Try to process, and handle expected errors. try: entry.state = u'processing' entry.save() _log.debug('Processing {0}'.format(entry)) proc_state = ProcessingState(entry) with mgg.workbench_manager.create() as workbench: proc_state.set_workbench(workbench) # run the processing code entry.media_manager.processor(proc_state) # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) entry.state = u'processed' entry.save() # Notify the PuSH servers as async task if mgg.app_config["push_urls"] and feed_url: handle_push_urls.subtask().delay(feed_url) json_processing_callback(entry) except BaseProcessingFail as exc: mark_entry_failed(entry.id, exc) json_processing_callback(entry) return except ImportError as exc: _log.error( 'Entry {0} failed to process due to an import error: {1}'\ .format( entry.title, exc)) mark_entry_failed(entry.id, exc) json_processing_callback(entry) except Exception as exc: _log.error('An unhandled exception was raised while' + ' processing {0}'.format( entry)) mark_entry_failed(entry.id, exc) json_processing_callback(entry) raise
def run(self, media_id): """ Pass the media entry off to the appropriate processing function (for now just process_image...) """ entry = mgg.database.MediaEntry.one( {'_id': ObjectId(media_id)}) # Try to process, and handle expected errors. try: manager = get_media_manager(entry.media_type) entry.state = u'processing' entry.save() _log.debug('Processing {0}'.format(entry)) manager['processor'](entry) entry.state = u'processed' entry.save() json_processing_callback(entry) except BaseProcessingFail as exc: mark_entry_failed(entry._id, exc) json_processing_callback(entry) return except ImportError as exc: _log.error( 'Entry {0} failed to process due to an import error: {1}'\ .format( entry.title, exc)) mark_entry_failed(entry._id, exc) json_processing_callback(entry) except Exception as exc: _log.error('An unhandled exception was raised while' + ' processing {0}'.format( entry)) mark_entry_failed(entry._id, exc) json_processing_callback(entry) raise
def run(self, media_id): """ Pass the media entry off to the appropriate processing function (for now just process_image...) """ entry = MediaEntry.query.get(media_id) # Try to process, and handle expected errors. try: entry.state = u'processing' entry.save() _log.debug('Processing {0}'.format(entry)) # run the processing code entry.media_manager['processor'](entry) # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) entry.state = u'processed' entry.save() json_processing_callback(entry) except BaseProcessingFail as exc: mark_entry_failed(entry.id, exc) json_processing_callback(entry) return except ImportError as exc: _log.error( 'Entry {0} failed to process due to an import error: {1}'\ .format( entry.title, exc)) mark_entry_failed(entry.id, exc) json_processing_callback(entry) except Exception as exc: _log.error('An unhandled exception was raised while' + ' processing {0}'.format( entry)) mark_entry_failed(entry.id, exc) json_processing_callback(entry) raise
def submit_start(request): """ First view for submitting a file. """ submit_form = submit_forms.SubmitStartForm(request.form) if request.method == 'POST' and submit_form.validate(): if not ('file' in request.files and isinstance(request.files['file'], FileStorage) and request.files['file'].stream): submit_form.file.errors.append( _(u'You must provide a file.')) else: try: filename = request.files['file'].filename # Sniff the submitted media to determine which # media plugin should handle processing media_type, media_manager = sniff_media( request.files['file']) # create entry and save in database entry = request.db.MediaEntry() entry.id = ObjectId() entry.media_type = unicode(media_type) entry.title = ( unicode(request.form['title']) or unicode(splitext(filename)[0])) entry.description = unicode(request.form.get('description')) entry.license = unicode(request.form.get('license', "")) or None entry.uploader = request.user._id # Process the user's folksonomy "tags" entry.tags = convert_to_tag_list_of_dicts( request.form.get('tags')) # Generate a slug from the title entry.generate_slug() # We generate this ourselves so we know what the taks id is for # retrieval later. # (If we got it off the task's auto-generation, there'd be # a risk of a race condition when we'd save after sending # off the task) task_id = unicode(uuid.uuid4()) # Now store generate the queueing related filename queue_filepath = request.app.queue_store.get_unique_filepath( ['media_entries', task_id, secure_filename(filename)]) # queue appropriately queue_file = request.app.queue_store.get_file( queue_filepath, 'wb') with queue_file: queue_file.write(request.files['file'].stream.read()) # Add queued filename to the entry entry.queued_media_file = queue_filepath entry.queued_task_id = task_id # Save now so we have this data before kicking off processing entry.save(validate=True) # Pass off to processing # # (... don't change entry after this point to avoid race # conditions with changes to the document via processing code) process_media = registry.tasks[ProcessMedia.name] try: process_media.apply_async( [unicode(entry._id)], {}, task_id=task_id) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that # the failure handling won't happen on Celery end. Since we # expect a lot of users to run things in this way we have to # capture stuff here. # # ... not completely the diaper pattern because the # exception is re-raised :) mark_entry_failed(entry._id, exc) # re-raise the exception raise if mg_globals.app_config["push_urls"]: feed_url = request.urlgen( 'mediagoblin.user_pages.atom_feed', qualified=True, user=request.user.username) hubparameters = { 'hub.mode': 'publish', 'hub.url': feed_url} hubdata = urllib.urlencode(hubparameters) hubheaders = { "Content-type": "application/x-www-form-urlencoded", "Connection": "close"} for huburl in mg_globals.app_config["push_urls"]: hubrequest = urllib2.Request(huburl, hubdata, hubheaders) try: hubresponse = urllib2.urlopen(hubrequest) except urllib2.HTTPError as exc: # This is not a big issue, the item will be fetched # by the PuSH server next time we hit it _log.warning( "push url %r gave error %r", huburl, exc.code) except urllib2.URLError as exc: _log.warning( "push url %r is unreachable %r", huburl, exc.reason) add_message(request, SUCCESS, _('Woohoo! Submitted!')) return redirect(request, "mediagoblin.user_pages.user_home", user=request.user.username) except Exception as e: ''' This section is intended to catch exceptions raised in mediagoblin.media_types ''' if isinstance(e, InvalidFileType) or \ isinstance(e, FileTypeNotSupported): submit_form.file.errors.append( e) else: raise return render_to_response( request, 'mediagoblin/submit/start.html', {'submit_form': submit_form, 'app_config': mg_globals.app_config})
def post_entry(request): _log.debug('Posting entry') if request.method == 'OPTIONS': return json_response({'status': 200}) if request.method != 'POST': _log.debug('Must POST against post_entry') return exc.HTTPBadRequest() if not 'file' in request.files \ or not isinstance(request.files['file'], FileStorage) \ or not request.files['file'].stream: _log.debug('File field not found') return exc.HTTPBadRequest() media_file = request.files['file'] media_type, media_manager = sniff_media(media_file) entry = request.db.MediaEntry() entry.id = ObjectId() entry.media_type = unicode(media_type) entry.title = unicode(request.form.get('title') or splitext(media_file.filename)[0]) entry.description = unicode(request.form.get('description')) entry.license = unicode(request.form.get('license', '')) entry.uploader = request.user.id entry.generate_slug() task_id = unicode(uuid.uuid4()) # Now store generate the queueing related filename queue_filepath = request.app.queue_store.get_unique_filepath( ['media_entries', task_id, secure_filename(media_file.filename)]) # queue appropriately queue_file = request.app.queue_store.get_file( queue_filepath, 'wb') with queue_file: queue_file.write(request.files['file'].stream.read()) # Add queued filename to the entry entry.queued_media_file = queue_filepath entry.queued_task_id = task_id # Save now so we have this data before kicking off processing entry.save(validate=True) if request.form.get('callback_url'): metadata = request.db.ProcessingMetaData() metadata.media_entry = entry metadata.callback_url = unicode(request.form['callback_url']) metadata.save() # Pass off to processing # # (... don't change entry after this point to avoid race # conditions with changes to the document via processing code) process_media = registry.tasks[ProcessMedia.name] try: process_media.apply_async( [unicode(entry._id)], {}, task_id=task_id) except BaseException as e: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that # the failure handling won't happen on Celery end. Since we # expect a lot of users to run things in this way we have to # capture stuff here. # # ... not completely the diaper pattern because the # exception is re-raised :) mark_entry_failed(entry._id, e) # re-raise the exception raise return json_response(get_entry_serializable(entry, request.urlgen))