def import_file(self, media):
try:
media_type, media_manager = sniff_media(media)
except (InvalidFileType, FileTypeNotSupported) as e:
print u"File error {0}: {1}".format(media.filename, repr(e)).encode("utf-8")
return
entry = self.db.MediaEntry()
entry.media_type = unicode(media_type)
entry.title = unicode(os.path.splitext(media.filename)[0])
entry.uploader = 1
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts("")
# Generate a slug from the title
entry.generate_slug()
task_id = unicode(uuid.uuid4())
entry.queued_media_file = media.filename.split("/")
entry.queued_task_id = task_id
entry.save()
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async( [unicode(entry.id)], {}, task_id=task_id)
except BaseException as exc:
mark_entry_failed(entry.id, exc)
raise
def test_garbage_collection_task(test_app):
""" Test old media entry are removed by GC task """
user = fixture_add_user()
# Create a media entry that's unprocessed and over an hour old.
entry_id = 72
now = datetime.datetime.now(pytz.UTC)
file_data = FileStorage(
stream=open(GOOD_JPG, "rb"),
filename="mah_test.jpg",
content_type="image/jpeg"
)
# Find media manager
media_type, media_manager = sniff_media(file_data, "mah_test.jpg")
entry = new_upload_entry(user)
entry.id = entry_id
entry.title = "Mah Image"
entry.slug = "slugy-slug-slug"
entry.media_type = 'image'
entry.created = now - datetime.timedelta(days=2)
entry.save()
# Validate the model exists
assert MediaEntry.query.filter_by(id=entry_id).first() is not None
# Call the garbage collection task
collect_garbage()
# Now validate the image has been deleted
assert MediaEntry.query.filter_by(id=entry_id).first() is None
def test_garbage_collection_task(test_app):
""" Test old media entry are removed by GC task """
user = fixture_add_user()
# Create a media entry that's unprocessed and over an hour old.
entry_id = 72
now = datetime.datetime.now(pytz.UTC)
file_data = FileStorage(stream=open(GOOD_JPG, "rb"),
filename="mah_test.jpg",
content_type="image/jpeg")
# Find media manager
media_type, media_manager = sniff_media(file_data, "mah_test.jpg")
entry = new_upload_entry(user)
entry.id = entry_id
entry.title = "Mah Image"
entry.slug = "slugy-slug-slug"
entry.media_type = 'image'
entry.created = now - datetime.timedelta(days=2)
entry.save()
# Validate the model exists
assert MediaEntry.query.filter_by(id=entry_id).first() is not None
# Call the garbage collection task
collect_garbage()
# Now validate the image has been deleted
assert MediaEntry.query.filter_by(id=entry_id).first() is None
def pwg_images_addSimple(request):
form = AddSimpleForm(request.form)
if not form.validate():
_log.error("addSimple: form failed")
raise BadRequest()
dump = []
for f in form:
dump.append("%s=%r" % (f.name, f.data))
_log.info("addSimple: %r %s %r", request.form, " ".join(dump), request.files)
if not check_file_field(request, "image"):
raise BadRequest()
filename = request.files["image"].filename
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(request.files["image"])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(form.name.data) or unicode(splitext(filename)[0])
entry.description = unicode(form.comment.data)
"""
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
form.tags.data)
"""
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
shutil.copyfileobj(request.files["image"].stream, queue_file, length=4 * 1048576)
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen("mediagoblin.user_pages.atom_feed", qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
collection_id = form.category.data
if collection_id > 0:
collection = Collection.query.get(collection_id)
if collection is not None and collection.creator == request.user.id:
add_media_to_collection(collection, entry, "")
return {"image_id": entry.id, "url": entry.url_for_self(request.urlgen, qualified=True)}
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
raise BadRequest()
if not 'file' in request.files \
or not isinstance(request.files['file'], FileStorage) \
or not request.files['file'].stream:
_log.debug('File field not found')
raise BadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = request.db.MediaEntry()
entry.media_type = unicode(media_type)
entry.title = unicode(request.form.get('title')
or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.uploader = request.user.id
entry.generate_slug()
# queue appropriately
queue_file = prepare_queue_task(request.app, entry, media_file.filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
run_process_media(entry)
return json_response(get_entry_serializable(entry, request.urlgen))
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
raise BadRequest()
if not check_file_field(request, 'file'):
_log.debug('File field not found')
raise BadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(
request.form.get('title') or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.generate_slug()
# queue appropriately
queue_file = prepare_queue_task(request.app, entry, media_file.filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen('mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
run_process_media(entry, feed_url)
return json_response(get_entry_serializable(entry, request.urlgen))
def post_entry(request):
_log.debug("Posting entry")
if request.method == "OPTIONS":
return json_response({"status": 200})
if request.method != "POST":
_log.debug("Must POST against post_entry")
raise BadRequest()
if not check_file_field(request, "file"):
_log.debug("File field not found")
raise BadRequest()
media_file = request.files["file"]
media_type, media_manager = sniff_media(media_file)
entry = request.db.MediaEntry()
entry.media_type = unicode(media_type)
entry.title = unicode(request.form.get("title") or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get("description"))
entry.license = unicode(request.form.get("license", ""))
entry.uploader = request.user.id
entry.generate_slug()
# queue appropriately
queue_file = prepare_queue_task(request.app, entry, media_file.filename)
with queue_file:
queue_file.write(request.files["file"].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
if request.form.get("callback_url"):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form["callback_url"])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen("mediagoblin.user_pages.atom_feed", qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
return json_response(get_entry_serializable(entry, request.urlgen))
def submit_start(request):
"""
First view for submitting a file.
"""
user = request.user
if user.upload_limit >= 0:
upload_limit = user.upload_limit
else:
upload_limit = mg_globals.app_config.get('upload_limit', None)
if upload_limit and user.uploaded >= upload_limit:
messages.add_message(
request,
messages.WARNING,
_('Sorry, you have reached your upload limit.'))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
max_file_size = mg_globals.app_config.get('max_file_size', None)
submit_form = submit_forms.get_submit_start_form(
request.form,
license=request.user.license_preference,
max_file_size=max_file_size,
upload_limit=upload_limit,
uploaded=user.uploaded)
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(request.files['file'].filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Get file size and round to 2 decimal places
file_size = request.app.queue_store.get_file_size(
entry.queued_media_file) / (1024.0 * 1024)
file_size = float('{0:.2f}'.format(file_size))
error = False
# Check if file size is over the limit
if max_file_size and file_size >= max_file_size:
submit_form.file.errors.append(
_(u'Sorry, the file size is too big.'))
error = True
# Check if user is over upload limit
if upload_limit and (user.uploaded + file_size) >= upload_limit:
submit_form.file.errors.append(
_('Sorry, uploading this file will put you over your'
' upload limit.'))
error = True
if not error:
user.uploaded = user.uploaded + file_size
user.save()
entry.file_size = file_size
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
add_comment_subscription(request.user, entry)
return redirect(request, "mediagoblin.user_pages.user_home",
user=user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def submit_media(mg_app, user, submitted_file, filename,
title=None, description=None,
license=None, tags_string=u"",
upload_limit=None, max_file_size=None,
callback_url=None,
# If provided we'll do the feed_url update, otherwise ignore
urlgen=None,):
"""
Args:
- mg_app: The MediaGoblinApp instantiated for this process
- user: the user object this media entry should be associated with
- submitted_file: the file-like object that has the
being-submitted file data in it (this object should really have
a .name attribute which is the filename on disk!)
- filename: the *original* filename of this. Not necessarily the
one on disk being referenced by submitted_file.
- title: title for this media entry
- description: description for this media entry
- license: license for this media entry
- tags_string: comma separated string of tags to be associated
with this entry
- upload_limit: size in megabytes that's the per-user upload limit
- max_file_size: maximum size each file can be that's uploaded
- callback_url: possible post-hook to call after submission
- urlgen: if provided, used to do the feed_url update
"""
if upload_limit and user.uploaded >= upload_limit:
raise UserPastUploadLimit()
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(user)
entry.media_type = media_type
entry.title = (title or unicode(splitext(filename)[0]))
entry.description = description or u""
entry.license = license or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(tags_string)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(mg_app, entry, filename)
with queue_file:
queue_file.write(submitted_file.read())
# Get file size and round to 2 decimal places
file_size = mg_app.queue_store.get_file_size(
entry.queued_media_file) / (1024.0 * 1024)
file_size = float('{0:.2f}'.format(file_size))
# Check if file size is over the limit
if max_file_size and file_size >= max_file_size:
raise FileUploadLimit()
# Check if user is over upload limit
if upload_limit and (user.uploaded + file_size) >= upload_limit:
raise UserUploadLimit()
user.uploaded = user.uploaded + file_size
user.save()
entry.file_size = file_size
# Save now so we have this data before kicking off processing
entry.save()
# Various "submit to stuff" things, callbackurl and this silly urlgen
# thing
if callback_url:
metadata = ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = callback_url
metadata.save()
if urlgen:
feed_url = urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=user.username)
else:
feed_url = None
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
run_process_media(entry, feed_url)
add_comment_subscription(user, entry)
return entry
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
return exc.HTTPBadRequest()
if not 'file' in request.files \
or not isinstance(request.files['file'], FileStorage) \
or not request.files['file'].stream:
_log.debug('File field not found')
return exc.HTTPBadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = request.db.MediaEntry()
entry.id = ObjectId()
entry.media_type = unicode(media_type)
entry.title = unicode(request.form.get('title')
or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.uploader = request.user.id
entry.generate_slug()
task_id = unicode(uuid.uuid4())
# Now store generate the queueing related filename
queue_filepath = request.app.queue_store.get_unique_filepath(
['media_entries',
task_id,
secure_filename(media_file.filename)])
# queue appropriately
queue_file = request.app.queue_store.get_file(
queue_filepath, 'wb')
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Add queued filename to the entry
entry.queued_media_file = queue_filepath
entry.queued_task_id = task_id
# Save now so we have this data before kicking off processing
entry.save(validate=True)
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async(
[unicode(entry._id)], {},
task_id=task_id)
except BaseException as e:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry._id, e)
# re-raise the exception
raise
return json_response(get_entry_serializable(entry, request.urlgen))
def multi_submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.get_submit_start_form(
request.form, license=request.user.license_preference)
users_collections = Collection.query.filter_by(
actor=request.user.id,
type=Collection.USER_DEFINED_TYPE).order_by(Collection.title)
if users_collections.count() > 0:
submit_form.collection.query = users_collections
else:
del submit_form.collection
# Below is what was used for mediagoblin 0.5.0-dev. Above is the new way.
# submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference)
filecount = 0
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide at least one file.'))
else:
for submitted_file in request.files.getlist('file'):
try:
if not submitted_file.filename:
# MOST likely an invalid file
continue # Skip the rest of the loop for this file
else:
filename = submitted_file.filename
_log.info("html5-multi-upload: Got filename: %s" %
filename)
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = str(
uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = str(media_type)
entry.title = (str(submit_form.title.data) or str(
splitext(submitted_file.filename)[0]))
entry.description = str(submit_form.description.data)
entry.license = str(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(
request.app, entry, filename)
with queue_file:
queue_file.write(submitted_file.stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
run_process_media(entry, feed_url)
if submit_form.collection and submit_form.collection.data:
add_media_to_collection(
submit_form.collection.data, entry)
create_activity("add",
entry,
request.user,
target=submit_form.collection.data)
add_comment_subscription(request.user, entry)
filecount = filecount + 1
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, TypeNotFound) or isinstance(
e, FileTypeNotSupported):
submit_form.file.errors.append(e)
else:
raise
add_message(request, SUCCESS,
_('Woohoo! Submitted %d Files!' % filecount))
return redirect(request,
"mediagoblin.user_pages.user_home",
user=request.user.username)
return render_to_response(request, 'start.html',
{'multi_submit_form': submit_form})
def submit_media(
mg_app,
user,
submitted_file,
filename,
title=None,
description=None,
collection_slug=None,
license=None,
metadata=None,
tags_string=u"",
callback_url=None,
urlgen=None,
):
"""
Args:
- mg_app: The MediaGoblinApp instantiated for this process
- user: the user object this media entry should be associated with
- submitted_file: the file-like object that has the
being-submitted file data in it (this object should really have
a .name attribute which is the filename on disk!)
- filename: the *original* filename of this. Not necessarily the
one on disk being referenced by submitted_file.
- title: title for this media entry
- description: description for this media entry
- collection_slug: collection for this media entry
- license: license for this media entry
- tags_string: comma separated string of tags to be associated
with this entry
- callback_url: possible post-hook to call after submission
- urlgen: if provided, used to do the feed_url update and assign a public
ID used in the API (very important).
"""
upload_limit, max_file_size = get_upload_file_limits(user)
if upload_limit and user.uploaded >= upload_limit:
raise UserPastUploadLimit()
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = six.text_type(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(user)
entry.media_type = media_type
entry.title = (title or six.text_type(splitext(filename)[0]))
entry.description = description or u""
entry.license = license or None
entry.media_metadata = metadata or {}
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(tags_string)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(mg_app, entry, filename)
with queue_file:
queue_file.write(submitted_file)
# Get file size and round to 2 decimal places
file_size = mg_app.queue_store.get_file_size(
entry.queued_media_file) / (1024.0 * 1024)
file_size = float('{0:.2f}'.format(file_size))
# Check if file size is over the limit
if max_file_size and file_size >= max_file_size:
raise FileUploadLimit()
# Check if user is over upload limit
if upload_limit and (user.uploaded + file_size) >= upload_limit:
raise UserUploadLimit()
user.uploaded = user.uploaded + file_size
user.save()
entry.file_size = file_size
# Save now so we have this data before kicking off processing
entry.save()
# Various "submit to stuff" things, callbackurl and this silly urlgen
# thing
if callback_url:
metadata = ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = callback_url
metadata.save()
if urlgen:
# Generate the public_id, this is very importent, especially relating
# to deletion, it allows the shell to be accessable post-delete!
entry.get_public_id(urlgen)
# Generate the feed URL
feed_url = urlgen('mediagoblin.user_pages.atom_feed',
qualified=True,
user=user.username)
else:
feed_url = None
add_comment_subscription(user, entry)
# Create activity
create_activity("post", entry, entry.actor)
entry.save()
# add to collection
if collection_slug:
collection = Collection.query.filter_by(slug=collection_slug,
actor=user.id).first()
if collection:
add_media_to_collection(collection, entry)
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
run_process_media(entry, feed_url)
return entry
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form)
if request.method == 'POST' and submit_form.validate():
if not ('file' in request.files
and isinstance(request.files['file'], FileStorage)
and request.files['file'].stream):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = request.db.MediaEntry()
entry.media_type = unicode(media_type)
entry.title = (
unicode(request.form['title'])
or unicode(splitext(filename)[0]))
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', "")) or None
entry.uploader = request.user.id
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
request.form.get('tags'))
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
run_process_media(entry)
handle_push_urls(request)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form,
license=request.user.license_preference)
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(request.files['file'].filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
add_comment_subscription(request.user, entry)
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def multi_submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.get_submit_start_form(request.form, license=request.user.license_preference)
users_collections = Collection.query.filter_by(
actor=request.user.id,
type=Collection.USER_DEFINED_TYPE
).order_by(Collection.title)
if users_collections.count() > 0:
submit_form.collection.query = users_collections
else:
del submit_form.collection
# Below is what was used for mediagoblin 0.5.0-dev. Above is the new way.
# submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference)
filecount = 0
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(_(u'You must provide at least one file.'))
else:
for submitted_file in request.files.getlist('file'):
try:
if not submitted_file.filename:
# MOST likely an invalid file
continue # Skip the rest of the loop for this file
else:
filename = submitted_file.filename
_log.info("html5-multi-upload: Got filename: %s" % filename)
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(submitted_file.filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(submitted_file.stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
if submit_form.collection and submit_form.collection.data:
add_media_to_collection(
submit_form.collection.data, entry)
create_activity(
"add", entry, request.user,
target=submit_form.collection.data)
add_comment_subscription(request.user, entry)
filecount = filecount + 1
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, TypeNotFound) or isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(e)
else:
raise
add_message(request, SUCCESS, _('Woohoo! Submitted %d Files!' % filecount))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
return render_to_response(
request,
'start.html',
{'multi_submit_form': submit_form})
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form,
license=request.user.license_preference)
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
# I've modified ../media_types/__init__.py so that a zip file can pass the checking system
# It will be seen as an image file
media_type, media_manager = sniff_media(
request.files['file'])
# Read the zip file and processing each photo with its metadata
zf = zipfile.ZipFile(request.files['file'], 'r')
for name in zf.namelist():
try:
data = zf.read(name)
if imghdr.what(name, data):
metadata = ElementTree.ElementTree(ElementTree.fromstring(zf.read(name.split(".")[0]+'.xml')))
# Fetch the data in metadata that matches the format in mediagoblin
# Not yet finish the whole checking problem, some data may not exist
img_title = metadata.find('title').text
img_description = metadata.find('description').text
img_tags = ''
for tag in metadata.find('tags'):
img_tags = img_tags + ', ' + tag.text
upload_data = data
upload_filename = name.lstrip('dst/')
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(img_title)#(
#unicode(submit_form.title.data)
#or unicode(splitext(request.files['file'].filename)[0]))
entry.description = unicode(img_description)#unicode(submit_form.description.data)
entry.license = unicode('http://creativecommons.org/publicdomain/mark/1.0/')#unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(img_tags)
# submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, upload_filename)#filename)
with queue_file:
queue_file.write(upload_data)#request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
except KeyError:
print 'ERROR: Did not find %s in zip file' % name
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
add_comment_subscription(request.user, entry)
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/import_flickr/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form)
if request.method == 'POST' and submit_form.validate():
if not ('file' in request.files
and isinstance(request.files['file'], FileStorage)
and request.files['file'].stream):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = request.db.MediaEntry()
entry.id = ObjectId()
entry.media_type = unicode(media_type)
entry.title = (
unicode(request.form['title'])
or unicode(splitext(filename)[0]))
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', "")) or None
entry.uploader = request.user._id
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
request.form.get('tags'))
# Generate a slug from the title
entry.generate_slug()
# We generate this ourselves so we know what the taks id is for
# retrieval later.
# (If we got it off the task's auto-generation, there'd be
# a risk of a race condition when we'd save after sending
# off the task)
task_id = unicode(uuid.uuid4())
# Now store generate the queueing related filename
queue_filepath = request.app.queue_store.get_unique_filepath(
['media_entries',
task_id,
secure_filename(filename)])
# queue appropriately
queue_file = request.app.queue_store.get_file(
queue_filepath, 'wb')
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Add queued filename to the entry
entry.queued_media_file = queue_filepath
entry.queued_task_id = task_id
# Save now so we have this data before kicking off processing
entry.save(validate=True)
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
process_media = registry.tasks[ProcessMedia.name]
try:
process_media.apply_async(
[unicode(entry._id)], {},
task_id=task_id)
except BaseException as exc:
# The purpose of this section is because when running in "lazy"
# or always-eager-with-exceptions-propagated celery mode that
# the failure handling won't happen on Celery end. Since we
# expect a lot of users to run things in this way we have to
# capture stuff here.
#
# ... not completely the diaper pattern because the
# exception is re-raised :)
mark_entry_failed(entry._id, exc)
# re-raise the exception
raise
if mg_globals.app_config["push_urls"]:
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
hubparameters = {
'hub.mode': 'publish',
'hub.url': feed_url}
hubdata = urllib.urlencode(hubparameters)
hubheaders = {
"Content-type": "application/x-www-form-urlencoded",
"Connection": "close"}
for huburl in mg_globals.app_config["push_urls"]:
hubrequest = urllib2.Request(huburl, hubdata, hubheaders)
try:
hubresponse = urllib2.urlopen(hubrequest)
except urllib2.HTTPError as exc:
# This is not a big issue, the item will be fetched
# by the PuSH server next time we hit it
_log.warning(
"push url %r gave error %r", huburl, exc.code)
except urllib2.URLError as exc:
_log.warning(
"push url %r is unreachable %r", huburl, exc.reason)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
