def test_garbage_collection_task(test_app):
""" Test old media entry are removed by GC task """
user = fixture_add_user()
# Create a media entry that's unprocessed and over an hour old.
entry_id = 72
now = datetime.datetime.now(pytz.UTC)
file_data = FileStorage(
stream=open(GOOD_JPG, "rb"),
filename="mah_test.jpg",
content_type="image/jpeg"
)
# Find media manager
media_type, media_manager = sniff_media(file_data, "mah_test.jpg")
entry = new_upload_entry(user)
entry.id = entry_id
entry.title = "Mah Image"
entry.slug = "slugy-slug-slug"
entry.media_type = 'image'
entry.created = now - datetime.timedelta(days=2)
entry.save()
# Validate the model exists
assert MediaEntry.query.filter_by(id=entry_id).first() is not None
# Call the garbage collection task
collect_garbage()
# Now validate the image has been deleted
assert MediaEntry.query.filter_by(id=entry_id).first() is None
def test_garbage_collection_task(test_app):
""" Test old media entry are removed by GC task """
user = fixture_add_user()
# Create a media entry that's unprocessed and over an hour old.
entry_id = 72
now = datetime.datetime.now(pytz.UTC)
file_data = FileStorage(stream=open(GOOD_JPG, "rb"),
filename="mah_test.jpg",
content_type="image/jpeg")
# Find media manager
media_type, media_manager = sniff_media(file_data, "mah_test.jpg")
entry = new_upload_entry(user)
entry.id = entry_id
entry.title = "Mah Image"
entry.slug = "slugy-slug-slug"
entry.media_type = 'image'
entry.created = now - datetime.timedelta(days=2)
entry.save()
# Validate the model exists
assert MediaEntry.query.filter_by(id=entry_id).first() is not None
# Call the garbage collection task
collect_garbage()
# Now validate the image has been deleted
assert MediaEntry.query.filter_by(id=entry_id).first() is None
def uploads_endpoint(request):
""" Endpoint for file uploads """
username = request.matchdict["username"]
requested_user = LocalUser.query.filter(
LocalUser.username == username).first()
if requested_user is None:
return json_error("No such 'user' with id '{0}'".format(username), 404)
if request.method == "POST":
# Ensure that the user is only able to upload to their own
# upload endpoint.
if requested_user.id != request.user.id:
return json_error("Not able to post to another users feed.",
status=403)
# Wrap the data in the werkzeug file wrapper
if "Content-Type" not in request.headers:
return json_error(
"Must supply 'Content-Type' header to upload media.")
mimetype = request.headers["Content-Type"]
filename = mimetypes.guess_all_extensions(mimetype)
filename = 'unknown' + filename[0] if filename else filename
file_data = FileStorage(stream=io.BytesIO(request.data),
filename=filename,
content_type=mimetype)
# Find media manager
entry = new_upload_entry(request.user)
entry.media_type = IMAGE_MEDIA_TYPE
return api_upload_request(request, file_data, entry)
return json_error("Not yet implemented", 501)
def uploads_endpoint(request):
""" Endpoint for file uploads """
username = request.matchdict["username"]
requested_user = User.query.filter_by(username=username).first()
if requested_user is None:
return json_error("No such 'user' with id '{0}'".format(username), 404)
if request.method == "POST":
# Ensure that the user is only able to upload to their own
# upload endpoint.
if requested_user.id != request.user.id:
return json_error("Not able to post to another users feed.", status=403)
# Wrap the data in the werkzeug file wrapper
if "Content-Type" not in request.headers:
return json_error("Must supply 'Content-Type' header to upload media.")
mimetype = request.headers["Content-Type"]
filename = mimetypes.guess_all_extensions(mimetype)
filename = "unknown" + filename[0] if filename else filename
file_data = FileStorage(stream=io.BytesIO(request.data), filename=filename, content_type=mimetype)
# Find media manager
entry = new_upload_entry(request.user)
entry.media_type = IMAGE_MEDIA_TYPE
return api_upload_request(request, file_data, entry)
return json_error("Not yet implemented", 501)
def pwg_images_addSimple(request):
form = AddSimpleForm(request.form)
if not form.validate():
_log.error("addSimple: form failed")
raise BadRequest()
dump = []
for f in form:
dump.append("%s=%r" % (f.name, f.data))
_log.info("addSimple: %r %s %r", request.form, " ".join(dump), request.files)
if not check_file_field(request, "image"):
raise BadRequest()
filename = request.files["image"].filename
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(request.files["image"])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(form.name.data) or unicode(splitext(filename)[0])
entry.description = unicode(form.comment.data)
"""
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
form.tags.data)
"""
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
shutil.copyfileobj(request.files["image"].stream, queue_file, length=4 * 1048576)
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen("mediagoblin.user_pages.atom_feed", qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
collection_id = form.category.data
if collection_id > 0:
collection = Collection.query.get(collection_id)
if collection is not None and collection.creator == request.user.id:
add_media_to_collection(collection, entry, "")
return {"image_id": entry.id, "url": entry.url_for_self(request.urlgen, qualified=True)}
def get_sample_entry(user, media_type):
entry = new_upload_entry(user)
entry.media_type = media_type
entry.title = 'testentry'
entry.description = ""
entry.license = None
entry.media_metadata = {}
entry.save()
return entry
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
raise BadRequest()
if not check_file_field(request, 'file'):
_log.debug('File field not found')
raise BadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(
request.form.get('title') or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.generate_slug()
# queue appropriately
queue_file = prepare_queue_task(request.app, entry, media_file.filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen('mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
run_process_media(entry, feed_url)
return json_response(get_entry_serializable(entry, request.urlgen))
def post_entry(request):
_log.debug('Posting entry')
if request.method == 'OPTIONS':
return json_response({'status': 200})
if request.method != 'POST':
_log.debug('Must POST against post_entry')
raise BadRequest()
if not check_file_field(request, 'file'):
_log.debug('File field not found')
raise BadRequest()
media_file = request.files['file']
media_type, media_manager = sniff_media(media_file)
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = unicode(request.form.get('title')
or splitext(media_file.filename)[0])
entry.description = unicode(request.form.get('description'))
entry.license = unicode(request.form.get('license', ''))
entry.generate_slug()
# queue appropriately
queue_file = prepare_queue_task(request.app, entry, media_file.filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
if request.form.get('callback_url'):
metadata = request.db.ProcessingMetaData()
metadata.media_entry = entry
metadata.callback_url = unicode(request.form['callback_url'])
metadata.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
return json_response(get_entry_serializable(entry, request.urlgen))
def multi_submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.get_submit_start_form(
request.form, license=request.user.license_preference)
users_collections = Collection.query.filter_by(
actor=request.user.id,
type=Collection.USER_DEFINED_TYPE).order_by(Collection.title)
if users_collections.count() > 0:
submit_form.collection.query = users_collections
else:
del submit_form.collection
# Below is what was used for mediagoblin 0.5.0-dev. Above is the new way.
# submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference)
filecount = 0
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide at least one file.'))
else:
for submitted_file in request.files.getlist('file'):
try:
if not submitted_file.filename:
# MOST likely an invalid file
continue # Skip the rest of the loop for this file
else:
filename = submitted_file.filename
_log.info("html5-multi-upload: Got filename: %s" %
filename)
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = str(
uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = str(media_type)
entry.title = (str(submit_form.title.data) or str(
splitext(submitted_file.filename)[0]))
entry.description = str(submit_form.description.data)
entry.license = str(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(
request.app, entry, filename)
with queue_file:
queue_file.write(submitted_file.stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True,
user=request.user.username)
run_process_media(entry, feed_url)
if submit_form.collection and submit_form.collection.data:
add_media_to_collection(
submit_form.collection.data, entry)
create_activity("add",
entry,
request.user,
target=submit_form.collection.data)
add_comment_subscription(request.user, entry)
filecount = filecount + 1
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, TypeNotFound) or isinstance(
e, FileTypeNotSupported):
submit_form.file.errors.append(e)
else:
raise
add_message(request, SUCCESS,
_('Woohoo! Submitted %d Files!' % filecount))
return redirect(request,
"mediagoblin.user_pages.user_home",
user=request.user.username)
return render_to_response(request, 'start.html',
{'multi_submit_form': submit_form})
def submit_start(request):
"""
First view for submitting a file.
"""
user = request.user
if user.upload_limit >= 0:
upload_limit = user.upload_limit
else:
upload_limit = mg_globals.app_config.get('upload_limit', None)
if upload_limit and user.uploaded >= upload_limit:
messages.add_message(
request,
messages.WARNING,
_('Sorry, you have reached your upload limit.'))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
max_file_size = mg_globals.app_config.get('max_file_size', None)
submit_form = submit_forms.get_submit_start_form(
request.form,
license=request.user.license_preference,
max_file_size=max_file_size,
upload_limit=upload_limit,
uploaded=user.uploaded)
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(request.files['file'].filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Get file size and round to 2 decimal places
file_size = request.app.queue_store.get_file_size(
entry.queued_media_file) / (1024.0 * 1024)
file_size = float('{0:.2f}'.format(file_size))
error = False
# Check if file size is over the limit
if max_file_size and file_size >= max_file_size:
submit_form.file.errors.append(
_(u'Sorry, the file size is too big.'))
error = True
# Check if user is over upload limit
if upload_limit and (user.uploaded + file_size) >= upload_limit:
submit_form.file.errors.append(
_('Sorry, uploading this file will put you over your'
' upload limit.'))
error = True
if not error:
user.uploaded = user.uploaded + file_size
user.save()
entry.file_size = file_size
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
add_comment_subscription(request.user, entry)
return redirect(request, "mediagoblin.user_pages.user_home",
user=user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
def multi_submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.get_submit_start_form(request.form, license=request.user.license_preference)
users_collections = Collection.query.filter_by(
actor=request.user.id,
type=Collection.USER_DEFINED_TYPE
).order_by(Collection.title)
if users_collections.count() > 0:
submit_form.collection.query = users_collections
else:
del submit_form.collection
# Below is what was used for mediagoblin 0.5.0-dev. Above is the new way.
# submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference)
filecount = 0
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(_(u'You must provide at least one file.'))
else:
for submitted_file in request.files.getlist('file'):
try:
if not submitted_file.filename:
# MOST likely an invalid file
continue # Skip the rest of the loop for this file
else:
filename = submitted_file.filename
_log.info("html5-multi-upload: Got filename: %s" % filename)
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
submitted_file, filename)
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(submitted_file.filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(submitted_file.stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
if submit_form.collection and submit_form.collection.data:
add_media_to_collection(
submit_form.collection.data, entry)
create_activity(
"add", entry, request.user,
target=submit_form.collection.data)
add_comment_subscription(request.user, entry)
filecount = filecount + 1
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, TypeNotFound) or isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(e)
else:
raise
add_message(request, SUCCESS, _('Woohoo! Submitted %d Files!' % filecount))
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
return render_to_response(
request,
'start.html',
{'multi_submit_form': submit_form})
def submit_start(request):
"""
First view for submitting a file.
"""
submit_form = submit_forms.SubmitStartForm(request.form,
license=request.user.license_preference)
if request.method == 'POST' and submit_form.validate():
if not check_file_field(request, 'file'):
submit_form.file.errors.append(
_(u'You must provide a file.'))
else:
try:
filename = request.files['file'].filename
# If the filename contains non ascii generate a unique name
if not all(ord(c) < 128 for c in filename):
filename = unicode(uuid.uuid4()) + splitext(filename)[-1]
# Sniff the submitted media to determine which
# media plugin should handle processing
media_type, media_manager = sniff_media(
request.files['file'])
# create entry and save in database
entry = new_upload_entry(request.user)
entry.media_type = unicode(media_type)
entry.title = (
unicode(submit_form.title.data)
or unicode(splitext(request.files['file'].filename)[0]))
entry.description = unicode(submit_form.description.data)
entry.license = unicode(submit_form.license.data) or None
# Process the user's folksonomy "tags"
entry.tags = convert_to_tag_list_of_dicts(
submit_form.tags.data)
# Generate a slug from the title
entry.generate_slug()
queue_file = prepare_queue_task(request.app, entry, filename)
with queue_file:
queue_file.write(request.files['file'].stream.read())
# Save now so we have this data before kicking off processing
entry.save()
# Pass off to async processing
#
# (... don't change entry after this point to avoid race
# conditions with changes to the document via processing code)
feed_url = request.urlgen(
'mediagoblin.user_pages.atom_feed',
qualified=True, user=request.user.username)
run_process_media(entry, feed_url)
add_message(request, SUCCESS, _('Woohoo! Submitted!'))
add_comment_subscription(request.user, entry)
return redirect(request, "mediagoblin.user_pages.user_home",
user=request.user.username)
except Exception as e:
'''
This section is intended to catch exceptions raised in
mediagoblin.media_types
'''
if isinstance(e, InvalidFileType) or \
isinstance(e, FileTypeNotSupported):
submit_form.file.errors.append(
e)
else:
raise
return render_to_response(
request,
'mediagoblin/submit/start.html',
{'submit_form': submit_form,
'app_config': mg_globals.app_config})
