def test_parse_content_range_stop():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
assert val.stop == ContentRange.parse("bytes 0-499/1234").stop
def test_parse_content_range_length():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.length, ContentRange.parse("bytes 0-499/1234").length)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange)
eq_(ContentRange.parse(None), None)
eq_(ContentRange.parse('0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99/xxx'), None)
eq_(ContentRange.parse('bytes 0 99/100'), None)
eq_(ContentRange.parse('bytes */100').__class__, ContentRange)
eq_(ContentRange.parse('bytes A-99/100'), None)
eq_(ContentRange.parse('bytes 0-B/100'), None)
eq_(ContentRange.parse('bytes 99-0/100'), None)
eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_cr_parse_range_star():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes */100").__class__, ContentRange)
def test_cr_parse_content_invalid():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 99-0/100"), None)
def test_cr_parse_no_bytes():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("0-99 100"), None)
def test_cr_parse_invalid_length():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-99/xxx"), None)
def test_cr_parse_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse(None), None)
def test_cr_parse_no_bytes():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('0-99 100'), None)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange)
eq_(ContentRange.parse(None), None)
eq_(ContentRange.parse('0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99/xxx'), None)
eq_(ContentRange.parse('bytes 0 99/100'), None)
eq_(ContentRange.parse('bytes */100').__class__, ContentRange)
eq_(ContentRange.parse('bytes A-99/100'), None)
eq_(ContentRange.parse('bytes 0-B/100'), None)
eq_(ContentRange.parse('bytes 99-0/100'), None)
eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_cr_parse_ok():
contentrange = ContentRange(0, 99, 100)
assert_true(contentrange.parse('bytes 0-99/100').__class__, ContentRange)
def test_parse_content_range_start():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.start, ContentRange.parse("bytes 0-499/1234").start)
def test_parse_content_range_length():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.length, ContentRange.parse("bytes 0-499/1234").length)
def post_repertoire_upload(request):
# create paths
create_paths(request)
# upload files
files = []
for name, fieldStorage in request.POST.items():
# check fieldStorage
if not isinstance(fieldStorage, FieldStorage):
continue
# configure upload
rank = (request.registry.settings['abuse_rank.active'] == 'true')
rank_max = int(request.registry.settings['abuse_rank.max'])
hostname = get_hostname()
descriptor = fieldStorage.file
filename = os.path.basename(fieldStorage.filename).encode('utf-8')
filename_hash = _hash_algorithm(filename).hexdigest()
temporary_path = get_path(request, _path_temporary, filename_hash)
contentrange = ContentRange.parse(
request.headers.get('Content-Range', None)
)
contentlength = request.headers.get('Content-Length', None)
# create checksum
with benchmark(request, name='checksum', uid=filename,
normalize=descriptor, scale=100*1024*1024):
checksum = create_checksum(
descriptor=descriptor,
algorithm=_checksum_algorithm
)
save_checksum(
path=temporary_path + _checksum_postfix,
algorithm=_checksum_algorithm.__name__,
checksum=checksum.hexdigest(),
contentrange=contentrange or (0, contentlength, contentlength)
)
# abuse rank
if rank:
if is_banned(request):
# TODO: number wont be replaced, also see
# BirthdateField line 300+ in register_webuser.py
files.append({
'name': fieldStorage.filename,
'error': _(
u"Abuse detected. Wait for {number}"
u" seconds before trying another"
u" upload.",
mapping={'number': int(still_banned_for(request))}
)})
continue
if is_collision(contentrange, checksum):
raise_abuse_rank(request)
current_rank = request.session['abuse_rank']['current']
if current_rank == rank_max:
ban(request)
# save to filesystem (-> temporary)
ok, complete = save_upload_to_fs(
descriptor=descriptor,
absolute_path=temporary_path,
contentrange=contentrange
)
if not ok:
pass
if not complete:
# client feedback
files.append({
'name': fieldStorage.filename,
'size': os.path.getsize(temporary_path)
})
continue
# get content uuid
content_uuid = get_content_uuid()
# get uuid paths
uploaded_path = get_path(request, _path_uploaded, content_uuid)
rejected_path = get_path(request, _path_rejected, content_uuid)
file_category = get_category_from_mimetype(temporary_path)
file_size = os.path.getsize(temporary_path)
mime_type = str(mime.from_file(temporary_path))
# validate file
error = validate_upload(filename, temporary_path)
if error:
# move files (temporary -> rejected)
ok = move_files_with_prefixes(
source=temporary_path, target=rejected_path
)
if not ok:
panic(
request,
reason="Files could not be moved.",
identifiers=[filename_hash, content_uuid]
)
# save file to database
_content = {
'uuid': content_uuid,
'processing_hostname': hostname,
'processing_state': "rejected",
'rejection_reason': "format_error",
'entity_origin': "direct",
'entity_creator': WebUser.current_web_user(request).party,
'name': str(name),
'category': file_category,
'mime_type': mime_type,
'size': file_size,
'path': rejected_path
}
content = save_upload_to_db(_content)
if not content:
panic(
request,
reason="Content could not be created.",
identifiers=[filename_hash, content_uuid]
)
# save checksums to database
# admin feedback
# 2DO: Mail
log.info(
(
"Content rejected (format error): %s\n"
) % (
rejected_path
)
)
# client feedback
files.append({
'name': fieldStorage.filename,
'error': error
})
continue
# we used to create a preview, now done in repertoire processing
# this is only for displaying some file properties
# audio = AudioSegment.from_file(temporary_path)
file_category = get_category_from_mimetype(temporary_path)
# move files (temporary -> uploaded)
ok = move_files_with_prefixes(
source=temporary_path, target=uploaded_path
)
if not ok:
panic(
request,
reason="Files could not be moved.",
identifiers=[filename_hash, content_uuid]
)
# save file to database
_content = {
'uuid': content_uuid,
'processing_hostname': hostname,
'processing_state': "uploaded",
'entity_origin': "direct",
'entity_creator': WebUser.current_web_user(request).party,
'name': str(filename),
'category': file_category,
'mime_type': str(mime.from_file(uploaded_path)),
'size': os.path.getsize(uploaded_path),
'path': uploaded_path,
# 'length': "%.6f" % audio.duration_seconds,
# 'channels': int(audio.channels),
# 'sample_rate': int(audio.frame_rate),
# 'sample_width': int(audio.sample_width * 8)
}
content = save_upload_to_db(_content)
if not content:
panic(
request,
reason="Content could not be created.",
identifiers=[filename_hash, content_uuid]
)
# save checksums to database
save_checksums_to_db(
content=content,
path=uploaded_path + _checksum_postfix
)
# client feedback
files.append(get_content_info(request, content))
# finally, see if there are old temporary files in the temp folder
# structure
cleanup_temp_directory(request)
# TODO: add timestamp file in temp folder to track if cleanup run
# was already started this day
return {'files': files}
def _parse_content_range(value):
if not value or not value.strip():
return None
# May still return None
return ContentRange.parse(value)
def test_cr_parse_missing_slash():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-99 100'), None)
def test_cr_parse_parse_problem_1():
contentrange = ContentRange( 0, 99, 100 )
assert_equal( contentrange.parse( 'bytes A-99/100' ), None )
def test_cr_parse_invalid_length():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-99/xxx'), None)
def test_cr_parse_missing_slash():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-99 100"), None)
def test_cr_parse_no_range():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0 99/100'), None)
def test_cr_parse_no_range():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0 99/100"), None)
def test_cr_parse_range_star():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes */100').__class__, ContentRange)
def test_cr_parse_parse_problem_2():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-B/100"), None)
def test_cr_parse_parse_problem_2():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-B/100'), None)
def test_contentrange_str_length_start():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0 99/*"), None)
def test_cr_parse_content_invalid():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 99-0/100'), None)
def test_parse_content_range_start():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.start, ContentRange.parse("bytes 0-499/1234").start)
def test_contentrange_str_length_start():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0 99/*'), None)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert isinstance(contentrange, Iterable)
assert ContentRange.parse("bytes 0-99/100").__class__ == ContentRange
assert ContentRange.parse(None) is None
assert ContentRange.parse("0-99 100") is None
assert ContentRange.parse("bytes 0-99 100") is None
assert ContentRange.parse("bytes 0-99/xxx") is None
assert ContentRange.parse("bytes 0 99/100") is None
assert ContentRange.parse("bytes */100").__class__ == ContentRange
assert ContentRange.parse("bytes A-99/100") is None
assert ContentRange.parse("bytes 0-B/100") is None
assert ContentRange.parse("bytes 99-0/100") is None
assert ContentRange.parse("bytes 0 99/*") is None
def test_cr_parse_ok():
contentrange = ContentRange(0, 99, 100)
assert_true(contentrange.parse("bytes 0-99/100").__class__, ContentRange)
def test_parse_content_range_stop():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
assert val.stop == ContentRange.parse("bytes 0-499/1234").stop
def test_cr_parse_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse(None), None)
def parse_content_range(value):
if not value or not value.strip():
return None
# May still return None
return ContentRange.parse(value)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
import collections
assert isinstance(contentrange, collections.Iterable)
assert ContentRange.parse('bytes 0-99/100').__class__ == ContentRange
assert ContentRange.parse(None) is None
assert ContentRange.parse('0-99 100') is None
assert ContentRange.parse('bytes 0-99 100') is None
assert ContentRange.parse('bytes 0-99/xxx') is None
assert ContentRange.parse('bytes 0 99/100') is None
assert ContentRange.parse('bytes */100').__class__ == ContentRange
assert ContentRange.parse('bytes A-99/100') is None
assert ContentRange.parse('bytes 0-B/100') is None
assert ContentRange.parse('bytes 99-0/100') is None
assert ContentRange.parse('bytes 0 99/*') is None
