def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert isinstance(contentrange, Iterable)
assert ContentRange.parse("bytes 0-99/100").__class__ == ContentRange
assert ContentRange.parse(None) is None
assert ContentRange.parse("0-99 100") is None
assert ContentRange.parse("bytes 0-99 100") is None
assert ContentRange.parse("bytes 0-99/xxx") is None
assert ContentRange.parse("bytes 0 99/100") is None
assert ContentRange.parse("bytes */100").__class__ == ContentRange
assert ContentRange.parse("bytes A-99/100") is None
assert ContentRange.parse("bytes 0-B/100") is None
assert ContentRange.parse("bytes 99-0/100") is None
assert ContentRange.parse("bytes 0 99/*") is None
def conditional_response_app(self, environ, start_response):
"""
Like the normal __call__ interface, but checks conditional headers:
* If-Modified-Since (304 Not Modified; only on GET, HEAD)
* If-None-Match (304 Not Modified; only on GET, HEAD)
* Range (406 Partial Content; only on GET, HEAD)
"""
req = BaseRequest(environ)
headerlist = self._abs_headerlist(environ)
method = environ.get('REQUEST_METHOD', 'GET')
if method in self._safe_methods:
status304 = False
if req.if_none_match and self.etag:
status304 = self.etag in req.if_none_match
elif req.if_modified_since and self.last_modified:
status304 = self.last_modified <= req.if_modified_since
if status304:
start_response('304 Not Modified', filter_headers(headerlist))
return EmptyResponse(self._app_iter)
if (req.range and self in req.if_range
and self.content_range is None
and method in ('HEAD', 'GET')
and self.status_code == 200
and self.content_length is not None
):
content_range = req.range.content_range(self.content_length)
if content_range is None:
iter_close(self._app_iter)
body = bytes_("Requested range not satisfiable: %s" % req.range)
headerlist = [
('Content-Length', str(len(body))),
('Content-Range', str(ContentRange(None, None,
self.content_length))),
('Content-Type', 'text/plain'),
] + filter_headers(headerlist)
start_response('416 Requested Range Not Satisfiable',
headerlist)
if method == 'HEAD':
return ()
return [body]
else:
app_iter = self.app_iter_range(content_range.start,
content_range.stop)
if app_iter is not None:
# the following should be guaranteed by
# Range.range_for_length(length)
assert content_range.start is not None
headerlist = [
('Content-Length',
str(content_range.stop - content_range.start)),
('Content-Range', str(content_range)),
] + filter_headers(headerlist, ('content-length',))
start_response('206 Partial Content', headerlist)
if method == 'HEAD':
return EmptyResponse(app_iter)
return app_iter
start_response(self.status, headerlist)
if method == 'HEAD':
return EmptyResponse(self._app_iter)
return self._app_iter
def test_cr_parse_content_invalid():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 99-0/100"), None)
def test_parse_content_range_stop():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
assert val.stop == ContentRange.parse("bytes 0-499/1234").stop
def test_cr_parse_ok():
contentrange = ContentRange(0, 99, 100)
assert_true(contentrange.parse("bytes 0-99/100").__class__, ContentRange)
def test_cr_parse_invalid_length():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-99/xxx"), None)
def test_cr_parse_invalid_length():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-99/xxx'), None)
def test_contentrange_repr():
contentrange = ContentRange(0, 99, 100)
assert_true(contentrange.__repr__(), "<ContentRange bytes 0-98/100>")
def test_cr_parse_no_bytes():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('0-99 100'), None)
def test_cr_parse_missing_slash():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-99 100'), None)
def test_cr_parse_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse(None), None)
def test_cr_parse_ok():
contentrange = ContentRange(0, 99, 100)
assert_true(contentrange.parse('bytes 0-99/100').__class__, ContentRange)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
def parse_content_range(value):
if not value or not value.strip():
return None
# May still return None
return ContentRange.parse(value)
def test_cr_parse_no_range():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0 99/100'), None)
def test_parse_content_range_length():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.length, ContentRange.parse("bytes 0-499/1234").length)
def test_cr_parse_range_star():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes */100').__class__, ContentRange)
def test_contentrange_str_start_none():
contentrange = ContentRange(0, 99, 100)
contentrange.start = None
contentrange.stop = None
assert_equal(contentrange.__str__(), "bytes */100")
def test_cr_parse_parse_problem_2():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0-B/100'), None)
def test_cr_parse_no_bytes():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("0-99 100"), None)
def test_cr_parse_content_invalid():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 99-0/100'), None)
def test_cr_parse_range_star():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes */100").__class__, ContentRange)
def test_contentrange_str_length_start():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse('bytes 0 99/*'), None)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange)
eq_(ContentRange.parse(None), None)
eq_(ContentRange.parse('0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99/xxx'), None)
eq_(ContentRange.parse('bytes 0 99/100'), None)
eq_(ContentRange.parse('bytes */100').__class__, ContentRange)
eq_(ContentRange.parse('bytes A-99/100'), None)
eq_(ContentRange.parse('bytes 0-B/100'), None)
eq_(ContentRange.parse('bytes 99-0/100'), None)
eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_is_content_range_valid_start_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(_is_content_range_valid(None, 99, 90), False)
def test_contentrange_str_start_none():
contentrange = ContentRange(0, 99, 100)
contentrange.start = None
contentrange.stop = None
assert_equal(contentrange.__str__(), 'bytes */100')
def test_is_content_range_valid_length_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(_is_content_range_valid(0, 99, None), True)
def test_parse_content_range_stop():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.stop, ContentRange.parse("bytes 0-499/1234").stop)
def test_is_content_range_valid_stop_greater_than_length_response():
contentrange = ContentRange(0, 99, 100)
assert_equal(_is_content_range_valid(0, 99, 90, response=True), False)
def post_repertoire_upload(request):
# create paths
create_paths(request)
# upload files
files = []
for name, fieldStorage in request.POST.items():
# check fieldStorage
if not isinstance(fieldStorage, FieldStorage):
continue
# configure upload
rank = (request.registry.settings['abuse_rank.active'] == 'true')
rank_max = int(request.registry.settings['abuse_rank.max'])
hostname = get_hostname()
descriptor = fieldStorage.file
filename = os.path.basename(fieldStorage.filename).encode('utf-8')
filename_hash = _hash_algorithm(filename).hexdigest()
temporary_path = get_path(request, _path_temporary, filename_hash)
contentrange = ContentRange.parse(
request.headers.get('Content-Range', None)
)
contentlength = request.headers.get('Content-Length', None)
# create checksum
with benchmark(request, name='checksum', uid=filename,
normalize=descriptor, scale=100*1024*1024):
checksum = create_checksum(
descriptor=descriptor,
algorithm=_checksum_algorithm
)
save_checksum(
path=temporary_path + _checksum_postfix,
algorithm=_checksum_algorithm.__name__,
checksum=checksum.hexdigest(),
contentrange=contentrange or (0, contentlength, contentlength)
)
# abuse rank
if rank:
if is_banned(request):
# TODO: number wont be replaced, also see
# BirthdateField line 300+ in register_webuser.py
files.append({
'name': fieldStorage.filename,
'error': _(
u"Abuse detected. Wait for {number}"
u" seconds before trying another"
u" upload.",
mapping={'number': int(still_banned_for(request))}
)})
continue
if is_collision(contentrange, checksum):
raise_abuse_rank(request)
current_rank = request.session['abuse_rank']['current']
if current_rank == rank_max:
ban(request)
# save to filesystem (-> temporary)
ok, complete = save_upload_to_fs(
descriptor=descriptor,
absolute_path=temporary_path,
contentrange=contentrange
)
if not ok:
pass
if not complete:
# client feedback
files.append({
'name': fieldStorage.filename,
'size': os.path.getsize(temporary_path)
})
continue
# get content uuid
content_uuid = get_content_uuid()
# get uuid paths
uploaded_path = get_path(request, _path_uploaded, content_uuid)
rejected_path = get_path(request, _path_rejected, content_uuid)
file_category = get_category_from_mimetype(temporary_path)
file_size = os.path.getsize(temporary_path)
mime_type = str(mime.from_file(temporary_path))
# validate file
error = validate_upload(filename, temporary_path)
if error:
# move files (temporary -> rejected)
ok = move_files_with_prefixes(
source=temporary_path, target=rejected_path
)
if not ok:
panic(
request,
reason="Files could not be moved.",
identifiers=[filename_hash, content_uuid]
)
# save file to database
_content = {
'uuid': content_uuid,
'processing_hostname': hostname,
'processing_state': "rejected",
'rejection_reason': "format_error",
'entity_origin': "direct",
'entity_creator': WebUser.current_web_user(request).party,
'name': str(name),
'category': file_category,
'mime_type': mime_type,
'size': file_size,
'path': rejected_path
}
content = save_upload_to_db(_content)
if not content:
panic(
request,
reason="Content could not be created.",
identifiers=[filename_hash, content_uuid]
)
# save checksums to database
# admin feedback
# 2DO: Mail
log.info(
(
"Content rejected (format error): %s\n"
) % (
rejected_path
)
)
# client feedback
files.append({
'name': fieldStorage.filename,
'error': error
})
continue
# we used to create a preview, now done in repertoire processing
# this is only for displaying some file properties
# audio = AudioSegment.from_file(temporary_path)
file_category = get_category_from_mimetype(temporary_path)
# move files (temporary -> uploaded)
ok = move_files_with_prefixes(
source=temporary_path, target=uploaded_path
)
if not ok:
panic(
request,
reason="Files could not be moved.",
identifiers=[filename_hash, content_uuid]
)
# save file to database
_content = {
'uuid': content_uuid,
'processing_hostname': hostname,
'processing_state': "uploaded",
'entity_origin': "direct",
'entity_creator': WebUser.current_web_user(request).party,
'name': str(filename),
'category': file_category,
'mime_type': str(mime.from_file(uploaded_path)),
'size': os.path.getsize(uploaded_path),
'path': uploaded_path,
# 'length': "%.6f" % audio.duration_seconds,
# 'channels': int(audio.channels),
# 'sample_rate': int(audio.frame_rate),
# 'sample_width': int(audio.sample_width * 8)
}
content = save_upload_to_db(_content)
if not content:
panic(
request,
reason="Content could not be created.",
identifiers=[filename_hash, content_uuid]
)
# save checksums to database
save_checksums_to_db(
content=content,
path=uploaded_path + _checksum_postfix
)
# client feedback
files.append(get_content_info(request, content))
# finally, see if there are old temporary files in the temp folder
# structure
cleanup_temp_directory(request)
# TODO: add timestamp file in temp folder to track if cleanup run
# was already started this day
return {'files': files}
def test_is_content_range_valid_stop_greater_than_length():
contentrange = ContentRange(0, 99, 100)
assert_equal(_is_content_range_valid(0, 99, 90), True)
def test_parse_content_range_length():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
assert val.length == ContentRange.parse("bytes 0-499/1234").length
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
import collections
assert isinstance(contentrange, collections.Iterable)
assert ContentRange.parse('bytes 0-99/100').__class__ == ContentRange
assert ContentRange.parse(None) is None
assert ContentRange.parse('0-99 100') is None
assert ContentRange.parse('bytes 0-99 100') is None
assert ContentRange.parse('bytes 0-99/xxx') is None
assert ContentRange.parse('bytes 0 99/100') is None
assert ContentRange.parse('bytes */100').__class__ == ContentRange
assert ContentRange.parse('bytes A-99/100') is None
assert ContentRange.parse('bytes 0-B/100') is None
assert ContentRange.parse('bytes 99-0/100') is None
assert ContentRange.parse('bytes 0 99/*') is None
def test_parse_content_range_start():
from webob.byterange import ContentRange
from webob.descriptors import parse_content_range
val = parse_content_range("bytes 0-499/1234")
eq_(val.start, ContentRange.parse("bytes 0-499/1234").start)
def test_contentrange_repr():
contentrange = ContentRange(0, 99, 100)
assert_true(repr(contentrange), '<ContentRange bytes 0-98/100>')
def test_contentrange_str_length_none():
contentrange = ContentRange(0, 99, 100)
contentrange.length = None
assert_equal(contentrange.__str__(), "bytes 0-98/*")
def test_contentrange_str():
contentrange = ContentRange(0, 99, None)
eq_(str(contentrange), 'bytes 0-98/*')
contentrange = ContentRange(None, None, 100)
eq_(str(contentrange), 'bytes */100')
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert_true(type(contentrange.__iter__()), iter)
assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange)
eq_(ContentRange.parse(None), None)
eq_(ContentRange.parse('0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99 100'), None)
eq_(ContentRange.parse('bytes 0-99/xxx'), None)
eq_(ContentRange.parse('bytes 0 99/100'), None)
eq_(ContentRange.parse('bytes */100').__class__, ContentRange)
eq_(ContentRange.parse('bytes A-99/100'), None)
eq_(ContentRange.parse('bytes 0-B/100'), None)
eq_(ContentRange.parse('bytes 99-0/100'), None)
eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_cr_parse_none():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse(None), None)
def test_contentrange_str_length_none():
contentrange = ContentRange( 0, 99, 100 )
contentrange.length = None
assert_equal( str(contentrange), 'bytes 0-98/*' )
def test_cr_parse_missing_slash():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-99 100"), None)
def test_contentrange_str_start_none():
contentrange = ContentRange( 0, 99, 100 )
contentrange.start = None
contentrange.stop = None
assert_equal( str(contentrange), 'bytes */100' )
def test_cr_parse_no_range():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0 99/100"), None)
def test_cr_parse_parse_problem_1():
contentrange = ContentRange( 0, 99, 100 )
assert_equal( contentrange.parse( 'bytes A-99/100' ), None )
def test_cr_parse_parse_problem_2():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0-B/100"), None)
def test_contentrange_bad_input():
with pytest.raises(ValueError):
ContentRange(None, 99, None)
def test_contentrange_str_length_start():
contentrange = ContentRange(0, 99, 100)
assert_equal(contentrange.parse("bytes 0 99/*"), None)
def test_contentrange_repr():
contentrange = ContentRange(0, 99, 100)
assert repr(contentrange) == "<ContentRange bytes 0-98/100>"
def test_contentrange_iter():
contentrange = ContentRange(0, 99, 100)
assert isinstance(contentrange, Iterable)
assert ContentRange.parse("bytes 0-99/100").__class__ == ContentRange
assert ContentRange.parse(None) is None
assert ContentRange.parse("0-99 100") is None
assert ContentRange.parse("bytes 0-99 100") is None
assert ContentRange.parse("bytes 0-99/xxx") is None
assert ContentRange.parse("bytes 0 99/100") is None
assert ContentRange.parse("bytes */100").__class__ == ContentRange
assert ContentRange.parse("bytes A-99/100") is None
assert ContentRange.parse("bytes 0-B/100") is None
assert ContentRange.parse("bytes 99-0/100") is None
assert ContentRange.parse("bytes 0 99/*") is None
def test_contentrange_str():
contentrange = ContentRange(0, 99, None)
assert str(contentrange) == "bytes 0-98/*"
contentrange = ContentRange(None, None, 100)
assert str(contentrange) == "bytes */100"
def _parse_content_range(value):
if not value or not value.strip():
return None
# May still return None
return ContentRange.parse(value)
def test_contentrange_str_length_none():
contentrange = ContentRange(0, 99, 100)
contentrange.length = None
assert_equal(contentrange.__str__(), 'bytes 0-98/*')
