def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) assert isinstance(contentrange, Iterable) assert ContentRange.parse("bytes 0-99/100").__class__ == ContentRange assert ContentRange.parse(None) is None assert ContentRange.parse("0-99 100") is None assert ContentRange.parse("bytes 0-99 100") is None assert ContentRange.parse("bytes 0-99/xxx") is None assert ContentRange.parse("bytes 0 99/100") is None assert ContentRange.parse("bytes */100").__class__ == ContentRange assert ContentRange.parse("bytes A-99/100") is None assert ContentRange.parse("bytes 0-B/100") is None assert ContentRange.parse("bytes 99-0/100") is None assert ContentRange.parse("bytes 0 99/*") is None
def conditional_response_app(self, environ, start_response): """ Like the normal __call__ interface, but checks conditional headers: * If-Modified-Since (304 Not Modified; only on GET, HEAD) * If-None-Match (304 Not Modified; only on GET, HEAD) * Range (406 Partial Content; only on GET, HEAD) """ req = BaseRequest(environ) headerlist = self._abs_headerlist(environ) method = environ.get('REQUEST_METHOD', 'GET') if method in self._safe_methods: status304 = False if req.if_none_match and self.etag: status304 = self.etag in req.if_none_match elif req.if_modified_since and self.last_modified: status304 = self.last_modified <= req.if_modified_since if status304: start_response('304 Not Modified', filter_headers(headerlist)) return EmptyResponse(self._app_iter) if (req.range and self in req.if_range and self.content_range is None and method in ('HEAD', 'GET') and self.status_code == 200 and self.content_length is not None ): content_range = req.range.content_range(self.content_length) if content_range is None: iter_close(self._app_iter) body = bytes_("Requested range not satisfiable: %s" % req.range) headerlist = [ ('Content-Length', str(len(body))), ('Content-Range', str(ContentRange(None, None, self.content_length))), ('Content-Type', 'text/plain'), ] + filter_headers(headerlist) start_response('416 Requested Range Not Satisfiable', headerlist) if method == 'HEAD': return () return [body] else: app_iter = self.app_iter_range(content_range.start, content_range.stop) if app_iter is not None: # the following should be guaranteed by # Range.range_for_length(length) assert content_range.start is not None headerlist = [ ('Content-Length', str(content_range.stop - content_range.start)), ('Content-Range', str(content_range)), ] + filter_headers(headerlist, ('content-length',)) start_response('206 Partial Content', headerlist) if method == 'HEAD': return EmptyResponse(app_iter) return app_iter start_response(self.status, headerlist) if method == 'HEAD': return EmptyResponse(self._app_iter) return self._app_iter
def test_cr_parse_content_invalid(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 99-0/100"), None)
def test_parse_content_range_stop(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") assert val.stop == ContentRange.parse("bytes 0-499/1234").stop
def test_cr_parse_ok(): contentrange = ContentRange(0, 99, 100) assert_true(contentrange.parse("bytes 0-99/100").__class__, ContentRange)
def test_cr_parse_invalid_length(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-99/xxx"), None)
def test_cr_parse_invalid_length(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-99/xxx'), None)
def test_contentrange_repr(): contentrange = ContentRange(0, 99, 100) assert_true(contentrange.__repr__(), "<ContentRange bytes 0-98/100>")
def test_cr_parse_no_bytes(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('0-99 100'), None)
def test_cr_parse_missing_slash(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-99 100'), None)
def test_cr_parse_none(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse(None), None)
def test_cr_parse_ok(): contentrange = ContentRange(0, 99, 100) assert_true(contentrange.parse('bytes 0-99/100').__class__, ContentRange)
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) assert_true(type(contentrange.__iter__()), iter)
def parse_content_range(value): if not value or not value.strip(): return None # May still return None return ContentRange.parse(value)
def test_cr_parse_no_range(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0 99/100'), None)
def test_parse_content_range_length(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") eq_(val.length, ContentRange.parse("bytes 0-499/1234").length)
def test_cr_parse_range_star(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes */100').__class__, ContentRange)
def test_contentrange_str_start_none(): contentrange = ContentRange(0, 99, 100) contentrange.start = None contentrange.stop = None assert_equal(contentrange.__str__(), "bytes */100")
def test_cr_parse_parse_problem_2(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-B/100'), None)
def test_cr_parse_no_bytes(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("0-99 100"), None)
def test_cr_parse_content_invalid(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 99-0/100'), None)
def test_cr_parse_range_star(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes */100").__class__, ContentRange)
def test_contentrange_str_length_start(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0 99/*'), None)
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) assert_true(type(contentrange.__iter__()), iter) assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange) eq_(ContentRange.parse(None), None) eq_(ContentRange.parse('0-99 100'), None) eq_(ContentRange.parse('bytes 0-99 100'), None) eq_(ContentRange.parse('bytes 0-99/xxx'), None) eq_(ContentRange.parse('bytes 0 99/100'), None) eq_(ContentRange.parse('bytes */100').__class__, ContentRange) eq_(ContentRange.parse('bytes A-99/100'), None) eq_(ContentRange.parse('bytes 0-B/100'), None) eq_(ContentRange.parse('bytes 99-0/100'), None) eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_is_content_range_valid_start_none(): contentrange = ContentRange(0, 99, 100) assert_equal(_is_content_range_valid(None, 99, 90), False)
def test_contentrange_str_start_none(): contentrange = ContentRange(0, 99, 100) contentrange.start = None contentrange.stop = None assert_equal(contentrange.__str__(), 'bytes */100')
def test_is_content_range_valid_length_none(): contentrange = ContentRange(0, 99, 100) assert_equal(_is_content_range_valid(0, 99, None), True)
def test_parse_content_range_stop(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") eq_(val.stop, ContentRange.parse("bytes 0-499/1234").stop)
def test_is_content_range_valid_stop_greater_than_length_response(): contentrange = ContentRange(0, 99, 100) assert_equal(_is_content_range_valid(0, 99, 90, response=True), False)
def post_repertoire_upload(request): # create paths create_paths(request) # upload files files = [] for name, fieldStorage in request.POST.items(): # check fieldStorage if not isinstance(fieldStorage, FieldStorage): continue # configure upload rank = (request.registry.settings['abuse_rank.active'] == 'true') rank_max = int(request.registry.settings['abuse_rank.max']) hostname = get_hostname() descriptor = fieldStorage.file filename = os.path.basename(fieldStorage.filename).encode('utf-8') filename_hash = _hash_algorithm(filename).hexdigest() temporary_path = get_path(request, _path_temporary, filename_hash) contentrange = ContentRange.parse( request.headers.get('Content-Range', None) ) contentlength = request.headers.get('Content-Length', None) # create checksum with benchmark(request, name='checksum', uid=filename, normalize=descriptor, scale=100*1024*1024): checksum = create_checksum( descriptor=descriptor, algorithm=_checksum_algorithm ) save_checksum( path=temporary_path + _checksum_postfix, algorithm=_checksum_algorithm.__name__, checksum=checksum.hexdigest(), contentrange=contentrange or (0, contentlength, contentlength) ) # abuse rank if rank: if is_banned(request): # TODO: number wont be replaced, also see # BirthdateField line 300+ in register_webuser.py files.append({ 'name': fieldStorage.filename, 'error': _( u"Abuse detected. Wait for {number}" u" seconds before trying another" u" upload.", mapping={'number': int(still_banned_for(request))} )}) continue if is_collision(contentrange, checksum): raise_abuse_rank(request) current_rank = request.session['abuse_rank']['current'] if current_rank == rank_max: ban(request) # save to filesystem (-> temporary) ok, complete = save_upload_to_fs( descriptor=descriptor, absolute_path=temporary_path, contentrange=contentrange ) if not ok: pass if not complete: # client feedback files.append({ 'name': fieldStorage.filename, 'size': os.path.getsize(temporary_path) }) continue # get content uuid content_uuid = get_content_uuid() # get uuid paths uploaded_path = get_path(request, _path_uploaded, content_uuid) rejected_path = get_path(request, _path_rejected, content_uuid) file_category = get_category_from_mimetype(temporary_path) file_size = os.path.getsize(temporary_path) mime_type = str(mime.from_file(temporary_path)) # validate file error = validate_upload(filename, temporary_path) if error: # move files (temporary -> rejected) ok = move_files_with_prefixes( source=temporary_path, target=rejected_path ) if not ok: panic( request, reason="Files could not be moved.", identifiers=[filename_hash, content_uuid] ) # save file to database _content = { 'uuid': content_uuid, 'processing_hostname': hostname, 'processing_state': "rejected", 'rejection_reason': "format_error", 'entity_origin': "direct", 'entity_creator': WebUser.current_web_user(request).party, 'name': str(name), 'category': file_category, 'mime_type': mime_type, 'size': file_size, 'path': rejected_path } content = save_upload_to_db(_content) if not content: panic( request, reason="Content could not be created.", identifiers=[filename_hash, content_uuid] ) # save checksums to database # admin feedback # 2DO: Mail log.info( ( "Content rejected (format error): %s\n" ) % ( rejected_path ) ) # client feedback files.append({ 'name': fieldStorage.filename, 'error': error }) continue # we used to create a preview, now done in repertoire processing # this is only for displaying some file properties # audio = AudioSegment.from_file(temporary_path) file_category = get_category_from_mimetype(temporary_path) # move files (temporary -> uploaded) ok = move_files_with_prefixes( source=temporary_path, target=uploaded_path ) if not ok: panic( request, reason="Files could not be moved.", identifiers=[filename_hash, content_uuid] ) # save file to database _content = { 'uuid': content_uuid, 'processing_hostname': hostname, 'processing_state': "uploaded", 'entity_origin': "direct", 'entity_creator': WebUser.current_web_user(request).party, 'name': str(filename), 'category': file_category, 'mime_type': str(mime.from_file(uploaded_path)), 'size': os.path.getsize(uploaded_path), 'path': uploaded_path, # 'length': "%.6f" % audio.duration_seconds, # 'channels': int(audio.channels), # 'sample_rate': int(audio.frame_rate), # 'sample_width': int(audio.sample_width * 8) } content = save_upload_to_db(_content) if not content: panic( request, reason="Content could not be created.", identifiers=[filename_hash, content_uuid] ) # save checksums to database save_checksums_to_db( content=content, path=uploaded_path + _checksum_postfix ) # client feedback files.append(get_content_info(request, content)) # finally, see if there are old temporary files in the temp folder # structure cleanup_temp_directory(request) # TODO: add timestamp file in temp folder to track if cleanup run # was already started this day return {'files': files}
def test_is_content_range_valid_stop_greater_than_length(): contentrange = ContentRange(0, 99, 100) assert_equal(_is_content_range_valid(0, 99, 90), True)
def test_parse_content_range_length(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") assert val.length == ContentRange.parse("bytes 0-499/1234").length
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) import collections assert isinstance(contentrange, collections.Iterable) assert ContentRange.parse('bytes 0-99/100').__class__ == ContentRange assert ContentRange.parse(None) is None assert ContentRange.parse('0-99 100') is None assert ContentRange.parse('bytes 0-99 100') is None assert ContentRange.parse('bytes 0-99/xxx') is None assert ContentRange.parse('bytes 0 99/100') is None assert ContentRange.parse('bytes */100').__class__ == ContentRange assert ContentRange.parse('bytes A-99/100') is None assert ContentRange.parse('bytes 0-B/100') is None assert ContentRange.parse('bytes 99-0/100') is None assert ContentRange.parse('bytes 0 99/*') is None
def test_parse_content_range_start(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") eq_(val.start, ContentRange.parse("bytes 0-499/1234").start)
def test_contentrange_repr(): contentrange = ContentRange(0, 99, 100) assert_true(repr(contentrange), '<ContentRange bytes 0-98/100>')
def test_contentrange_str_length_none(): contentrange = ContentRange(0, 99, 100) contentrange.length = None assert_equal(contentrange.__str__(), "bytes 0-98/*")
def test_contentrange_str(): contentrange = ContentRange(0, 99, None) eq_(str(contentrange), 'bytes 0-98/*') contentrange = ContentRange(None, None, 100) eq_(str(contentrange), 'bytes */100')
def test_contentrange_str_length_none(): contentrange = ContentRange( 0, 99, 100 ) contentrange.length = None assert_equal( str(contentrange), 'bytes 0-98/*' )
def test_cr_parse_missing_slash(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-99 100"), None)
def test_contentrange_str_start_none(): contentrange = ContentRange( 0, 99, 100 ) contentrange.start = None contentrange.stop = None assert_equal( str(contentrange), 'bytes */100' )
def test_cr_parse_no_range(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0 99/100"), None)
def test_cr_parse_parse_problem_1(): contentrange = ContentRange( 0, 99, 100 ) assert_equal( contentrange.parse( 'bytes A-99/100' ), None )
def test_cr_parse_parse_problem_2(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-B/100"), None)
def test_contentrange_bad_input(): with pytest.raises(ValueError): ContentRange(None, 99, None)
def test_contentrange_str_length_start(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0 99/*"), None)
def test_contentrange_repr(): contentrange = ContentRange(0, 99, 100) assert repr(contentrange) == "<ContentRange bytes 0-98/100>"
def test_contentrange_str(): contentrange = ContentRange(0, 99, None) assert str(contentrange) == "bytes 0-98/*" contentrange = ContentRange(None, None, 100) assert str(contentrange) == "bytes */100"
def _parse_content_range(value): if not value or not value.strip(): return None # May still return None return ContentRange.parse(value)
def test_contentrange_str_length_none(): contentrange = ContentRange(0, 99, 100) contentrange.length = None assert_equal(contentrange.__str__(), 'bytes 0-98/*')