def test_parse_content_range_stop(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") assert val.stop == ContentRange.parse("bytes 0-499/1234").stop
def test_parse_content_range_length(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") eq_(val.length, ContentRange.parse("bytes 0-499/1234").length)
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) assert_true(type(contentrange.__iter__()), iter) assert_true(ContentRange.parse('bytes 0-99/100').__class__, ContentRange) eq_(ContentRange.parse(None), None) eq_(ContentRange.parse('0-99 100'), None) eq_(ContentRange.parse('bytes 0-99 100'), None) eq_(ContentRange.parse('bytes 0-99/xxx'), None) eq_(ContentRange.parse('bytes 0 99/100'), None) eq_(ContentRange.parse('bytes */100').__class__, ContentRange) eq_(ContentRange.parse('bytes A-99/100'), None) eq_(ContentRange.parse('bytes 0-B/100'), None) eq_(ContentRange.parse('bytes 99-0/100'), None) eq_(ContentRange.parse('bytes 0 99/*'), None)
def test_cr_parse_range_star(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes */100").__class__, ContentRange)
def test_cr_parse_content_invalid(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 99-0/100"), None)
def test_cr_parse_no_bytes(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("0-99 100"), None)
def test_cr_parse_invalid_length(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-99/xxx"), None)
def test_cr_parse_none(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse(None), None)
def test_cr_parse_no_bytes(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('0-99 100'), None)
def test_cr_parse_ok(): contentrange = ContentRange(0, 99, 100) assert_true(contentrange.parse('bytes 0-99/100').__class__, ContentRange)
def test_parse_content_range_start(): from webob.byterange import ContentRange from webob.descriptors import parse_content_range val = parse_content_range("bytes 0-499/1234") eq_(val.start, ContentRange.parse("bytes 0-499/1234").start)
def post_repertoire_upload(request): # create paths create_paths(request) # upload files files = [] for name, fieldStorage in request.POST.items(): # check fieldStorage if not isinstance(fieldStorage, FieldStorage): continue # configure upload rank = (request.registry.settings['abuse_rank.active'] == 'true') rank_max = int(request.registry.settings['abuse_rank.max']) hostname = get_hostname() descriptor = fieldStorage.file filename = os.path.basename(fieldStorage.filename).encode('utf-8') filename_hash = _hash_algorithm(filename).hexdigest() temporary_path = get_path(request, _path_temporary, filename_hash) contentrange = ContentRange.parse( request.headers.get('Content-Range', None) ) contentlength = request.headers.get('Content-Length', None) # create checksum with benchmark(request, name='checksum', uid=filename, normalize=descriptor, scale=100*1024*1024): checksum = create_checksum( descriptor=descriptor, algorithm=_checksum_algorithm ) save_checksum( path=temporary_path + _checksum_postfix, algorithm=_checksum_algorithm.__name__, checksum=checksum.hexdigest(), contentrange=contentrange or (0, contentlength, contentlength) ) # abuse rank if rank: if is_banned(request): # TODO: number wont be replaced, also see # BirthdateField line 300+ in register_webuser.py files.append({ 'name': fieldStorage.filename, 'error': _( u"Abuse detected. Wait for {number}" u" seconds before trying another" u" upload.", mapping={'number': int(still_banned_for(request))} )}) continue if is_collision(contentrange, checksum): raise_abuse_rank(request) current_rank = request.session['abuse_rank']['current'] if current_rank == rank_max: ban(request) # save to filesystem (-> temporary) ok, complete = save_upload_to_fs( descriptor=descriptor, absolute_path=temporary_path, contentrange=contentrange ) if not ok: pass if not complete: # client feedback files.append({ 'name': fieldStorage.filename, 'size': os.path.getsize(temporary_path) }) continue # get content uuid content_uuid = get_content_uuid() # get uuid paths uploaded_path = get_path(request, _path_uploaded, content_uuid) rejected_path = get_path(request, _path_rejected, content_uuid) file_category = get_category_from_mimetype(temporary_path) file_size = os.path.getsize(temporary_path) mime_type = str(mime.from_file(temporary_path)) # validate file error = validate_upload(filename, temporary_path) if error: # move files (temporary -> rejected) ok = move_files_with_prefixes( source=temporary_path, target=rejected_path ) if not ok: panic( request, reason="Files could not be moved.", identifiers=[filename_hash, content_uuid] ) # save file to database _content = { 'uuid': content_uuid, 'processing_hostname': hostname, 'processing_state': "rejected", 'rejection_reason': "format_error", 'entity_origin': "direct", 'entity_creator': WebUser.current_web_user(request).party, 'name': str(name), 'category': file_category, 'mime_type': mime_type, 'size': file_size, 'path': rejected_path } content = save_upload_to_db(_content) if not content: panic( request, reason="Content could not be created.", identifiers=[filename_hash, content_uuid] ) # save checksums to database # admin feedback # 2DO: Mail log.info( ( "Content rejected (format error): %s\n" ) % ( rejected_path ) ) # client feedback files.append({ 'name': fieldStorage.filename, 'error': error }) continue # we used to create a preview, now done in repertoire processing # this is only for displaying some file properties # audio = AudioSegment.from_file(temporary_path) file_category = get_category_from_mimetype(temporary_path) # move files (temporary -> uploaded) ok = move_files_with_prefixes( source=temporary_path, target=uploaded_path ) if not ok: panic( request, reason="Files could not be moved.", identifiers=[filename_hash, content_uuid] ) # save file to database _content = { 'uuid': content_uuid, 'processing_hostname': hostname, 'processing_state': "uploaded", 'entity_origin': "direct", 'entity_creator': WebUser.current_web_user(request).party, 'name': str(filename), 'category': file_category, 'mime_type': str(mime.from_file(uploaded_path)), 'size': os.path.getsize(uploaded_path), 'path': uploaded_path, # 'length': "%.6f" % audio.duration_seconds, # 'channels': int(audio.channels), # 'sample_rate': int(audio.frame_rate), # 'sample_width': int(audio.sample_width * 8) } content = save_upload_to_db(_content) if not content: panic( request, reason="Content could not be created.", identifiers=[filename_hash, content_uuid] ) # save checksums to database save_checksums_to_db( content=content, path=uploaded_path + _checksum_postfix ) # client feedback files.append(get_content_info(request, content)) # finally, see if there are old temporary files in the temp folder # structure cleanup_temp_directory(request) # TODO: add timestamp file in temp folder to track if cleanup run # was already started this day return {'files': files}
def _parse_content_range(value): if not value or not value.strip(): return None # May still return None return ContentRange.parse(value)
def test_cr_parse_missing_slash(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-99 100'), None)
def test_cr_parse_parse_problem_1(): contentrange = ContentRange( 0, 99, 100 ) assert_equal( contentrange.parse( 'bytes A-99/100' ), None )
def test_cr_parse_invalid_length(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-99/xxx'), None)
def test_cr_parse_missing_slash(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-99 100"), None)
def test_cr_parse_no_range(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0 99/100'), None)
def test_cr_parse_no_range(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0 99/100"), None)
def test_cr_parse_range_star(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes */100').__class__, ContentRange)
def test_cr_parse_parse_problem_2(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0-B/100"), None)
def test_cr_parse_parse_problem_2(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0-B/100'), None)
def test_contentrange_str_length_start(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse("bytes 0 99/*"), None)
def test_cr_parse_content_invalid(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 99-0/100'), None)
def test_contentrange_str_length_start(): contentrange = ContentRange(0, 99, 100) assert_equal(contentrange.parse('bytes 0 99/*'), None)
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) assert isinstance(contentrange, Iterable) assert ContentRange.parse("bytes 0-99/100").__class__ == ContentRange assert ContentRange.parse(None) is None assert ContentRange.parse("0-99 100") is None assert ContentRange.parse("bytes 0-99 100") is None assert ContentRange.parse("bytes 0-99/xxx") is None assert ContentRange.parse("bytes 0 99/100") is None assert ContentRange.parse("bytes */100").__class__ == ContentRange assert ContentRange.parse("bytes A-99/100") is None assert ContentRange.parse("bytes 0-B/100") is None assert ContentRange.parse("bytes 99-0/100") is None assert ContentRange.parse("bytes 0 99/*") is None
def test_cr_parse_ok(): contentrange = ContentRange(0, 99, 100) assert_true(contentrange.parse("bytes 0-99/100").__class__, ContentRange)
def parse_content_range(value): if not value or not value.strip(): return None # May still return None return ContentRange.parse(value)
def test_contentrange_iter(): contentrange = ContentRange(0, 99, 100) import collections assert isinstance(contentrange, collections.Iterable) assert ContentRange.parse('bytes 0-99/100').__class__ == ContentRange assert ContentRange.parse(None) is None assert ContentRange.parse('0-99 100') is None assert ContentRange.parse('bytes 0-99 100') is None assert ContentRange.parse('bytes 0-99/xxx') is None assert ContentRange.parse('bytes 0 99/100') is None assert ContentRange.parse('bytes */100').__class__ == ContentRange assert ContentRange.parse('bytes A-99/100') is None assert ContentRange.parse('bytes 0-B/100') is None assert ContentRange.parse('bytes 99-0/100') is None assert ContentRange.parse('bytes 0 99/*') is None