def handle(self, *args, **options): """ Regenerates the thumbnails of all image user files. If the USER_THUMBNAILS setting ever changes then this file can be used to fix all the thumbnails. """ i = 0 handler = UserFileHandler() buffer_size = 100 queryset = UserFile.objects.filter(is_image=True) count = queryset.count() while i < count: user_files = queryset[i:min(count, i + buffer_size)] for user_file in user_files: i += 1 full_path = handler.user_file_path(user_file) stream = default_storage.open(full_path) try: image = Image.open(stream) handler.generate_and_save_image_thumbnails( image, user_file, storage=default_storage) image.close() except IOError: pass stream.close() self.stdout.write( self.style.SUCCESS(f"{i} thumbnails have been regenerated."))
def test_user_file_path(data_fixture): handler = UserFileHandler() assert handler.user_file_path("test.jpg") == "user_files/test.jpg" assert handler.user_file_path("another_file.png") == "user_files/another_file.png" user_file = data_fixture.create_user_file() assert handler.user_file_path(user_file) == f"user_files/{user_file.name}"
def get_export_serialized_value(self, row, field_name, cache, files_zip, storage): file_names = [] user_file_handler = UserFileHandler() for file in getattr(row, field_name): # Check if the user file object is already in the cache and if not, # it must be fetched and added to to it. cache_entry = f"user_file_{file['name']}" if cache_entry not in cache: try: user_file = UserFile.objects.all().name(file["name"]).get() except UserFile.DoesNotExist: continue if file["name"] not in files_zip.namelist(): # Load the user file from the content and write it to the zip file # because it might not exist in the environment that it is going # to be imported in. file_path = user_file_handler.user_file_path( user_file.name) with storage.open(file_path, mode="rb") as storage_file: files_zip.writestr(file["name"], storage_file.read()) cache[cache_entry] = user_file file_names.append({ "name": file["name"], "visible_name": file["visible_name"], "original_name": cache[cache_entry].original_name, }) return file_names
def test_user_file_path(data_fixture): handler = UserFileHandler() assert handler.user_file_path('test.jpg') == 'user_files/test.jpg' assert handler.user_file_path('another_file.png') == 'user_files/another_file.png' user_file = data_fixture.create_user_file() assert handler.user_file_path(user_file) == f'user_files/{user_file.name}'
def test_user_file_thumbnail_path(data_fixture): handler = UserFileHandler() assert handler.user_file_thumbnail_path( 'test.jpg', 'tiny') == 'thumbnails/tiny/test.jpg' assert handler.user_file_thumbnail_path( 'another_file.png', 'small') == 'thumbnails/small/another_file.png' user_file = data_fixture.create_user_file() assert handler.user_file_thumbnail_path( user_file, 'tiny') == f'thumbnails/tiny/{user_file.name}'
def post(self, request, data): """Uploads a user file by downloading it from the provided URL.""" url = data['url'] user_file = UserFileHandler().upload_user_file_by_url(request.user, url) serializer = UserFileSerializer(user_file) return Response(serializer.data)
def test_user_file_thumbnail_path(data_fixture): handler = UserFileHandler() assert ( handler.user_file_thumbnail_path("test.jpg", "tiny") == "thumbnails/tiny/test.jpg" ) assert ( handler.user_file_thumbnail_path("another_file.png", "small") == "thumbnails/small/another_file.png" ) user_file = data_fixture.create_user_file() assert ( handler.user_file_thumbnail_path(user_file, "tiny") == f"thumbnails/tiny/{user_file.name}" )
def set_import_serialized_value(self, row, field_name, value, id_mapping, files_zip, storage): user_file_handler = UserFileHandler() files = [] for file in value: with files_zip.open(file["name"]) as stream: # Try to upload the user file with the original name to make sure # that if the was already uploaded, it will not be uploaded again. user_file = user_file_handler.upload_user_file( None, file["original_name"], stream, storage=storage) value = user_file.serialize() value["visible_name"] = file["visible_name"] files.append(value) setattr(row, field_name, files)
def post(self, request): """Uploads a file by uploading the contents directly.""" if 'file' not in request.FILES: raise InvalidFileStreamError('No file was provided.') file = request.FILES.get('file') user_file = UserFileHandler().upload_user_file(request.user, file.name, file) serializer = UserFileSerializer(user_file) return Response(serializer.data)
def test_upload_user_file_by_url(data_fixture, tmpdir): user = data_fixture.create_user() storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost") handler = UserFileHandler() responses.add( responses.GET, "https://baserow.io/test.txt", body=b"Hello World", status=200, content_type="text/plain", stream=True, ) responses.add( responses.GET, "https://baserow.io/not-found.pdf", status=404, ) # Could not be reached because it it responds with a 404 with pytest.raises(FileURLCouldNotBeReached): handler.upload_user_file_by_url( user, "https://baserow.io/not-found.pdf", storage=storage ) # Only the http and https protocol are supported. with pytest.raises(InvalidFileURLError): handler.upload_user_file_by_url( user, "ftp://baserow.io/not-found.pdf", storage=storage ) with freeze_time("2020-01-01 12:00"): user_file = handler.upload_user_file_by_url( user, "https://baserow.io/test.txt", storage=storage ) assert user_file.original_name == "test.txt" assert user_file.original_extension == "txt" assert len(user_file.unique) == 32 assert user_file.size == 11 assert user_file.mime_type == "text/plain" assert user_file.uploaded_by_id == user.id assert user_file.uploaded_at.year == 2020 assert user_file.uploaded_at.month == 1 assert user_file.uploaded_at.day == 1 assert user_file.is_image is False assert user_file.image_width is None assert user_file.image_height is None assert user_file.sha256_hash == ( "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" ) file_path = tmpdir.join("user_files", user_file.name) assert file_path.isfile() assert file_path.open().read() == "Hello World"
def test_upload_user_file_by_url(data_fixture, tmpdir): user = data_fixture.create_user() storage = FileSystemStorage(location=str(tmpdir), base_url='http://localhost') handler = UserFileHandler() responses.add( responses.GET, 'http://localhost/test.txt', body=b'Hello World', status=200, content_type="text/plain", stream=True, ) responses.add( responses.GET, 'http://localhost/not-found.pdf', body=b'Hello World', status=404, content_type="application/pdf", stream=True, ) with pytest.raises(FileURLCouldNotBeReached): handler.upload_user_file_by_url(user, 'http://localhost/test2.txt', storage=storage) with freeze_time('2020-01-01 12:00'): user_file = handler.upload_user_file_by_url( user, 'http://localhost/test.txt', storage=storage) with pytest.raises(FileURLCouldNotBeReached): handler.upload_user_file_by_url(user, 'http://localhost/not-found.pdf', storage=storage) assert user_file.original_name == 'test.txt' assert user_file.original_extension == 'txt' assert len(user_file.unique) == 32 assert user_file.size == 11 assert user_file.mime_type == 'text/plain' assert user_file.uploaded_by_id == user.id assert user_file.uploaded_at.year == 2020 assert user_file.uploaded_at.month == 1 assert user_file.uploaded_at.day == 1 assert user_file.is_image is False assert user_file.image_width is None assert user_file.image_height is None assert user_file.sha256_hash == ( 'a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e') file_path = tmpdir.join('user_files', user_file.name) assert file_path.isfile() assert file_path.open().read() == 'Hello World'
def test_generate_unique(data_fixture): user = data_fixture.create_user() handler = UserFileHandler() assert len(handler.generate_unique("test", "txt", 32)) == 32 assert len(handler.generate_unique("test", "txt", 10)) == 10 assert handler.generate_unique("test", "txt", 32) != handler.generate_unique( "test", "txt", 32 ) unique = handler.generate_unique("test", "txt", 32) assert not UserFile.objects.filter(unique=unique).exists() for char in string.ascii_letters + string.digits: data_fixture.create_user_file( uploaded_by=user, unique=char, original_extension="txt", sha256_hash="test" ) with pytest.raises(MaximumUniqueTriesError): handler.generate_unique("test", "txt", 1, 3) handler.generate_unique("test2", "txt", 1, 3) handler.generate_unique("test", "txt2", 1, 3)
def get_thumbnails(self, instance): if not self.get_instance_attr(instance, 'is_image'): return None name = self.get_instance_attr(instance, 'name') return { thumbnail_name: { 'url': default_storage.url(UserFileHandler().user_file_thumbnail_path( name, thumbnail_name)), 'width': size[0], 'height': size[1] } for thumbnail_name, size in settings.USER_THUMBNAILS.items() }
def get_thumbnails(self, instance): if not self.get_instance_attr(instance, "is_image"): return None name = self.get_instance_attr(instance, "name") return { thumbnail_name: { "url": default_storage.url(UserFileHandler().user_file_thumbnail_path( name, thumbnail_name)), "width": size[0], "height": size[1], } for thumbnail_name, size in settings.USER_THUMBNAILS.items() }
def test_upload_user_file_by_url_within_private_network(data_fixture, tmpdir): user = data_fixture.create_user() storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost") handler = UserFileHandler() # Could not be reached because it is an internal private URL. with pytest.raises(FileURLCouldNotBeReached): handler.upload_user_file_by_url( user, "http://localhost/test.txt", storage=storage ) with pytest.raises(FileURLCouldNotBeReached): handler.upload_user_file_by_url( user, "http://192.168.1.1/test.txt", storage=storage )
def test_upload_user_file(data_fixture, tmpdir): user = data_fixture.create_user() storage = FileSystemStorage(location=str(tmpdir), base_url='http://localhost') handler = UserFileHandler() with pytest.raises(InvalidFileStreamError): handler.upload_user_file( user, 'test.txt', 'NOT A STREAM!', storage=storage ) with pytest.raises(InvalidFileStreamError): handler.upload_user_file( user, 'test.txt', None, storage=storage ) old_limit = settings.USER_FILE_SIZE_LIMIT settings.USER_FILE_SIZE_LIMIT = 6 with pytest.raises(FileSizeTooLargeError): handler.upload_user_file( user, 'test.txt', ContentFile(b'Hello World') ) settings.USER_FILE_SIZE_LIMIT = old_limit with freeze_time('2020-01-01 12:00'): user_file = handler.upload_user_file( user, 'test.txt', ContentFile(b'Hello World'), storage=storage ) assert user_file.original_name == 'test.txt' assert user_file.original_extension == 'txt' assert len(user_file.unique) == 32 assert user_file.size == 11 assert user_file.mime_type == 'text/plain' assert user_file.uploaded_by_id == user.id assert user_file.uploaded_at.year == 2020 assert user_file.uploaded_at.month == 1 assert user_file.uploaded_at.day == 1 assert user_file.is_image is False assert user_file.image_width is None assert user_file.image_height is None assert user_file.sha256_hash == ( 'a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e' ) file_path = tmpdir.join('user_files', user_file.name) assert file_path.isfile() assert file_path.open().read() == 'Hello World' user_file = handler.upload_user_file( user, 'another.txt', BytesIO(b'Hello'), storage=storage ) assert user_file.original_name == 'another.txt' assert user_file.original_extension == 'txt' assert user_file.mime_type == 'text/plain' assert user_file.size == 5 assert user_file.sha256_hash == ( '185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969' ) file_path = tmpdir.join('user_files', user_file.name) assert file_path.isfile() assert file_path.open().read() == 'Hello' assert ( handler.upload_user_file( user, 'another.txt', ContentFile(b'Hello'), storage=storage ).id == user_file.id ) assert handler.upload_user_file( user, 'another_name.txt', ContentFile(b'Hello'), storage=storage ).id != user_file.id image = Image.new('RGB', (100, 140), color='red') image_bytes = BytesIO() image.save(image_bytes, format='PNG') user_file = handler.upload_user_file( user, 'some image.png', image_bytes, storage=storage ) assert user_file.mime_type == 'image/png' assert user_file.is_image is True assert user_file.image_width == 100 assert user_file.image_height == 140 file_path = tmpdir.join('user_files', user_file.name) assert file_path.isfile() file_path = tmpdir.join('thumbnails', 'tiny', user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open('rb')) assert thumbnail.height == 21 assert thumbnail.width == 21 old_thumbnail_settings = settings.USER_THUMBNAILS settings.USER_THUMBNAILS = {'tiny': [None, 100]} image = Image.new('RGB', (1920, 1080), color='red') image_bytes = BytesIO() image.save(image_bytes, format='PNG') user_file = handler.upload_user_file( user, 'red.png', image_bytes, storage=storage ) file_path = tmpdir.join('thumbnails', 'tiny', user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open('rb')) assert thumbnail.width == 178 assert thumbnail.height == 100 image = Image.new('RGB', (400, 400), color='red') image_bytes = BytesIO() image.save(image_bytes, format='PNG') user_file = handler.upload_user_file( user, 'red2.png', image_bytes, storage=storage ) file_path = tmpdir.join('thumbnails', 'tiny', user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open('rb')) assert thumbnail.width == 100 assert thumbnail.height == 100 settings.USER_THUMBNAILS = {'tiny': [21, None]} image = Image.new('RGB', (1400, 1000), color='red') image_bytes = BytesIO() image.save(image_bytes, format='PNG') user_file = handler.upload_user_file( user, 'red3.png', image_bytes, storage=storage ) file_path = tmpdir.join('thumbnails', 'tiny', user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open('rb')) assert thumbnail.width == 21 assert thumbnail.height == 15 settings.USER_THUMBNAILS = old_thumbnail_settings assert UserFile.objects.all().count() == 7 image = Image.new('RGB', (1, 1), color='red') image_bytes = BytesIO() image.save(image_bytes, format='PNG') user_file = handler.upload_user_file( user, 'this_file_has_an_extreme_long_file_name_that_should_not_make_the_system_' 'fail_hard_when_trying_to_upload.png', image_bytes, storage=storage ) assert user_file.original_name == 'this_file_has_an_extreme_long_f...hard_when_' \ 'trying_to_upload.png'
def get_url(self, instance): name = self.get_instance_attr(instance, 'name') path = UserFileHandler().user_file_path(name) url = default_storage.url(path) return url
def test_upload_user_file(data_fixture, tmpdir): user = data_fixture.create_user() storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost") handler = UserFileHandler() with pytest.raises(InvalidFileStreamError): handler.upload_user_file(user, "test.txt", "NOT A STREAM!", storage=storage) with pytest.raises(InvalidFileStreamError): handler.upload_user_file(user, "test.txt", None, storage=storage) old_limit = settings.USER_FILE_SIZE_LIMIT settings.USER_FILE_SIZE_LIMIT = 6 with pytest.raises(FileSizeTooLargeError): handler.upload_user_file(user, "test.txt", ContentFile(b"Hello World")) settings.USER_FILE_SIZE_LIMIT = old_limit with freeze_time("2020-01-01 12:00"): user_file = handler.upload_user_file( user, "test.txt", ContentFile(b"Hello World"), storage=storage ) assert user_file.original_name == "test.txt" assert user_file.original_extension == "txt" assert len(user_file.unique) == 32 assert user_file.size == 11 assert user_file.mime_type == "text/plain" assert user_file.uploaded_by_id == user.id assert user_file.uploaded_at.year == 2020 assert user_file.uploaded_at.month == 1 assert user_file.uploaded_at.day == 1 assert user_file.is_image is False assert user_file.image_width is None assert user_file.image_height is None assert user_file.sha256_hash == ( "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e" ) file_path = tmpdir.join("user_files", user_file.name) assert file_path.isfile() assert file_path.open().read() == "Hello World" user_file = handler.upload_user_file( user, "another.txt", BytesIO(b"Hello"), storage=storage ) assert user_file.original_name == "another.txt" assert user_file.original_extension == "txt" assert user_file.mime_type == "text/plain" assert user_file.size == 5 assert user_file.sha256_hash == ( "185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969" ) file_path = tmpdir.join("user_files", user_file.name) assert file_path.isfile() assert file_path.open().read() == "Hello" assert ( handler.upload_user_file( user, "another.txt", ContentFile(b"Hello"), storage=storage ).id == user_file.id ) assert ( handler.upload_user_file( user, "another_name.txt", ContentFile(b"Hello"), storage=storage ).id != user_file.id ) image = Image.new("RGB", (100, 140), color="red") image_bytes = BytesIO() image.save(image_bytes, format="PNG") user_file = handler.upload_user_file( user, "some image.png", image_bytes, storage=storage ) assert user_file.mime_type == "image/png" assert user_file.is_image is True assert user_file.image_width == 100 assert user_file.image_height == 140 file_path = tmpdir.join("user_files", user_file.name) assert file_path.isfile() file_path = tmpdir.join("thumbnails", "tiny", user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open("rb")) assert thumbnail.height == 21 assert thumbnail.width == 21 old_thumbnail_settings = settings.USER_THUMBNAILS settings.USER_THUMBNAILS = {"tiny": [None, 100]} image = Image.new("RGB", (1920, 1080), color="red") image_bytes = BytesIO() image.save(image_bytes, format="PNG") user_file = handler.upload_user_file(user, "red.png", image_bytes, storage=storage) file_path = tmpdir.join("thumbnails", "tiny", user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open("rb")) assert thumbnail.width == 178 assert thumbnail.height == 100 image = Image.new("RGB", (400, 400), color="red") image_bytes = BytesIO() image.save(image_bytes, format="PNG") user_file = handler.upload_user_file(user, "red2.png", image_bytes, storage=storage) file_path = tmpdir.join("thumbnails", "tiny", user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open("rb")) assert thumbnail.width == 100 assert thumbnail.height == 100 settings.USER_THUMBNAILS = {"tiny": [21, None]} image = Image.new("RGB", (1400, 1000), color="red") image_bytes = BytesIO() image.save(image_bytes, format="PNG") user_file = handler.upload_user_file(user, "red3.png", image_bytes, storage=storage) file_path = tmpdir.join("thumbnails", "tiny", user_file.name) assert file_path.isfile() thumbnail = Image.open(file_path.open("rb")) assert thumbnail.width == 21 assert thumbnail.height == 15 settings.USER_THUMBNAILS = old_thumbnail_settings assert UserFile.objects.all().count() == 7 image = Image.new("RGB", (1, 1), color="red") image_bytes = BytesIO() image.save(image_bytes, format="PNG") user_file = handler.upload_user_file( user, "this_file_has_an_extreme_long_file_name_that_should_not_make_the_system_" "fail_hard_when_trying_to_upload.png", image_bytes, storage=storage, ) assert ( user_file.original_name == "this_file_has_an_extreme_long_f...hard_when_" "trying_to_upload.png" )
def test_import_export_file_field(data_fixture, tmpdir, user_tables_in_separate_db): user = data_fixture.create_user() imported_group = data_fixture.create_group(user=user) database = data_fixture.create_database_application(user=user) table = data_fixture.create_database_table(database=database) field = data_fixture.create_file_field(table=table, name="File") storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost") handler = UserFileHandler() user_file = handler.upload_user_file(user, "test.txt", ContentFile(b"Hello World"), storage=storage) core_handler = CoreHandler() row_handler = RowHandler() model = table.get_model() row_1 = row_handler.create_row( user=user, table=table, values={ f"field_{field.id}": [{ "name": user_file.name, "visible_name": "a.txt" }] }, model=model, ) row_2 = row_handler.create_row( user=user, table=table, values={}, model=model, ) row_3 = row_handler.create_row( user=user, table=table, values={f"field_{field.id}": [{ "name": user_file.name }]}, model=model, ) files_buffer = BytesIO() exported_applications = core_handler.export_group_applications( database.group, files_buffer=files_buffer, storage=storage) # We expect that the exported zip file contains the user file used in the created # rows. with ZipFile(files_buffer, "r", ZIP_DEFLATED, False) as zip_file: assert zip_file.read(user_file.name) == b"Hello World" assert (exported_applications[0]["tables"][0]["rows"][0] [f"field_{field.id}"][0]["name"] == user_file.name) assert ( exported_applications[0]["tables"][0]["rows"][0][f"field_{field.id}"] [0]["original_name"] == user_file.original_name) assert exported_applications[0]["tables"][0]["rows"][1][ f"field_{field.id}"] == [] assert (exported_applications[0]["tables"][0]["rows"][2] [f"field_{field.id}"][0]["name"] == user_file.name) # Change the original name for enforce that the file is re-uploaded when saved. exported_applications[0]["tables"][0]["rows"][0][f"field_{field.id}"][0][ "original_name"] = "test2.txt" exported_applications[0]["tables"][0]["rows"][2][f"field_{field.id}"][0][ "original_name"] = "test2.txt" imported_applications, id_mapping = core_handler.import_applications_to_group( imported_group, exported_applications, files_buffer, storage) imported_database = imported_applications[0] imported_tables = imported_database.table_set.all() imported_table = imported_tables[0] imported_field = imported_table.field_set.all().first().specific imported_user_file = UserFile.objects.all()[1] import_row_1 = row_handler.get_row(user=user, table=imported_table, row_id=row_1.id) import_row_2 = row_handler.get_row(user=user, table=imported_table, row_id=row_2.id) import_row_3 = row_handler.get_row(user=user, table=imported_table, row_id=row_3.id) assert len(getattr(import_row_1, f"field_{imported_field.id}")) == 1 assert (getattr( import_row_1, f"field_{imported_field.id}")[0]["name"] == imported_user_file.name) assert (getattr( import_row_1, f"field_{imported_field.id}")[0]["visible_name"] == "a.txt") assert len(getattr(import_row_2, f"field_{imported_field.id}")) == 0 assert len(getattr(import_row_3, f"field_{imported_field.id}")) == 1 assert (getattr( import_row_3, f"field_{imported_field.id}")[0]["name"] == imported_user_file.name) assert (getattr( import_row_3, f"field_{imported_field.id}")[0]["visible_name"] == "test.txt") assert UserFile.objects.all().count() == 2 assert user_file.name != imported_user_file.name file_path = tmpdir.join("user_files", imported_user_file.name) assert file_path.isfile() assert file_path.open().read() == "Hello World"