def handle(self, *args, **options):
"""
Regenerates the thumbnails of all image user files. If the USER_THUMBNAILS
setting ever changes then this file can be used to fix all the thumbnails.
"""
i = 0
handler = UserFileHandler()
buffer_size = 100
queryset = UserFile.objects.filter(is_image=True)
count = queryset.count()
while i < count:
user_files = queryset[i:min(count, i + buffer_size)]
for user_file in user_files:
i += 1
full_path = handler.user_file_path(user_file)
stream = default_storage.open(full_path)
try:
image = Image.open(stream)
handler.generate_and_save_image_thumbnails(
image, user_file, storage=default_storage)
image.close()
except IOError:
pass
stream.close()
self.stdout.write(
self.style.SUCCESS(f"{i} thumbnails have been regenerated."))
def test_user_file_path(data_fixture):
handler = UserFileHandler()
assert handler.user_file_path("test.jpg") == "user_files/test.jpg"
assert handler.user_file_path("another_file.png") == "user_files/another_file.png"
user_file = data_fixture.create_user_file()
assert handler.user_file_path(user_file) == f"user_files/{user_file.name}"
def get_export_serialized_value(self, row, field_name, cache, files_zip,
storage):
file_names = []
user_file_handler = UserFileHandler()
for file in getattr(row, field_name):
# Check if the user file object is already in the cache and if not,
# it must be fetched and added to to it.
cache_entry = f"user_file_{file['name']}"
if cache_entry not in cache:
try:
user_file = UserFile.objects.all().name(file["name"]).get()
except UserFile.DoesNotExist:
continue
if file["name"] not in files_zip.namelist():
# Load the user file from the content and write it to the zip file
# because it might not exist in the environment that it is going
# to be imported in.
file_path = user_file_handler.user_file_path(
user_file.name)
with storage.open(file_path, mode="rb") as storage_file:
files_zip.writestr(file["name"], storage_file.read())
cache[cache_entry] = user_file
file_names.append({
"name":
file["name"],
"visible_name":
file["visible_name"],
"original_name":
cache[cache_entry].original_name,
})
return file_names
def test_user_file_path(data_fixture):
handler = UserFileHandler()
assert handler.user_file_path('test.jpg') == 'user_files/test.jpg'
assert handler.user_file_path('another_file.png') == 'user_files/another_file.png'
user_file = data_fixture.create_user_file()
assert handler.user_file_path(user_file) == f'user_files/{user_file.name}'
def test_user_file_thumbnail_path(data_fixture):
handler = UserFileHandler()
assert handler.user_file_thumbnail_path(
'test.jpg', 'tiny') == 'thumbnails/tiny/test.jpg'
assert handler.user_file_thumbnail_path(
'another_file.png', 'small') == 'thumbnails/small/another_file.png'
user_file = data_fixture.create_user_file()
assert handler.user_file_thumbnail_path(
user_file, 'tiny') == f'thumbnails/tiny/{user_file.name}'
def post(self, request, data):
"""Uploads a user file by downloading it from the provided URL."""
url = data['url']
user_file = UserFileHandler().upload_user_file_by_url(request.user, url)
serializer = UserFileSerializer(user_file)
return Response(serializer.data)
def test_user_file_thumbnail_path(data_fixture):
handler = UserFileHandler()
assert (
handler.user_file_thumbnail_path("test.jpg", "tiny")
== "thumbnails/tiny/test.jpg"
)
assert (
handler.user_file_thumbnail_path("another_file.png", "small")
== "thumbnails/small/another_file.png"
)
user_file = data_fixture.create_user_file()
assert (
handler.user_file_thumbnail_path(user_file, "tiny")
== f"thumbnails/tiny/{user_file.name}"
)
def set_import_serialized_value(self, row, field_name, value, id_mapping,
files_zip, storage):
user_file_handler = UserFileHandler()
files = []
for file in value:
with files_zip.open(file["name"]) as stream:
# Try to upload the user file with the original name to make sure
# that if the was already uploaded, it will not be uploaded again.
user_file = user_file_handler.upload_user_file(
None, file["original_name"], stream, storage=storage)
value = user_file.serialize()
value["visible_name"] = file["visible_name"]
files.append(value)
setattr(row, field_name, files)
def post(self, request):
"""Uploads a file by uploading the contents directly."""
if 'file' not in request.FILES:
raise InvalidFileStreamError('No file was provided.')
file = request.FILES.get('file')
user_file = UserFileHandler().upload_user_file(request.user, file.name, file)
serializer = UserFileSerializer(user_file)
return Response(serializer.data)
def test_upload_user_file_by_url(data_fixture, tmpdir):
user = data_fixture.create_user()
storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost")
handler = UserFileHandler()
responses.add(
responses.GET,
"https://baserow.io/test.txt",
body=b"Hello World",
status=200,
content_type="text/plain",
stream=True,
)
responses.add(
responses.GET,
"https://baserow.io/not-found.pdf",
status=404,
)
# Could not be reached because it it responds with a 404
with pytest.raises(FileURLCouldNotBeReached):
handler.upload_user_file_by_url(
user, "https://baserow.io/not-found.pdf", storage=storage
)
# Only the http and https protocol are supported.
with pytest.raises(InvalidFileURLError):
handler.upload_user_file_by_url(
user, "ftp://baserow.io/not-found.pdf", storage=storage
)
with freeze_time("2020-01-01 12:00"):
user_file = handler.upload_user_file_by_url(
user, "https://baserow.io/test.txt", storage=storage
)
assert user_file.original_name == "test.txt"
assert user_file.original_extension == "txt"
assert len(user_file.unique) == 32
assert user_file.size == 11
assert user_file.mime_type == "text/plain"
assert user_file.uploaded_by_id == user.id
assert user_file.uploaded_at.year == 2020
assert user_file.uploaded_at.month == 1
assert user_file.uploaded_at.day == 1
assert user_file.is_image is False
assert user_file.image_width is None
assert user_file.image_height is None
assert user_file.sha256_hash == (
"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
)
file_path = tmpdir.join("user_files", user_file.name)
assert file_path.isfile()
assert file_path.open().read() == "Hello World"
def test_upload_user_file_by_url(data_fixture, tmpdir):
user = data_fixture.create_user()
storage = FileSystemStorage(location=str(tmpdir),
base_url='http://localhost')
handler = UserFileHandler()
responses.add(
responses.GET,
'http://localhost/test.txt',
body=b'Hello World',
status=200,
content_type="text/plain",
stream=True,
)
responses.add(
responses.GET,
'http://localhost/not-found.pdf',
body=b'Hello World',
status=404,
content_type="application/pdf",
stream=True,
)
with pytest.raises(FileURLCouldNotBeReached):
handler.upload_user_file_by_url(user,
'http://localhost/test2.txt',
storage=storage)
with freeze_time('2020-01-01 12:00'):
user_file = handler.upload_user_file_by_url(
user, 'http://localhost/test.txt', storage=storage)
with pytest.raises(FileURLCouldNotBeReached):
handler.upload_user_file_by_url(user,
'http://localhost/not-found.pdf',
storage=storage)
assert user_file.original_name == 'test.txt'
assert user_file.original_extension == 'txt'
assert len(user_file.unique) == 32
assert user_file.size == 11
assert user_file.mime_type == 'text/plain'
assert user_file.uploaded_by_id == user.id
assert user_file.uploaded_at.year == 2020
assert user_file.uploaded_at.month == 1
assert user_file.uploaded_at.day == 1
assert user_file.is_image is False
assert user_file.image_width is None
assert user_file.image_height is None
assert user_file.sha256_hash == (
'a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e')
file_path = tmpdir.join('user_files', user_file.name)
assert file_path.isfile()
assert file_path.open().read() == 'Hello World'
def test_generate_unique(data_fixture):
user = data_fixture.create_user()
handler = UserFileHandler()
assert len(handler.generate_unique("test", "txt", 32)) == 32
assert len(handler.generate_unique("test", "txt", 10)) == 10
assert handler.generate_unique("test", "txt", 32) != handler.generate_unique(
"test", "txt", 32
)
unique = handler.generate_unique("test", "txt", 32)
assert not UserFile.objects.filter(unique=unique).exists()
for char in string.ascii_letters + string.digits:
data_fixture.create_user_file(
uploaded_by=user, unique=char, original_extension="txt", sha256_hash="test"
)
with pytest.raises(MaximumUniqueTriesError):
handler.generate_unique("test", "txt", 1, 3)
handler.generate_unique("test2", "txt", 1, 3)
handler.generate_unique("test", "txt2", 1, 3)
def get_thumbnails(self, instance):
if not self.get_instance_attr(instance, 'is_image'):
return None
name = self.get_instance_attr(instance, 'name')
return {
thumbnail_name: {
'url':
default_storage.url(UserFileHandler().user_file_thumbnail_path(
name, thumbnail_name)),
'width':
size[0],
'height':
size[1]
}
for thumbnail_name, size in settings.USER_THUMBNAILS.items()
}
def get_thumbnails(self, instance):
if not self.get_instance_attr(instance, "is_image"):
return None
name = self.get_instance_attr(instance, "name")
return {
thumbnail_name: {
"url":
default_storage.url(UserFileHandler().user_file_thumbnail_path(
name, thumbnail_name)),
"width":
size[0],
"height":
size[1],
}
for thumbnail_name, size in settings.USER_THUMBNAILS.items()
}
def test_upload_user_file_by_url_within_private_network(data_fixture, tmpdir):
user = data_fixture.create_user()
storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost")
handler = UserFileHandler()
# Could not be reached because it is an internal private URL.
with pytest.raises(FileURLCouldNotBeReached):
handler.upload_user_file_by_url(
user, "http://localhost/test.txt", storage=storage
)
with pytest.raises(FileURLCouldNotBeReached):
handler.upload_user_file_by_url(
user, "http://192.168.1.1/test.txt", storage=storage
)
def test_upload_user_file(data_fixture, tmpdir):
user = data_fixture.create_user()
storage = FileSystemStorage(location=str(tmpdir), base_url='http://localhost')
handler = UserFileHandler()
with pytest.raises(InvalidFileStreamError):
handler.upload_user_file(
user,
'test.txt',
'NOT A STREAM!',
storage=storage
)
with pytest.raises(InvalidFileStreamError):
handler.upload_user_file(
user,
'test.txt',
None,
storage=storage
)
old_limit = settings.USER_FILE_SIZE_LIMIT
settings.USER_FILE_SIZE_LIMIT = 6
with pytest.raises(FileSizeTooLargeError):
handler.upload_user_file(
user,
'test.txt',
ContentFile(b'Hello World')
)
settings.USER_FILE_SIZE_LIMIT = old_limit
with freeze_time('2020-01-01 12:00'):
user_file = handler.upload_user_file(
user,
'test.txt',
ContentFile(b'Hello World'),
storage=storage
)
assert user_file.original_name == 'test.txt'
assert user_file.original_extension == 'txt'
assert len(user_file.unique) == 32
assert user_file.size == 11
assert user_file.mime_type == 'text/plain'
assert user_file.uploaded_by_id == user.id
assert user_file.uploaded_at.year == 2020
assert user_file.uploaded_at.month == 1
assert user_file.uploaded_at.day == 1
assert user_file.is_image is False
assert user_file.image_width is None
assert user_file.image_height is None
assert user_file.sha256_hash == (
'a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e'
)
file_path = tmpdir.join('user_files', user_file.name)
assert file_path.isfile()
assert file_path.open().read() == 'Hello World'
user_file = handler.upload_user_file(
user,
'another.txt',
BytesIO(b'Hello'),
storage=storage
)
assert user_file.original_name == 'another.txt'
assert user_file.original_extension == 'txt'
assert user_file.mime_type == 'text/plain'
assert user_file.size == 5
assert user_file.sha256_hash == (
'185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969'
)
file_path = tmpdir.join('user_files', user_file.name)
assert file_path.isfile()
assert file_path.open().read() == 'Hello'
assert (
handler.upload_user_file(
user,
'another.txt',
ContentFile(b'Hello'),
storage=storage
).id == user_file.id
)
assert handler.upload_user_file(
user,
'another_name.txt',
ContentFile(b'Hello'),
storage=storage
).id != user_file.id
image = Image.new('RGB', (100, 140), color='red')
image_bytes = BytesIO()
image.save(image_bytes, format='PNG')
user_file = handler.upload_user_file(
user,
'some image.png',
image_bytes,
storage=storage
)
assert user_file.mime_type == 'image/png'
assert user_file.is_image is True
assert user_file.image_width == 100
assert user_file.image_height == 140
file_path = tmpdir.join('user_files', user_file.name)
assert file_path.isfile()
file_path = tmpdir.join('thumbnails', 'tiny', user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open('rb'))
assert thumbnail.height == 21
assert thumbnail.width == 21
old_thumbnail_settings = settings.USER_THUMBNAILS
settings.USER_THUMBNAILS = {'tiny': [None, 100]}
image = Image.new('RGB', (1920, 1080), color='red')
image_bytes = BytesIO()
image.save(image_bytes, format='PNG')
user_file = handler.upload_user_file(
user,
'red.png',
image_bytes,
storage=storage
)
file_path = tmpdir.join('thumbnails', 'tiny', user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open('rb'))
assert thumbnail.width == 178
assert thumbnail.height == 100
image = Image.new('RGB', (400, 400), color='red')
image_bytes = BytesIO()
image.save(image_bytes, format='PNG')
user_file = handler.upload_user_file(
user,
'red2.png',
image_bytes,
storage=storage
)
file_path = tmpdir.join('thumbnails', 'tiny', user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open('rb'))
assert thumbnail.width == 100
assert thumbnail.height == 100
settings.USER_THUMBNAILS = {'tiny': [21, None]}
image = Image.new('RGB', (1400, 1000), color='red')
image_bytes = BytesIO()
image.save(image_bytes, format='PNG')
user_file = handler.upload_user_file(
user,
'red3.png',
image_bytes,
storage=storage
)
file_path = tmpdir.join('thumbnails', 'tiny', user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open('rb'))
assert thumbnail.width == 21
assert thumbnail.height == 15
settings.USER_THUMBNAILS = old_thumbnail_settings
assert UserFile.objects.all().count() == 7
image = Image.new('RGB', (1, 1), color='red')
image_bytes = BytesIO()
image.save(image_bytes, format='PNG')
user_file = handler.upload_user_file(
user,
'this_file_has_an_extreme_long_file_name_that_should_not_make_the_system_'
'fail_hard_when_trying_to_upload.png',
image_bytes,
storage=storage
)
assert user_file.original_name == 'this_file_has_an_extreme_long_f...hard_when_' \
'trying_to_upload.png'
def get_url(self, instance):
name = self.get_instance_attr(instance, 'name')
path = UserFileHandler().user_file_path(name)
url = default_storage.url(path)
return url
def test_upload_user_file(data_fixture, tmpdir):
user = data_fixture.create_user()
storage = FileSystemStorage(location=str(tmpdir), base_url="http://localhost")
handler = UserFileHandler()
with pytest.raises(InvalidFileStreamError):
handler.upload_user_file(user, "test.txt", "NOT A STREAM!", storage=storage)
with pytest.raises(InvalidFileStreamError):
handler.upload_user_file(user, "test.txt", None, storage=storage)
old_limit = settings.USER_FILE_SIZE_LIMIT
settings.USER_FILE_SIZE_LIMIT = 6
with pytest.raises(FileSizeTooLargeError):
handler.upload_user_file(user, "test.txt", ContentFile(b"Hello World"))
settings.USER_FILE_SIZE_LIMIT = old_limit
with freeze_time("2020-01-01 12:00"):
user_file = handler.upload_user_file(
user, "test.txt", ContentFile(b"Hello World"), storage=storage
)
assert user_file.original_name == "test.txt"
assert user_file.original_extension == "txt"
assert len(user_file.unique) == 32
assert user_file.size == 11
assert user_file.mime_type == "text/plain"
assert user_file.uploaded_by_id == user.id
assert user_file.uploaded_at.year == 2020
assert user_file.uploaded_at.month == 1
assert user_file.uploaded_at.day == 1
assert user_file.is_image is False
assert user_file.image_width is None
assert user_file.image_height is None
assert user_file.sha256_hash == (
"a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e"
)
file_path = tmpdir.join("user_files", user_file.name)
assert file_path.isfile()
assert file_path.open().read() == "Hello World"
user_file = handler.upload_user_file(
user, "another.txt", BytesIO(b"Hello"), storage=storage
)
assert user_file.original_name == "another.txt"
assert user_file.original_extension == "txt"
assert user_file.mime_type == "text/plain"
assert user_file.size == 5
assert user_file.sha256_hash == (
"185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969"
)
file_path = tmpdir.join("user_files", user_file.name)
assert file_path.isfile()
assert file_path.open().read() == "Hello"
assert (
handler.upload_user_file(
user, "another.txt", ContentFile(b"Hello"), storage=storage
).id
== user_file.id
)
assert (
handler.upload_user_file(
user, "another_name.txt", ContentFile(b"Hello"), storage=storage
).id
!= user_file.id
)
image = Image.new("RGB", (100, 140), color="red")
image_bytes = BytesIO()
image.save(image_bytes, format="PNG")
user_file = handler.upload_user_file(
user, "some image.png", image_bytes, storage=storage
)
assert user_file.mime_type == "image/png"
assert user_file.is_image is True
assert user_file.image_width == 100
assert user_file.image_height == 140
file_path = tmpdir.join("user_files", user_file.name)
assert file_path.isfile()
file_path = tmpdir.join("thumbnails", "tiny", user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open("rb"))
assert thumbnail.height == 21
assert thumbnail.width == 21
old_thumbnail_settings = settings.USER_THUMBNAILS
settings.USER_THUMBNAILS = {"tiny": [None, 100]}
image = Image.new("RGB", (1920, 1080), color="red")
image_bytes = BytesIO()
image.save(image_bytes, format="PNG")
user_file = handler.upload_user_file(user, "red.png", image_bytes, storage=storage)
file_path = tmpdir.join("thumbnails", "tiny", user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open("rb"))
assert thumbnail.width == 178
assert thumbnail.height == 100
image = Image.new("RGB", (400, 400), color="red")
image_bytes = BytesIO()
image.save(image_bytes, format="PNG")
user_file = handler.upload_user_file(user, "red2.png", image_bytes, storage=storage)
file_path = tmpdir.join("thumbnails", "tiny", user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open("rb"))
assert thumbnail.width == 100
assert thumbnail.height == 100
settings.USER_THUMBNAILS = {"tiny": [21, None]}
image = Image.new("RGB", (1400, 1000), color="red")
image_bytes = BytesIO()
image.save(image_bytes, format="PNG")
user_file = handler.upload_user_file(user, "red3.png", image_bytes, storage=storage)
file_path = tmpdir.join("thumbnails", "tiny", user_file.name)
assert file_path.isfile()
thumbnail = Image.open(file_path.open("rb"))
assert thumbnail.width == 21
assert thumbnail.height == 15
settings.USER_THUMBNAILS = old_thumbnail_settings
assert UserFile.objects.all().count() == 7
image = Image.new("RGB", (1, 1), color="red")
image_bytes = BytesIO()
image.save(image_bytes, format="PNG")
user_file = handler.upload_user_file(
user,
"this_file_has_an_extreme_long_file_name_that_should_not_make_the_system_"
"fail_hard_when_trying_to_upload.png",
image_bytes,
storage=storage,
)
assert (
user_file.original_name == "this_file_has_an_extreme_long_f...hard_when_"
"trying_to_upload.png"
)
def test_import_export_file_field(data_fixture, tmpdir,
user_tables_in_separate_db):
user = data_fixture.create_user()
imported_group = data_fixture.create_group(user=user)
database = data_fixture.create_database_application(user=user)
table = data_fixture.create_database_table(database=database)
field = data_fixture.create_file_field(table=table, name="File")
storage = FileSystemStorage(location=str(tmpdir),
base_url="http://localhost")
handler = UserFileHandler()
user_file = handler.upload_user_file(user,
"test.txt",
ContentFile(b"Hello World"),
storage=storage)
core_handler = CoreHandler()
row_handler = RowHandler()
model = table.get_model()
row_1 = row_handler.create_row(
user=user,
table=table,
values={
f"field_{field.id}": [{
"name": user_file.name,
"visible_name": "a.txt"
}]
},
model=model,
)
row_2 = row_handler.create_row(
user=user,
table=table,
values={},
model=model,
)
row_3 = row_handler.create_row(
user=user,
table=table,
values={f"field_{field.id}": [{
"name": user_file.name
}]},
model=model,
)
files_buffer = BytesIO()
exported_applications = core_handler.export_group_applications(
database.group, files_buffer=files_buffer, storage=storage)
# We expect that the exported zip file contains the user file used in the created
# rows.
with ZipFile(files_buffer, "r", ZIP_DEFLATED, False) as zip_file:
assert zip_file.read(user_file.name) == b"Hello World"
assert (exported_applications[0]["tables"][0]["rows"][0]
[f"field_{field.id}"][0]["name"] == user_file.name)
assert (
exported_applications[0]["tables"][0]["rows"][0][f"field_{field.id}"]
[0]["original_name"] == user_file.original_name)
assert exported_applications[0]["tables"][0]["rows"][1][
f"field_{field.id}"] == []
assert (exported_applications[0]["tables"][0]["rows"][2]
[f"field_{field.id}"][0]["name"] == user_file.name)
# Change the original name for enforce that the file is re-uploaded when saved.
exported_applications[0]["tables"][0]["rows"][0][f"field_{field.id}"][0][
"original_name"] = "test2.txt"
exported_applications[0]["tables"][0]["rows"][2][f"field_{field.id}"][0][
"original_name"] = "test2.txt"
imported_applications, id_mapping = core_handler.import_applications_to_group(
imported_group, exported_applications, files_buffer, storage)
imported_database = imported_applications[0]
imported_tables = imported_database.table_set.all()
imported_table = imported_tables[0]
imported_field = imported_table.field_set.all().first().specific
imported_user_file = UserFile.objects.all()[1]
import_row_1 = row_handler.get_row(user=user,
table=imported_table,
row_id=row_1.id)
import_row_2 = row_handler.get_row(user=user,
table=imported_table,
row_id=row_2.id)
import_row_3 = row_handler.get_row(user=user,
table=imported_table,
row_id=row_3.id)
assert len(getattr(import_row_1, f"field_{imported_field.id}")) == 1
assert (getattr(
import_row_1,
f"field_{imported_field.id}")[0]["name"] == imported_user_file.name)
assert (getattr(
import_row_1,
f"field_{imported_field.id}")[0]["visible_name"] == "a.txt")
assert len(getattr(import_row_2, f"field_{imported_field.id}")) == 0
assert len(getattr(import_row_3, f"field_{imported_field.id}")) == 1
assert (getattr(
import_row_3,
f"field_{imported_field.id}")[0]["name"] == imported_user_file.name)
assert (getattr(
import_row_3,
f"field_{imported_field.id}")[0]["visible_name"] == "test.txt")
assert UserFile.objects.all().count() == 2
assert user_file.name != imported_user_file.name
file_path = tmpdir.join("user_files", imported_user_file.name)
assert file_path.isfile()
assert file_path.open().read() == "Hello World"
