def new_file(self, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
# If file size is larger than the maximum allowed, then abort
if MAX_UPLOAD_SIZE:
if self.content_length > MAX_UPLOAD_SIZE:
raise UploadFailed('File too large')
super().new_file(*args, **kwargs)
self.parts = []
self.bucket_name = settings.AWS_STORAGE_BUCKET_NAME
self.s3_key = generate_object_key(self.request, self.file_name)
self.client = s3_client()
self.multipart = self.client.create_multipart_upload(
Bucket=self.bucket_name,
Key=self.s3_key
)
self.upload_id = self.multipart['UploadId']
self.executor = ThreadedS3ChunkUploader(
self.client,
self.bucket_name,
key=self.s3_key,
upload_id=self.upload_id
)
# prepare a storages object as a file placeholder
self.storage = S3Boto3Storage()
self.file = S3Boto3StorageFile(self.s3_key, 'w', self.storage)
self.file.original_name = self.file_name
def isfile(self, name):
try:
name = self._normalize_name(self._clean_name(name))
f = S3Boto3StorageFile(name, 'rb', self)
if not f:
return False
return True
except:
return False
def isfile(self, name):
try:
name = self._normalize_name(self._clean_name(name))
f = S3Boto3StorageFile(name, 'rb', self)
if "directory" in f.obj.content_type:
return False
return True
except Exception:
return False
def read_theme_file_from_s3(file_path):
"""
Read a theme file from S3.
The file_path should be the relative path in the media directory.
Example:
file_content = read_theme_file_from_s3('themename/templates/default.html')
"""
# the DEFAULT_S3_PATH is where the media files are stored.
file_path = '%s/%s' % (settings.THEME_S3_PATH, str(file_path).lstrip('/'))
storage = S3Boto3Storage()
f = S3Boto3StorageFile(file_path, 'r', storage)
content = f.read()
f.close()
return content
def read_media_file_from_s3(file_path):
"""
Read a media file from S3.
The file_path should be the relative path in the media directory.
Example:
file_content = read_media_file_from_s3('/files/99/Earth-and-Moon.gif')
"""
# the DEFAULT_S3_PATH is where the media files are stored.
file_path = '%s/%s' % (settings.DEFAULT_S3_PATH,
str(file_path).lstrip('/'))
storage = S3Boto3Storage()
f = S3Boto3StorageFile(file_path, 'r', storage)
content = f.read()
f.close()
return content
def test_image_from_hash(self):
storage = S3Boto3Storage(bucket=self.bucket)
s3_file = S3Boto3StorageFile(name=self.s3_image_key,
mode='r',
storage=storage)
# Mock model Image field
# S3Boto3StorageFile store storage information in ._storage wich is not checked by
# ImageFile during storage identification
s3_file.storage = storage
image_s3 = ImageFile(s3_file)
# Test local part
options = {}
thumbnail = get_thumbnail(s3_file, self.geometry_string, **options)
print('Generated thumbnail url: {}'.format(thumbnail.url))
print('Thumbnail cache key: {}'.format(thumbnail.name))
# Now, test shrinkmeister server (should be up and running)
resp = requests.get(thumbnail.url)
image = Image(blob=resp.content)
self.assertEqual(image.width, 50)
self.assertEqual(image.height, 38)
thumbnail_from_cache = self.cache.get(thumbnail.name)
self.assertNotEqual(thumbnail_from_cache,
None,
msg="No image in cache detected :(")
image = Image(blob=resp.content)
self.assertEqual(image.width, 50)
self.assertEqual(image.height, 38)
resp = requests.get(thumbnail_from_cache.url)
url, ext = thumbnail_from_cache.url.rsplit('.', 1)
x2_url = '{}@2x.{}'.format(url, ext)
print('x2 url {}'.format(x2_url))
resp = requests.get(x2_url)
image = Image(blob=resp.content)
self.assertEqual(image.width, 100)
self.assertEqual(image.height, 75)
def get_redirect_url(self, *args, **kwargs):
try:
data = signing.loads(kwargs['hash'],
key=settings.THUMBNAIL_SECRET_KEY)
bucket = data['bucket']
key = data['key']
geometry_string = data['geometry_string']
except KeyError:
raise Http404()
if 'options' in data:
options = merge_with_defaults(data['options'])
else:
options = {}
if 'cache_key' in data:
cache_key = data['cache_key']
else:
# !!! WARNING !!!
# Generating same cache_key on both sides may be tricky
# Preferred way is to transfer cache_key from client side
cache_key = generate_cache_key(bucket=bucket,
key=key,
geometry_string=geometry_string,
**options)
# Multiple calls for same image during thumbnail generation scenario
cached_thumbnail = shrinkmeister_cache.get(cache_key, None)
# if cached_thumbnail:
# return cached_thumbnail.url
client = boto3.client('s3')
storage = S3Boto3Storage(bucket=bucket)
s3_file = S3Boto3StorageFile(name=key, mode='r', storage=storage)
# Mock model Image field
# S3Boto3StorageFile store storage information in ._storage wich is not checked by
# ImageFile during storage identification
s3_file.storage = storage
options['cache_key'] = cache_key
thumbnail = get_thumbnail(s3_file, geometry_string, **options)
return thumbnail.url
def new_file(self, *args, **kwargs):
"""
Create the file object to append to as data is coming in.
"""
# If content_length is provided and file size is larger than the maximum allowed, then abort
# Note that as per Django docs, content_length is not guaranteed to be provided.
if MAX_UPLOAD_SIZE and self.content_length:
if self.content_length > MAX_UPLOAD_SIZE:
raise UploadFailed('File too large')
super().new_file(*args, **kwargs)
self.parts = []
self.bucket_name = AWS_STORAGE_BUCKET_NAME
file_name = self.file_name
if CLEAN_FILE_NAME:
file_name = slugify(self.file_name)
self.s3_key = generate_object_key(self.request, file_name)
self.client = s3_client()
self.multipart = self.client.create_multipart_upload(
Bucket=self.bucket_name,
Key=self.s3_key,
ContentType=self.content_type,
)
self.upload_id = self.multipart['UploadId']
self.executor = ThreadedS3ChunkUploader(
self.client,
self.bucket_name,
key=self.s3_key,
upload_id=self.upload_id
)
# prepare a storages object as a file placeholder
self.storage = S3Boto3Storage()
self.file = S3Boto3StorageFile(self.s3_key, 'w', self.storage)
self.file.original_name = self.file_name
self.file.content_type = self.content_type
def file_complete(self, file_size):
self.executor.add(None)
# Wait for all threads to complete
wait(
self.executor.futures, return_when=concurrent.futures.ALL_COMPLETED
)
parts = self.executor.get_parts()
self.s3_client.complete_multipart_upload(
Bucket=AWS_STORAGE_BUCKET_NAME,
Key=self.s3_key,
UploadId=self.upload_id,
MultipartUpload={"Parts": parts},
)
self.s3_client.copy_object(
Bucket=AWS_STORAGE_BUCKET_NAME,
CopySource=f"{AWS_STORAGE_BUCKET_NAME}/{self.s3_key}",
Key=self.new_file_name,
ContentType=self.content_type,
)
self.s3_client.delete_object(
Bucket=AWS_STORAGE_BUCKET_NAME,
Key=self.s3_key,
)
if "clam_av_results" in self.content_type_extra:
for result in self.content_type_extra["clam_av_results"]:
if result["file_name"] == self.file_name:
# Set AV headers
if result["av_passed"]:
self.s3_client.copy_object(
Bucket=AWS_STORAGE_BUCKET_NAME,
CopySource=f"{AWS_STORAGE_BUCKET_NAME}/{self.new_file_name}",
Key=self.new_file_name,
Metadata={
"av-scanned-at": result["scanned_at"].strftime(
"%Y-%m-%d %H:%M:%S"
),
"av-passed": "True",
},
ContentType=self.content_type,
MetadataDirective="REPLACE",
)
else:
# Remove file with virus from S3
self.s3_client.delete_object(
Bucket=AWS_STORAGE_BUCKET_NAME,
Key=self.new_file_name,
)
if CHUNK_UPLOADER_RAISE_EXCEPTION_ON_VIRUS_FOUND:
raise VirusFoundInFileException()
else:
return FileWithVirus(field_name=self.field_name)
storage = S3Boto3Storage()
file = S3Boto3StorageFile(self.new_file_name, "rb", storage)
file.content_type = self.content_type
file.original_name = self.file_name
file.file_size = file_size
file.close()
return file