def setUpClass(cls): super(TestS3Storage, cls).setUpClass() cls.key_prefix = uuid.uuid4().hex cls.storage = S3Storage(aws_s3_key_prefix=cls.key_prefix) cls.storage_metadata = S3Storage( aws_s3_key_prefix=cls.key_prefix, aws_s3_metadata={ "Content-Disposition": lambda name: "attachment;filename={}".format( posixpath.basename(name)), "Content-Language": "fr", }) cls.insecure_storage = S3Storage(aws_s3_key_prefix=cls.key_prefix, aws_s3_bucket_auth=False, aws_s3_max_age_seconds=60 * 60 * 24 * 365) cls.key_prefix_static = uuid.uuid4().hex cls.static_storage = StaticS3Storage( aws_s3_key_prefix=cls.key_prefix_static) cls.upload_base = uuid.uuid4().hex cls.file_contents = force_bytes(uuid.uuid4().hex * 1000, "ascii") cls.file = ContentFile(cls.file_contents) cls.upload_dirname = uuid.uuid4().hex cls.upload_dir = posixpath.join(cls.upload_base, cls.upload_dirname) cls.upload_basename = cls.generateUploadBasename() cls.upload_path = cls.generateUploadPath(cls.upload_basename) cls.upload_time = datetime.datetime.now() # Save a file to the upload path. cls.saveTestFile()
def testCannotUseBucketAuthWithPublicUrl(self): with self.assertRaises(ImproperlyConfigured) as cm: S3Storage(aws_s3_bucket_auth=True, aws_s3_public_url="http://www.example.com/foo/") self.assertEqual( force_text(cm.exception), "Cannot use AWS_S3_BUCKET_AUTH with AWS_S3_PUBLIC_URL.")
def get_file_storage(): if settings.AWS_CONFIGURED: from django_s3_storage.storage import S3Storage return S3Storage() else: return default_storage
class BlogPost(TranslatableModel): #todo: this should be named "slug" and be translated... id = models.CharField(max_length=255, primary_key=True) header_image = models.ImageField(null=True, blank=True, upload_to="blog-images", storage=S3Storage()) author = models.CharField(max_length=200, null=True, blank=True) date = models.DateField(blank=True, null=True) published = models.BooleanField(default=False) translations = TranslatedFields(title=models.CharField(max_length=255), intro=models.TextField(max_length=1000, null=True, blank=True), body=models.TextField(null=True, blank=True)) def html_body(self): return markdown.markdown( replace_cmz_files(self.body), extensions=['markdown.extensions.fenced_code']) def html_intro(self): return markdown.markdown( self.intro, extensions=['markdown.extensions.fenced_code']) def __unicode__(self): return self.title class Meta: ordering = ['-date']
def _cleanup_folders(parent_dir: str = None, storage_key: str = None) -> None: # Since we use AWS in prod and a normal file structure in the media folder # in development, check if debug is active and if parent key is sent in. if settings.DEBUG and parent_dir: # Check if parent_dir is a folder, and that the folder is empty if os.path.isdir(parent_dir) and len(os.listdir(parent_dir)) == 0: # If so, delete the folder os.rmdir(parent_dir) # If we're not in debug, check if storage_key was sent int elif storage_key: # Assert that the key actually exists assert ( storage_key is not None ), "Storage key is none, not able to cleanup remote folder" # Create a new storage instance based in s3 bucket storage = S3Storage(aws_s3_bucket_name=settings.AWS_S3_BUCKET_NAME) # Get the "dir" key of file deleted parent_dir_key = storage_key.rsplit("/", 1)[ 0 ] # Get everything before last / # Check if ket exists if storage.exists(parent_dir_key): # Extract dirs and files within the key dirs, files = storage.listdir(parent_dir_key) # If both are empty, delete the folder if len(dirs) == 0 and len(files) == 0: storage.delete(parent_dir_key)
class NamedFile(models.Model): name = models.CharField(max_length=300) upload = models.FileField(null=True, blank=True, upload_to="cmz-files", storage=S3Storage()) thumb = models.FileField(null=True, blank=True, editable=False, upload_to="cmz-files", storage=S3Storage()) def thumb_preview(self): if self.thumb: return mark_safe('<img src="%s" width="100" height="auto" />' % (self.thumb.url)) return "" thumb_preview.short_description = 'Thumb' def save(self, *args, **kwargs): self.thumb.delete(False) try: im = Image.open(self.upload.file) size = (100, 100 * im.size[0] / im.size[1]) im.thumbnail(size, Image.ANTIALIAS) thumb_io = StringIO.StringIO() im.save(thumb_io, format='JPEG') # Create a new Django file-like object to be used in models as ImageField using # InMemoryUploadedFile. If you look at the source in Django, a # SimpleUploadedFile is essentially instantiated similarly to what is shown here thumb_file = InMemoryUploadedFile( thumb_io, None, '%s.thumb.jpg' % self.upload.name, 'image/jpeg', thumb_io.len, None) self.thumb = thumb_file except: pass return super(NamedFile, self).save(*args, **kwargs) def __unicode__(self): return self.name
class S3StoragePresignedURLUnitTestCase(StudioTestCase): """ Test cases for generating presigned URLs for S3 storage, i.e. Minio. """ STORAGE = S3Storage() def setUp(self): self.client = MagicMock() super().setUp() def test_returns_string_if_inputs_are_valid(self): """ Sanity check that get_presigned_upload_url returns a string if all arguments are valid. """ # use a real connection here as a sanity check ret = get_presigned_upload_url("a/b/abc.jpg", "aBc", 10, 1, storage=self.STORAGE, client=None) url = ret["uploadURL"] assert isinstance(url, str) def test_can_upload_file_to_presigned_url(self): """ Test that we can get a 200 OK when we upload a file to the URL returned by get_presigned_upload_url. """ file_contents = b"blahfilecontents" file = BytesIO(file_contents) # S3 expects a base64-encoded MD5 checksum md5 = hashlib.md5(file_contents) md5_checksum = md5.hexdigest() md5_checksum_base64 = codecs.encode(codecs.decode(md5_checksum, "hex"), "base64").decode() filename = "blahfile.jpg" filepath = generate_object_storage_name(md5_checksum, filename) ret = get_presigned_upload_url(filepath, md5_checksum_base64, 1000, len(file_contents)) url = ret["uploadURL"] content_type = ret["mimetype"] resp = requests.put(url, data=file, headers={ "Content-Type": content_type, }) resp.raise_for_status()
def store_file(file): """Stores File Object in file system. Arguments: file {FileObject} -- The file to be saved. Returns: {str} -- File path on file system. """ if (settings.DEFAULT_FILE_STORAGE != "django.core.files.storage.FileSystemStorage"): fs = S3Storage() return fs.save(file.name, file) else: fs = FileSystemStorage() filename = fs.save(file.name, file) return fs.url(filename)
class Tech(TranslatableModel): icon = models.ImageField(null=True, blank=True, upload_to="technology-icons", storage=S3Storage()) translations = TranslatedFields(title=models.CharField(max_length=255), description=models.TextField(null=True, blank=True)) order = models.IntegerField(blank=True, default=0) class Meta: ordering = ['order'] def __unicode__(self): return self.title
class PortfolioItem(TranslatableModel): header_image = models.ImageField(null=True, blank=True, upload_to="portfolio-images", storage=S3Storage()) author = models.CharField(max_length=200, null=True, blank=True) created = models.DateField(auto_now_add=True) blue_tags = models.CharField(max_length=200, null=True, blank=True) red_tags = models.CharField(max_length=200, null=True, blank=True) published = models.BooleanField(default=False) translations = TranslatedFields(title=models.CharField(max_length=255), intro=models.TextField(max_length=1000, null=True, blank=True), body=models.TextField(null=True, blank=True)) order = models.IntegerField(blank=True, default=0) class Meta: ordering = ['order'] def html_body(self): return markdown.markdown(replace_cmz_files(self.body)) def html_intro(self): return markdown.markdown(self.intro) def blue_tags_list(self): return self.blue_tags.split(",") def red_tags_list(self): return self.red_tags.split(",") def __unicode__(self): return self.title
def testGeneratePublicUrl(self): storage = S3Storage(aws_s3_bucket_auth=False, aws_s3_public_url="http://www.example.com/foo/") self.assertEqual(storage.url("bar.png"), "http://www.example.com/foo/bar.png")
import momoko import uuid from tornado import ioloop from tornado import httpclient from tornado.web import RequestHandler from tornado import gen from .authentication import SocketJWTAuthentication from doiq.socket.raw_queries import QUERIES, ResultIter from django.conf import settings from django.utils import timezone from django_s3_storage.storage import S3Storage from zipfile import ZipFile, ZIP_DEFLATED from StringIO import StringIO s3_storage = S3Storage() dsn = 'dbname=%s user=%s password=%s host=%s port=%s' % ( settings.DATABASES['default']['NAME'], settings.DATABASES['default']['USER'], settings.DATABASES['default']['PASSWORD'], settings.DATABASES['default']['HOST'], settings.DATABASES['default']['PORT']) db = momoko.Pool( dsn=dsn, size=2, max_size=5, ioloop=ioloop.IOLoop.current(), # setsession=("SET TIME ZONE UTC",), raise_connect_errors=False, reconnect_interval=5 * 1000,
from django.db import models # from django.core.validators import validate_email from django.contrib import messages # from django.contrib.messages import get_messages import re from datetime import date, datetime, timedelta import timeago # import pytz from django.contrib.sites.models import Site from PIL import Image from django_s3_storage.storage import S3Storage storage = S3Storage(aws_s3_bucket_name='django-skate-spots') class UserManager(models.Manager): def is_reg_valid(self, request): NAME_REGEX = re.compile(r'^[a-zA-Z ]+$') if not NAME_REGEX.match(request.POST['first_name']): messages.error(request, "Please enter your First Name") if not NAME_REGEX.match(request.POST['last_name']): messages.error(request, "Please enter your Last Name") if len(request.POST['first_name']) < 2: messages.error( request, "First Name should be at least 2 characters") if len(request.POST['last_name']) < 2: messages.error( request, "Last Name should be at least 2 characters")
def testSettingsUnknown(self): self.assertRaises(ImproperlyConfigured, lambda: S3Storage(foo=True, ))
def testSettingsCannotUsePublicUrlAndBucketAuth(self): self.assertRaises( ImproperlyConfigured, lambda: S3Storage( aws_s3_bucket_auth=True, aws_s3_public_url="/foo/", ))
def testSettingsOverwrittenByKwargs(self): self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE, "") self.assertEqual( S3Storage(aws_s3_content_language="foo").settings. AWS_S3_CONTENT_LANGUAGE, "foo")
from django.db import models from django_s3_storage.storage import S3Storage storage = S3Storage(aws_s3_bucket_name='vaccme') # Create your models here. class MedOrganization(models.Model): name = models.CharField(max_length=255, null=False) address = models.CharField(max_length=255, null=False) has_type_1 = models.BooleanField(default=True) has_type_2 = models.BooleanField(default=True) type_1_stock = models.BooleanField(default=True) type_2_stock = models.BooleanField(default=True) schedule_weekday = models.CharField(max_length=255, null=False) schedule_time = models.CharField(max_length=255, null=False) phone_number = models.CharField(max_length=128, null=False) website = models.URLField(max_length=128, null=True, blank=True) twogis_link = models.URLField(max_length=128, null=True, blank=True) rayon = models.CharField(max_length=255, null=True) photo = models.ImageField(storage=storage, null=True, blank=True) documents_needed = models.CharField(max_length=255, null=True) rooms = models.CharField(max_length=255, null=True, blank=True) extra = models.CharField(max_length=255, null=True, blank=True) city = models.CharField(max_length=255, null=False) contact_information = models.CharField(max_length=255, null=True, blank=True)
import re from collections import Counter from datetime import datetime, date, timedelta, timezone from django.conf import settings from django.db import models from django.contrib import messages import timeago from django.contrib.sites.models import Site from PIL import Image from django_s3_storage.storage import S3Storage import bcrypt storage = S3Storage(aws_s3_bucket_name='greek-restaurant') class UserManager(models.Manager): # def is_reg_valid(self, request): def registration_validator(self, postData): errors = {} if len(postData['first_name']) < 2: errors['first_name'] = "first name: at least 2 characters required" if len(postData['last_name']) < 2: errors['last_name'] = "last name: at least 2 characters required" if not postData['first_name'].isalpha( ) or not postData['last_name'].isalpha(): errors['name'] = "invalid name" EMAIL_REGEX = re.compile( r'^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9._-]+\.[a-zA-Z]+$') if not EMAIL_REGEX.match(postData['email']): errors['email'] = 'invalid email address!'
from django.db import models from accounts.models import FileShareAppUser from django.core.files.storage import FileSystemStorage from django_s3_storage.storage import S3Storage storage = S3Storage(aws_s3_bucket_name='s3-bucket-name') # storage = FileSystemStorage(location='/media/files') def content_file_name(instance, filename): return '/'.join(['files', instance.file_name, filename]) class FileItem(models.Model): file_uploader = models.ForeignKey(FileShareAppUser, on_delete=models.CASCADE,related_name= 'file_uploader') file_downloader = models.ForeignKey(FileShareAppUser, on_delete=models.CASCADE, related_name= 'file_downloader') file_file = models.FileField(storage=storage) file_desc = models.TextField(blank=True) file_name = models.CharField(max_length=20,blank=False,default="file_test") def __str__(self): return self.file_name
from django.utils import timezone from django.contrib.auth.models import User from PIL import Image, ExifTags from apps.project.models import Category from django.db.models.signals import post_save from django.dispatch import receiver from django_countries.fields import CountryField import os from django.utils import timezone from apps.project.models import Project from django.core.signals import request_finished from django.db.models.signals import post_save from django_s3_storage.storage import S3Storage storage = S3Storage( aws_s3_bucket_name=os.environ.get('AWS_STORAGE_BUCKET_NAME')) class Gronner(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) extract = models.TextField(default='', blank=True, null=True) dedication = models.CharField(default='Gronner', max_length=40) points = models.IntegerField(default=0) image = models.ImageField(default='default.jpg', storage=storage) birth = models.DateField() categories_followed = models.ManyToManyField(Category, blank=True) country = CountryField(default='Argentina', blank=True, null=True) facebook = models.CharField(max_length=50, default='', blank=True) instagram = models.CharField(max_length=50, default='', blank=True) linkedin = models.CharField(max_length=50, default='', blank=True) twitter = models.CharField(max_length=50, default='', blank=True)
from django.conf import settings from django.contrib.auth.models import User from django.core.files.base import ContentFile, File from django.db import models from django.urls import reverse from django.utils.timezone import now, utc from PIL import Image from utils.helper import tz_at_coords, random_key, time_base64, country_at_coords from utils.validators import (validate_corners_coordinates, validate_latitude, validate_longitude) from django_s3_storage.storage import S3Storage map_storage = S3Storage(aws_s3_bucket_name='drawmyroute-maps') def map_upload_path(instance=None, file_name=None): tmp_path = ['maps'] time_hash = time_base64() basename = instance.uid + '_' + time_hash tmp_path.append(basename[0]) tmp_path.append(basename[1]) tmp_path.append(basename) return os.path.join(*tmp_path) def route_upload_path(instance=None, file_name=None): tmp_path = ['routes'] basename = instance.uid
from django.db import models from datetime import date, timedelta from django.contrib.auth.models import User from django.db.models.signals import post_save from django.urls import reverse from django_s3_storage.storage import S3Storage from django.core.validators import MinValueValidator storage = S3Storage(aws_s3_bucket_name='exchange-2') # Create your models here. # Can reorder or make more TAGS = ( ('A', 'Animals'), ('B', 'Books'), ('C', 'Clothes'), ('E', 'Electronics'), ('F', 'Furniture'), ('H', 'Household Goods'), ('J', 'Jewelry'), ('M', 'Makeup'), ('S', 'Sports'), ('V', 'Vehicles'), ) class Profile(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) bio = models.TextField(max_length=1000, blank=True) avatar = models.ImageField(upload_to=storage, default='img/user.svg')
from django.contrib.postgres.fields import ArrayField from django_s3_storage.storage import S3Storage from users.models import User from django.conf import settings from django.db import models storage = S3Storage(aws_s3_bucket_name=settings.AWS_STORAGE_BUCKET_NAME) SEX_CHOICES = [ ('woman', 'woman'), ('man', 'man'), ] class Advert(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='adverts') title = models.CharField(max_length=30) animal_type = models.CharField(max_length=15, blank=True, null=True) breed = models.CharField(max_length=30, blank=True, null=True) color = models.CharField(max_length=15, blank=True, null=True) sex = models.CharField( max_length=max(len(el[0]) for el in SEX_CHOICES), choices=SEX_CHOICES, default="man", blank=True, null=True, ) description = models.CharField(max_length=600, blank=True, null=True) animal_features = models.JSONField(blank=True, default=dict)
from django.db import models from django.db.models.signals import post_delete, pre_delete from django.dispatch import receiver from django.utils import timezone from django_s3_storage.storage import S3Storage from django.conf import settings s3_storage = S3Storage(aws_s3_bucket_name=settings.AWS_S3_BUCKET_NAME_STATIC) # class S3MediaStorage(S3Boto3Storage): # location = "media" # default_acl = "public-read" # file_overwrite = False class SliderImages(models.Model): image = models.ImageField(upload_to="images/", storage=s3_storage) class Meta: verbose_name_plural = "صور الموقع" class Category(models.Model): """ ex(nagdy) """ name = models.CharField(max_length=128, blank=False) image = models.ImageField(upload_to="images/", blank=True, storage=s3_storage)
def testSettingsImported(self): self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE, "") with self.settings(AWS_S3_CONTENT_LANGUAGE="foo"): self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE, "foo")
class Analyze(BaseModel, TimeStampedModel): STATES = Choices( ('requested', 'requested', _("Requested")), ('registration_found', 'registration_found', _("Regisration found")), ('analyzed', 'analyzed', _("Perished")) ) TYPES = Choices( ('apartment', 'apartment', _("Apartment")), ('house', 'house', _("House")), ('ground', 'ground', _("Ground")), ('site', 'site', _("Site")) ) state = models.CharField(max_length=30, choices=STATES, default=STATES.requested, verbose_name=_("State")) user = models.ForeignKey('core.User', related_name='analyses', on_delete=models.CASCADE, null=True) address = models.CharField(max_length=255, null=True, verbose_name=_("Address")) number = models.CharField(max_length=255, null=True, verbose_name=_("Number")) zipcode = models.CharField(max_length=10, null=True, verbose_name=_("CEP")) block = models.CharField(max_length=20, null=True, verbose_name=_("Block")) lot = models.CharField(max_length=20, null=True, verbose_name=_("Lot")) registration_number = models.CharField(max_length=20, null=True, verbose_name=_("Registration Number")) registration = models.FileField(null=True, verbose_name=_("Registration"), storage=S3Storage()) type = models.CharField(max_length=30, choices=TYPES, default=TYPES.house, verbose_name=_("Type")) complement = models.CharField(max_length=100, verbose_name=_("complement"), null=True, blank=True) tracker = FieldTracker() @property def code(self): return self.id.hex[:10] @property def is_analyzed(self): return self.state == self.STATES.analyzed @property def type_display(self): _ = self.TYPES.for_constant(self.type) return _.display @property def state_display(self): state = self.STATES.for_constant(self.state) return state.display @property def state_badge_class(self): if self.state == 'requested': return 'badge-info' if self.state == 'registration_found': return 'badge-warning' if self.state == 'analyzed': return 'badge-success' def __str__(self): return self.user.email if self.user else 'N/A' class Meta: verbose_name = _("Analyse") verbose_name_plural = _("Analyses")