def setUpClass(cls):
super(TestS3Storage, cls).setUpClass()
cls.key_prefix = uuid.uuid4().hex
cls.storage = S3Storage(aws_s3_key_prefix=cls.key_prefix)
cls.storage_metadata = S3Storage(
aws_s3_key_prefix=cls.key_prefix,
aws_s3_metadata={
"Content-Disposition":
lambda name: "attachment;filename={}".format(
posixpath.basename(name)),
"Content-Language":
"fr",
})
cls.insecure_storage = S3Storage(aws_s3_key_prefix=cls.key_prefix,
aws_s3_bucket_auth=False,
aws_s3_max_age_seconds=60 * 60 * 24 *
365)
cls.key_prefix_static = uuid.uuid4().hex
cls.static_storage = StaticS3Storage(
aws_s3_key_prefix=cls.key_prefix_static)
cls.upload_base = uuid.uuid4().hex
cls.file_contents = force_bytes(uuid.uuid4().hex * 1000, "ascii")
cls.file = ContentFile(cls.file_contents)
cls.upload_dirname = uuid.uuid4().hex
cls.upload_dir = posixpath.join(cls.upload_base, cls.upload_dirname)
cls.upload_basename = cls.generateUploadBasename()
cls.upload_path = cls.generateUploadPath(cls.upload_basename)
cls.upload_time = datetime.datetime.now()
# Save a file to the upload path.
cls.saveTestFile()
def testCannotUseBucketAuthWithPublicUrl(self):
with self.assertRaises(ImproperlyConfigured) as cm:
S3Storage(aws_s3_bucket_auth=True,
aws_s3_public_url="http://www.example.com/foo/")
self.assertEqual(
force_text(cm.exception),
"Cannot use AWS_S3_BUCKET_AUTH with AWS_S3_PUBLIC_URL.")
def get_file_storage():
if settings.AWS_CONFIGURED:
from django_s3_storage.storage import S3Storage
return S3Storage()
else:
return default_storage
class BlogPost(TranslatableModel):
#todo: this should be named "slug" and be translated...
id = models.CharField(max_length=255, primary_key=True)
header_image = models.ImageField(null=True,
blank=True,
upload_to="blog-images",
storage=S3Storage())
author = models.CharField(max_length=200, null=True, blank=True)
date = models.DateField(blank=True, null=True)
published = models.BooleanField(default=False)
translations = TranslatedFields(title=models.CharField(max_length=255),
intro=models.TextField(max_length=1000,
null=True,
blank=True),
body=models.TextField(null=True,
blank=True))
def html_body(self):
return markdown.markdown(
replace_cmz_files(self.body),
extensions=['markdown.extensions.fenced_code'])
def html_intro(self):
return markdown.markdown(
self.intro, extensions=['markdown.extensions.fenced_code'])
def __unicode__(self):
return self.title
class Meta:
ordering = ['-date']
def _cleanup_folders(parent_dir: str = None, storage_key: str = None) -> None:
# Since we use AWS in prod and a normal file structure in the media folder
# in development, check if debug is active and if parent key is sent in.
if settings.DEBUG and parent_dir:
# Check if parent_dir is a folder, and that the folder is empty
if os.path.isdir(parent_dir) and len(os.listdir(parent_dir)) == 0:
# If so, delete the folder
os.rmdir(parent_dir)
# If we're not in debug, check if storage_key was sent int
elif storage_key:
# Assert that the key actually exists
assert (
storage_key is not None
), "Storage key is none, not able to cleanup remote folder"
# Create a new storage instance based in s3 bucket
storage = S3Storage(aws_s3_bucket_name=settings.AWS_S3_BUCKET_NAME)
# Get the "dir" key of file deleted
parent_dir_key = storage_key.rsplit("/", 1)[
0
] # Get everything before last /
# Check if ket exists
if storage.exists(parent_dir_key):
# Extract dirs and files within the key
dirs, files = storage.listdir(parent_dir_key)
# If both are empty, delete the folder
if len(dirs) == 0 and len(files) == 0:
storage.delete(parent_dir_key)
class NamedFile(models.Model):
name = models.CharField(max_length=300)
upload = models.FileField(null=True,
blank=True,
upload_to="cmz-files",
storage=S3Storage())
thumb = models.FileField(null=True,
blank=True,
editable=False,
upload_to="cmz-files",
storage=S3Storage())
def thumb_preview(self):
if self.thumb:
return mark_safe('<img src="%s" width="100" height="auto" />' %
(self.thumb.url))
return ""
thumb_preview.short_description = 'Thumb'
def save(self, *args, **kwargs):
self.thumb.delete(False)
try:
im = Image.open(self.upload.file)
size = (100, 100 * im.size[0] / im.size[1])
im.thumbnail(size, Image.ANTIALIAS)
thumb_io = StringIO.StringIO()
im.save(thumb_io, format='JPEG')
# Create a new Django file-like object to be used in models as ImageField using
# InMemoryUploadedFile. If you look at the source in Django, a
# SimpleUploadedFile is essentially instantiated similarly to what is shown here
thumb_file = InMemoryUploadedFile(
thumb_io, None, '%s.thumb.jpg' % self.upload.name,
'image/jpeg', thumb_io.len, None)
self.thumb = thumb_file
except:
pass
return super(NamedFile, self).save(*args, **kwargs)
def __unicode__(self):
return self.name
class S3StoragePresignedURLUnitTestCase(StudioTestCase):
"""
Test cases for generating presigned URLs for S3 storage, i.e. Minio.
"""
STORAGE = S3Storage()
def setUp(self):
self.client = MagicMock()
super().setUp()
def test_returns_string_if_inputs_are_valid(self):
"""
Sanity check that get_presigned_upload_url returns a string if all arguments
are valid.
"""
# use a real connection here as a sanity check
ret = get_presigned_upload_url("a/b/abc.jpg",
"aBc",
10,
1,
storage=self.STORAGE,
client=None)
url = ret["uploadURL"]
assert isinstance(url, str)
def test_can_upload_file_to_presigned_url(self):
"""
Test that we can get a 200 OK when we upload a file to the URL returned by get_presigned_upload_url.
"""
file_contents = b"blahfilecontents"
file = BytesIO(file_contents)
# S3 expects a base64-encoded MD5 checksum
md5 = hashlib.md5(file_contents)
md5_checksum = md5.hexdigest()
md5_checksum_base64 = codecs.encode(codecs.decode(md5_checksum, "hex"),
"base64").decode()
filename = "blahfile.jpg"
filepath = generate_object_storage_name(md5_checksum, filename)
ret = get_presigned_upload_url(filepath, md5_checksum_base64, 1000,
len(file_contents))
url = ret["uploadURL"]
content_type = ret["mimetype"]
resp = requests.put(url,
data=file,
headers={
"Content-Type": content_type,
})
resp.raise_for_status()
def store_file(file):
"""Stores File Object in file system.
Arguments:
file {FileObject} -- The file to be saved.
Returns:
{str} -- File path on file system.
"""
if (settings.DEFAULT_FILE_STORAGE !=
"django.core.files.storage.FileSystemStorage"):
fs = S3Storage()
return fs.save(file.name, file)
else:
fs = FileSystemStorage()
filename = fs.save(file.name, file)
return fs.url(filename)
class Tech(TranslatableModel):
icon = models.ImageField(null=True,
blank=True,
upload_to="technology-icons",
storage=S3Storage())
translations = TranslatedFields(title=models.CharField(max_length=255),
description=models.TextField(null=True,
blank=True))
order = models.IntegerField(blank=True, default=0)
class Meta:
ordering = ['order']
def __unicode__(self):
return self.title
class PortfolioItem(TranslatableModel):
header_image = models.ImageField(null=True,
blank=True,
upload_to="portfolio-images",
storage=S3Storage())
author = models.CharField(max_length=200, null=True, blank=True)
created = models.DateField(auto_now_add=True)
blue_tags = models.CharField(max_length=200, null=True, blank=True)
red_tags = models.CharField(max_length=200, null=True, blank=True)
published = models.BooleanField(default=False)
translations = TranslatedFields(title=models.CharField(max_length=255),
intro=models.TextField(max_length=1000,
null=True,
blank=True),
body=models.TextField(null=True,
blank=True))
order = models.IntegerField(blank=True, default=0)
class Meta:
ordering = ['order']
def html_body(self):
return markdown.markdown(replace_cmz_files(self.body))
def html_intro(self):
return markdown.markdown(self.intro)
def blue_tags_list(self):
return self.blue_tags.split(",")
def red_tags_list(self):
return self.red_tags.split(",")
def __unicode__(self):
return self.title
def testGeneratePublicUrl(self):
storage = S3Storage(aws_s3_bucket_auth=False,
aws_s3_public_url="http://www.example.com/foo/")
self.assertEqual(storage.url("bar.png"),
"http://www.example.com/foo/bar.png")
import momoko
import uuid
from tornado import ioloop
from tornado import httpclient
from tornado.web import RequestHandler
from tornado import gen
from .authentication import SocketJWTAuthentication
from doiq.socket.raw_queries import QUERIES, ResultIter
from django.conf import settings
from django.utils import timezone
from django_s3_storage.storage import S3Storage
from zipfile import ZipFile, ZIP_DEFLATED
from StringIO import StringIO
s3_storage = S3Storage()
dsn = 'dbname=%s user=%s password=%s host=%s port=%s' % (
settings.DATABASES['default']['NAME'],
settings.DATABASES['default']['USER'],
settings.DATABASES['default']['PASSWORD'],
settings.DATABASES['default']['HOST'],
settings.DATABASES['default']['PORT'])
db = momoko.Pool(
dsn=dsn,
size=2,
max_size=5,
ioloop=ioloop.IOLoop.current(),
# setsession=("SET TIME ZONE UTC",),
raise_connect_errors=False,
reconnect_interval=5 * 1000,
from django.db import models
# from django.core.validators import validate_email
from django.contrib import messages
# from django.contrib.messages import get_messages
import re
from datetime import date, datetime, timedelta
import timeago
# import pytz
from django.contrib.sites.models import Site
from PIL import Image
from django_s3_storage.storage import S3Storage
storage = S3Storage(aws_s3_bucket_name='django-skate-spots')
class UserManager(models.Manager):
def is_reg_valid(self, request):
NAME_REGEX = re.compile(r'^[a-zA-Z ]+$')
if not NAME_REGEX.match(request.POST['first_name']):
messages.error(request, "Please enter your First Name")
if not NAME_REGEX.match(request.POST['last_name']):
messages.error(request, "Please enter your Last Name")
if len(request.POST['first_name']) < 2:
messages.error(
request, "First Name should be at least 2 characters")
if len(request.POST['last_name']) < 2:
messages.error(
request, "Last Name should be at least 2 characters")
def testSettingsUnknown(self):
self.assertRaises(ImproperlyConfigured, lambda: S3Storage(foo=True, ))
def testSettingsCannotUsePublicUrlAndBucketAuth(self):
self.assertRaises(
ImproperlyConfigured, lambda: S3Storage(
aws_s3_bucket_auth=True,
aws_s3_public_url="/foo/",
))
def testSettingsOverwrittenByKwargs(self):
self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE, "")
self.assertEqual(
S3Storage(aws_s3_content_language="foo").settings.
AWS_S3_CONTENT_LANGUAGE, "foo")
from django.db import models
from django_s3_storage.storage import S3Storage
storage = S3Storage(aws_s3_bucket_name='vaccme')
# Create your models here.
class MedOrganization(models.Model):
name = models.CharField(max_length=255, null=False)
address = models.CharField(max_length=255, null=False)
has_type_1 = models.BooleanField(default=True)
has_type_2 = models.BooleanField(default=True)
type_1_stock = models.BooleanField(default=True)
type_2_stock = models.BooleanField(default=True)
schedule_weekday = models.CharField(max_length=255, null=False)
schedule_time = models.CharField(max_length=255, null=False)
phone_number = models.CharField(max_length=128, null=False)
website = models.URLField(max_length=128, null=True, blank=True)
twogis_link = models.URLField(max_length=128, null=True, blank=True)
rayon = models.CharField(max_length=255, null=True)
photo = models.ImageField(storage=storage, null=True, blank=True)
documents_needed = models.CharField(max_length=255, null=True)
rooms = models.CharField(max_length=255, null=True, blank=True)
extra = models.CharField(max_length=255, null=True, blank=True)
city = models.CharField(max_length=255, null=False)
contact_information = models.CharField(max_length=255,
null=True,
blank=True)
import re
from collections import Counter
from datetime import datetime, date, timedelta, timezone
from django.conf import settings
from django.db import models
from django.contrib import messages
import timeago
from django.contrib.sites.models import Site
from PIL import Image
from django_s3_storage.storage import S3Storage
import bcrypt
storage = S3Storage(aws_s3_bucket_name='greek-restaurant')
class UserManager(models.Manager):
# def is_reg_valid(self, request):
def registration_validator(self, postData):
errors = {}
if len(postData['first_name']) < 2:
errors['first_name'] = "first name: at least 2 characters required"
if len(postData['last_name']) < 2:
errors['last_name'] = "last name: at least 2 characters required"
if not postData['first_name'].isalpha(
) or not postData['last_name'].isalpha():
errors['name'] = "invalid name"
EMAIL_REGEX = re.compile(
r'^[a-zA-Z0-9.+_-]+@[a-zA-Z0-9._-]+\.[a-zA-Z]+$')
if not EMAIL_REGEX.match(postData['email']):
errors['email'] = 'invalid email address!'
from django.db import models
from accounts.models import FileShareAppUser
from django.core.files.storage import FileSystemStorage
from django_s3_storage.storage import S3Storage
storage = S3Storage(aws_s3_bucket_name='s3-bucket-name')
# storage = FileSystemStorage(location='/media/files')
def content_file_name(instance, filename):
return '/'.join(['files', instance.file_name, filename])
class FileItem(models.Model):
file_uploader = models.ForeignKey(FileShareAppUser, on_delete=models.CASCADE,related_name= 'file_uploader')
file_downloader = models.ForeignKey(FileShareAppUser, on_delete=models.CASCADE, related_name= 'file_downloader')
file_file = models.FileField(storage=storage)
file_desc = models.TextField(blank=True)
file_name = models.CharField(max_length=20,blank=False,default="file_test")
def __str__(self):
return self.file_name
from django.utils import timezone
from django.contrib.auth.models import User
from PIL import Image, ExifTags
from apps.project.models import Category
from django.db.models.signals import post_save
from django.dispatch import receiver
from django_countries.fields import CountryField
import os
from django.utils import timezone
from apps.project.models import Project
from django.core.signals import request_finished
from django.db.models.signals import post_save
from django_s3_storage.storage import S3Storage
storage = S3Storage(
aws_s3_bucket_name=os.environ.get('AWS_STORAGE_BUCKET_NAME'))
class Gronner(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
extract = models.TextField(default='', blank=True, null=True)
dedication = models.CharField(default='Gronner', max_length=40)
points = models.IntegerField(default=0)
image = models.ImageField(default='default.jpg', storage=storage)
birth = models.DateField()
categories_followed = models.ManyToManyField(Category, blank=True)
country = CountryField(default='Argentina', blank=True, null=True)
facebook = models.CharField(max_length=50, default='', blank=True)
instagram = models.CharField(max_length=50, default='', blank=True)
linkedin = models.CharField(max_length=50, default='', blank=True)
twitter = models.CharField(max_length=50, default='', blank=True)
from django.conf import settings
from django.contrib.auth.models import User
from django.core.files.base import ContentFile, File
from django.db import models
from django.urls import reverse
from django.utils.timezone import now, utc
from PIL import Image
from utils.helper import tz_at_coords, random_key, time_base64, country_at_coords
from utils.validators import (validate_corners_coordinates, validate_latitude,
validate_longitude)
from django_s3_storage.storage import S3Storage
map_storage = S3Storage(aws_s3_bucket_name='drawmyroute-maps')
def map_upload_path(instance=None, file_name=None):
tmp_path = ['maps']
time_hash = time_base64()
basename = instance.uid + '_' + time_hash
tmp_path.append(basename[0])
tmp_path.append(basename[1])
tmp_path.append(basename)
return os.path.join(*tmp_path)
def route_upload_path(instance=None, file_name=None):
tmp_path = ['routes']
basename = instance.uid
from django.db import models
from datetime import date, timedelta
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.urls import reverse
from django_s3_storage.storage import S3Storage
from django.core.validators import MinValueValidator
storage = S3Storage(aws_s3_bucket_name='exchange-2')
# Create your models here.
# Can reorder or make more
TAGS = (
('A', 'Animals'),
('B', 'Books'),
('C', 'Clothes'),
('E', 'Electronics'),
('F', 'Furniture'),
('H', 'Household Goods'),
('J', 'Jewelry'),
('M', 'Makeup'),
('S', 'Sports'),
('V', 'Vehicles'),
)
class Profile(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
bio = models.TextField(max_length=1000, blank=True)
avatar = models.ImageField(upload_to=storage, default='img/user.svg')
from django.contrib.postgres.fields import ArrayField
from django_s3_storage.storage import S3Storage
from users.models import User
from django.conf import settings
from django.db import models
storage = S3Storage(aws_s3_bucket_name=settings.AWS_STORAGE_BUCKET_NAME)
SEX_CHOICES = [
('woman', 'woman'),
('man', 'man'),
]
class Advert(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, related_name='adverts')
title = models.CharField(max_length=30)
animal_type = models.CharField(max_length=15, blank=True, null=True)
breed = models.CharField(max_length=30, blank=True, null=True)
color = models.CharField(max_length=15, blank=True, null=True)
sex = models.CharField(
max_length=max(len(el[0]) for el in SEX_CHOICES),
choices=SEX_CHOICES,
default="man",
blank=True,
null=True,
)
description = models.CharField(max_length=600, blank=True, null=True)
animal_features = models.JSONField(blank=True, default=dict)
from django.db import models
from django.db.models.signals import post_delete, pre_delete
from django.dispatch import receiver
from django.utils import timezone
from django_s3_storage.storage import S3Storage
from django.conf import settings
s3_storage = S3Storage(aws_s3_bucket_name=settings.AWS_S3_BUCKET_NAME_STATIC)
# class S3MediaStorage(S3Boto3Storage):
# location = "media"
# default_acl = "public-read"
# file_overwrite = False
class SliderImages(models.Model):
image = models.ImageField(upload_to="images/", storage=s3_storage)
class Meta:
verbose_name_plural = "صور الموقع"
class Category(models.Model):
"""
ex(nagdy)
"""
name = models.CharField(max_length=128, blank=False)
image = models.ImageField(upload_to="images/",
blank=True,
storage=s3_storage)
def testSettingsImported(self):
self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE, "")
with self.settings(AWS_S3_CONTENT_LANGUAGE="foo"):
self.assertEqual(S3Storage().settings.AWS_S3_CONTENT_LANGUAGE,
"foo")
class Analyze(BaseModel, TimeStampedModel):
STATES = Choices(
('requested', 'requested', _("Requested")),
('registration_found', 'registration_found', _("Regisration found")),
('analyzed', 'analyzed', _("Perished"))
)
TYPES = Choices(
('apartment', 'apartment', _("Apartment")),
('house', 'house', _("House")),
('ground', 'ground', _("Ground")),
('site', 'site', _("Site"))
)
state = models.CharField(max_length=30, choices=STATES, default=STATES.requested, verbose_name=_("State"))
user = models.ForeignKey('core.User', related_name='analyses', on_delete=models.CASCADE, null=True)
address = models.CharField(max_length=255, null=True, verbose_name=_("Address"))
number = models.CharField(max_length=255, null=True, verbose_name=_("Number"))
zipcode = models.CharField(max_length=10, null=True, verbose_name=_("CEP"))
block = models.CharField(max_length=20, null=True, verbose_name=_("Block"))
lot = models.CharField(max_length=20, null=True, verbose_name=_("Lot"))
registration_number = models.CharField(max_length=20, null=True, verbose_name=_("Registration Number"))
registration = models.FileField(null=True, verbose_name=_("Registration"), storage=S3Storage())
type = models.CharField(max_length=30, choices=TYPES, default=TYPES.house, verbose_name=_("Type"))
complement = models.CharField(max_length=100, verbose_name=_("complement"), null=True, blank=True)
tracker = FieldTracker()
@property
def code(self):
return self.id.hex[:10]
@property
def is_analyzed(self):
return self.state == self.STATES.analyzed
@property
def type_display(self):
_ = self.TYPES.for_constant(self.type)
return _.display
@property
def state_display(self):
state = self.STATES.for_constant(self.state)
return state.display
@property
def state_badge_class(self):
if self.state == 'requested':
return 'badge-info'
if self.state == 'registration_found':
return 'badge-warning'
if self.state == 'analyzed':
return 'badge-success'
def __str__(self):
return self.user.email if self.user else 'N/A'
class Meta:
verbose_name = _("Analyse")
verbose_name_plural = _("Analyses")
