def PrivateS3Boto3Storage():
return S3Boto3Storage(default_acl='private')
def download_tarball(key, local_filename):
connection = S3Boto3Storage().connection
bucket = connection.Bucket("arxiv")
bucket.download_file(key, local_filename, {"RequestPayer": "requester"})
def download_tarball(key, local_filename):
connection = S3Boto3Storage().connection
bucket = connection.Bucket('arxiv')
bucket.download_file(key, local_filename, {'RequestPayer': 'requester'})
from storages.backends.s3boto3 import S3Boto3Storage #StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media')
def MediaRootS3BotoStorage():
return S3Boto3Storage(location='media') # noqa
def setUpClass(cls):
if six.PY2:
raise SkipTest("Skipping redundant test")
cls.pixelmatch_bin = os.environ.get('PIXELMATCH_BIN')
if not cls.pixelmatch_bin:
cls.pixelmatch_bin = find_executable('pixelmatch')
if not cls.pixelmatch_bin or not os.path.exists(cls.pixelmatch_bin):
raise SkipTest("pixelmatch not installed")
cls.screenshot_output_dir = os.environ.get('SCREENSHOT_OUTPUT_DIR',
'/tmp/djn-tests')
super(BaseNestedAdminTestCase, cls).setUpClass()
cls.root_temp_dir = tempfile.mkdtemp()
if os.environ.get('TRAVIS_BUILD_NUMBER'):
# For some reason these tests fail on travis when Django > 1.11
if django.VERSION > (1, 11):
raise SkipTest("Issue with travis and Django >= 1.11")
cls.path_prefix = "travis_%s" % os.environ['TRAVIS_BUILD_NUMBER']
else:
cls.path_prefix = "local"
cls.temp_dir = tempfile.mkdtemp(dir=cls.root_temp_dir)
os.makedirs(os.path.join(cls.temp_dir, cls.path_prefix))
if cls.screenshot_output_dir:
screenshot_path = os.path.join(cls.screenshot_output_dir,
cls.path_prefix)
if not os.path.exists(screenshot_path):
os.makedirs(screenshot_path)
if all(
os.environ.get(k)
for k in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']):
try:
storage = S3Boto3Storage()
bucket = storage.bucket # noqa
except:
pass
else:
cls.storage = storage
cls.all_models = {}
cls.all_model_names = {}
for root_model in cls.root_models:
root_admin = admin_site._registry[root_model]
def descend_admin_inlines(admin):
data = [admin.model, []]
for inline in (getattr(admin, 'inlines', None) or []):
data[1].append(descend_admin_inlines(inline))
return data
cls.all_models[root_model] = models = descend_admin_inlines(
root_admin)
def recursive_map_model_names(data):
if isinstance(data, list):
return [m for m in map(recursive_map_model_names, data)]
else:
return get_model_name(data)
cls.all_model_names[root_model] = recursive_map_model_names(models)
from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') ProtectedS3Storage = lambda: S3Boto3Storage(location='protected')
class Resort(models.Model):
resort_pk = models.AutoField(primary_key=True)
resort_id = models.UUIDField(verbose_name="resort unique id", default=uuid.uuid4, editable=False)
resort_name = models.CharField(verbose_name="resort name", max_length=255)
website = models.URLField(verbose_name="resort website", blank=True)
location = models.ForeignKey(Location, verbose_name="resort location", blank=True, null=True)
network_key = models.CharField(max_length=255, verbose_name="resort network key", default=get_random_key,
unique=True)
license_expiry_date = models.DateTimeField(verbose_name="resort expiry expiration date", blank=True, null=True)
licenses = models.IntegerField(verbose_name="licenses owned by resort", blank=True, null=True)
map_kml = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, blank=True, max_length=255)
map_type = models.IntegerField(choices=MapType, blank=True, default=GOOGLE_MAP)
report_form = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, verbose_name="report form for resort", blank=True,
max_length=255)
print_on_device = models.IntegerField(choices=Choice, blank=True, default=NO)
map_lat = models.FloatField(blank=True, null=True)
map_lng = models.FloatField(blank=True, null=True)
initial_map_zoom_level = models.IntegerField(default=18, validators=[validate_zoom_level])
domain_id = models.ForeignKey(Domains, null=True)
unit_format = models.IntegerField(choices=UnitType, default=IMPERIAL, blank=True)
default_unit_paper = models.IntegerField(choices=PaperSize, blank=True, default=A4)
timezone = models.CharField(max_length=50, blank=True, choices=timezones)
datetime_format = models.IntegerField(blank=True, choices=DATETIME_FORMAT, default=DDMMYYYY)
resort_logo = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, verbose_name="resort logo", blank=True, max_length=255)
dt_modified = models.DateTimeField(auto_now=True)
incident_template = JSONField(verbose_name="Incident template for resort", blank=True)
season_start_date = models.DateField(verbose_name='season start date', default=dt.date.today)
use_sequential_incident_id = models.IntegerField(verbose_name='Use sequential incident id',
choices=SequentialIncidentIdChoice, default=OFF)
dispatch_field_choice = JSONField(verbose_name='choice for dispatch screen',
default=[{"field_key": "name"},
{"field_key": "phone"},
{"field_key": "field_52d4798f6d227____body_part"},
{"field_key": "field_52d48077a16be"}])
resort_controlled_substances = models.BooleanField(verbose_name="resort controlled substances", default=False)
resort_asset_management = models.BooleanField(verbose_name="resort asset management", default=False)
kms_enabled = models.BooleanField(verbose_name="KMS enabled for resort", default=False)
kms_cmk = models.CharField(max_length=255, verbose_name="Customer Master Key for Resort", null=True, blank=True)
enc_data_key = models.TextField(default="")
def __unicode__(self):
return self.resort_name
def save(self, *args, **kwargs):
from apps.incidents.utils import get_data_key
from apps.resorts.utils import template_update
if self.domain_id is None:
self.domain_id = Domains.objects.get(pk=1)
# If this is a new resort then do the following
# Update the template
# If KMS is enabled then create new customer master key and generate new data_key using that CMK
if self.pk is None:
self.incident_template = template_update(self, True)
client = create_client()
if self.kms_enabled is True:
response = client.create_key(Description=self.resort_name, KeyUsage='ENCRYPT_DECRYPT')
self.kms_cmk = response['KeyMetadata']['Arn']
response = client.generate_data_key(
KeyId=self.kms_cmk,
KeySpec='AES_256',
)
self.enc_data_key = base64.b64encode(response['CiphertextBlob'])
else:
self.kms_cmk = settings.GLOBAL_KMS_KEY_ID
response = client.generate_data_key(
KeyId=self.kms_cmk,
KeySpec='AES_256',
)
self.enc_data_key = base64.b64encode(response['CiphertextBlob'])
super(Resort, self).save(*args, **kwargs) # Call the "real" save() method.
update_resort_discovery(self, self.domain_id)
# If the resort is already created then
# Check if sequential numbering is on (or) not and perform operation accordingly
# if KMS is enabled then create new master key and also create new data key using that CMK
else:
self.incident_template = template_update(self, False)
resort = Resort.objects.get(resort_pk=self.pk)
if resort.use_sequential_incident_id == OFF and self.use_sequential_incident_id == ON:
if self.season_start_date:
startDate = dt.datetime.combine(self.season_start_date, dt.time())
endDate = startDate + dt.timedelta(days=365)
while startDate.date() <= dt.date.today():
for index, incident in enumerate(
Incident.objects.filter(dt_created__gte=startDate, dt_created__lte=endDate,
resort=self).exclude(incident_status__order=9).order_by(
'dt_created')):
incident.incident_sequence = index + 1
incident.save()
startDate = startDate + dt.timedelta(days=365)
endDate = startDate + dt.timedelta(days=365)
if (self.kms_enabled is True) and ((resort.kms_cmk == '') or (resort.kms_cmk is None)):
client = create_client()
response = client.create_key(Description=self.resort_name, KeyUsage='ENCRYPT_DECRYPT')
self.kms_cmk = response['KeyMetadata']['Arn']
pre_data_key = get_data_key(self)
response = client.generate_data_key(
KeyId=self.kms_cmk,
KeySpec='AES_256',
)
self.enc_data_key = base64.b64encode(response['CiphertextBlob'])
encrypt_patient_data_kms(self, pre_data_key, base64.b64encode(response['Plaintext']))
upload_status, message = encrypt_s3_files(self, self.kms_cmk)
if not upload_status:
raise Exception
super(Resort, self).save(*args, **kwargs) # Call the "real" save() method.
if((resort.resort_name != self.resort_name) or (resort.domain_id != self.domain_id) or (resort.network_key != self.network_key)):
update_resort_discovery(self, self.domain_id)
from storages.backends.s3boto3 import S3Boto3Storage MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='mediafiles') StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static')
def get_file_storage():
if settings.AWS_CONFIGURED:
from storages.backends.s3boto3 import S3Boto3Storage
return S3Boto3Storage()
else:
return default_storage
class PageBlockElement(Element):
block = models.ForeignKey(PageBlock,
on_delete=models.CASCADE,
related_name="elements")
markdown = models.TextField(blank=True, default="")
connection = models.URLField(blank=True, default="", max_length=1000)
internal_connection = models.ForeignKey(Page,
on_delete=models.SET_NULL,
null=True,
related_name="connections")
plain_text = models.TextField(blank=True, default="", max_length=1000)
code = models.TextField(blank=True, default="", max_length=1000)
embed_video = models.TextField(blank=True, default="", max_length=1000)
file = models.FileField(
null=True,
storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_PAGES_BUCKET_NAME),
upload_to=file_upload_path,
max_length=200,
)
image = models.ImageField(
null=True,
storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_PAGES_BUCKET_NAME),
upload_to=file_upload_path,
max_length=200,
)
parent = models.ForeignKey("PageBlockElement",
on_delete=models.CASCADE,
related_name="sets_elements",
null=True)
custom_element_set = models.ForeignKey("CustomElementSet",
on_delete=models.CASCADE,
related_name="elements",
null=True)
observable_hq = models.OneToOneField("PageBlockObservableElement",
on_delete=models.CASCADE,
null=True)
state = models.ForeignKey("states.State",
null=True,
related_name="elements",
on_delete=models.SET_NULL)
_clone_one_to_one_fields = ["observable_hq"]
objects = managers.PageBlockElementManager()
def natural_key(self):
if hasattr(self.block, "page"):
return (self.name, self.order, self.custom_element_set,
self.parent) + self.block.natural_key()
if hasattr(self.block, "block"):
return self.name, self.order + self.block.project.title, self.block.name
natural_key.dependencies = ["states.state", "pages.pageblock"]
def relative_path_to_save(self, filename):
base_path = self.image.storage.location
if not self.block_id:
raise ValueError("Page is not set")
return os.path.join(
base_path,
f"{self.block.page_id}/blocks/{self.block_id}/{filename}")
def get_original_file_name(self, file_name=None, image=True):
if not file_name:
file_name = self.image.name if image else self.file.name
name, ext = os.path.splitext(os.path.basename(file_name))
return name, os.path.basename(file_name)
def update_or_create_custom_element_sets(self, elements_sets_list):
for elements_set in elements_sets_list:
element_set_order = elements_set.get("order")
custom_element_set, _ = CustomElementSet.objects.update_or_create(
id=elements_set.pop("id", None),
defaults=dict(order=element_set_order),
)
set_elements = elements_set.get("elements", [])
self.update_or_create_custom_element_set_elements(
set_elements, custom_element_set)
def update_or_create_custom_element_set_elements(self, elements,
custom_element_set):
for element in elements:
element_type = element.get("type")
element_value = element.pop("value")
if (element_type in [
constants.ElementType.IMAGE, constants.ElementType.FILE
] and element_value is not False):
element[element_type] = element_value
if element_type in [
constants.ElementType.MARKDOWN,
constants.ElementType.PLAIN_TEXT,
constants.ElementType.CODE,
constants.ElementType.INTERNAL_CONNECTION,
constants.ElementType.CONNECTION,
constants.ElementType.EMBED_VIDEO,
constants.ElementType.STATE,
]:
element[element_type] = element_value
element, _ = PageBlockElement.objects.update_or_create(
id=element.pop("id", None),
defaults=dict(block=self.block,
parent=self,
custom_element_set=custom_element_set,
**element),
)
if element_type == constants.ElementType.OBSERVABLE_HQ:
self.create_update_observable_element(element_value, element)
def delete_custom_elements_sets(self, ids_to_delete):
CustomElementSet.objects.filter(id__in=ids_to_delete).delete()
def create_update_observable_element(self, value, element=None):
element_instance = element if element else self
hq_element, is_create = PageBlockObservableElement.objects.update_or_create(
id=value.pop("id", None), defaults=dict(**value))
if is_create:
element_instance.observable_hq = hq_element
element_instance.save(update_fields=["observable_hq"])
def clone(self, block=None):
block = block if block else self.block
if self.custom_element_set:
return None
if self.type == constants.ElementType.OBSERVABLE_HQ:
return self.clone_observable_element(block)
elif self.type == constants.ElementType.CUSTOM_ELEMENT:
return self.clone_custom_element(block)
else:
return self.clone_simple_element(block)
def clone_simple_element(self, block=None, parent=None, elements_set=None):
attrs = {"block": block}
if elements_set and parent:
attrs["custom_element_set"] = elements_set
attrs["parent"] = parent
return self.make_clone(attrs=attrs)
def clone_observable_element(self,
block=None,
parent=None,
elements_set=None):
new_obs = self.observable_hq.make_clone()
attrs = {"block": block, "observable_hq": new_obs}
if elements_set and parent:
attrs["custom_element_set"] = elements_set
attrs["parent"] = parent
return self.make_clone(attrs=attrs)
def clone_custom_element(self, block=None):
block = block if block else self.block
new_ele = self.make_clone(attrs={"block": block})
elements_sets = {
element.custom_element_set
for element in self.sets_elements.all()
}
for element_set in elements_sets:
new_set = element_set.make_clone()
for set_element in element_set.elements.all():
if set_element.type == constants.ElementType.OBSERVABLE_HQ:
set_element.clone_observable_element(block=block,
parent=new_ele,
elements_set=new_set)
else:
set_element.clone_simple_element(block=block,
parent=new_ele,
elements_set=new_set)
return new_ele
def s3_client():
storage = S3Boto3Storage(bucket_name=BUCKET)
return storage.connection.meta.client
f"max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate"
}
# STATIC
# ------------------------
STATICFILES_STORAGE = "config.settings.production.StaticRootS3BotoStorage"
STATIC_URL = f"http://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/static/"
# MEDIA
# ------------------------------------------------------------------------------
# region http://stackoverflow.com/questions/10390244/
from storages.backends.s3boto3 import S3Boto3Storage # noqa E402
StaticRootS3BotoStorage = lambda: S3Boto3Storage(location="static") # noqa
MediaRootS3BotoStorage = lambda: S3Boto3Storage(location="media",
file_overwrite=False) # noqa
# endregion
DEFAULT_FILE_STORAGE = "config.settings.production.MediaRootS3BotoStorage"
MEDIA_URL = f"http://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/media/"
# TEMPLATES
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES[0]["OPTIONS"]["loaders"] = [ # noqa F405
(
"django.template.loaders.cached.Loader",
[
"django.template.loaders.filesystem.Loader",
"django.template.loaders.app_directories.Loader",
class LeafletUploadWizzard(NamedUrlSessionWizardView):
extra_added = False
TEMPLATES = {
"front": "leaflets/upload_form/image_front.html",
"back": "leaflets/upload_form/image_back.html",
"inside": "leaflets/upload_form/image_inside.html",
"postcode": "leaflets/upload_form/postcode.html",
"people": "leaflets/upload_form/people.html",
}
if os.environ.get("AWS_SESSION_TOKEN", None):
file_storage = S3Boto3Storage(location="leaflets_tmp")
else:
file_storage = FileSystemStorage(
location=os.path.join(settings.MEDIA_ROOT, "images/leaflets_tmp"))
def get_template_names(self):
if self.steps.current.startswith("inside"):
step_name = "inside"
else:
step_name = self.steps.current
return [self.TEMPLATES[step_name]]
def get_form_initial(self, step):
if step == "people":
api = DevsDCAPIHelper()
postcode = self.get_cleaned_data_for_step("postcode")
if postcode:
results = api.postcode_request(postcode["postcode"])
if results.status_code == 200:
return {"postcode_results": results}
return {}
@property
def extra_inside_forms(self):
return self.storage.extra_data.get("extra_inside", 0)
def get_context_data(self, **kwargs):
context = super(LeafletUploadWizzard, self).get_context_data(**kwargs)
context["hide_footer"] = True
return context
def add_extra_inside_forms(self):
self.storage.extra_data["extra_inside"] = self.extra_inside_forms + 1
self.extra_added = True
def get(self, *args, **kwargs):
if "reset" in self.request.GET:
self.storage.reset()
return HttpResponseRedirect("/")
self._insert_extra_inside_forms()
return super(LeafletUploadWizzard, self).get(*args, **kwargs)
def post(self, *args, **kwargs):
# Add forms to the form_list that we should have.
# We need to do this on every request, as the extra forms
# Aren't stored betweet requests, but are held in the session.
self._insert_extra_inside_forms()
# The user has finished uploading all image, move to the postcode form
if self.request.POST.get("skip", None):
if self.request.FILES:
form = self.get_form(data=self.request.POST,
files=self.request.FILES)
if form.is_valid():
super(LeafletUploadWizzard, self).post(*args, **kwargs)
else:
if "extra_inside" in self.storage.extra_data:
self.storage.extra_data["extra_inside"] -= 1
self._insert_extra_inside_forms(force=True)
self.storage.extra_data["skip_to_postcode"] = True
return self.render_goto_step("postcode")
# If there are more pages, add them to the form_list
# Validate the form first, though
if self.request.POST.get("add_extra_inside", None):
form = self.get_form(data=self.request.POST,
files=self.request.FILES)
if form.is_valid():
self.add_extra_inside_forms()
self._insert_extra_inside_forms(force=True)
return super(LeafletUploadWizzard, self).post(*args, **kwargs)
def _insert_extra_inside_forms(self, force=False):
"""
Adds to self.form_list, inserting an extra 'inside page' form
after the last inside page form we have.
"""
# We've not added any new forms, so the form list is as set in urls.py
if not self.extra_inside_forms:
return self.form_list
# If we've already added the extra forms, don't them again.
if self.extra_added and not force:
return self.form_list
# self.form_list is an ordered dict. Convert it to a list of tuples.
form_list = [(k, v) for k, v in list(self.form_list.items())]
# Reverse the list, so the the first step starting with 'inside'
# is the last inside page form.
form_list.reverse()
for index, (name, form) in enumerate(form_list):
if name.startswith("inside"):
# For every extra inside form in self.extra_inside_forms
# add to the form_list
for step in range(self.extra_inside_forms):
new_name = "inside-%s" % step
form_list.insert(index, (new_name, InsidePageImageForm))
# We're finished going back through the form_list now
break
form_list.reverse()
self.form_list = OrderedDict()
for k, v in form_list:
self.form_list[k] = v
self.extra_added = True
return self.form_list
def done(self, form_list, **kwargs):
# Create a new leaflet
leaflet = Leaflet()
leaflet.save()
for form in form_list:
form_prefix = form.prefix.split("-")[0]
if form_prefix in ["front", "back", "inside"]:
# Dealing with an image form
image_type = None
if form.prefix == "front":
image_type = "1_front"
if form.prefix == "back":
image_type = "2_back"
if form.prefix == "inside":
image_type = "3_inside"
image = LeafletImage(leaflet=leaflet, image_type=image_type)
image.image = form.cleaned_data["image"]
image.save()
if form_prefix == "postcode":
leaflet.postcode = form.cleaned_data["postcode"]
if form_prefix == "people":
if ("people" in form.cleaned_data
and isinstance(form.cleaned_data["people"], str)
and form.cleaned_data["people"] != ""):
signer = Signer()
data = json.loads(
signer.unsign(form.cleaned_data["people"]))
leaflet.ynr_party_id = data["ynr_party_id"]
leaflet.ynr_party_name = data["ynr_party_name"]
leaflet.ballot_id = data["ballot_id"]
person, _ = Person.objects.get_or_create(
remote_id=data["ynr_person_id"],
defaults={
"name":
data["ynr_person_name"],
"source_url":
"https://candidates.democracyclub.org.uk/person/{}"
.format(data["ynr_person_id"]),
"source_name":
"YNR2017",
},
)
leaflet.publisher_person = person
elif (isinstance(form.cleaned_data["parties"], str)
and form.cleaned_data["parties"] != ""):
signer = Signer()
(
leaflet.ynr_party_id,
leaflet.ynr_party_name,
) = signer.unsign(form.cleaned_data["parties"]).split("--")
else:
person = form.cleaned_data.get("people")
if person:
if person.current_party:
leaflet.publisher_party = person.current_party.party
leaflet.publisher_person = person
leaflet.election = person.current_election
leaflet.save()
messages.success(self.request,
random.sample(settings.THANKYOU_MESSAGES, 1)[0])
return redirect(reverse("leaflet", kwargs={"pk": leaflet.pk}))
from django.core.files.storage import get_storage_class
from storages.backends.s3boto3 import S3Boto3Storage
"""
Documentation on:
and https://stackoverflow.com/questions/35417502/django-aws-s3-using-boto-with-compressor-fails-to-compress-uncompressablefileerr
"""
class CachedS3Boto3Storage(S3Boto3Storage):
"""
S3 storage backend that saves files locally too.
"""
location = "static"
def __init__(self, *args, **kwargs):
super(CachedS3Boto3Storage, self).__init__(*args, **kwargs)
self.local_storage = get_storage_class(
"compressor.storage.CompressorFileStorage")()
def save(self, name, content):
self.local_storage._save(name, content)
super(CachedS3Boto3Storage, self).save(name,
self.local_storage._open(name))
return name
# StaticRootS3Boto3Storage = lambda: S3Boto3Storage(location='staticcollection')
MediaRootS3BotoStorage = lambda: S3Boto3Storage(location="media",
file_overwrite="False")
from django.db.models import *
# from django.db import transaction, models
from django.utils import timezone
import django_mysql.models
from django.contrib.auth.models import User
from django.dispatch import receiver
from mptt.models import MPTTModel, TreeForeignKey
from theguide.user.models import *
from storages.backends.s3boto3 import S3Boto3Storage
MediaNoOverwriteS3BotoStorage = lambda: S3Boto3Storage(location='uploads',
file_overwrite=False)
MediaS3BotoStorage = lambda: S3Boto3Storage(location='uploads',
file_overwrite=True)
class ModuleType(MPTTModel):
name = CharField(max_length=50)
parent = TreeForeignKey('self',
on_delete=CASCADE,
null=True,
blank=True,
related_name='children')
class Module(Model):
author = ForeignKey(User, ondelete=PROTECT)
title = CharField(max_length=100)
description = TextField()
edit_copy = OneToOneField("self",
on_delete=CASCADE,
from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') PremProRootS3BotoStorage = lambda: S3Boto3Storage(location='prem_pro_root')
from storages.backends.s3boto3 import S3Boto3Storage
from decouple import config
StaticRootS3BotoStorage = lambda: S3Boto3Storage(location=config('STATIC_PATH')
)
MediaRootS3BotoStorage = lambda: S3Boto3Storage(location=config('MEDIA_PATH'))
VIDEO_STORAGE = GoogleCloudStorage(location="%svideo" % ROOT_URL,
file_overwrite=False)
FILE_STORAGE = GoogleCloudStorage(location="%sfile" % ROOT_URL,
file_overwrite=False)
AVATAR_STORAGE = GoogleCloudStorage(location="%spicture/avatar" % ROOT_URL,
file_overwrite=False)
COVER_STORAGE = GoogleCloudStorage(location="%spicture/cover" % ROOT_URL,
file_overwrite=False)
LOGO_STORAGE = GoogleCloudStorage(location="%spicture/logo" % ROOT_URL,
file_overwrite=False)
PICTURE_STORAGE = GoogleCloudStorage(location="%spicture/others" %
ROOT_URL,
file_overwrite=False)
elif USE_S3:
VIDEO_STORAGE = S3Boto3Storage(location='%svideo' % ROOT_URL,
file_overwrite=False)
FILE_STORAGE = S3Boto3Storage(location='%sfile' % ROOT_URL,
file_overwrite=False)
AVATAR_STORAGE = S3Boto3Storage(location='%spicture/avatar' % ROOT_URL,
file_overwrite=False)
COVER_STORAGE = S3Boto3Storage(location='%spicture/cover' % ROOT_URL,
file_overwrite=False)
LOGO_STORAGE = S3Boto3Storage(location='%spicture/logo' % ROOT_URL,
file_overwrite=False)
PICTURE_STORAGE = S3Boto3Storage(location='%spicture/others' % ROOT_URL,
file_overwrite=False)
else:
VIDEO_STORAGE = FileSystemStorage(location="%s/video" % MEDIA_ROOT,
base_url="%svideo/" % UPLOAD_ROOT)
FILE_STORAGE = FileSystemStorage(location="%s/file" % MEDIA_ROOT,
def write_manifest(self, manifest):
s3 = S3Boto3Storage().connection
manifest_json = json.dumps(manifest, indent=2)
s3.Object(self.output_bucket, "manifest.json").put(Body=manifest_json)
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media') def StaticRootS3BotoStorage(): return S3Boto3Storage(location='static')
class CustomUser(AbstractUser):
# username = None
email = models.EmailField(_('email address'), unique=True)
alphanumeric = RegexValidator(
r'^[0-9a-zA-Z]*$',
'Only alphanumeric characters are allowed for Username.')
alpha = RegexValidator(r'^[a-zA-Z]*$',
'Only alpha characters are allowed for Username.')
username = models.CharField(max_length=100,
unique=True,
validators=[alphanumeric])
is_active = models.BooleanField(default=False)
is_maillist = models.BooleanField(default=False)
is_liked_email = models.BooleanField(default=True)
mapillary_access_token = models.TextField(default='',
null=True,
blank=True)
verify_email_key = models.CharField(max_length=100, default='')
avatar = models.ImageField(
upload_to=image_directory_path,
null=True,
blank=True,
storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_BUCKET_NAME))
first_name = models.CharField(max_length=30,
null=True,
blank=True,
validators=[alpha])
last_name = models.CharField(max_length=30,
null=True,
blank=True,
validators=[alpha])
website_url = models.TextField(null=True, blank=True)
description = models.TextField(null=True, blank=True)
user_grade = models.ForeignKey(Grade,
on_delete=models.CASCADE,
null=True,
blank=True)
USERNAME_FIELD = 'email'
REQUIRED_FIELDS = ['username']
objects = CustomUserManager()
def __str__(self):
return self.username
# return self.email
def get_absolute_url(self):
from django.urls import reverse
return reverse('account.profile', kwargs={'username': self.username})
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
if self._password is not None:
password_validation.password_changed(self._password, self)
self._password = None
def get_short_web_url(self):
website_url = self.website_url
if len(website_url) > 30:
return website_url[0:30] + '...'
else:
return website_url
def get_sequence_limit(self):
if self.user_grade is None:
return 5
else:
return self.user_grade.sequence_limit_count
def get_custom_banner(self):
custom_banner = CustomBanner.objects.filter(
user=self).order_by('-updated_at').first()
if custom_banner is None:
return None
else:
return custom_banner
def StaticRootS3BotoStorage():
return S3Boto3Storage(location='static') # noqa
from storages.backends.s3boto3 import S3Boto3Storage
StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static')
# MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media')
class MediaStorage(S3Boto3Storage):
location = 'media'
file_overwrite = False
AWS_EXPIRY = 60 * 60 * 24 * 7
# TODO See: https://github.com/jschneier/django-storages/issues/47
# Revert the following and use str after the above-mentioned bug is fixed in
# either django-storage-redux or boto
control = 'max-age=%d, s-maxage=%d, must-revalidate' % (AWS_EXPIRY, AWS_EXPIRY)
AWS_HEADERS = {
'Cache-Control': bytes(control, encoding='latin-1')
}
# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
# See:http://stackoverflow.com/questions/10390244/
from storages.backends.s3boto3 import S3Boto3Storage
StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') # noqa
MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') # noqa
DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage'
MEDIA_URL = 'https://s3.amazonaws.com/%s/media/' % AWS_STORAGE_BUCKET_NAME
# Static Assets
# ------------------------
STATIC_URL = 'https://s3.amazonaws.com/%s/static/' % AWS_STORAGE_BUCKET_NAME
STATICFILES_STORAGE = 'config.settings.production.StaticRootS3BotoStorage'
# See: https://github.com/antonagestam/collectfast
# For Django 1.7+, 'collectfast' should come before
# 'django.contrib.staticfiles'
AWS_PRELOAD_METADATA = True
INSTALLED_APPS = ['collectfast', ] + INSTALLED_APPS
def MediaRootS3BotoStorage():
return S3Boto3Storage(location='media', file_overwrite=False)
class Scene(models.Model):
unique_id = models.UUIDField(default=uuid.uuid4,
editable=False,
unique=True)
guidebook = models.ForeignKey(Guidebook, on_delete=models.CASCADE)
image_key = models.CharField(max_length=100)
title = models.CharField(max_length=255)
description = models.TextField(null=True)
lat = models.FloatField(default=0)
lng = models.FloatField(default=0)
point = models.PointField(null=True, blank=True)
start_x = models.FloatField(default=0.5)
start_y = models.FloatField(default=0.5)
sort = models.IntegerField(default=1, null=True)
image_url = models.CharField(max_length=100, null=True, blank=True)
username = models.CharField(
max_length=100,
default='',
null=True,
blank=True,
verbose_name="Mapillary Username",
)
video_url = models.TextField(default='', blank=True)
image = models.ImageField(
upload_to=scene_image_directory_path,
max_length=255,
null=True,
blank=True,
storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_BUCKET_NAME))
external_url = models.ManyToManyField(SceneExternalURL, blank=True)
objects = models.Manager()
vector_tiles = CustomSceneMVTManager(
geo_col='point',
select_columns=['image_key', 'unique_id'],
is_show_id=False,
source_layer='mtp-scenes')
def get_poi_count(self):
points_of_interest = PointOfInterest.objects.filter(scene=self)
return points_of_interest.count()
def get_poi_categories(self):
points_of_interest = PointOfInterest.objects.filter(scene=self)
categories = []
if points_of_interest.count() > 0:
for poi in points_of_interest:
if poi.category.name not in categories:
categories.append(poi.category.name)
if len(categories) > 0:
return ', '.join(categories)
else:
return ''
def get_sequence(self):
sequences = Sequence.objects.filter(
coordinates_image__contains=[self.image_key])
if sequences.count() == 0:
return None
else:
return sequences[0]
def get_external_urls(self):
external_urls = self.external_url.all()
return external_urls
def _save(self, name, content):
"""
We're going to delete the file before we save it.
"""
full_path = self.path(name)
try:
os.remove(full_path)
except OSError:
pass
return super(OverwritingFileSystemStorage, self)._save(name, content)
if settings.AWS_S3_ENABLED:
ImageStorage = lambda: S3Boto3Storage()
else:
ImageStorage = lambda: OverwritingFileSystemStorage(location=settings.
UPLOADS_DIRECTORY)
class S3PipelineStorage(PipelineMixin, ManifestFilesMixin, S3Boto3Storage):
pass
StaticRootS3BotoStorage = lambda: S3PipelineStorage(location=settings.
STATIC_ROOT)
class LocalPipelineStorage(PipelineMixin, ManifestFilesMixin,
FileSystemStorage):
AWS_S3_OBJECT_PARAMETERS = {
'CacheControl': f'max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate',
}
# STATIC
# ------------------------
STATICFILES_STORAGE = 'config.settings.production.StaticRootS3BotoStorage'
STATIC_URL = 'https://s3.amazonaws.com/%s/static/' % AWS_STORAGE_BUCKET_NAME
# MEDIA
# ------------------------------------------------------------------------------
# region http://stackoverflow.com/questions/10390244/
from storages.backends.s3boto3 import S3Boto3Storage
StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') # noqa
MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media', file_overwrite=False) # noqa
# endregion
DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage'
MEDIA_URL = f'https://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/media/'
# TEMPLATES
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#templates
TEMPLATES[0]['OPTIONS']['loaders'] = [ # noqa F405
(
'django.template.loaders.cached.Loader',
[
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
]
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media') DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage'
