def PrivateS3Boto3Storage(): return S3Boto3Storage(default_acl='private')
def download_tarball(key, local_filename): connection = S3Boto3Storage().connection bucket = connection.Bucket("arxiv") bucket.download_file(key, local_filename, {"RequestPayer": "requester"})
def download_tarball(key, local_filename): connection = S3Boto3Storage().connection bucket = connection.Bucket('arxiv') bucket.download_file(key, local_filename, {'RequestPayer': 'requester'})
from storages.backends.s3boto3 import S3Boto3Storage #StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media')
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media') # noqa
def setUpClass(cls): if six.PY2: raise SkipTest("Skipping redundant test") cls.pixelmatch_bin = os.environ.get('PIXELMATCH_BIN') if not cls.pixelmatch_bin: cls.pixelmatch_bin = find_executable('pixelmatch') if not cls.pixelmatch_bin or not os.path.exists(cls.pixelmatch_bin): raise SkipTest("pixelmatch not installed") cls.screenshot_output_dir = os.environ.get('SCREENSHOT_OUTPUT_DIR', '/tmp/djn-tests') super(BaseNestedAdminTestCase, cls).setUpClass() cls.root_temp_dir = tempfile.mkdtemp() if os.environ.get('TRAVIS_BUILD_NUMBER'): # For some reason these tests fail on travis when Django > 1.11 if django.VERSION > (1, 11): raise SkipTest("Issue with travis and Django >= 1.11") cls.path_prefix = "travis_%s" % os.environ['TRAVIS_BUILD_NUMBER'] else: cls.path_prefix = "local" cls.temp_dir = tempfile.mkdtemp(dir=cls.root_temp_dir) os.makedirs(os.path.join(cls.temp_dir, cls.path_prefix)) if cls.screenshot_output_dir: screenshot_path = os.path.join(cls.screenshot_output_dir, cls.path_prefix) if not os.path.exists(screenshot_path): os.makedirs(screenshot_path) if all( os.environ.get(k) for k in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']): try: storage = S3Boto3Storage() bucket = storage.bucket # noqa except: pass else: cls.storage = storage cls.all_models = {} cls.all_model_names = {} for root_model in cls.root_models: root_admin = admin_site._registry[root_model] def descend_admin_inlines(admin): data = [admin.model, []] for inline in (getattr(admin, 'inlines', None) or []): data[1].append(descend_admin_inlines(inline)) return data cls.all_models[root_model] = models = descend_admin_inlines( root_admin) def recursive_map_model_names(data): if isinstance(data, list): return [m for m in map(recursive_map_model_names, data)] else: return get_model_name(data) cls.all_model_names[root_model] = recursive_map_model_names(models)
from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') ProtectedS3Storage = lambda: S3Boto3Storage(location='protected')
class Resort(models.Model): resort_pk = models.AutoField(primary_key=True) resort_id = models.UUIDField(verbose_name="resort unique id", default=uuid.uuid4, editable=False) resort_name = models.CharField(verbose_name="resort name", max_length=255) website = models.URLField(verbose_name="resort website", blank=True) location = models.ForeignKey(Location, verbose_name="resort location", blank=True, null=True) network_key = models.CharField(max_length=255, verbose_name="resort network key", default=get_random_key, unique=True) license_expiry_date = models.DateTimeField(verbose_name="resort expiry expiration date", blank=True, null=True) licenses = models.IntegerField(verbose_name="licenses owned by resort", blank=True, null=True) map_kml = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, blank=True, max_length=255) map_type = models.IntegerField(choices=MapType, blank=True, default=GOOGLE_MAP) report_form = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, verbose_name="report form for resort", blank=True, max_length=255) print_on_device = models.IntegerField(choices=Choice, blank=True, default=NO) map_lat = models.FloatField(blank=True, null=True) map_lng = models.FloatField(blank=True, null=True) initial_map_zoom_level = models.IntegerField(default=18, validators=[validate_zoom_level]) domain_id = models.ForeignKey(Domains, null=True) unit_format = models.IntegerField(choices=UnitType, default=IMPERIAL, blank=True) default_unit_paper = models.IntegerField(choices=PaperSize, blank=True, default=A4) timezone = models.CharField(max_length=50, blank=True, choices=timezones) datetime_format = models.IntegerField(blank=True, choices=DATETIME_FORMAT, default=DDMMYYYY) resort_logo = models.FileField(storage=S3Boto3Storage(bucket=settings.BUCKET_NAME), upload_to=get_upload_path, verbose_name="resort logo", blank=True, max_length=255) dt_modified = models.DateTimeField(auto_now=True) incident_template = JSONField(verbose_name="Incident template for resort", blank=True) season_start_date = models.DateField(verbose_name='season start date', default=dt.date.today) use_sequential_incident_id = models.IntegerField(verbose_name='Use sequential incident id', choices=SequentialIncidentIdChoice, default=OFF) dispatch_field_choice = JSONField(verbose_name='choice for dispatch screen', default=[{"field_key": "name"}, {"field_key": "phone"}, {"field_key": "field_52d4798f6d227____body_part"}, {"field_key": "field_52d48077a16be"}]) resort_controlled_substances = models.BooleanField(verbose_name="resort controlled substances", default=False) resort_asset_management = models.BooleanField(verbose_name="resort asset management", default=False) kms_enabled = models.BooleanField(verbose_name="KMS enabled for resort", default=False) kms_cmk = models.CharField(max_length=255, verbose_name="Customer Master Key for Resort", null=True, blank=True) enc_data_key = models.TextField(default="") def __unicode__(self): return self.resort_name def save(self, *args, **kwargs): from apps.incidents.utils import get_data_key from apps.resorts.utils import template_update if self.domain_id is None: self.domain_id = Domains.objects.get(pk=1) # If this is a new resort then do the following # Update the template # If KMS is enabled then create new customer master key and generate new data_key using that CMK if self.pk is None: self.incident_template = template_update(self, True) client = create_client() if self.kms_enabled is True: response = client.create_key(Description=self.resort_name, KeyUsage='ENCRYPT_DECRYPT') self.kms_cmk = response['KeyMetadata']['Arn'] response = client.generate_data_key( KeyId=self.kms_cmk, KeySpec='AES_256', ) self.enc_data_key = base64.b64encode(response['CiphertextBlob']) else: self.kms_cmk = settings.GLOBAL_KMS_KEY_ID response = client.generate_data_key( KeyId=self.kms_cmk, KeySpec='AES_256', ) self.enc_data_key = base64.b64encode(response['CiphertextBlob']) super(Resort, self).save(*args, **kwargs) # Call the "real" save() method. update_resort_discovery(self, self.domain_id) # If the resort is already created then # Check if sequential numbering is on (or) not and perform operation accordingly # if KMS is enabled then create new master key and also create new data key using that CMK else: self.incident_template = template_update(self, False) resort = Resort.objects.get(resort_pk=self.pk) if resort.use_sequential_incident_id == OFF and self.use_sequential_incident_id == ON: if self.season_start_date: startDate = dt.datetime.combine(self.season_start_date, dt.time()) endDate = startDate + dt.timedelta(days=365) while startDate.date() <= dt.date.today(): for index, incident in enumerate( Incident.objects.filter(dt_created__gte=startDate, dt_created__lte=endDate, resort=self).exclude(incident_status__order=9).order_by( 'dt_created')): incident.incident_sequence = index + 1 incident.save() startDate = startDate + dt.timedelta(days=365) endDate = startDate + dt.timedelta(days=365) if (self.kms_enabled is True) and ((resort.kms_cmk == '') or (resort.kms_cmk is None)): client = create_client() response = client.create_key(Description=self.resort_name, KeyUsage='ENCRYPT_DECRYPT') self.kms_cmk = response['KeyMetadata']['Arn'] pre_data_key = get_data_key(self) response = client.generate_data_key( KeyId=self.kms_cmk, KeySpec='AES_256', ) self.enc_data_key = base64.b64encode(response['CiphertextBlob']) encrypt_patient_data_kms(self, pre_data_key, base64.b64encode(response['Plaintext'])) upload_status, message = encrypt_s3_files(self, self.kms_cmk) if not upload_status: raise Exception super(Resort, self).save(*args, **kwargs) # Call the "real" save() method. if((resort.resort_name != self.resort_name) or (resort.domain_id != self.domain_id) or (resort.network_key != self.network_key)): update_resort_discovery(self, self.domain_id)
from storages.backends.s3boto3 import S3Boto3Storage MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='mediafiles') StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static')
def get_file_storage(): if settings.AWS_CONFIGURED: from storages.backends.s3boto3 import S3Boto3Storage return S3Boto3Storage() else: return default_storage
class PageBlockElement(Element): block = models.ForeignKey(PageBlock, on_delete=models.CASCADE, related_name="elements") markdown = models.TextField(blank=True, default="") connection = models.URLField(blank=True, default="", max_length=1000) internal_connection = models.ForeignKey(Page, on_delete=models.SET_NULL, null=True, related_name="connections") plain_text = models.TextField(blank=True, default="", max_length=1000) code = models.TextField(blank=True, default="", max_length=1000) embed_video = models.TextField(blank=True, default="", max_length=1000) file = models.FileField( null=True, storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_PAGES_BUCKET_NAME), upload_to=file_upload_path, max_length=200, ) image = models.ImageField( null=True, storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_PAGES_BUCKET_NAME), upload_to=file_upload_path, max_length=200, ) parent = models.ForeignKey("PageBlockElement", on_delete=models.CASCADE, related_name="sets_elements", null=True) custom_element_set = models.ForeignKey("CustomElementSet", on_delete=models.CASCADE, related_name="elements", null=True) observable_hq = models.OneToOneField("PageBlockObservableElement", on_delete=models.CASCADE, null=True) state = models.ForeignKey("states.State", null=True, related_name="elements", on_delete=models.SET_NULL) _clone_one_to_one_fields = ["observable_hq"] objects = managers.PageBlockElementManager() def natural_key(self): if hasattr(self.block, "page"): return (self.name, self.order, self.custom_element_set, self.parent) + self.block.natural_key() if hasattr(self.block, "block"): return self.name, self.order + self.block.project.title, self.block.name natural_key.dependencies = ["states.state", "pages.pageblock"] def relative_path_to_save(self, filename): base_path = self.image.storage.location if not self.block_id: raise ValueError("Page is not set") return os.path.join( base_path, f"{self.block.page_id}/blocks/{self.block_id}/{filename}") def get_original_file_name(self, file_name=None, image=True): if not file_name: file_name = self.image.name if image else self.file.name name, ext = os.path.splitext(os.path.basename(file_name)) return name, os.path.basename(file_name) def update_or_create_custom_element_sets(self, elements_sets_list): for elements_set in elements_sets_list: element_set_order = elements_set.get("order") custom_element_set, _ = CustomElementSet.objects.update_or_create( id=elements_set.pop("id", None), defaults=dict(order=element_set_order), ) set_elements = elements_set.get("elements", []) self.update_or_create_custom_element_set_elements( set_elements, custom_element_set) def update_or_create_custom_element_set_elements(self, elements, custom_element_set): for element in elements: element_type = element.get("type") element_value = element.pop("value") if (element_type in [ constants.ElementType.IMAGE, constants.ElementType.FILE ] and element_value is not False): element[element_type] = element_value if element_type in [ constants.ElementType.MARKDOWN, constants.ElementType.PLAIN_TEXT, constants.ElementType.CODE, constants.ElementType.INTERNAL_CONNECTION, constants.ElementType.CONNECTION, constants.ElementType.EMBED_VIDEO, constants.ElementType.STATE, ]: element[element_type] = element_value element, _ = PageBlockElement.objects.update_or_create( id=element.pop("id", None), defaults=dict(block=self.block, parent=self, custom_element_set=custom_element_set, **element), ) if element_type == constants.ElementType.OBSERVABLE_HQ: self.create_update_observable_element(element_value, element) def delete_custom_elements_sets(self, ids_to_delete): CustomElementSet.objects.filter(id__in=ids_to_delete).delete() def create_update_observable_element(self, value, element=None): element_instance = element if element else self hq_element, is_create = PageBlockObservableElement.objects.update_or_create( id=value.pop("id", None), defaults=dict(**value)) if is_create: element_instance.observable_hq = hq_element element_instance.save(update_fields=["observable_hq"]) def clone(self, block=None): block = block if block else self.block if self.custom_element_set: return None if self.type == constants.ElementType.OBSERVABLE_HQ: return self.clone_observable_element(block) elif self.type == constants.ElementType.CUSTOM_ELEMENT: return self.clone_custom_element(block) else: return self.clone_simple_element(block) def clone_simple_element(self, block=None, parent=None, elements_set=None): attrs = {"block": block} if elements_set and parent: attrs["custom_element_set"] = elements_set attrs["parent"] = parent return self.make_clone(attrs=attrs) def clone_observable_element(self, block=None, parent=None, elements_set=None): new_obs = self.observable_hq.make_clone() attrs = {"block": block, "observable_hq": new_obs} if elements_set and parent: attrs["custom_element_set"] = elements_set attrs["parent"] = parent return self.make_clone(attrs=attrs) def clone_custom_element(self, block=None): block = block if block else self.block new_ele = self.make_clone(attrs={"block": block}) elements_sets = { element.custom_element_set for element in self.sets_elements.all() } for element_set in elements_sets: new_set = element_set.make_clone() for set_element in element_set.elements.all(): if set_element.type == constants.ElementType.OBSERVABLE_HQ: set_element.clone_observable_element(block=block, parent=new_ele, elements_set=new_set) else: set_element.clone_simple_element(block=block, parent=new_ele, elements_set=new_set) return new_ele
def s3_client(): storage = S3Boto3Storage(bucket_name=BUCKET) return storage.connection.meta.client
f"max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate" } # STATIC # ------------------------ STATICFILES_STORAGE = "config.settings.production.StaticRootS3BotoStorage" STATIC_URL = f"http://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/static/" # MEDIA # ------------------------------------------------------------------------------ # region http://stackoverflow.com/questions/10390244/ from storages.backends.s3boto3 import S3Boto3Storage # noqa E402 StaticRootS3BotoStorage = lambda: S3Boto3Storage(location="static") # noqa MediaRootS3BotoStorage = lambda: S3Boto3Storage(location="media", file_overwrite=False) # noqa # endregion DEFAULT_FILE_STORAGE = "config.settings.production.MediaRootS3BotoStorage" MEDIA_URL = f"http://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/media/" # TEMPLATES # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#templates TEMPLATES[0]["OPTIONS"]["loaders"] = [ # noqa F405 ( "django.template.loaders.cached.Loader", [ "django.template.loaders.filesystem.Loader", "django.template.loaders.app_directories.Loader",
class LeafletUploadWizzard(NamedUrlSessionWizardView): extra_added = False TEMPLATES = { "front": "leaflets/upload_form/image_front.html", "back": "leaflets/upload_form/image_back.html", "inside": "leaflets/upload_form/image_inside.html", "postcode": "leaflets/upload_form/postcode.html", "people": "leaflets/upload_form/people.html", } if os.environ.get("AWS_SESSION_TOKEN", None): file_storage = S3Boto3Storage(location="leaflets_tmp") else: file_storage = FileSystemStorage( location=os.path.join(settings.MEDIA_ROOT, "images/leaflets_tmp")) def get_template_names(self): if self.steps.current.startswith("inside"): step_name = "inside" else: step_name = self.steps.current return [self.TEMPLATES[step_name]] def get_form_initial(self, step): if step == "people": api = DevsDCAPIHelper() postcode = self.get_cleaned_data_for_step("postcode") if postcode: results = api.postcode_request(postcode["postcode"]) if results.status_code == 200: return {"postcode_results": results} return {} @property def extra_inside_forms(self): return self.storage.extra_data.get("extra_inside", 0) def get_context_data(self, **kwargs): context = super(LeafletUploadWizzard, self).get_context_data(**kwargs) context["hide_footer"] = True return context def add_extra_inside_forms(self): self.storage.extra_data["extra_inside"] = self.extra_inside_forms + 1 self.extra_added = True def get(self, *args, **kwargs): if "reset" in self.request.GET: self.storage.reset() return HttpResponseRedirect("/") self._insert_extra_inside_forms() return super(LeafletUploadWizzard, self).get(*args, **kwargs) def post(self, *args, **kwargs): # Add forms to the form_list that we should have. # We need to do this on every request, as the extra forms # Aren't stored betweet requests, but are held in the session. self._insert_extra_inside_forms() # The user has finished uploading all image, move to the postcode form if self.request.POST.get("skip", None): if self.request.FILES: form = self.get_form(data=self.request.POST, files=self.request.FILES) if form.is_valid(): super(LeafletUploadWizzard, self).post(*args, **kwargs) else: if "extra_inside" in self.storage.extra_data: self.storage.extra_data["extra_inside"] -= 1 self._insert_extra_inside_forms(force=True) self.storage.extra_data["skip_to_postcode"] = True return self.render_goto_step("postcode") # If there are more pages, add them to the form_list # Validate the form first, though if self.request.POST.get("add_extra_inside", None): form = self.get_form(data=self.request.POST, files=self.request.FILES) if form.is_valid(): self.add_extra_inside_forms() self._insert_extra_inside_forms(force=True) return super(LeafletUploadWizzard, self).post(*args, **kwargs) def _insert_extra_inside_forms(self, force=False): """ Adds to self.form_list, inserting an extra 'inside page' form after the last inside page form we have. """ # We've not added any new forms, so the form list is as set in urls.py if not self.extra_inside_forms: return self.form_list # If we've already added the extra forms, don't them again. if self.extra_added and not force: return self.form_list # self.form_list is an ordered dict. Convert it to a list of tuples. form_list = [(k, v) for k, v in list(self.form_list.items())] # Reverse the list, so the the first step starting with 'inside' # is the last inside page form. form_list.reverse() for index, (name, form) in enumerate(form_list): if name.startswith("inside"): # For every extra inside form in self.extra_inside_forms # add to the form_list for step in range(self.extra_inside_forms): new_name = "inside-%s" % step form_list.insert(index, (new_name, InsidePageImageForm)) # We're finished going back through the form_list now break form_list.reverse() self.form_list = OrderedDict() for k, v in form_list: self.form_list[k] = v self.extra_added = True return self.form_list def done(self, form_list, **kwargs): # Create a new leaflet leaflet = Leaflet() leaflet.save() for form in form_list: form_prefix = form.prefix.split("-")[0] if form_prefix in ["front", "back", "inside"]: # Dealing with an image form image_type = None if form.prefix == "front": image_type = "1_front" if form.prefix == "back": image_type = "2_back" if form.prefix == "inside": image_type = "3_inside" image = LeafletImage(leaflet=leaflet, image_type=image_type) image.image = form.cleaned_data["image"] image.save() if form_prefix == "postcode": leaflet.postcode = form.cleaned_data["postcode"] if form_prefix == "people": if ("people" in form.cleaned_data and isinstance(form.cleaned_data["people"], str) and form.cleaned_data["people"] != ""): signer = Signer() data = json.loads( signer.unsign(form.cleaned_data["people"])) leaflet.ynr_party_id = data["ynr_party_id"] leaflet.ynr_party_name = data["ynr_party_name"] leaflet.ballot_id = data["ballot_id"] person, _ = Person.objects.get_or_create( remote_id=data["ynr_person_id"], defaults={ "name": data["ynr_person_name"], "source_url": "https://candidates.democracyclub.org.uk/person/{}" .format(data["ynr_person_id"]), "source_name": "YNR2017", }, ) leaflet.publisher_person = person elif (isinstance(form.cleaned_data["parties"], str) and form.cleaned_data["parties"] != ""): signer = Signer() ( leaflet.ynr_party_id, leaflet.ynr_party_name, ) = signer.unsign(form.cleaned_data["parties"]).split("--") else: person = form.cleaned_data.get("people") if person: if person.current_party: leaflet.publisher_party = person.current_party.party leaflet.publisher_person = person leaflet.election = person.current_election leaflet.save() messages.success(self.request, random.sample(settings.THANKYOU_MESSAGES, 1)[0]) return redirect(reverse("leaflet", kwargs={"pk": leaflet.pk}))
from django.core.files.storage import get_storage_class from storages.backends.s3boto3 import S3Boto3Storage """ Documentation on: and https://stackoverflow.com/questions/35417502/django-aws-s3-using-boto-with-compressor-fails-to-compress-uncompressablefileerr """ class CachedS3Boto3Storage(S3Boto3Storage): """ S3 storage backend that saves files locally too. """ location = "static" def __init__(self, *args, **kwargs): super(CachedS3Boto3Storage, self).__init__(*args, **kwargs) self.local_storage = get_storage_class( "compressor.storage.CompressorFileStorage")() def save(self, name, content): self.local_storage._save(name, content) super(CachedS3Boto3Storage, self).save(name, self.local_storage._open(name)) return name # StaticRootS3Boto3Storage = lambda: S3Boto3Storage(location='staticcollection') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location="media", file_overwrite="False")
from django.db.models import * # from django.db import transaction, models from django.utils import timezone import django_mysql.models from django.contrib.auth.models import User from django.dispatch import receiver from mptt.models import MPTTModel, TreeForeignKey from theguide.user.models import * from storages.backends.s3boto3 import S3Boto3Storage MediaNoOverwriteS3BotoStorage = lambda: S3Boto3Storage(location='uploads', file_overwrite=False) MediaS3BotoStorage = lambda: S3Boto3Storage(location='uploads', file_overwrite=True) class ModuleType(MPTTModel): name = CharField(max_length=50) parent = TreeForeignKey('self', on_delete=CASCADE, null=True, blank=True, related_name='children') class Module(Model): author = ForeignKey(User, ondelete=PROTECT) title = CharField(max_length=100) description = TextField() edit_copy = OneToOneField("self", on_delete=CASCADE,
from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') PremProRootS3BotoStorage = lambda: S3Boto3Storage(location='prem_pro_root')
from storages.backends.s3boto3 import S3Boto3Storage from decouple import config StaticRootS3BotoStorage = lambda: S3Boto3Storage(location=config('STATIC_PATH') ) MediaRootS3BotoStorage = lambda: S3Boto3Storage(location=config('MEDIA_PATH'))
VIDEO_STORAGE = GoogleCloudStorage(location="%svideo" % ROOT_URL, file_overwrite=False) FILE_STORAGE = GoogleCloudStorage(location="%sfile" % ROOT_URL, file_overwrite=False) AVATAR_STORAGE = GoogleCloudStorage(location="%spicture/avatar" % ROOT_URL, file_overwrite=False) COVER_STORAGE = GoogleCloudStorage(location="%spicture/cover" % ROOT_URL, file_overwrite=False) LOGO_STORAGE = GoogleCloudStorage(location="%spicture/logo" % ROOT_URL, file_overwrite=False) PICTURE_STORAGE = GoogleCloudStorage(location="%spicture/others" % ROOT_URL, file_overwrite=False) elif USE_S3: VIDEO_STORAGE = S3Boto3Storage(location='%svideo' % ROOT_URL, file_overwrite=False) FILE_STORAGE = S3Boto3Storage(location='%sfile' % ROOT_URL, file_overwrite=False) AVATAR_STORAGE = S3Boto3Storage(location='%spicture/avatar' % ROOT_URL, file_overwrite=False) COVER_STORAGE = S3Boto3Storage(location='%spicture/cover' % ROOT_URL, file_overwrite=False) LOGO_STORAGE = S3Boto3Storage(location='%spicture/logo' % ROOT_URL, file_overwrite=False) PICTURE_STORAGE = S3Boto3Storage(location='%spicture/others' % ROOT_URL, file_overwrite=False) else: VIDEO_STORAGE = FileSystemStorage(location="%s/video" % MEDIA_ROOT, base_url="%svideo/" % UPLOAD_ROOT) FILE_STORAGE = FileSystemStorage(location="%s/file" % MEDIA_ROOT,
def write_manifest(self, manifest): s3 = S3Boto3Storage().connection manifest_json = json.dumps(manifest, indent=2) s3.Object(self.output_bucket, "manifest.json").put(Body=manifest_json)
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media') def StaticRootS3BotoStorage(): return S3Boto3Storage(location='static')
class CustomUser(AbstractUser): # username = None email = models.EmailField(_('email address'), unique=True) alphanumeric = RegexValidator( r'^[0-9a-zA-Z]*$', 'Only alphanumeric characters are allowed for Username.') alpha = RegexValidator(r'^[a-zA-Z]*$', 'Only alpha characters are allowed for Username.') username = models.CharField(max_length=100, unique=True, validators=[alphanumeric]) is_active = models.BooleanField(default=False) is_maillist = models.BooleanField(default=False) is_liked_email = models.BooleanField(default=True) mapillary_access_token = models.TextField(default='', null=True, blank=True) verify_email_key = models.CharField(max_length=100, default='') avatar = models.ImageField( upload_to=image_directory_path, null=True, blank=True, storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_BUCKET_NAME)) first_name = models.CharField(max_length=30, null=True, blank=True, validators=[alpha]) last_name = models.CharField(max_length=30, null=True, blank=True, validators=[alpha]) website_url = models.TextField(null=True, blank=True) description = models.TextField(null=True, blank=True) user_grade = models.ForeignKey(Grade, on_delete=models.CASCADE, null=True, blank=True) USERNAME_FIELD = 'email' REQUIRED_FIELDS = ['username'] objects = CustomUserManager() def __str__(self): return self.username # return self.email def get_absolute_url(self): from django.urls import reverse return reverse('account.profile', kwargs={'username': self.username}) def save(self, *args, **kwargs): super().save(*args, **kwargs) if self._password is not None: password_validation.password_changed(self._password, self) self._password = None def get_short_web_url(self): website_url = self.website_url if len(website_url) > 30: return website_url[0:30] + '...' else: return website_url def get_sequence_limit(self): if self.user_grade is None: return 5 else: return self.user_grade.sequence_limit_count def get_custom_banner(self): custom_banner = CustomBanner.objects.filter( user=self).order_by('-updated_at').first() if custom_banner is None: return None else: return custom_banner
def StaticRootS3BotoStorage(): return S3Boto3Storage(location='static') # noqa
from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') # MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') class MediaStorage(S3Boto3Storage): location = 'media' file_overwrite = False
AWS_EXPIRY = 60 * 60 * 24 * 7 # TODO See: https://github.com/jschneier/django-storages/issues/47 # Revert the following and use str after the above-mentioned bug is fixed in # either django-storage-redux or boto control = 'max-age=%d, s-maxage=%d, must-revalidate' % (AWS_EXPIRY, AWS_EXPIRY) AWS_HEADERS = { 'Cache-Control': bytes(control, encoding='latin-1') } # URL that handles the media served from MEDIA_ROOT, used for managing # stored files. # See:http://stackoverflow.com/questions/10390244/ from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') # noqa MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media') # noqa DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage' MEDIA_URL = 'https://s3.amazonaws.com/%s/media/' % AWS_STORAGE_BUCKET_NAME # Static Assets # ------------------------ STATIC_URL = 'https://s3.amazonaws.com/%s/static/' % AWS_STORAGE_BUCKET_NAME STATICFILES_STORAGE = 'config.settings.production.StaticRootS3BotoStorage' # See: https://github.com/antonagestam/collectfast # For Django 1.7+, 'collectfast' should come before # 'django.contrib.staticfiles' AWS_PRELOAD_METADATA = True INSTALLED_APPS = ['collectfast', ] + INSTALLED_APPS
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media', file_overwrite=False)
class Scene(models.Model): unique_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) guidebook = models.ForeignKey(Guidebook, on_delete=models.CASCADE) image_key = models.CharField(max_length=100) title = models.CharField(max_length=255) description = models.TextField(null=True) lat = models.FloatField(default=0) lng = models.FloatField(default=0) point = models.PointField(null=True, blank=True) start_x = models.FloatField(default=0.5) start_y = models.FloatField(default=0.5) sort = models.IntegerField(default=1, null=True) image_url = models.CharField(max_length=100, null=True, blank=True) username = models.CharField( max_length=100, default='', null=True, blank=True, verbose_name="Mapillary Username", ) video_url = models.TextField(default='', blank=True) image = models.ImageField( upload_to=scene_image_directory_path, max_length=255, null=True, blank=True, storage=S3Boto3Storage(bucket=settings.AWS_STORAGE_BUCKET_NAME)) external_url = models.ManyToManyField(SceneExternalURL, blank=True) objects = models.Manager() vector_tiles = CustomSceneMVTManager( geo_col='point', select_columns=['image_key', 'unique_id'], is_show_id=False, source_layer='mtp-scenes') def get_poi_count(self): points_of_interest = PointOfInterest.objects.filter(scene=self) return points_of_interest.count() def get_poi_categories(self): points_of_interest = PointOfInterest.objects.filter(scene=self) categories = [] if points_of_interest.count() > 0: for poi in points_of_interest: if poi.category.name not in categories: categories.append(poi.category.name) if len(categories) > 0: return ', '.join(categories) else: return '' def get_sequence(self): sequences = Sequence.objects.filter( coordinates_image__contains=[self.image_key]) if sequences.count() == 0: return None else: return sequences[0] def get_external_urls(self): external_urls = self.external_url.all() return external_urls
def _save(self, name, content): """ We're going to delete the file before we save it. """ full_path = self.path(name) try: os.remove(full_path) except OSError: pass return super(OverwritingFileSystemStorage, self)._save(name, content) if settings.AWS_S3_ENABLED: ImageStorage = lambda: S3Boto3Storage() else: ImageStorage = lambda: OverwritingFileSystemStorage(location=settings. UPLOADS_DIRECTORY) class S3PipelineStorage(PipelineMixin, ManifestFilesMixin, S3Boto3Storage): pass StaticRootS3BotoStorage = lambda: S3PipelineStorage(location=settings. STATIC_ROOT) class LocalPipelineStorage(PipelineMixin, ManifestFilesMixin, FileSystemStorage):
AWS_S3_OBJECT_PARAMETERS = { 'CacheControl': f'max-age={_AWS_EXPIRY}, s-maxage={_AWS_EXPIRY}, must-revalidate', } # STATIC # ------------------------ STATICFILES_STORAGE = 'config.settings.production.StaticRootS3BotoStorage' STATIC_URL = 'https://s3.amazonaws.com/%s/static/' % AWS_STORAGE_BUCKET_NAME # MEDIA # ------------------------------------------------------------------------------ # region http://stackoverflow.com/questions/10390244/ from storages.backends.s3boto3 import S3Boto3Storage StaticRootS3BotoStorage = lambda: S3Boto3Storage(location='static') # noqa MediaRootS3BotoStorage = lambda: S3Boto3Storage(location='media', file_overwrite=False) # noqa # endregion DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage' MEDIA_URL = f'https://s3.amazonaws.com/{AWS_STORAGE_BUCKET_NAME}/media/' # TEMPLATES # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#templates TEMPLATES[0]['OPTIONS']['loaders'] = [ # noqa F405 ( 'django.template.loaders.cached.Loader', [ 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]
def MediaRootS3BotoStorage(): return S3Boto3Storage(location='media') DEFAULT_FILE_STORAGE = 'config.settings.production.MediaRootS3BotoStorage'