def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = DropboxClient(oauth2_access_token)
def __init__(self, host=None, params=None, interactive=None, file_mode=None, dir_mode=None, uid=None, gid=None, known_host_file=None, root_path=None, base_url=None): self._host = host or setting('SFTP_STORAGE_HOST') self._params = params or setting('SFTP_STORAGE_PARAMS', {}) self._interactive = setting('SFTP_STORAGE_INTERACTIVE', False) \ if interactive is None else interactive self._file_mode = setting('SFTP_STORAGE_FILE_MODE') \ if file_mode is None else file_mode self._dir_mode = setting('SFTP_STORAGE_DIR_MODE') if \ dir_mode is None else dir_mode self._uid = setting('SFTP_STORAGE_UID') if uid is None else uid self._gid = setting('SFTP_STORAGE_GID') if gid is None else gid self._known_host_file = setting('SFTP_KNOWN_HOST_FILE') \ if known_host_file is None else known_host_file self._root_path = setting('SFTP_STORAGE_ROOT', '') \ if root_path is None else root_path self._base_url = setting('MEDIA_URL') if base_url is None else base_url # for now it's all posix paths. Maybe someday we'll support figuring # out if the remote host is windows. self._pathmod = posixpath
def url(self, name, headers=None, response_headers=None): url = '{}/{}'.format(self.base_url, name) expires = int(time.time()) + setting('AWS_CLOUDFRONT_LINK_EXPIRES_TIME') if setting('AWS_CLOUDFRONT_SIGNED_URL'): return self.cf_dist.create_signed_url( url, setting('AWS_CLOUDFRONT_KEY_PAIR_ID'), expires, private_key_file=setting('AWS_CLOUDFRONT_PRIV_KEY_FILE') ) return url
def __init__(self, acl=None, bucket=None, bucket_alias=None, **settings): # check if some of the settings we've provided as class attributes # need to be overwritten with values passed in here for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) # For backward-compatibility of old differing parameter names if acl is not None: self.default_acl = acl if bucket is not None: self.bucket_name = bucket if bucket_alias is not None: self.bucket_name = setting('AWS_STORAGE_BUCKET_ALIASES', {})[bucket_alias] self.location = (self.location or '').lstrip('/') # Backward-compatibility: given the anteriority of the SECURE_URL setting # we fall back to https if specified in order to avoid the construction # of unsecure urls. if self.secure_urls: self.url_protocol = 'https:' self._entries = {} self._bucket = None self._connection = None if not self.access_key and not self.secret_key: self.access_key, self.secret_key = self._get_access_keys()
def url(self, name): if hasattr(self.connection, 'make_blob_url'): return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
def url(self, name): cdn_url = setting('AZURE_CDN_URL') if cdn_url: full_url = "{}{}/{}".format(cdn_url, self.azure_container, name) elif hasattr(self.connection, 'make_blob_url'): return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, ) else: full_url = "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name) key = setting('AZURE_CDN_TOKEN_KEY') timeout = setting('AZURE_CDN_TOKEN_TIMEOUT') if key and timeout: # Get GMT timestamp. timestamp = int(datetime(*(time.gmtime()[:-3])).strftime("%s")) full_url += '?{}'.format(encrypt_v3(key, 'ec_expire={}'.format(timestamp + timeout))) return full_url
def __init__(self, location=None, base_url=None): location = location or setting("FTP_STORAGE_LOCATION") if location is None: raise ImproperlyConfigured( "You must set a location at " "instanciation or at " " settings.FTP_STORAGE_LOCATION'." ) self.location = location base_url = base_url or settings.MEDIA_URL self._config = self._decode_location(location) self._base_url = base_url self._connection = None
def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None) ) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file
def __init__(self, *args, **kwargs): super(S3CloudFrontStorage, self).__init__(*args, **kwargs) if setting('AWS_CLOUDFRONT_SIGNED_URL'): self.cf_connection = cloudfront.CloudFrontConnection( setting('AWS_ACCESS_KEY_ID'), setting('AWS_SECRET_ACCESS_KEY') ) self.cf_dist = self.cf_connection.get_distribution_info(setting('AWS_CLOUDFRONT_DISTRIBUTION_ID')) self.base_url = "%s://%s" % (setting('AWS_CLOUDFRONT_DISTRIBUTION_PROTOCOL'), self.cf_dist.domain_name) else: self.base_url = "%s://%s" % (setting('AWS_CLOUDFRONT_DISTRIBUTION_PROTOCOL'), setting('AWS_CLOUDFRONT_DISTRIBUTION_DOMAIN'))
def __init__(self, host=None, params=None, interactive=None, file_mode=None, dir_mode=None, uid=None, gid=None, known_host_file=None, root_path=None, base_url=None): self._host = host or setting('SFTP_STORAGE_HOST') self._params = params or setting('SFTP_STORAGE_PARAMS', {}) self._interactive = setting('SFTP_STORAGE_INTERACTIVE', False) \ if interactive is None else interactive self._file_mode = setting('SFTP_STORAGE_FILE_MODE') \ if file_mode is None else file_mode self._dir_mode = setting('SFTP_STORAGE_DIR_MODE') if \ dir_mode is None else dir_mode self._uid = setting('SFTP_STORAGE_UID') if uid is None else uid self._gid = setting('SFTP_STORAGE_GID') if gid is None else gid self._known_host_file = setting('SFTP_KNOWN_HOST_FILE') \ if known_host_file is None else known_host_file self._root_path = setting('SFTP_STORAGE_ROOT', '') \ if root_path is None else root_path self._base_url = setting('MEDIA_URL') if base_url is None else base_url self._sftp = None
def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".S3Boto3StorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None) ) if 'r' in self._mode: self._is_dirty = False self._file.write(self.obj.get()['Body'].read()) self._file.seek(0) if self._storage.gzip and self.obj.content_encoding == 'gzip': self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0) return self._file
def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".S3BotoStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None), ) if "r" in self._mode: self._is_dirty = False self.key.get_contents_to_file(self._file) self._file.seek(0) if self._storage.gzip and self.key.content_encoding == "gzip": self._file = GzipFile(mode=self._mode, fileobj=self._file) return self._file
def get_modified_time(self, name): """ Returns an (aware) datetime object containing the last modified time if USE_TZ is True, otherwise returns a naive datetime in the local timezone. """ name = self._normalize_name(self._clean_name(name)) entry = self.entries.get(name) # only call self.bucket.Object() if the key is not found # in the preloaded metadata. if entry is None: entry = self.bucket.Object(self._encode_name(name)) if setting('USE_TZ'): # boto3 returns TZ aware timestamps return entry.last_modified else: return localtime(entry.last_modified).replace(tzinfo=None)
class MediaBoto3Storage(S3Boto3Storage): # pragma: no cover """ Custom storage to allow an S3 bucket to be configured specifically for uploaded media. """ bucket_name = setting( "MEDIA_STORAGE_BUCKET_NAME", setting("AWS_STORAGE_BUCKET_NAME") ) location = setting("MEDIA_LOCATION", setting("AWS_LOCATION", "")) custom_domain = setting("MEDIA_S3_CUSTOM_DOMAIN", setting("AWS_S3_CUSTOM_DOMAIN"))
def url(self, name): """ Return public url or a signed url for the Blob. This DOES NOT check for existance of Blob - that makes codes too slow for many use cases. """ name = self._normalize_name(clean_name(name)) blob = self.bucket.blob(name) custom_endpoint = setting("GS_CUSTOM_ENDPOINT") if custom_endpoint == None: return blob.public_url else: custom_url = '{storage_base_url}/{quoted_name}'.format( storage_base_url=custom_endpoint, quoted_name=_quote(name, safe=b"/~")) print(custom_url) return custom_url
def get_modified_time(self, name): """ Returns an (aware) datetime object containing the last modified time if USE_TZ is True, otherwise returns a naive datetime in the local timezone. """ properties = self.client.get_blob_properties( self._get_valid_path(name), timeout=self.timeout) if not setting('USE_TZ', False): return timezone.make_naive(properties.last_modified) tz = timezone.get_current_timezone() if timezone.is_naive(properties.last_modified): return timezone.make_aware(properties.last_modified, tz) # `last_modified` is in UTC time_zone, we # must convert it to settings time_zone return properties.last_modified.astimezone(tz)
def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".AzureBoto3StorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None) ) if 'r' in self._mode: self._is_dirty = False # I set max connection to 1 since spooledtempfile is not seekable which is required if we use # max_conection > 1 self._storage.connection.get_blob_to_stream(container_name=self._storage.azure_container, blob_name=self._name, stream=self._file, max_connections=1) self._file.seek(0) return self._file
def _get_file(self): if self._file is not None: return self._file file = SpooledTemporaryFile(max_size=self._storage.max_memory_size, suffix=".AzureStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode or 'a' in self._mode: download_stream = self._storage.client.download_blob( self._path, timeout=self._storage.timeout) download_stream.readinto(file) if 'r' in self._mode: file.seek(0) self._file = file return self._file
class CustomPublicGoogleCloudStorage(GoogleCloudStorage): """ Override the `write` method of `GoogleCloudStorage` to set the paths that match a glob in the `GS_PUBLIC_READABLE_PATHS` setting list to readable after upload """ public_readable_paths = setting('GS_PUBLIC_READABLE_PATHS', []) def _save(self, name, content): cleaned_name = super()._save(name, content) for public_readable_path in self.public_readable_paths: if fnmatch.fnmatch(cleaned_name, public_readable_path): encoded_name = self._encode_name(name) blob = self.bucket.blob(encoded_name) blob.make_public() break return cleaned_name
def get_modified_time(self, name): """ Returns an (aware) datetime object containing the last modified time if USE_TZ is True, otherwise returns a naive datetime in the local timezone. """ properties = self.service.get_blob_properties( self.azure_container, self._get_valid_path(name), timeout=self.timeout).properties if not setting('USE_TZ', False): return timezone.make_naive(properties.last_modified) tz = timezone.get_current_timezone() if timezone.is_naive(properties.last_modified): return timezone.make_aware(properties.last_modified, tz) # `last_modified` is in UTC time_zone, we # must convert it to settings time_zone return properties.last_modified.astimezone(tz)
def url(self, name): """ Return public url or a signed url for the Blob. This DOES NOT check for existance of Blob - that makes codes too slow for many use cases. """ name = self._normalize_name(clean_name(name)) blob = self.bucket.blob(self._encode_name(name)) name = self._normalize_name(clean_name(name)) if self.default_acl == 'publicRead': object_url = blob.public_url filepath = urlparse(object_url).path.lstrip('/') prefix = 'tophatch/media/' if filepath.startswith(prefix): filepath = filepath[len(prefix):] return setting('MEDIA_URL') + filepath return blob.generate_signed_url(self.expiration)
def _get_file(self): if self._file is not None: return self._file file = SpooledTemporaryFile(max_size=self._storage.max_memory_size, suffix=".AzureStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode or 'a' in self._mode: # I set max connection to 1 since spooledtempfile is # not seekable which is required if we use max_connections > 1 download_stream = self._storage.client.download_blob( self._path, timeout=self._storage.timeout) download_stream.download_to_stream(file, max_concurrency=1) if 'r' in self._mode: file.seek(0) self._file = file return self._file
class CustomAzureStorage(AzureStorage): # This is a workaround for AzureStorage to support custom domains custom_domain = setting("AZURE_CUSTOM_DOMAIN", None) @property def service(self): if self._service is None: self._service = BlockBlobService( self.account_name, self.account_key, is_emulated=self.is_emulated, custom_domain=self.custom_domain, ) return self._service def url(self, name, expire=None): url = super().url(name, expire=expire) if self.is_emulated and url.startswith("https://azurite:"): url = url.replace("https://azurite:", "http://localhost:") return url
def url(self, name, expire=None, mode='r'): if hasattr(self.connection, 'make_blob_url'): sas_token = None make_blob_url_kwargs = {} if expire: now, now_plus_delta = self._expire_at(expire) sas_token = self.connection.generate_blob_shared_access_signature(self.azure_container, name, 'r', expiry=now_plus_delta) make_blob_url_kwargs['sas_token'] = sas_token if self.azure_protocol: make_blob_url_kwargs['protocol'] = self.azure_protocol return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, **make_blob_url_kwargs ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
def __init__(self, name, mode, storage, buffer_size=None): if 'r' in mode and 'w' in mode: raise ValueError("Can't combine 'r' and 'w' in mode.") self._storage = storage self.name = name[len(self._storage.location):].lstrip('/') self._mode = mode self._force_mode = (lambda b: b) if 'b' in mode else (lambda b: b.decode()) self.obj = storage.bucket.Object(name) if 'w' not in mode: # Force early RAII-style exception if object does not exist self.obj.load() self._is_dirty = False self._raw_bytes_written = 0 self._file = None self._multipart = None # 5 MB is the minimum part size (if there is more than one part). # Amazon allows up to 10,000 parts. The default supports uploads # up to roughly 50 GB. Increase the part size to accommodate # for files larger than this. self.buffer_size = buffer_size or setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) self._write_counter = 0
class CustomPublicGoogleCloudStorage(GoogleCloudStorage): """ Override the `_save` method of `GoogleCloudStorage` to set readable permission after upload. This is for objects that should be world readable, e.g. avatar images. The paths are those that match the `GS_PUBLIC_READABLE_PATHS` setting glob.:w """ public_readable_paths = setting("GS_PUBLIC_READABLE_PATHS", []) def _save(self, name, content): cleaned_name = super()._save(name, content) for public_readable_path in self.public_readable_paths: if fnmatch.fnmatch(cleaned_name, public_readable_path): encoded_name = self._encode_name(name) blob = self.bucket.blob(encoded_name) blob.make_public() break return cleaned_name
def _get_file(self): if self._file is not None: return self._file file = SpooledTemporaryFile(max_size=self._storage.max_memory_size, suffix=".AzureStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode or 'a' in self._mode: # I set max connection to 1 since spooledtempfile is # not seekable which is required if we use max_connections > 1 self._storage.service.get_blob_to_stream( container_name=self._storage.azure_container, blob_name=self._path, stream=file, max_connections=1, timeout=self._storage.timeout) if 'r' in self._mode: file.seek(0) self._file = file return self._file
def url(self, name, expire=None): if hasattr(self.service, 'make_blob_url'): if self.auto_sign: start = (datetime.utcnow() + timedelta(seconds=-120)).strftime('%Y-%m-%dT%H:%M:%SZ') expiry = (datetime.utcnow() + timedelta(seconds=self.ap_expiry)).strftime('%Y-%m-%dT%H:%M:%SZ') sas_token = self.service.generate_blob_shared_access_signature( self.azure_container, name, permission=self.azure_access_policy_permission, expiry=expiry, start=start, ) else: sas_token = None return self.service.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, sas_token=sas_token, ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
class EToolsAzureStorage(AzureStorage): account_name = setting("AZURE_ACCOUNT_NAME") account_key = setting("AZURE_ACCOUNT_KEY") azure_container = setting("AZURE_CONTAINER") azure_ssl = setting("AZURE_SSL") auto_sign = setting("AZURE_AUTO_SIGN") azure_access_policy_permission = setting("AZURE_ACCESS_POLICY_PERMISSION") ap_expiry = setting("AZURE_ACCESS_POLICY_EXPIRY") def __init__(self, *args, **kwargs): super(EToolsAzureStorage, self).__init__(*args, **kwargs) self._connection = None def url(self, name): if hasattr(self.connection, 'make_blob_url'): if self.auto_sign: access_policy = AccessPolicy() access_policy.start = ( datetime.utcnow() + timedelta(seconds=-120)).strftime('%Y-%m-%dT%H:%M:%SZ') access_policy.expiry = (datetime.utcnow() + timedelta( seconds=self.ap_expiry)).strftime('%Y-%m-%dT%H:%M:%SZ') access_policy.permission = self.azure_access_policy_permission sap = SharedAccessPolicy(access_policy) sas_token = self.connection.generate_shared_access_signature( self.azure_container, blob_name=name, shared_access_policy=sap, ) else: sas_token = None return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, sas_token=sas_token, ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
def _get_file(self): if self._file is not None: return self._file file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".AzureStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode or 'a' in self._mode: # I set max connection to 1 since spooledtempfile is # not seekable which is required if we use max_connections > 1 self._storage.service.get_blob_to_stream( container_name=self._storage.azure_container, blob_name=self._path, stream=file, max_connections=1, timeout=10) if 'r' in self._mode: file.seek(0) self._file = file return self._file
def url(self, name): if hasattr(self.connection, 'make_blob_url'): if self.auto_sign: access_policy = AccessPolicy() access_policy.start = (datetime.utcnow() + timedelta(seconds=-120)).strftime('%Y-%m-%dT%H:%M:%SZ') access_policy.expiry = (datetime.utcnow() + timedelta(seconds=self.ap_expiry)).strftime('%Y-%m-%dT%H:%M:%SZ') access_policy.permission = self.azure_access_policy_permission sap = SharedAccessPolicy(access_policy) sas_token = self.connection.generate_shared_access_signature( self.azure_container, blob_name=name, shared_access_policy=sap, ) else: sas_token = None return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, sas_token=sas_token ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
def __init__(self, file_id, name, original_name, size, metadata=None, **kwargs): boto_storage = S3Boto3Storage( bucket_name=setting("AWS_STORAGE_BUCKET_NAME"), endpoint_url=setting("AWS_S3_ENDPOINT_URL"), access_key=setting( "AWS_S3_ACCESS_KEY_ID", setting("AWS_ACCESS_KEY_ID") ), secret_key=setting( "AWS_S3_SECRET_ACCESS_KEY", setting("AWS_SECRET_ACCESS_KEY") ), ) super().__init__(name=name, mode="rb", storage=boto_storage, **kwargs) self.file_id = file_id self.original_name = original_name # self.size is derived from S3boto3StorageFile # but size is passed for consistency, and potentially # for validation self.is_placeholder = False self.is_s3direct = True self.metadata = metadata
class AzureStorage(Storage): account_name = setting("AZURE_ACCOUNT_NAME") account_key = setting("AZURE_ACCOUNT_KEY") azure_container = setting("AZURE_CONTAINER") azure_ssl = setting("AZURE_SSL") def __init__(self, *args, **kwargs): super(AzureStorage, self).__init__(*args, **kwargs) self._connection = None @property def connection(self): if self._connection is None: self._connection = BlockBlobService( account_name = self.account_name, account_key = self.account_key) #self._connection = BlobService( # self.account_name, self.account_key) return self._connection @property def azure_protocol(self): if self.azure_ssl: return 'https' return 'http' if self.azure_ssl is not None else None def __get_blob_properties(self, name): try: return self.connection.get_blob_properties( self.azure_container, name ) except AzureMissingResourceHttpError: return None def _open(self, name, mode="rb"): contents = self.connection.get_blob(self.azure_container, name) return ContentFile(contents) def exists(self, name): return self.__get_blob_properties(name) is not None def delete(self, name): try: self.connection.delete_blob(self.azure_container, name) except AzureMissingResourceHttpError: pass def size(self, name): properties = self.connection.get_blob_properties( self.azure_container, name) return properties["content-length"] def _save(self, name, content): if hasattr(content.file, 'content_type'): content_type = content.file.content_type else: content_type = mimetypes.guess_type(name)[0] if hasattr(content, 'chunks'): content_data = b''.join(chunk for chunk in content.chunks()) else: content_data = content.read() self.connection.create_blob_from_bytes(self.azure_container, name, content_data, content_settings=ContentSettings(content_type=content_type)) #self.connection.put_blob(self.azure_container, name, # content_data, "BlockBlob", # x_ms_blob_content_type=content_type) return name def url(self, name): if hasattr(self.connection, 'make_blob_url'): return self.connection.make_blob_url( container_name=self.azure_container, blob_name=name, protocol=self.azure_protocol, ) else: return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name) def modified_time(self, name): try: modified = self.__get_blob_properties(name)['last-modified'] except (TypeError, KeyError): return super(AzureStorage, self).modified_time(name) modified = time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z') modified = datetime.fromtimestamp(mktime(modified)) return modified
def __init__(self): bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME) super(ImportExportS3Storage, self).__init__(bucket=bucket, custom_domain=None, querystring_auth=True)
class GoogleCloudStorage(Storage): project_id = setting('GS_PROJECT_ID', None) credentials = setting('GS_CREDENTIALS', None) bucket_name = setting('GS_BUCKET_NAME', None) location = setting('GS_LOCATION', '') auto_create_bucket = setting('GS_AUTO_CREATE_BUCKET', False) auto_create_acl = setting('GS_AUTO_CREATE_ACL', 'projectPrivate') file_name_charset = setting('GS_FILE_NAME_CHARSET', 'utf-8') file_overwrite = setting('GS_FILE_OVERWRITE', True) # The max amount of memory a returned file can take up before being # rolled over into a temporary file on disk. Default is 0: Do not roll over. max_memory_size = setting('GS_MAX_MEMORY_SIZE', 0) base_url = setting('GS_CDN_BASE_URL', None) def __init__(self, **settings): # check if some of the settings we've provided as class attributes # need to be overwritten with values passed in here for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) self.location = (self.location or '').lstrip('/') self._bucket = None self._client = None @property def client(self): if self._client is None: self._client = Client( project=self.project_id, credentials=self.credentials ) return self._client @property def bucket(self): if self._bucket is None: self._bucket = self._get_or_create_bucket(self.bucket_name) return self._bucket def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. """ try: return self.client.get_bucket(name) except NotFound: if self.auto_create_bucket: bucket = self.client.create_bucket(name) bucket.acl.save_predefined(self.auto_create_acl) return bucket raise ImproperlyConfigured("Bucket %s does not exist. Buckets " "can be automatically created by " "setting GS_AUTO_CREATE_BUCKET to " "``True``." % name) def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so they need to be fixed here. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name) def _encode_name(self, name): return smart_str(name, encoding=self.file_name_charset) def _open(self, name, mode='rb'): name = self._normalize_name(clean_name(name)) file_object = GoogleCloudFile(name, mode, self) if not file_object.blob: raise IOError(u'File does not exist: %s' % name) return file_object def _save(self, name, content): cleaned_name = clean_name(name) name = self._normalize_name(cleaned_name) content.name = cleaned_name encoded_name = self._encode_name(name) file = GoogleCloudFile(encoded_name, 'rw', self) file.blob.upload_from_file(content, size=content.size, content_type=file.mime_type) return cleaned_name def delete(self, name): name = self._normalize_name(clean_name(name)) self.bucket.delete_blob(self._encode_name(name)) def exists(self, name): if not name: # root element aka the bucket try: self.bucket return True except ImproperlyConfigured: return False name = self._normalize_name(clean_name(name)) return bool(self.bucket.get_blob(self._encode_name(name))) def listdir(self, name): name = self._normalize_name(clean_name(name)) # for the bucket.list and logic below name needs to end in / # But for the root path "" we leave it as an empty string if name and not name.endswith('/'): name += '/' files_list = list(self.bucket.list_blobs(prefix=self._encode_name(name))) files = [] dirs = set() base_parts = name.split("/")[:-1] for item in files_list: parts = item.name.split("/") parts = parts[len(base_parts):] if len(parts) == 1 and parts[0]: # File files.append(parts[0]) elif len(parts) > 1 and parts[0]: # Directory dirs.add(parts[0]) return list(dirs), files def _get_blob(self, name): # Wrap google.cloud.storage's blob to raise if the file doesn't exist blob = self.bucket.get_blob(name) if blob is None: raise NotFound(u'File does not exist: {}'.format(name)) return blob def size(self, name): name = self._normalize_name(clean_name(name)) blob = self._get_blob(self._encode_name(name)) return blob.size def modified_time(self, name): name = self._normalize_name(clean_name(name)) blob = self._get_blob(self._encode_name(name)) return timezone.make_naive(blob.updated) def get_modified_time(self, name): name = self._normalize_name(clean_name(name)) blob = self._get_blob(self._encode_name(name)) updated = blob.updated return updated if setting('USE_TZ') else timezone.make_naive(updated) def get_created_time(self, name): """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ name = self._normalize_name(clean_name(name)) blob = self._get_blob(self._encode_name(name)) created = blob.time_created return created if setting('USE_TZ') else timezone.make_naive(created) def url(self, name): # Preserve the trailing slash after normalizing the path. name = self._normalize_name(clean_name(name)) if self.base_url: return self.base_url + name else: blob = self._get_blob(self._encode_name(name)) return blob.public_url def get_available_name(self, name, max_length=None): if self.file_overwrite: name = clean_name(name) return name return super(GoogleCloudStorage, self).get_available_name(name, max_length)
class GoogleCloudStorage(StorageWithTransactionSupportMixin, gcloud.GoogleCloudStorage): preserve_acl = setting('GS_PRESERVE_ACL', True) def __init__(self, location=settings.LOCATION, storage_url=None, **settings): self._location = location self._storage_url = storage_url super().__init__(**settings) def internal_url(self, name): if self._storage_url: return self._storage_url + filepath_to_uri(name) # Remove unnecessary :443 from url that is created by boto for python > 2.7 url = super().url(name) if ':443' in url: return url.replace(':443', '') return url def copy(self, src_name, dest_name): src_name = self._normalize_name(gcloud.clean_name(src_name)) dest_name = self._normalize_name(gcloud.clean_name(dest_name)) bucket = self.bucket source_blob = bucket.blob(self._encode_name(src_name)) destination_blob = bucket.copy_blob(source_blob, bucket, self._encode_name(dest_name)) if self.preserve_acl and self.default_acl: destination_blob.acl.save(acl=source_blob.acl) return dest_name def size(self, name): name = self._normalize_name(gcloud.clean_name(name)) return self.bucket.get_blob(self._encode_name(name)).size def delete(self, name): name = self._normalize_name(gcloud.clean_name(name)) self.bucket.delete_blobs([self._encode_name(name)], on_error=lambda blob: None) def delete_files(self, prefix, buckets, **kwargs): if not prefix.endswith('/'): prefix += '/' for bucket_name in buckets: bucket_name = get_loc_env(self._location, bucket_name) bucket = self.client.get_bucket(bucket_name) for blob in bucket.list_blobs(prefix=prefix): blob.delete() return def _save(self, name, content): storage = getattr(content, '_storage', None) if storage and storage == self: # If already exists, just copy it. return self.copy(content.name, name) return super()._save(name, content)
class S3Boto3StorageFile(File): """ The default file object used by the S3Boto3Storage backend. This file implements file streaming using boto's multipart uploading functionality. The file can be opened in read or write mode. This class extends Django's File class. However, the contained data is only the data contained in the current buffer. So you should not access the contained file object directly. You should access the data via this class. Warning: This file *must* be closed using the close() method in order to properly write the file to S3. Be sure to close the file in your application. """ buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) def __init__(self, name, mode, storage, buffer_size=None): if 'r' in mode and 'w' in mode: raise ValueError("Can't combine 'r' and 'w' in mode.") self._storage = storage self.name = name[len(self._storage.location):].lstrip('/') self._mode = mode self._force_mode = (lambda b: b) if 'b' in mode else ( lambda b: b.decode()) self.obj = storage.bucket.Object(name) if 'w' not in mode: # Force early RAII-style exception if object does not exist self.obj.load() self._is_dirty = False self._raw_bytes_written = 0 self._file = None self._multipart = None # 5 MB is the minimum part size (if there is more than one part). # Amazon allows up to 10,000 parts. The default supports uploads # up to roughly 50 GB. Increase the part size to accommodate # for files larger than this. if buffer_size is not None: self.buffer_size = buffer_size self._write_counter = 0 @property def size(self): return self.obj.content_length def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".S3Boto3StorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR")) if 'r' in self._mode: self._is_dirty = False self.obj.download_fileobj(self._file) self._file.seek(0) if self._storage.gzip and self.obj.content_encoding == 'gzip': self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, *args, **kwargs): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") return self._force_mode(super().read(*args, **kwargs)) def readline(self, *args, **kwargs): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") return self._force_mode(super().readline(*args, **kwargs)) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True if self._multipart is None: self._multipart = self.obj.initiate_multipart_upload( **self._storage._get_write_parameters(self.obj.key)) if self.buffer_size <= self._buffer_file_size: self._flush_write_buffer() bstr = force_bytes(content) self._raw_bytes_written += len(bstr) return super().write(bstr) @property def _buffer_file_size(self): pos = self.file.tell() self.file.seek(0, os.SEEK_END) length = self.file.tell() self.file.seek(pos) return length def _flush_write_buffer(self): """ Flushes the write buffer. """ if self._buffer_file_size: self._write_counter += 1 self.file.seek(0) part = self._multipart.Part(self._write_counter) part.upload(Body=self.file.read()) self.file.seek(0) self.file.truncate() def _create_empty_on_close(self): """ Attempt to create an empty file for this key when this File is closed if no bytes have been written and no object already exists on S3 for this key. This behavior is meant to mimic the behavior of Django's builtin FileSystemStorage, where files are always created after they are opened in write mode: f = storage.open("file.txt", mode="w") f.close() """ assert "w" in self._mode assert self._raw_bytes_written == 0 try: # Check if the object exists on the server; if so, don't do anything self.obj.load() except ClientError as err: if err.response["ResponseMetadata"]["HTTPStatusCode"] == 404: self.obj.put(Body=b"", **self._storage._get_write_parameters( self.obj.key)) else: raise def close(self): if self._is_dirty: self._flush_write_buffer() # TODO: Possibly cache the part ids as they're being uploaded # instead of requesting parts from server. For now, emulating # s3boto's behavior. parts = [{ 'ETag': part.e_tag, 'PartNumber': part.part_number } for part in self._multipart.parts.all()] self._multipart.complete(MultipartUpload={'Parts': parts}) else: if self._multipart is not None: self._multipart.abort() if 'w' in self._mode and self._raw_bytes_written == 0: self._create_empty_on_close() if self._file is not None: self._file.close() self._file = None
def test_get_setting(self): value = utils.setting('SECRET_KEY') self.assertEqual(settings.SECRET_KEY, value)
class S3Boto3Storage(Storage): """ Amazon Simple Storage Service using Boto3 This storage backend supports opening files in read or write mode and supports streaming(buffering) data in chunks to S3 when writing. """ connection_service_name = 's3' default_content_type = 'application/octet-stream' connection_response_error = ClientError file_class = S3Boto3StorageFile # If config provided in init, signature_version and addressing_style settings/args are ignored. config = None # used for looking up the access and secret key from env vars access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) file_overwrite = setting('AWS_S3_FILE_OVERWRITE', True) object_parameters = setting('AWS_S3_OBJECT_PARAMETERS', {}) bucket_name = setting('AWS_STORAGE_BUCKET_NAME') auto_create_bucket = setting('AWS_AUTO_CREATE_BUCKET', False) default_acl = setting('AWS_DEFAULT_ACL', 'public-read') bucket_acl = setting('AWS_BUCKET_ACL', default_acl) querystring_auth = setting('AWS_QUERYSTRING_AUTH', True) querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600) signature_version = setting('AWS_S3_SIGNATURE_VERSION') reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False) location = setting('AWS_LOCATION', '') encryption = setting('AWS_S3_ENCRYPTION', False) custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') addressing_style = setting('AWS_S3_ADDRESSING_STYLE') secure_urls = setting('AWS_S3_SECURE_URLS', True) file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8') gzip = setting('AWS_IS_GZIPPED', False) preload_metadata = setting('AWS_PRELOAD_METADATA', False) gzip_content_types = setting('GZIP_CONTENT_TYPES', ( 'text/css', 'text/javascript', 'application/javascript', 'application/x-javascript', 'image/svg+xml', )) url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:') endpoint_url = setting('AWS_S3_ENDPOINT_URL', None) region_name = setting('AWS_S3_REGION_NAME', None) use_ssl = setting('AWS_S3_USE_SSL', True) # The max amount of memory a returned file can take up before being # rolled over into a temporary file on disk. Default is 0: Do not roll over. max_memory_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0) def __init__(self, acl=None, bucket=None, **settings): # check if some of the settings we've provided as class attributes # need to be overwritten with values passed in here for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) # For backward-compatibility of old differing parameter names if acl is not None: self.default_acl = acl if bucket is not None: self.bucket_name = bucket self.location = (self.location or '').lstrip('/') # Backward-compatibility: given the anteriority of the SECURE_URL setting # we fall back to https if specified in order to avoid the construction # of unsecure urls. if self.secure_urls: self.url_protocol = 'https:' self._entries = {} self._bucket = None self._connection = None if not self.access_key and not self.secret_key: self.access_key, self.secret_key = self._get_access_keys() if not self.config: self.config = Config( s3={'addressing_style': self.addressing_style}, signature_version=self.signature_version) @property def connection(self): # TODO: Support host, port like in s3boto # Note that proxies are handled by environment variables that the underlying # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338 # and http://docs.python-requests.org/en/latest/user/advanced/#proxies if self._connection is None: session = boto3.session.Session() self._connection = session.resource( self.connection_service_name, aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, region_name=self.region_name, use_ssl=self.use_ssl, endpoint_url=self.endpoint_url, config=self.config) return self._connection @property def bucket(self): """ Get the current bucket. If there is no current bucket object create it. """ if self._bucket is None: self._bucket = self._get_or_create_bucket(self.bucket_name) return self._bucket @property def entries(self): """ Get the locally cached files for the bucket. """ if self.preload_metadata and not self._entries: self._entries = dict( (self._decode_name(entry.key), entry) for entry in self.bucket.objects.filter(Prefix=self.location)) return self._entries def _get_access_keys(self): """ Gets the access keys to use when accessing S3. If none are provided to the class in the constructor or in the settings then get them from the environment variables. """ def lookup_env(names): for name in names: value = os.environ.get(name) if value: return value access_key = self.access_key or lookup_env(self.access_key_names) secret_key = self.secret_key or lookup_env(self.secret_key_names) return access_key, secret_key def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. """ bucket = self.connection.Bucket(name) if self.auto_create_bucket: try: # Directly call head_bucket instead of bucket.load() because head_bucket() # fails on wrong region, while bucket.load() does not. bucket.meta.client.head_bucket(Bucket=name) except self.connection_response_error as err: if err.response['ResponseMetadata']['HTTPStatusCode'] == 301: raise ImproperlyConfigured( "Bucket %s exists, but in a different " "region than we are connecting to. Set " "the region to connect to by setting " "AWS_S3_REGION_NAME to the correct region." % name) elif err.response['ResponseMetadata']['HTTPStatusCode'] == 404: # Notes: When using the us-east-1 Standard endpoint, you can create # buckets in other regions. The same is not true when hitting region specific # endpoints. However, when you create the bucket not in the same region, the # connection will fail all future requests to the Bucket after the creation # (301 Moved Permanently). # # For simplicity, we enforce in S3Boto3Storage that any auto-created # bucket must match the region that the connection is for. # # Also note that Amazon specifically disallows "us-east-1" when passing bucket # region names; LocationConstraint *must* be blank to create in US Standard. bucket_params = {'ACL': self.bucket_acl} region_name = self.connection.meta.client.meta.region_name if region_name != 'us-east-1': bucket_params['CreateBucketConfiguration'] = { 'LocationConstraint': region_name } bucket.create(ACL=self.bucket_acl) else: raise ImproperlyConfigured( "Bucket %s does not exist. Buckets " "can be automatically created by " "setting AWS_AUTO_CREATE_BUCKET to " "``True``." % name) return bucket def _clean_name(self, name): """ Cleans the name so that Windows style paths work """ # Normalize Windows style paths clean_name = posixpath.normpath(name).replace('\\', '/') # os.path.normpath() can strip trailing slashes so we implement # a workaround here. if name.endswith('/') and not clean_name.endswith('/'): # Add a trailing slash as it was stripped. return clean_name + '/' else: return clean_name def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt work. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name) def _encode_name(self, name): return smart_str(name, encoding=self.file_name_charset) def _decode_name(self, name): return force_text(name, encoding=self.file_name_charset) def _compress_content(self, content): """Gzip a given string content.""" zbuf = BytesIO() zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) try: zfile.write(force_bytes(content.read())) finally: zfile.close() zbuf.seek(0) # Boto 2 returned the InMemoryUploadedFile with the file pointer replaced, # but Boto 3 seems to have issues with that. No need for fp.name in Boto3 # so just returning the BytesIO directly return zbuf def _open(self, name, mode='rb'): name = self._normalize_name(self._clean_name(name)) try: f = self.file_class(name, mode, self) except self.connection_response_error as err: if err.response['ResponseMetadata']['HTTPStatusCode'] == 404: raise IOError('File does not exist: %s' % name) raise # Let it bubble up if it was some other error return f def _save(self, name, content): cleaned_name = self._clean_name(name) name = self._normalize_name(cleaned_name) parameters = self.object_parameters.copy() content_type = getattr( content, 'content_type', mimetypes.guess_type(name)[0] or self.default_content_type) # setting the content_type in the key object is not enough. parameters.update({'ContentType': content_type}) if self.gzip and content_type in self.gzip_content_types: content = self._compress_content(content) parameters.update({'ContentEncoding': 'gzip'}) encoded_name = self._encode_name(name) obj = self.bucket.Object(encoded_name) if self.preload_metadata: self._entries[encoded_name] = obj self._save_content(obj, content, parameters=parameters) # Note: In boto3, after a put, last_modified is automatically reloaded # the next time it is accessed; no need to specifically reload it. return cleaned_name def _save_content(self, obj, content, parameters): # only pass backwards incompatible arguments if they vary from the default put_parameters = parameters.copy() if parameters else {} if self.encryption: put_parameters['ServerSideEncryption'] = 'AES256' if self.reduced_redundancy: put_parameters['StorageClass'] = 'REDUCED_REDUNDANCY' if self.default_acl: put_parameters['ACL'] = self.default_acl content.seek(0, os.SEEK_SET) obj.upload_fileobj(content, ExtraArgs=put_parameters) def delete(self, name): name = self._normalize_name(self._clean_name(name)) self.bucket.Object(self._encode_name(name)).delete() def exists(self, name): if not name: try: self.bucket return True except ImproperlyConfigured: return False name = self._normalize_name(self._clean_name(name)) if self.entries: return name in self.entries obj = self.bucket.Object(self._encode_name(name)) try: obj.load() return True except self.connection_response_error: return False def listdir(self, name): name = self._normalize_name(self._clean_name(name)) # for the bucket.objects.filter and logic below name needs to end in / # But for the root path "" we leave it as an empty string if name and not name.endswith('/'): name += '/' files = [] dirs = set() base_parts = name.split("/")[:-1] for item in self.bucket.objects.filter(Prefix=self._encode_name(name)): parts = item.key.split("/") parts = parts[len(base_parts):] if len(parts) == 1: # File files.append(parts[0]) elif len(parts) > 1: # Directory dirs.add(parts[0]) return list(dirs), files def size(self, name): name = self._normalize_name(self._clean_name(name)) if self.entries: entry = self.entries.get(name) if entry: return entry.content_length return 0 return self.bucket.Object(self._encode_name(name)).content_length def get_modified_time(self, name): """ Returns an (aware) datetime object containing the last modified time if USE_TZ is True, otherwise returns a naive datetime in the local timezone. """ name = self._normalize_name(self._clean_name(name)) entry = self.entries.get(name) # only call self.bucket.Object() if the key is not found # in the preloaded metadata. if entry is None: entry = self.bucket.Object(self._encode_name(name)) if setting('USE_TZ'): # boto3 returns TZ aware timestamps return entry.last_modified else: return localtime(entry.last_modified).replace(tzinfo=None) def modified_time(self, name): """Returns a naive datetime object containing the last modified time.""" # If USE_TZ=False then get_modified_time will return a naive datetime # so we just return that, else we have to localize and strip the tz mtime = self.get_modified_time(name) return mtime if is_naive(mtime) else localtime(mtime).replace( tzinfo=None) def _strip_signing_parameters(self, url): # Boto3 does not currently support generating URLs that are unsigned. Instead we # take the signed URLs and strip any querystring params related to signing and expiration. # Note that this may end up with URLs that are still invalid, especially if params are # passed in that only work with signed URLs, e.g. response header params. # The code attempts to strip all query parameters that match names of known parameters # from v2 and v4 signatures, regardless of the actual signature version used. split_url = urlparse.urlsplit(url) qs = urlparse.parse_qsl(split_url.query, keep_blank_values=True) blacklist = set([ 'x-amz-algorithm', 'x-amz-credential', 'x-amz-date', 'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature', 'x-amz-security-token', 'awsaccesskeyid', 'expires', 'signature' ]) filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist) # Note: Parameters that did not have a value in the original query string will have # an '=' sign appended to it, e.g ?foo&bar becomes ?foo=&bar= joined_qs = ('='.join(keyval) for keyval in filtered_qs) split_url = split_url._replace(query="&".join(joined_qs)) return split_url.geturl() def url(self, name, parameters=None, expire=None): # Preserve the trailing slash after normalizing the path. # TODO: Handle force_http=not self.secure_urls like in s3boto name = self._normalize_name(self._clean_name(name)) if self.custom_domain: return "%s//%s/%s" % (self.url_protocol, self.custom_domain, filepath_to_uri(name)) if expire is None: expire = self.querystring_expire params = parameters.copy() if parameters else {} params['Bucket'] = self.bucket.name params['Key'] = self._encode_name(name) url = self.bucket.meta.client.generate_presigned_url('get_object', Params=params, ExpiresIn=expire) if self.querystring_auth: return url return self._strip_signing_parameters(url) def get_available_name(self, name, max_length=None): """Overwrite existing file with the same name.""" if self.file_overwrite: name = self._clean_name(name) return name return super(S3Boto3Storage, self).get_available_name(name, max_length)
class S3BotoStorageFile(File): """ The default file object used by the S3BotoStorage backend. This file implements file streaming using boto's multipart uploading functionality. The file can be opened in read or write mode. This class extends Django's File class. However, the contained data is only the data contained in the current buffer. So you should not access the contained file object directly. You should access the data via this class. Warning: This file *must* be closed using the close() method in order to properly write the file to S3. Be sure to close the file in your application. """ # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing. # TODO: When Django drops support for Python 2.5, rewrite to use the # BufferedIO streams in the Python 2.6 io module. buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) def __init__(self, name, mode, storage, buffer_size=None): self._storage = storage self.name = name[len(self._storage.location):].lstrip('/') self._mode = mode self.key = storage.bucket.get_key(self._storage._encode_name(name)) if not self.key and 'w' in mode: self.key = storage.bucket.new_key(storage._encode_name(name)) self._is_dirty = False self._file = None self._multipart = None # 5 MB is the minimum part size (if there is more than one part). # Amazon allows up to 10,000 parts. The default supports uploads # up to roughly 50 GB. Increase the part size to accommodate # for files larger than this. if buffer_size is not None: self.buffer_size = buffer_size self._write_counter = 0 if not hasattr(django_settings, 'AWS_DEFAULT_ACL'): warnings.warn( "The default behavior of S3BotoStorage is insecure. By default files " "and new buckets are saved with an ACL of 'public-read' (globally " "publicly readable). To change to using the bucket's default ACL " "set AWS_DEFAULT_ACL = None, otherwise to silence this warning " "explicitly set AWS_DEFAULT_ACL." ) @property def size(self): return self.key.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix='.S3BotoStorageFile', dir=setting('FILE_UPLOAD_TEMP_DIR') ) if 'r' in self._mode: self._is_dirty = False self.key.get_contents_to_file(self._file) self._file.seek(0) if self._storage.gzip and self.key.content_encoding == 'gzip': self._file = GzipFile(mode=self._mode, fileobj=self._file) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, *args, **kwargs): if 'r' not in self._mode: raise AttributeError('File was not opened in read mode.') return super(S3BotoStorageFile, self).read(*args, **kwargs) def write(self, content, *args, **kwargs): if 'w' not in self._mode: raise AttributeError('File was not opened in write mode.') self._is_dirty = True if self._multipart is None: provider = self.key.bucket.connection.provider upload_headers = {} if self._storage.default_acl: upload_headers[provider.acl_header] = self._storage.default_acl upload_headers.update({ 'Content-Type': mimetypes.guess_type(self.key.name)[0] or self._storage.key_class.DefaultContentType }) upload_headers.update(self._storage.headers) self._multipart = self._storage.bucket.initiate_multipart_upload( self.key.name, headers=upload_headers, reduced_redundancy=self._storage.reduced_redundancy, encrypt_key=self._storage.encryption, ) if self.buffer_size <= self._buffer_file_size: self._flush_write_buffer() return super(S3BotoStorageFile, self).write(force_bytes(content), *args, **kwargs) @property def _buffer_file_size(self): pos = self.file.tell() self.file.seek(0, os.SEEK_END) length = self.file.tell() self.file.seek(pos) return length def _flush_write_buffer(self): if self._buffer_file_size: self._write_counter += 1 self.file.seek(0) headers = self._storage.headers.copy() self._multipart.upload_part_from_file( self.file, self._write_counter, headers=headers) self.file.seek(0) self.file.truncate() def close(self): if self._is_dirty: self._flush_write_buffer() self._multipart.complete_upload() else: if self._multipart is not None: self._multipart.cancel_upload() self.key.close() if self._file is not None: self._file.close() self._file = None
def url(self, name): return u"{}/{}".format(setting('MEDIA_URL').rstrip('/'), name)
class DigitalOceanSpacesPrivateMediaStorage(DigitalOceanSpacesStorage): location = setting('DO_SPACES_PRIVATE_MEDIA_LOCATION') default_acl = 'private' file_overwrite = False custom_domain = False
def get_modified_time(self, name): name = self._normalize_name(clean_name(name)) blob = self._get_blob(self._encode_name(name)) updated = blob.updated return updated if setting('USE_TZ') else timezone.make_naive(updated)
def get_modified_time(self, name): dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) return dt if setting('USE_TZ') else tz.make_naive(dt)
def test_setting_unfound(self): self.assertIsNone(utils.setting('FOO')) self.assertEqual(utils.setting('FOO', 'bar'), 'bar') with self.assertRaises(ImproperlyConfigured): utils.setting('FOO', strict=True)
class S3BotoStorage(Storage): """ Amazon Simple Storage Service using Boto This storage backend supports opening files in read or write mode and supports streaming(buffering) data in chunks to S3 when writing. """ connection_class = S3Connection connection_response_error = S3ResponseError file_class = S3BotoStorageFile key_class = S3Key # used for looking up the access and secret key from env vars access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] security_token = None access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) file_overwrite = setting('AWS_S3_FILE_OVERWRITE', True) headers = setting('AWS_HEADERS', {}) bucket_name = setting('AWS_STORAGE_BUCKET_NAME') auto_create_bucket = setting('AWS_AUTO_CREATE_BUCKET', False) default_acl = setting('AWS_DEFAULT_ACL', 'public-read') bucket_acl = setting('AWS_BUCKET_ACL', default_acl) querystring_auth = setting('AWS_QUERYSTRING_AUTH', True) querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600) reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False) location = setting('AWS_LOCATION', '') origin = setting('AWS_ORIGIN', Location.DEFAULT) encryption = setting('AWS_S3_ENCRYPTION', False) custom_domain = setting('AWS_S3_CUSTOM_DOMAIN') calling_format = setting('AWS_S3_CALLING_FORMAT', SubdomainCallingFormat()) secure_urls = setting('AWS_S3_SECURE_URLS', True) file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8') gzip = setting('AWS_IS_GZIPPED', False) preload_metadata = setting('AWS_PRELOAD_METADATA', False) gzip_content_types = setting('GZIP_CONTENT_TYPES', ( 'text/css', 'text/javascript', 'application/javascript', 'application/x-javascript', 'image/svg+xml', )) url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:') host = setting('AWS_S3_HOST', S3Connection.DefaultHost) use_ssl = setting('AWS_S3_USE_SSL', True) port = setting('AWS_S3_PORT') proxy = setting('AWS_S3_PROXY_HOST') proxy_port = setting('AWS_S3_PROXY_PORT') max_memory_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0) def __init__(self, acl=None, bucket=None, **settings): # check if some of the settings we've provided as class attributes # need to be overwritten with values passed in here for name, value in settings.items(): if hasattr(self, name): setattr(self, name, value) # For backward-compatibility of old differing parameter names if acl is not None: self.default_acl = acl if bucket is not None: self.bucket_name = bucket check_location(self) # Backward-compatibility: given the anteriority of the SECURE_URL setting # we fall back to https if specified in order to avoid the construction # of unsecure urls. if self.secure_urls: self.url_protocol = 'https:' self._entries = {} self._bucket = None self._connection = None self._loaded_meta = False self.access_key, self.secret_key = self._get_access_keys() self.security_token = self._get_security_token() @property def connection(self): if self._connection is None: kwargs = self._get_connection_kwargs() self._connection = self.connection_class( self.access_key, self.secret_key, **kwargs ) return self._connection def _get_connection_kwargs(self): return dict( security_token=self.security_token, is_secure=self.use_ssl, calling_format=self.calling_format, host=self.host, port=self.port, proxy=self.proxy, proxy_port=self.proxy_port ) @property def bucket(self): """ Get the current bucket. If there is no current bucket object create it. """ if self._bucket is None: self._bucket = self._get_or_create_bucket(self.bucket_name) return self._bucket @property def entries(self): """ Get the locally cached files for the bucket. """ if self.preload_metadata and not self._loaded_meta: self._entries.update({ self._decode_name(entry.key): entry for entry in self.bucket.list(prefix=self.location) }) self._loaded_meta = True return self._entries def _get_access_keys(self): """ Gets the access keys to use when accessing S3. If none is provided in the settings then get them from the environment variables. """ access_key = self.access_key or lookup_env(S3BotoStorage.access_key_names) secret_key = self.secret_key or lookup_env(S3BotoStorage.secret_key_names) return access_key, secret_key def _get_security_token(self): """ Gets the security token to use when accessing S3. Get it from the environment variables. """ security_token = self.security_token or lookup_env(S3BotoStorage.security_token_names) return security_token def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. """ try: return self.connection.get_bucket(name, validate=self.auto_create_bucket) except self.connection_response_error: if self.auto_create_bucket: bucket = self.connection.create_bucket(name, location=self.origin) if not hasattr(django_settings, 'AWS_BUCKET_ACL'): warnings.warn( "The default behavior of S3BotoStorage is insecure. By default new buckets " "are saved with an ACL of 'public-read' (globally publicly readable). To change " "to using Amazon's default of the bucket owner set AWS_DEFAULT_ACL = None, " "otherwise to silence this warning explicitly set AWS_DEFAULT_ACL." ) if self.bucket_acl: bucket.set_acl(self.bucket_acl) return bucket raise ImproperlyConfigured('Bucket %s does not exist. Buckets ' 'can be automatically created by ' 'setting AWS_AUTO_CREATE_BUCKET to ' '``True``.' % name) def _clean_name(self, name): """ Cleans the name so that Windows style paths work """ return clean_name(name) def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt work. We check to make sure that the path pointed to is not outside the directory specified by the LOCATION setting. """ try: return safe_join(self.location, name) except ValueError: raise SuspiciousOperation("Attempted access to '%s' denied." % name) def _encode_name(self, name): return smart_str(name, encoding=self.file_name_charset) def _decode_name(self, name): return force_text(name, encoding=self.file_name_charset) def _compress_content(self, content): """Gzip a given string content.""" zbuf = io.BytesIO() # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html) # This means each time a file is compressed it changes even if the other contents don't change # For S3 this defeats detection of changes using MD5 sums on gzipped files # Fixing the mtime at 0.0 at compression time avoids this problem zfile = GzipFile(mode='wb', fileobj=zbuf, mtime=0.0) try: zfile.write(force_bytes(content.read())) finally: zfile.close() zbuf.seek(0) content.file = zbuf content.seek(0) return content def _open(self, name, mode='rb'): name = self._normalize_name(self._clean_name(name)) f = self.file_class(name, mode, self) if not f.key: raise IOError('File does not exist: %s' % name) return f def _save(self, name, content): cleaned_name = self._clean_name(name) name = self._normalize_name(cleaned_name) headers = self.headers.copy() _type, encoding = mimetypes.guess_type(name) content_type = getattr(content, 'content_type', None) content_type = content_type or _type or self.key_class.DefaultContentType # setting the content_type in the key object is not enough. headers.update({'Content-Type': content_type}) if self.gzip and content_type in self.gzip_content_types: content = self._compress_content(content) headers.update({'Content-Encoding': 'gzip'}) elif encoding: # If the content already has a particular encoding, set it headers.update({'Content-Encoding': encoding}) content.name = cleaned_name encoded_name = self._encode_name(name) key = self.bucket.get_key(encoded_name) if not key: key = self.bucket.new_key(encoded_name) if self.preload_metadata: self._entries[encoded_name] = key key.last_modified = datetime.utcnow().strftime(ISO8601) key.set_metadata('Content-Type', content_type) self._save_content(key, content, headers=headers) return cleaned_name def _save_content(self, key, content, headers): # only pass backwards incompatible arguments if they vary from the default kwargs = {} if self.encryption: kwargs['encrypt_key'] = self.encryption key.set_contents_from_file(content, headers=headers, policy=self.default_acl, reduced_redundancy=self.reduced_redundancy, rewind=True, **kwargs) def _get_key(self, name): name = self._normalize_name(self._clean_name(name)) if self.entries: return self.entries.get(name) return self.bucket.get_key(self._encode_name(name)) def delete(self, name): name = self._normalize_name(self._clean_name(name)) self.bucket.delete_key(self._encode_name(name)) def exists(self, name): if not name: # root element aka the bucket try: self.bucket return True except ImproperlyConfigured: return False return self._get_key(name) is not None def listdir(self, name): name = self._normalize_name(self._clean_name(name)) # for the bucket.list and logic below name needs to end in / # But for the root path "" we leave it as an empty string if name and not name.endswith('/'): name += '/' dirlist = self.bucket.list(self._encode_name(name)) files = [] dirs = set() base_parts = name.split('/')[:-1] for item in dirlist: parts = item.name.split('/') parts = parts[len(base_parts):] if len(parts) == 1: # File files.append(parts[0]) elif len(parts) > 1: # Directory dirs.add(parts[0]) return list(dirs), files def size(self, name): return self._get_key(name).size def get_modified_time(self, name): dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) return dt if setting('USE_TZ') else tz.make_naive(dt) def modified_time(self, name): dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) return tz.make_naive(dt) def url(self, name, headers=None, response_headers=None, expire=None): # Preserve the trailing slash after normalizing the path. name = self._normalize_name(self._clean_name(name)) if self.custom_domain: return '{}//{}/{}'.format(self.url_protocol, self.custom_domain, filepath_to_uri(name)) if expire is None: expire = self.querystring_expire return self.connection.generate_url( expire, method='GET', bucket=self.bucket.name, key=self._encode_name(name), headers=headers, query_auth=self.querystring_auth, force_http=not self.secure_urls, response_headers=response_headers, ) def get_available_name(self, name, max_length=None): """ Overwrite existing file with the same name. """ name = self._clean_name(name) if self.file_overwrite: return get_available_overwrite_name(name, max_length) return super(S3BotoStorage, self).get_available_name(name, max_length)
class S3Boto3StorageFile(File): """ The default file object used by the S3Boto3Storage backend. This file implements file streaming using boto's multipart uploading functionality. The file can be opened in read or write mode. This class extends Django's File class. However, the contained data is only the data contained in the current buffer. So you should not access the contained file object directly. You should access the data via this class. Warning: This file *must* be closed using the close() method in order to properly write the file to S3. Be sure to close the file in your application. """ # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing. # TODO: When Django drops support for Python 2.5, rewrite to use the # BufferedIO streams in the Python 2.6 io module. buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880) def __init__(self, name, mode, storage, buffer_size=None): self._storage = storage self.name = name[len(self._storage.location):].lstrip('/') self._mode = mode self.obj = storage.bucket.Object(storage._encode_name(name)) if 'w' not in mode: # Force early RAII-style exception if object does not exist self.obj.load() self._is_dirty = False self._file = None self._multipart = None # 5 MB is the minimum part size (if there is more than one part). # Amazon allows up to 10,000 parts. The default supports uploads # up to roughly 50 GB. Increase the part size to accommodate # for files larger than this. if buffer_size is not None: self.buffer_size = buffer_size self._write_counter = 0 @property def size(self): return self.obj.content_length def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".S3Boto3StorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR", None)) if 'r' in self._mode: self._is_dirty = False self._file.write(self.obj.get()['Body'].read()) self._file.seek(0) if self._storage.gzip and self.obj.content_encoding == 'gzip': self._file = GzipFile(mode=self._mode, fileobj=self._file, mtime=0.0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, *args, **kwargs): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") return super(S3Boto3StorageFile, self).read(*args, **kwargs) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True if self._multipart is None: parameters = self._storage.object_parameters.copy() parameters['ACL'] = self._storage.default_acl parameters['ContentType'] = (mimetypes.guess_type(self.obj.key)[0] or self._storage.default_content_type) if self._storage.reduced_redundancy: parameters['StorageClass'] = 'REDUCED_REDUNDANCY' if self._storage.encryption: parameters['ServerSideEncryption'] = 'AES256' self._multipart = self.obj.initiate_multipart_upload(**parameters) if self.buffer_size <= self._buffer_file_size: self._flush_write_buffer() return super(S3Boto3StorageFile, self).write(force_bytes(content)) @property def _buffer_file_size(self): pos = self.file.tell() self.file.seek(0, os.SEEK_END) length = self.file.tell() self.file.seek(pos) return length def _flush_write_buffer(self): """ Flushes the write buffer. """ if self._buffer_file_size: self._write_counter += 1 self.file.seek(0) part = self._multipart.Part(self._write_counter) part.upload(Body=self.file.read()) def close(self): if self._is_dirty: self._flush_write_buffer() # TODO: Possibly cache the part ids as they're being uploaded # instead of requesting parts from server. For now, emulating # s3boto's behavior. parts = [{ 'ETag': part.e_tag, 'PartNumber': part.part_number } for part in self._multipart.parts.all()] self._multipart.complete(MultipartUpload={'Parts': parts}) else: if self._multipart is not None: self._multipart.abort() if self._file is not None: self._file.close() self._file = None
def url(self, name): return "{}{}/{}".format(setting('MEDIA_URL'), self.azure_container, name)
def get_default_settings(self): cloudfront_key_id = setting('AWS_CLOUDFRONT_KEY_ID') cloudfront_key = setting('AWS_CLOUDFRONT_KEY') if bool(cloudfront_key_id) ^ bool(cloudfront_key): raise ImproperlyConfigured( 'Both AWS_CLOUDFRONT_KEY_ID and AWS_CLOUDFRONT_KEY must be ' 'provided together.') if cloudfront_key_id: cloudfront_signer = self.get_cloudfront_signer( cloudfront_key_id, cloudfront_key) else: cloudfront_signer = None return { "access_key": setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')), "secret_key": setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')), "file_overwrite": setting('AWS_S3_FILE_OVERWRITE', True), "object_parameters": setting('AWS_S3_OBJECT_PARAMETERS', {}), "bucket_name": setting('AWS_STORAGE_BUCKET_NAME'), "querystring_auth": setting('AWS_QUERYSTRING_AUTH', True), "querystring_expire": setting('AWS_QUERYSTRING_EXPIRE', 3600), "signature_version": setting('AWS_S3_SIGNATURE_VERSION'), "location": setting('AWS_LOCATION', ''), "custom_domain": setting('AWS_S3_CUSTOM_DOMAIN'), "cloudfront_signer": cloudfront_signer, "addressing_style": setting('AWS_S3_ADDRESSING_STYLE'), "secure_urls": setting('AWS_S3_SECURE_URLS', True), "file_name_charset": setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8'), "gzip": setting('AWS_IS_GZIPPED', False), "gzip_content_types": setting('GZIP_CONTENT_TYPES', ( 'text/css', 'text/javascript', 'application/javascript', 'application/x-javascript', 'image/svg+xml', )), "url_protocol": setting('AWS_S3_URL_PROTOCOL', 'http:'), "endpoint_url": setting('AWS_S3_ENDPOINT_URL'), "proxies": setting('AWS_S3_PROXIES'), "region_name": setting('AWS_S3_REGION_NAME'), "use_ssl": setting('AWS_S3_USE_SSL', True), "verify": setting('AWS_S3_VERIFY', None), "max_memory_size": setting('AWS_S3_MAX_MEMORY_SIZE', 0), }
def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')): if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") self.client = DropboxClient(oauth2_access_token)
def __init__(self, *args, **kwargs): self.account_name = kwargs.pop('account_name', setting("AZURE_ACCOUNT_NAME")) self.account_key = kwargs.pop('account_key', setting("AZURE_ACCOUNT_KEY")) self.azure_container = kwargs.pop('azure_container', setting("AZURE_CONTAINER")) super(AzureStorage, self).__init__(*args, **kwargs) self._connection = None
def __init__(self, *args, **kwargs): if not settings.MEDIA_URL: raise Exception('MEDIA_URL has not been configured') kwargs['bucket_name'] = setting('GS_MEDIA_BUCKET_NAME', strict=True) super(GoogleCloudMediaStorage, self).__init__(*args, **kwargs)
def __init__(self, base_url=setting("AWS_CLOUDFRONT_URL"), *args, **kwargs): super(S3CloudFrontStorage, self).__init__(*args, **kwargs) self.base_url = base_url