def __init__(self,
bucket=None,
access_key=None,
secret_key=None,
headers=None,
calling_format=None,
cache=None,
base_url=None):
if bucket is None:
bucket = settings.AWS_STORAGE_BUCKET_NAME
if calling_format is None:
calling_format = getattr(settings, 'AWS_CALLING_FORMAT',
CallingFormat.SUBDOMAIN)
self.bucket = bucket
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
self.connection = AWSAuthConnection(access_key,
secret_key,
calling_format=calling_format)
default_headers = getattr(settings, HEADERS, [])
# Backwards compatibility for original format from django-storages
if isinstance(default_headers, dict):
default_headers = [('.*', default_headers)]
if headers:
# Headers passed to __init__ take precedence over headers from
# settings file.
default_headers = list(headers) + list(default_headers)
self.headers = []
for value in default_headers:
self.headers.append((re.compile(value[0]), value[1]))
if cache is not None:
self.cache = cache
else:
cache = getattr(settings, 'CUDDLYBUDDLY_STORAGE_S3_CACHE', None)
if cache is not None:
self.cache = self._get_cache_class(cache)()
else:
self.cache = None
if base_url is None:
if not self.static:
base_url = settings.MEDIA_URL
else:
base_url = settings.STATIC_URL
self.base_url = base_url
def __init__(self, bucket=None, access_key=None, secret_key=None,
headers=None, calling_format=None, cache=None, base_url=None):
if bucket is None:
bucket = settings.AWS_STORAGE_BUCKET_NAME
if calling_format is None:
calling_format = getattr(settings, 'AWS_CALLING_FORMAT',
CallingFormat.SUBDOMAIN)
self.bucket = bucket
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
self.connection = AWSAuthConnection(access_key, secret_key,
calling_format=calling_format)
default_headers = getattr(settings, HEADERS, [])
# Backwards compatibility for original format from django-storages
if isinstance(default_headers, dict):
default_headers = [('.*', default_headers)]
if headers:
# Headers passed to __init__ take precedence over headers from
# settings file.
default_headers = list(headers) + list(default_headers)
self.headers = []
for value in default_headers:
self.headers.append((re.compile(value[0]), value[1]))
if cache is not None:
self.cache = cache
else:
cache = getattr(settings, 'CUDDLYBUDDLY_STORAGE_S3_CACHE', None)
if cache is not None:
self.cache = self._get_cache_class(cache)()
else:
self.cache = None
if base_url is None:
if not self.static:
base_url = settings.MEDIA_URL
else:
base_url = settings.STATIC_URL
self.base_url = base_url
def _get_connection(self):
return AWSAuthConnection(*self._get_access_keys())
class S3Storage(Storage):
"""Amazon Simple Storage Service"""
static = False
def __init__(self, bucket=None, access_key=None, secret_key=None,
headers=None, calling_format=None, cache=None, base_url=None):
if bucket is None:
bucket = settings.AWS_STORAGE_BUCKET_NAME
if calling_format is None:
calling_format = getattr(settings, 'AWS_CALLING_FORMAT',
CallingFormat.SUBDOMAIN)
self.bucket = bucket
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
self.connection = AWSAuthConnection(access_key, secret_key,
calling_format=calling_format)
default_headers = getattr(settings, HEADERS, [])
# Backwards compatibility for original format from django-storages
if isinstance(default_headers, dict):
default_headers = [('.*', default_headers)]
if headers:
# Headers passed to __init__ take precedence over headers from
# settings file.
default_headers = list(headers) + list(default_headers)
self.headers = []
for value in default_headers:
self.headers.append((re.compile(value[0]), value[1]))
if cache is not None:
self.cache = cache
else:
cache = getattr(settings, 'CUDDLYBUDDLY_STORAGE_S3_CACHE', None)
if cache is not None:
self.cache = self._get_cache_class(cache)()
else:
self.cache = None
if base_url is None:
if not self.static:
base_url = settings.MEDIA_URL
else:
base_url = settings.STATIC_URL
self.base_url = base_url
def _get_cache_class(self, import_path=None):
try:
dot = import_path.rindex('.')
except ValueError:
raise ImproperlyConfigured("%s isn't a cache module." % import_path)
module, classname = import_path[:dot], import_path[dot+1:]
try:
mod = import_module(module)
except ImportError as e:
raise ImproperlyConfigured('Error importing cache module %s: "%s"' % (module, e))
try:
return getattr(mod, classname)
except AttributeError:
raise ImproperlyConfigured('Cache module "%s" does not define a "%s" class.' % (module, classname))
def _store_in_cache(self, name, response):
size = int(response.getheader('Content-Length'))
date = response.getheader('Last-Modified')
date = timegm(parsedate(date))
self.cache.save(name, size=size, mtime=date)
def _get_access_keys(self):
access_key = getattr(settings, ACCESS_KEY_NAME, None)
secret_key = getattr(settings, SECRET_KEY_NAME, None)
if (access_key or secret_key) and (not access_key or not secret_key):
access_key = os.environ.get(ACCESS_KEY_NAME)
secret_key = os.environ.get(SECRET_KEY_NAME)
if access_key and secret_key:
# Both were provided, so use them
return access_key, secret_key
return None, None
def _get_connection(self):
return AWSAuthConnection(*self._get_access_keys())
def _put_file(self, name, content):
name = self._path(name)
placeholder = False
if self.cache:
if not self.cache.exists(name):
self.cache.save(name, 0, 0)
placedholder = True
content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
headers = {}
for pattern in self.headers:
if pattern[0].match(name):
headers = pattern[1].copy()
break
file_pos = content.tell()
content.seek(0, 2)
content_length = content.tell()
content.seek(0)
gz_cts = getattr(
settings,
'CUDDLYBUDDLY_STORAGE_S3_GZIP_CONTENT_TYPES',
(
'text/css',
'application/javascript',
'application/x-javascript'
)
)
gz_content = None
if content_length > 1024 and content_type in gz_cts:
gz_content = StringIO()
gzf = GzipFile(mode='wb', fileobj=gz_content)
gzf.write(content.read())
content.seek(0)
gzf.close()
gz_content.seek(0, 2)
gz_content_length = gz_content.tell()
gz_content.seek(0)
if gz_content_length < content_length:
content_length = gz_content_length
headers.update({
'Content-Encoding': 'gzip'
})
else:
gz_content = None
headers.update({
'Content-Type': content_type,
'Content-Length': str(content_length)
})
# Httplib in < 2.6 doesn't accept file like objects. Meanwhile in
# >= 2.7 it will try to join a content str object with the headers which
# results in encoding problems.
if sys.version_info[0] == 2 and sys.version_info[1] < 6:
content_to_send = gz_content.read() if gz_content is not None else content.read()
else:
content_to_send = gz_content if gz_content is not None else content
response = self.connection.put(self.bucket, name, content_to_send, headers)
content.seek(file_pos)
if response.http_response.status != 200:
if placeholder:
self.cache.remove(name)
raise S3Error(response.message)
if self.cache:
date = response.http_response.getheader('Date')
date = timegm(parsedate(date))
self.cache.save(name, size=content_length, mtime=date)
def _open(self, name, mode='rb'):
remote_file = S3StorageFile(name, self, mode=mode)
return remote_file
def _read(self, name, start_range=None, end_range=None):
name = self._path(name)
headers, range_ = {}, None
if start_range is not None and end_range is not None:
range_ = '%s-%s' % (start_range, end_range)
elif start_range is not None:
range_ = '%s' % start_range
if range_ is not None:
headers = {'Range': 'bytes=%s' % range_}
response = self.connection.get(self.bucket, name, headers)
valid_responses = [200]
if start_range is not None or end_range is not None:
valid_responses.append(206)
if response.http_response.status not in valid_responses:
raise S3Error(response.message)
headers = response.http_response.msg
data = response.object.data
if headers.get('Content-Encoding') == 'gzip':
gzf = GzipFile(mode='rb', fileobj=StringIO(data))
data = gzf.read()
gzf.close()
return data, headers.get('etag', None), headers.get('content-range', None)
def _save(self, name, content):
self._put_file(name, content)
return name
def delete(self, name):
name = self._path(name)
response = self.connection.delete(self.bucket, name)
if response.http_response.status != 204:
raise S3Error(response.message)
if self.cache:
self.cache.remove(name)
def exists(self, name, force_check=False):
if not name:
return False
name = self._path(name)
if self.cache and not force_check:
exists = self.cache.exists(name)
if exists is not None:
return exists
response = self.connection._make_request('HEAD', self.bucket, name)
exists = response.status == 200
if self.cache and exists:
self._store_in_cache(name, response)
return exists
def size(self, name, force_check=False):
name = self._path(name)
if self.cache and not force_check:
size = self.cache.size(name)
if size is not None:
return size
response = self.connection._make_request('HEAD', self.bucket, name)
content_length = response.getheader('Content-Length')
if self.cache:
self._store_in_cache(name, response)
return content_length and int(content_length) or 0
def modified_time(self, name, force_check=False):
name = self._path(name)
if self.cache and not force_check:
last_modified = self.cache.modified_time(name)
if last_modified:
return datetime.fromtimestamp(last_modified)
response = self.connection._make_request('HEAD', self.bucket, name)
if response.status == 404:
raise S3Error("Cannot find the file specified: '%s'" % name)
last_modified = timegm(parsedate(response.getheader('Last-Modified')))
if self.cache:
self._store_in_cache(name, response)
return datetime.fromtimestamp(last_modified)
def url(self, name):
if self.base_url is None:
raise ValueError("This file is not accessible via a URL.")
name = self._path(name)
if request_is_secure():
if hasattr(self.base_url, 'https'):
url = self.base_url.https()
else:
if hasattr(self.base_url, 'match'):
url = self.base_url.match(name)
else:
url = self.base_url
url = url.replace('http://', 'https://')
else:
if hasattr(self.base_url, 'match'):
url = self.base_url.match(name)
else:
url = self.base_url
url = url.replace('https://', 'http://')
return urljoin(url, iri_to_uri(name))
def listdir(self, path):
path = self._path(path)
if not path.endswith('/'):
path = path+'/'
directories, files = [], []
options = {'prefix': path, 'delimiter': '/'}
response = self.connection.list_bucket(self.bucket, options=options)
for prefix in response.common_prefixes:
directories.append(prefix.prefix.replace(path, '').strip('/'))
for entry in response.entries:
files.append(entry.key.replace(path, ''))
return directories, files
def _path(self, name):
name = name.replace('\\', '/')
# Because the S3 lib just loves to add slashes
if name.startswith('/'):
name = name[1:]
return name
class S3Storage(Storage):
"""Amazon Simple Storage Service"""
static = False
def __init__(self, bucket=None, access_key=None, secret_key=None,
headers=None, calling_format=None, cache=None, base_url=None):
if bucket is None:
bucket = settings.AWS_STORAGE_BUCKET_NAME
if calling_format is None:
calling_format = getattr(settings, 'AWS_CALLING_FORMAT',
CallingFormat.SUBDOMAIN)
self.bucket = bucket
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
self.connection = AWSAuthConnection(access_key, secret_key,
calling_format=calling_format)
default_headers = getattr(settings, HEADERS, [])
# Backwards compatibility for original format from django-storages
if isinstance(default_headers, dict):
default_headers = [('.*', default_headers)]
if headers:
# Headers passed to __init__ take precedence over headers from
# settings file.
default_headers = list(headers) + list(default_headers)
self.headers = []
for value in default_headers:
self.headers.append((re.compile(value[0]), value[1]))
if cache is not None:
self.cache = cache
else:
cache = getattr(settings, 'CUDDLYBUDDLY_STORAGE_S3_CACHE', None)
if cache is not None:
self.cache = self._get_cache_class(cache)()
else:
self.cache = None
if base_url is None:
if not self.static:
base_url = settings.MEDIA_URL
else:
base_url = settings.STATIC_URL
self.base_url = base_url
def _get_cache_class(self, import_path=None):
try:
dot = import_path.rindex('.')
except ValueError:
raise ImproperlyConfigured("%s isn't a cache module." % import_path)
module, classname = import_path[:dot], import_path[dot+1:]
try:
mod = import_module(module)
except ImportError as e:
raise ImproperlyConfigured('Error importing cache module %s: "%s"' % (module, e))
try:
return getattr(mod, classname)
except AttributeError:
raise ImproperlyConfigured('Cache module "%s" does not define a "%s" class.' % (module, classname))
def _store_in_cache(self, name, response):
size = int(response.getheader('Content-Length'))
date = response.getheader('Last-Modified')
date = timegm(parsedate(date))
self.cache.save(name, size=size, mtime=date)
def _get_access_keys(self):
access_key = getattr(settings, ACCESS_KEY_NAME, None)
secret_key = getattr(settings, SECRET_KEY_NAME, None)
if (access_key or secret_key) and (not access_key or not secret_key):
access_key = os.environ.get(ACCESS_KEY_NAME)
secret_key = os.environ.get(SECRET_KEY_NAME)
if access_key and secret_key:
# Both were provided, so use them
return access_key, secret_key
return None, None
def _get_connection(self):
return AWSAuthConnection(*self._get_access_keys())
def _put_file(self, name, content):
name = self._path(name)
placeholder = False
if self.cache:
if not self.cache.exists(name):
self.cache.save(name, 0, 0)
placedholder = True
content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
headers = {}
for pattern in self.headers:
if pattern[0].match(name):
headers = pattern[1].copy()
break
file_pos = content.tell()
content.seek(0, 2)
content_length = content.tell()
content.seek(0)
gz_cts = getattr(
settings,
'CUDDLYBUDDLY_STORAGE_S3_GZIP_CONTENT_TYPES',
(
'text/css',
'application/javascript',
'application/x-javascript'
)
)
gz_content = None
if content_length > 1024 and content_type in gz_cts:
gz_content = StringIO()
gzf = GzipFile(mode='wb', fileobj=gz_content)
gzf.write(content.read())
content.seek(0)
gzf.close()
gz_content.seek(0, 2)
gz_content_length = gz_content.tell()
gz_content.seek(0)
if gz_content_length < content_length:
content_length = gz_content_length
headers.update({
'Content-Encoding': 'gzip'
})
else:
gz_content = None
headers.update({
'Content-Type': content_type,
'Content-Length': str(content_length)
})
# Httplib in < 2.6 doesn't accept file like objects. Meanwhile in
# >= 2.7 it will try to join a content str object with the headers which
# results in encoding problems.
if sys.version_info[0] == 2 and sys.version_info[1] < 6:
content_to_send = gz_content.read() if gz_content is not None else content.read()
else:
content_to_send = gz_content if gz_content is not None else content
response = self.connection.put(self.bucket, name, content_to_send, headers)
content.seek(file_pos)
if response.http_response.status != 200:
if placeholder:
self.cache.remove(name)
raise S3Error(response.message)
if self.cache:
date = response.http_response.getheader('Date')
date = timegm(parsedate(date))
self.cache.save(name, size=content_length, mtime=date)
def _open(self, name, mode='rb'):
remote_file = S3StorageFile(name, self, mode=mode)
return remote_file
def _read(self, name, start_range=None, end_range=None):
name = self._path(name)
headers, range_ = {}, None
if start_range is not None and end_range is not None:
range_ = '%s-%s' % (start_range, end_range)
elif start_range is not None:
range_ = '%s' % start_range
if range_ is not None:
headers = {'Range': 'bytes=%s' % range_}
response = self.connection.get(self.bucket, name, headers)
valid_responses = [200]
if start_range is not None or end_range is not None:
valid_responses.append(206)
if response.http_response.status not in valid_responses:
raise S3Error(response.message)
headers = response.http_response.msg
data = response.object.data
if headers.get('Content-Encoding') == 'gzip':
gzf = GzipFile(mode='rb', fileobj=StringIO(data))
data = gzf.read()
gzf.close()
return data, headers.get('etag', None), headers.get('content-range', None)
def _save(self, name, content):
self._put_file(name, content)
return name
def delete(self, name):
name = self._path(name)
response = self.connection.delete(self.bucket, name)
if response.http_response.status != 204:
raise S3Error(response.message)
if self.cache:
self.cache.remove(name)
def exists(self, name, force_check=False):
if not name:
return False
name = self._path(name)
if self.cache and not force_check:
exists = self.cache.exists(name)
if exists is not None:
return exists
response = self.connection._make_request('HEAD', self.bucket, name)
exists = response.status == 200
if self.cache and exists:
self._store_in_cache(name, response)
return exists
def size(self, name, force_check=False):
name = self._path(name)
if self.cache and not force_check:
size = self.cache.size(name)
if size is not None:
return size
response = self.connection._make_request('HEAD', self.bucket, name)
content_length = response.getheader('Content-Length')
if self.cache:
self._store_in_cache(name, response)
return content_length and int(content_length) or 0
def modified_time(self, name, force_check=False):
name = self._path(name)
if self.cache and not force_check:
last_modified = self.cache.modified_time(name)
if last_modified:
return datetime.fromtimestamp(last_modified)
response = self.connection._make_request('HEAD', self.bucket, name)
if response.status == 404:
raise S3Error("Cannot find the file specified: '%s'" % name)
last_modified = timegm(parsedate(response.getheader('Last-Modified')))
if self.cache:
self._store_in_cache(name, response)
return datetime.fromtimestamp(last_modified)
def url(self, name):
if self.base_url is None:
raise ValueError("This file is not accessible via a URL.")
name = self._path(name)
if request_is_secure():
if hasattr(self.base_url, 'https'):
url = self.base_url.https()
else:
if hasattr(self.base_url, 'match'):
url = self.base_url.match(name)
else:
url = self.base_url
url = url.replace('http://', 'https://')
else:
if hasattr(self.base_url, 'match'):
url = self.base_url.match(name)
else:
url = self.base_url
url = url.replace('https://', 'http://')
return urljoin(url, iri_to_uri(name))
def listdir(self, path):
path = self._path(path)
if not path.endswith('/'):
path = path+'/'
directories, files = [], []
options = {'prefix': path, 'delimiter': '/'}
response = self.connection.list_bucket(self.bucket, options=options)
for prefix in response.common_prefixes:
directories.append(prefix.prefix.replace(path, '').strip('/'))
for entry in response.entries:
files.append(entry.key.replace(path, ''))
return directories, files
def _path(self, name):
name = name.replace('\\', '/')
# Because the S3 lib just loves to add slashes
if name.startswith('/'):
name = name[1:]
return name