class S3LocalCachedMixin(object): """ Mixin that adds local caching to S3 storage backend """ def __init__(self, *args, **kwargs): super(S3LocalCachedMixin, self).__init__(*args, **kwargs) self._local = StaticFilesStorage(location=CONVOY_LOCAL_CACHE_ROOT) def save(self, name, content, *args, **kwargs): if not hasattr(content, 'chunks'): content = ContentFile(content) sname = self._save(name, content, *args, **kwargs) return sname def _save(self, name, content, *args, **kwargs): # some configurations of s3 backend mutate the content in place # Esp when AWS_IS_GZIPPED = True # keep a pre-mutation copy for the local cache so we don't save garbage to disk orig_content = copy.copy(content) sname = super(S3LocalCachedMixin, self)._save(name, content, *args, **kwargs) if self._local.exists(name): self._local.delete(name) lname = self._local._save(name, orig_content, *args, **kwargs) return name def delete(self, *args, **kwargs): if self._local.exists(*args, **kwargs): self._local.delete(*args, **kwargs) return super(S3LocalCachedMixin, self).delete(*args, **kwargs) def open(self, name, *args, **kwargs): if self._local.exists(name): #print "reading %s from cache" % name return self._local.open(name, *args, **kwargs) else: #print "reading %s from network" % name the_file = super(S3LocalCachedMixin, self).open(name, *args, **kwargs) #we had a cache miss, save it locally for the future self._local.save(name, the_file) if hasattr(the_file, "seek"): the_file.seek(0) return the_file def local_path(self, *args, **kwargs): return self._local.path(*args, **kwargs)