class SubStream(object): def __init__(self, dbfile, buffersize): self._dbfile = dbfile self._buffersize = buffersize self._buffer = BytesIO() self.blocks = [] def tell(self): return sum(b[2] for b in self.blocks) + self._buffer.tell() def write(self, inbytes): bio = self._buffer buflen = bio.tell() length = buflen + len(inbytes) if length >= self._buffersize: offset = self._dbfile.tell() self._dbfile.write(bio.getvalue()[:buflen]) self._dbfile.write(inbytes) self.blocks.append((None, offset, length)) self._buffer.seek(0) else: bio.write(inbytes) def close(self): bio = self._buffer length = bio.tell() if length: self.blocks.append((bio, 0, length))
class DatastoreFile(db.Model): """A file-like object that is backed by a BytesIO() object whose contents is loaded from a BlobProperty in the app engine datastore. """ value = db.BlobProperty() mtime = db.IntegerProperty(default=0) def __init__(self, *args, **kwargs): super(DatastoreFile, self).__init__(*args, **kwargs) self.data = BytesIO() @classmethod def loadfile(cls, name): value = memcache.get(name, namespace="DatastoreFile") if value is None: file = cls.get_by_key_name(name) memcache.set(name, file.value, namespace="DatastoreFile") else: file = cls(value=value) file.data = BytesIO(file.value) return file def close(self): oldvalue = self.value self.value = self.getvalue() if oldvalue != self.value: self.mtime = int(time.time()) self.put() memcache.set(self.key().id_or_name(), self.value, namespace="DatastoreFile") def tell(self): return self.data.tell() def write(self, data): return self.data.write(data) def read(self, length): return self.data.read(length) def seek(self, *args): return self.data.seek(*args) def readline(self): return self.data.readline() def getvalue(self): return self.data.getvalue()
class SAEFile(File): def __init__(self, name, mode, bucket, *args, **kwargs): self.name = name self.mode = mode self.data = BytesIO() self.bucket = bucket self._is_dirty = False self.value = kwargs.get('value', '') @classmethod def loadfile(cls, name, mode, bucket): value = bucket.get_object_contents(name) file = cls(name, mode, bucket=bucket, value=value) file.data = BytesIO(file.value) return file def close(self): self.value = self.getvalue() self.bucket.put_object(self.name, self.value) def tell(self): return self.data.tell() def write(self, data): return self.data.write(data) def read(self, length): return self.data.read(length) def seek(self, *args): return self.data.seek(*args) def readline(self): return self.data.readline() def getvalue(self): return self.data.getvalue()