def __init__(self, name, mode, bucket, *args, **kwargs):
self.name = name
self.mode = mode
self.data = BytesIO()
self.bucket = bucket
self._is_dirty = False
self.value = kwargs.get('value', '')
class SubStream(object):
def __init__(self, dbfile, buffersize):
self._dbfile = dbfile
self._buffersize = buffersize
self._buffer = BytesIO()
self.blocks = []
def tell(self):
return sum(b[2] for b in self.blocks) + self._buffer.tell()
def write(self, inbytes):
bio = self._buffer
buflen = bio.tell()
length = buflen + len(inbytes)
if length >= self._buffersize:
offset = self._dbfile.tell()
self._dbfile.write(bio.getvalue()[:buflen])
self._dbfile.write(inbytes)
self.blocks.append((None, offset, length))
self._buffer.seek(0)
else:
bio.write(inbytes)
def close(self):
bio = self._buffer
length = bio.tell()
if length:
self.blocks.append((bio, 0, length))
class SubStream(object):
def __init__(self, dbfile, buffersize):
self._dbfile = dbfile
self._buffersize = buffersize
self._buffer = BytesIO()
self.blocks = []
def tell(self):
return sum(b[2] for b in self.blocks) + self._buffer.tell()
def write(self, inbytes):
bio = self._buffer
buflen = bio.tell()
length = buflen + len(inbytes)
if length >= self._buffersize:
offset = self._dbfile.tell()
self._dbfile.write(bio.getvalue()[:buflen])
self._dbfile.write(inbytes)
self.blocks.append((None, offset, length))
self._buffer.seek(0)
else:
bio.write(inbytes)
def close(self):
bio = self._buffer
length = bio.tell()
if length:
self.blocks.append((bio, 0, length))
def __getitem__(self, docnum):
bio = BytesIO(self._child[docnum])
count = bio.read_varint()
out = []
for _ in xrange(count):
vlen = bio.read_varint()
v = bio.read(vlen)
out.append(v)
return out
def __getitem__(self, docnum):
bio = BytesIO(self._child[docnum])
count = bio.read_varint()
out = []
for _ in xrange(count):
vlen = bio.read_varint()
v = bio.read(vlen)
out.append(v)
return out
def __getitem__(self, docnum):
data = self._child[docnum]
if not data:
return []
bio = BytesIO(data)
count = read_varint(bio.read)
out = []
for _ in xrange(count):
vlen = read_varint(bio.read)
v = bio.read(vlen)
out.append(v)
return out
def __getitem__(self, docnum):
data = self._child[docnum]
if not data:
return []
bio = BytesIO(data)
count = read_varint(bio.read)
out = []
for _ in xrange(count):
vlen = read_varint(bio.read)
v = bio.read(vlen)
out.append(v)
return out
def __init__(self, name, mode, bucket, *args, **kwargs):
self.name = name
self.mode = mode
self.data = BytesIO()
self.bucket = bucket
self._is_dirty = False
self.value = kwargs.get('value', '')
def open_file(self, name, *args, **kwargs):
if name not in self.files:
raise NameError("No such file %r" % name)
return StructFile(BytesIO(self.files[name]),
name=name,
*args,
**kwargs)
def test_random_multistream():
letters = "abcdefghijklmnopqrstuvwxyz"
def randstring(n):
s = "".join(random.choice(letters) for _ in xrange(n))
return s.encode("latin1")
domain = {}
for _ in xrange(100):
name = randstring(random.randint(5, 10))
value = randstring(2500)
domain[name] = value
outfiles = dict((name, BytesIO(value)) for name, value in domain.items())
with TempStorage() as st:
msw = compound.CompoundWriter(st, buffersize=1024)
mfiles = {}
for name in domain:
mfiles[name] = msw.create_file(name)
while outfiles:
name = random.choice(list(outfiles.keys()))
v = outfiles[name].read(1000)
mfiles[name].write(v)
if len(v) < 1000:
del outfiles[name]
f = st.create_file("test")
msw.save_as_compound(f)
f = st.open_file("test")
msr = compound.CompoundStorage(f)
for name, value in domain.items():
assert msr.open_file(name).read() == value
msr.close()
def __init__(self, buf, name=None, onclose=None):
self._buf = buf
self._name = name
self.file = BytesIO(buf)
self.onclose = onclose
self.is_real = False
self.is_closed = False
def loadfile(cls, name):
value = memcache.get(name, namespace="DatastoreFile")
if value is None:
file = cls.get_by_key_name(name)
memcache.set(name, file.value, namespace="DatastoreFile")
else:
file = cls(value=value)
file.data = BytesIO(file.value)
return file
def open_file(self, name, **kwargs):
if self._bucket.stat_object(self._fpath(name)) is None:
raise NameError(name)
content = self._bucket.get_object_contents(self._fpath(name))
def onclose_fn(sfile):
self._bucket.put_object(self._fpath(name), sfile.file.getvalue())
return StructFile(BytesIO(content), name=name, onclose=onclose_fn)
def open_file(self, name, *args, **kwargs):
info = self.dir[name]
offset = info["offset"]
length = info["length"]
if self.source:
# Create a memoryview/buffer from the mmap
buf = memoryview_(self.source, offset, length)
f = BytesIO(buf)
else:
# If mmap is not available, use the slower sub-file implementation
f = SubFile(self.file, offset, length)
return StructFile(f, name=name)
class DatastoreFile(db.Model):
"""A file-like object that is backed by a BytesIO() object whose contents
is loaded from a BlobProperty in the app engine datastore.
"""
value = db.BlobProperty()
mtime = db.IntegerProperty(default=0)
def __init__(self, *args, **kwargs):
super(DatastoreFile, self).__init__(*args, **kwargs)
self.data = BytesIO()
@classmethod
def loadfile(cls, name):
value = memcache.get(name, namespace="DatastoreFile")
if value is None:
file = cls.get_by_key_name(name)
memcache.set(name, file.value, namespace="DatastoreFile")
else:
file = cls(value=value)
file.data = BytesIO(file.value)
return file
def close(self):
oldvalue = self.value
self.value = self.getvalue()
if oldvalue != self.value:
self.mtime = int(time.time())
self.put()
memcache.set(self.key().id_or_name(),
self.value,
namespace="DatastoreFile")
def tell(self):
return self.data.tell()
def write(self, data):
return self.data.write(data)
def read(self, length):
return self.data.read(length)
def seek(self, *args):
return self.data.seek(*args)
def readline(self):
return self.data.readline()
def getvalue(self):
return self.data.getvalue()
class DatastoreFile(db.Model):
"""A file-like object that is backed by a BytesIO() object whose contents
is loaded from a BlobProperty in the app engine datastore.
"""
value = db.BlobProperty()
mtime = db.IntegerProperty(default=0)
def __init__(self, *args, **kwargs):
super(DatastoreFile, self).__init__(*args, **kwargs)
self.data = BytesIO()
@classmethod
def loadfile(cls, name):
value = memcache.get(name, namespace="DatastoreFile")
if value is None:
file = cls.get_by_key_name(name)
memcache.set(name, file.value, namespace="DatastoreFile")
else:
file = cls(value=value)
file.data = BytesIO(file.value)
return file
def close(self):
oldvalue = self.value
self.value = self.getvalue()
if oldvalue != self.value:
self.mtime = int(time.time())
self.put()
memcache.set(self.key().id_or_name(), self.value,
namespace="DatastoreFile")
def tell(self):
return self.data.tell()
def write(self, data):
return self.data.write(data)
def read(self, length):
return self.data.read(length)
def seek(self, *args):
return self.data.seek(*args)
def readline(self):
return self.data.readline()
def getvalue(self):
return self.data.getvalue()
class SAEFile(File):
def __init__(self, name, mode, bucket, *args, **kwargs):
self.name = name
self.mode = mode
self.data = BytesIO()
self.bucket = bucket
self._is_dirty = False
self.value = kwargs.get('value', '')
@classmethod
def loadfile(cls, name, mode, bucket):
value = bucket.get_object_contents(name)
file = cls(name, mode, bucket=bucket, value=value)
file.data = BytesIO(file.value)
return file
def close(self):
self.value = self.getvalue()
self.bucket.put_object(self.name, self.value)
def tell(self):
return self.data.tell()
def write(self, data):
return self.data.write(data)
def read(self, length):
return self.data.read(length)
def seek(self, *args):
return self.data.seek(*args)
def readline(self):
return self.data.readline()
def getvalue(self):
return self.data.getvalue()
class SAEFile(File):
def __init__(self, name, mode, bucket, *args, **kwargs):
self.name = name
self.mode = mode
self.data = BytesIO()
self.bucket = bucket
self._is_dirty = False
self.value = kwargs.get('value', '')
@classmethod
def loadfile(cls, name, mode, bucket):
value = bucket.get_object_contents(name)
file = cls(name, mode, bucket=bucket, value=value)
file.data = BytesIO(file.value)
return file
def close(self):
self.value = self.getvalue()
self.bucket.put_object(self.name, self.value)
def tell(self):
return self.data.tell()
def write(self, data):
return self.data.write(data)
def read(self, length):
return self.data.read(length)
def seek(self, *args):
return self.data.seek(*args)
def readline(self):
return self.data.readline()
def getvalue(self):
return self.data.getvalue()
def __init__(self, dbfile, buffersize):
self._dbfile = dbfile
self._buffersize = buffersize
self._buffer = BytesIO()
self.blocks = []
def create_file(self, name, **kwargs):
def onclose_fn(sfile):
self.files[name] = sfile.file.getvalue()
f = StructFile(BytesIO(), name=name, onclose=onclose_fn)
return f
def create_file(self, name, **kwargs):
def onclose_fn(sfile):
self._bucket.put_object(self._fpath(name), sfile.file.getvalue())
f = StructFile(BytesIO(), name=name, onclose=onclose_fn)
return f
def loadfile(cls, name, mode, bucket):
value = bucket.get_object_contents(name)
file = cls(name, mode, bucket=bucket, value=value)
file.data = BytesIO(file.value)
return file
def __init__(self, dbfile, buffersize):
self._dbfile = dbfile
self._buffersize = buffersize
self._buffer = BytesIO()
self.blocks = []
def __init__(self, *args, **kwargs):
super(DatastoreFile, self).__init__(*args, **kwargs)
self.data = BytesIO()
def __init__(self, *args, **kwargs):
super(DatastoreFile, self).__init__(*args, **kwargs)
self.data = BytesIO()
def _write_node(self, uncnode):
vtype = self.vtype
dbfile = self.dbfile
arcs = uncnode.arcs
numarcs = len(arcs)
if not numarcs:
if uncnode.accept:
return None
else:
# What does it mean for an arc to stop but not be accepted?
raise Exception
self.node_count += 1
buf = StructFile(BytesIO())
nodestart = dbfile.tell()
#self.count += 1
#self.arccount += numarcs
fixedsize = -1
arcstart = buf.tell()
for i, arc in enumerate(arcs):
self.arc_count += 1
target = arc.target
label = arc.label
flags = 0
if len(label) > 1:
flags += MULTIBYTE_LABEL
if i == numarcs - 1:
flags += ARC_LAST
if arc.accept:
flags += ARC_ACCEPT
if target is None:
flags += ARC_STOP
if arc.value is not None:
flags += ARC_HAS_VAL
if arc.acceptval is not None:
flags += ARC_HAS_ACCEPT_VAL
buf.write(pack_byte(flags))
if len(label) > 1:
buf.write(varint(len(label)))
buf.write(label)
if target is not None:
buf.write(pack_uint(target))
if arc.value is not None:
vtype.write(buf, arc.value)
if arc.acceptval is not None:
vtype.write(buf, arc.acceptval)
here = buf.tell()
thissize = here - arcstart
arcstart = here
if fixedsize == -1:
fixedsize = thissize
elif fixedsize > 0 and thissize != fixedsize:
fixedsize = 0
if fixedsize > 0:
# Write a fake arc containing the fixed size and number of arcs
dbfile.write_byte(255) # FIXED_SIZE
dbfile.write_int(fixedsize)
dbfile.write_int(numarcs)
self.fixed_count += 1
dbfile.write(buf.file.getvalue())
return nodestart
