def write_gints(dbfile, nums):
"""Write the integers in the iterator nums to bytes stream dbfile.
"""
buf = array("c")
count = 0
key = 0
for v in nums:
shift = count * 2
if v < 256:
buf.append(chr(v))
elif v < 65536:
key |= 1 << shift
buf.extend(pack_ushort_le(v))
elif v < 16777216:
key |= 2 << shift
buf.extend(pack_uint_le(v)[:3])
else:
key |= 3 << shift
buf.extend(pack_uint_le(v))
count += 1
if count == 4:
#print bin(key), repr(buf)
dbfile.write(chr(key))
dbfile.write(array_tobytes(buf))
count = 0
key = 0
del buf[0:len(buf)]
if count:
dbfile.write(chr(key))
dbfile.write(array_tobytes(buf))
def write_gints(dbfile, nums):
"""Write the integers in the iterator nums to bytes stream dbfile.
"""
buf = array("c")
count = 0
key = 0
for v in nums:
shift = count * 2
if v < 256:
buf.append(chr(v))
elif v < 65536:
key |= 1 << shift
buf.extend(pack_ushort_le(v))
elif v < 16777216:
key |= 2 << shift
buf.extend(pack_uint_le(v)[:3])
else:
key |= 3 << shift
buf.extend(pack_uint_le(v))
count += 1
if count == 4:
#print bin(key), repr(buf)
dbfile.write(chr(key))
dbfile.write(array_tobytes(buf))
count = 0
key = 0
del buf[0:len(buf)]
if count:
dbfile.write(chr(key))
dbfile.write(array_tobytes(buf))
def _varint(i):
a = array("B")
while (i & ~0x7F) != 0:
a.append((i & 0x7F) | 0x80)
i = i >> 7
a.append(i)
return array_tobytes(a)
def _varint(i):
a = array("B")
while (i & ~0x7F) != 0:
a.append((i & 0x7F) | 0x80)
i = i >> 7
a.append(i)
return array_tobytes(a)
def write_array(self, arry):
if IS_LITTLE:
arry = copy(arry)
arry.byteswap()
if self.is_real:
arry.tofile(self.file)
else:
self.file.write(array_tobytes(arry))
def write_array(self, arry):
if IS_LITTLE:
arry = copy(arry)
arry.byteswap()
if self.is_real:
arry.tofile(self.file)
else:
self.write(array_tobytes(arry))
def minimize_weights(weights, compression=0):
if all(w == 1.0 for w in weights):
string = b("")
else:
if not IS_LITTLE:
weights.byteswap()
string = array_tobytes(weights)
if string and compression:
string = compress(string, compression)
return string
def minimize_weights(weights, compression=0):
if all(w == 1.0 for w in weights):
string = b("")
else:
if not IS_LITTLE:
weights.byteswap()
string = array_tobytes(weights)
if string and compression:
string = compress(string, compression)
return string
def finish(self, doccount):
dbfile = self._dbfile
bits = self._bitset.bits
if zlib and len(bits) <= self._compressat:
compressed = zlib.compress(array_tobytes(bits), 3)
dbfile.write(compressed)
dbfile.write_byte(1)
else:
dbfile.write_array(bits)
dbfile.write_byte(0)
def finish(self, doccount):
dbfile = self._dbfile
bits = self._bitset.bits
if zlib and len(bits) <= self._compressat:
compressed = zlib.compress(array_tobytes(bits), 3)
dbfile.write(compressed)
dbfile.write_byte(1)
else:
dbfile.write_array(bits)
dbfile.write_byte(0)
def to_7bit(x, islong):
if not islong:
shift = 31
nchars = 5
else:
shift = 63
nchars = 10
buffer = array("c", "\x00" * nchars)
x += (1 << shift) - 1
while x:
buffer[nchars - 1] = chr(x & 0x7f)
x >>= 7
nchars -= 1
return array_tobytes(buffer)
def to_7bit(x, islong):
if not islong:
shift = 31
nchars = 5
else:
shift = 63
nchars = 10
buffer = array("c", "\x00" * nchars)
x += (1 << shift) - 1
while x:
buffer[nchars - 1] = chr(x & 0x7f)
x >>= 7
nchars -= 1
return array_tobytes(buffer)
def minimize_ids(arry, stringids, compression=0):
amax = arry[-1]
if stringids:
typecode = ''
string = dumps(arry)
else:
typecode = arry.typecode
if amax <= 255:
typecode = "B"
elif amax <= 65535:
typecode = "H"
if typecode != arry.typecode:
arry = array(typecode, iter(arry))
if not IS_LITTLE:
arry.byteswap()
string = array_tobytes(arry)
if compression:
string = compress(string, compression)
return (typecode, string)
def minimize_ids(arry, stringids, compression=0):
amax = arry[-1]
if stringids:
typecode = ''
string = dumps(arry)
else:
typecode = arry.typecode
if amax <= 255:
typecode = "B"
elif amax <= 65535:
typecode = "H"
if typecode != arry.typecode:
arry = array(typecode, iter(arry))
if not IS_LITTLE:
arry.byteswap()
string = array_tobytes(arry)
if compression:
string = compress(string, compression)
return (typecode, string)
def random_btext():
a = array("H", (random.randint(0, 0xd7ff) for _ in xrange(1, 20)))
return array_tobytes(a).decode("utf-16")
def randstring():
length = random.randint(1, 5)
a = array("B", (random.randint(0, 255) for _ in xrange(length)))
return array_tobytes(a)
def to_bytes(v):
return array_tobytes(v)
def to_bytes(v):
return array_tobytes(v)
def randstring():
length = random.randint(1, 5)
a = array("B", (random.randint(0, 255) for _ in xrange(length)))
return array_tobytes(a)
def random_btext():
a = array("H", (random.randint(0, 0xd7ff) for _ in xrange(1, 20)))
return array_tobytes(a).decode("utf-16")
