def write_gints(dbfile, nums): """Write the integers in the iterator nums to bytes stream dbfile. """ buf = array("c") count = 0 key = 0 for v in nums: shift = count * 2 if v < 256: buf.append(chr(v)) elif v < 65536: key |= 1 << shift buf.extend(pack_ushort_le(v)) elif v < 16777216: key |= 2 << shift buf.extend(pack_uint_le(v)[:3]) else: key |= 3 << shift buf.extend(pack_uint_le(v)) count += 1 if count == 4: #print bin(key), repr(buf) dbfile.write(chr(key)) dbfile.write(array_tobytes(buf)) count = 0 key = 0 del buf[0:len(buf)] if count: dbfile.write(chr(key)) dbfile.write(array_tobytes(buf))
def _varint(i): a = array("B") while (i & ~0x7F) != 0: a.append((i & 0x7F) | 0x80) i = i >> 7 a.append(i) return array_tobytes(a)
def write_array(self, arry): if IS_LITTLE: arry = copy(arry) arry.byteswap() if self.is_real: arry.tofile(self.file) else: self.file.write(array_tobytes(arry))
def write_array(self, arry): if IS_LITTLE: arry = copy(arry) arry.byteswap() if self.is_real: arry.tofile(self.file) else: self.write(array_tobytes(arry))
def minimize_weights(weights, compression=0): if all(w == 1.0 for w in weights): string = b("") else: if not IS_LITTLE: weights.byteswap() string = array_tobytes(weights) if string and compression: string = compress(string, compression) return string
def finish(self, doccount): dbfile = self._dbfile bits = self._bitset.bits if zlib and len(bits) <= self._compressat: compressed = zlib.compress(array_tobytes(bits), 3) dbfile.write(compressed) dbfile.write_byte(1) else: dbfile.write_array(bits) dbfile.write_byte(0)
def to_7bit(x, islong): if not islong: shift = 31 nchars = 5 else: shift = 63 nchars = 10 buffer = array("c", "\x00" * nchars) x += (1 << shift) - 1 while x: buffer[nchars - 1] = chr(x & 0x7f) x >>= 7 nchars -= 1 return array_tobytes(buffer)
def minimize_ids(arry, stringids, compression=0): amax = arry[-1] if stringids: typecode = '' string = dumps(arry) else: typecode = arry.typecode if amax <= 255: typecode = "B" elif amax <= 65535: typecode = "H" if typecode != arry.typecode: arry = array(typecode, iter(arry)) if not IS_LITTLE: arry.byteswap() string = array_tobytes(arry) if compression: string = compress(string, compression) return (typecode, string)
def random_btext(): a = array("H", (random.randint(0, 0xd7ff) for _ in xrange(1, 20))) return array_tobytes(a).decode("utf-16")
def randstring(): length = random.randint(1, 5) a = array("B", (random.randint(0, 255) for _ in xrange(length))) return array_tobytes(a)
def to_bytes(v): return array_tobytes(v)