def __init__(self, byte_array=None, meta=b'\x00', bitstring=b'\x00'):
self.meta = BitArray()
self.bitstring = BitArray()
if byte_array is None:
self.meta.frombytes(meta)
self.bitstring.frombytes(bitstring)
else:
self.meta.frombytes(byte_array[0:1])
self.bitstring.frombytes(byte_array[1:])
def get_row(self, index):
b = BitArray()
_row_bytes = self.get(index, b'')
# if not _row_bytes:
# logger.warning(
# "There is no row %i. Run `bigsi init` and `bigsi build` before `insert` or `search`. Creating row regardless." % index)
b.frombytes(_row_bytes)
return b
def to_dense(self):
if self.is_sparse():
new_bitstring = BitArray()
new_bitstring.frombytes(b'\x00')
for i in self._bit_1_indexes():
setbit(new_bitstring, i, 1)
self.bitstring = new_bitstring
self.meta[0] = False
def to_sparse(self):
if self.is_dense():
indexes = self.indexes()
self.meta[0] = True
bo = choose_int_encoding(indexes)
self._set_sparse_byte_length(bo)
_bytes = b''.join([
int(i).to_bytes(self.sparse_byte_length, byteorder='big')
for i in indexes
])
self.bitstring = BitArray()
self.bitstring.frombytes(_bytes)
def _setbit_sparse(self, pos, i):
if i == 0:
self.to_dense()
self._setbit_dense(pos, i)
self.to_sparse()
else:
if not pos in self.colours():
if choose_int_encoding([pos]) > self.sparse_byte_length:
# lazy option
self.to_dense()
self._setbit_dense(pos, i)
self.to_sparse()
else:
_append_bytes = int(pos).to_bytes(self.sparse_byte_length,
byteorder='big')
b = b''.join([self.bitstring.tobytes(), _append_bytes])
self.bitstring = BitArray()
self.bitstring.frombytes(b)
def _batch_insert_prob_redis(conn, names, all_hashes, colour, count=0):
r = conn
with r.pipeline() as pipe:
try:
pipe.watch(names)
vals = get_vals(r, names, all_hashes)
pipe.multi()
for name, values, hs in zip(names, vals, all_hashes):
for val, h in zip(values, hs):
ba = BitArray()
if val is None:
val = b''
ba.frombytes(val)
ba.setbit(colour, 1)
pipe.hset(name, h, ba.tobytes())
pipe.execute()
except redis.WatchError:
logger.warning("Retrying %s %s " % (r, name))
if count < 5:
self._batch_insert(conn, hk, colour, count=count + 1)
else:
logger.warning(
"Failed %s %s. Too many retries. Contining regardless." %
(r, name))
def get_column(self, colour):
bf = BitArray()
for i in range(self.size):
bf.extend([self._getbit(i, colour)])
return bf
def get_row(self, index):
b = BitArray()
b.frombytes(self.get(index, b''))
return b