def _addRows(self, rows):
'''
Adds a bunch of rows to the database
Take care: this was written this way for performance, in particular when len(rows) is
large.
'''
encs = []
with self._getTxn(write=True) as txn:
next_pk = self.next_pk
# First, we encode all the i, p, v, t for all rows
for i, p, v, t in rows:
if next_pk > MAX_PK:
raise s_common.HitCoreLimit(
name='MAX_PK',
size=MAX_PK,
mesg='Out of primary key values')
if len(p) > MAX_PROP_LEN:
raise s_common.HitCoreLimit(
name='MAX_PROP_LEN',
size=MAX_PROP_LEN,
mesg='Property length too large')
i_enc = _encIden(i)
p_enc = _encProp(p)
v_key_enc = _encValKey(v)
t_enc = s_msgpack.en(t)
pk_enc = _encPk(next_pk)
row_enc = s_msgpack.en((i, p, v, t))
# idx 0 1 2 3 4 5
encs.append((i_enc, p_enc, row_enc, t_enc, v_key_enc, pk_enc))
next_pk += 1
# An iterator of what goes into the main table: key=pk_enc, val=encoded(i, p, v, t)
kvs = ((x[5], x[2]) for x in encs)
# Shove it all in at once
consumed, added = txn.cursor(self.rows).putmulti(kvs,
overwrite=False,
append=True)
if consumed != added or consumed != len(encs):
# Will only fail if record already exists, which should never happen
raise s_common.BadCoreStore(store='lmdb',
mesg='unexpected pk in DB')
# Update the indices for all rows
kvs = ((x[0] + x[1], x[5]) for x in encs)
txn.cursor(self.index_ip).putmulti(kvs, dupdata=True)
kvs = ((x[1] + x[4] + x[3], x[5]) for x in encs)
txn.cursor(self.index_pvt).putmulti(kvs, dupdata=True)
kvs = ((x[1] + x[3], x[5]) for x in encs)
txn.cursor(self.index_pt).putmulti(kvs, dupdata=True)
# self.next_pk should be protected from multiple writers. Luckily lmdb write lock does
# that for us.
self.next_pk = next_pk
def _setMaxKey(self):
'''
Put 1 max key sentinel at the end of each index table. This avoids unfortunate behavior
where the cursor moves backwards after deleting the final record.
'''
with self._getTxn(write=True) as txn:
for db in (self.index_ip, self.index_pvt, self.index_pt):
txn.put(MAX_INDEX_KEY, b'', db=db)
# One more sentinel for going backwards through the pvt table.
txn.put(b'\x00', b'', db=self.index_pvt)
# Find the largest stored pk. We just track this in memory from now on.
largest_pk = self._getLargestPk()
if largest_pk == MAX_PK:
raise s_common.HitCoreLimit(name='MAX_PK', size=MAX_PK, mesg='Out of primary key values')
self.next_pk = largest_pk + 1