def _delRowsByProp(self, prop, valu=None, mintime=None, maxtime=None):
indx = self.index_pt if valu is None else self.index_pvt
first_key, last_key, v_is_hashed, do_fast_compare = _calcFirstLastKeys(prop, valu,
mintime, maxtime)
with self._getTxn(write=True) as txn, txn.cursor(indx) as cursor:
if not cursor.set_range(first_key):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
while True:
key, pk_enc = cursor.item()
if do_fast_compare:
if key[:len(first_key)] != first_key:
break
else:
if s_compat.memToBytes(key) >= last_key:
break
if self._delRowAndIndices(txn, pk_enc,
delete_pt=(valu is not None),
delete_pvt=(valu is None),
only_if_val=(valu if v_is_hashed else None)):
# Delete did go through: delete entry at cursor
if not cursor.delete():
raise s_common.BadCoreStore(store='lmdb', mesg='Delete failure')
else:
# Delete didn't go through: advance to next
if not cursor.next():
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
def getRowsByProp(self, prop, valu=None, limit=None, mintime=None, maxtime=None,
do_count_only=False):
indx = self.index_pt if valu is None else self.index_pvt
first_key, last_key, v_is_hashed, do_fast_compare = _calcFirstLastKeys(prop, valu,
mintime, maxtime)
count = 0
rows = []
with self._getTxn() as txn, txn.cursor(indx) as cursor:
if not cursor.set_range(first_key):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
while True:
key, pk_enc = cursor.item()
if do_fast_compare:
if key[:len(first_key)] != first_key:
break
else:
if s_compat.memToBytes(key) >= last_key:
break
if v_is_hashed or not do_count_only:
row = self._getRowByPkValEnc(txn, pk_enc)
if v_is_hashed:
if row[2] != valu:
continue
if not do_count_only:
rows.append(row)
count += 1
if limit is not None and count >= limit:
break
if not cursor.next():
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
return count if do_count_only else rows
def _delRowsByIdProp(self, iden, prop, valu=None):
i_enc = _encIden(iden)
p_enc = _encProp(prop)
first_key = i_enc + p_enc
with self._getTxn(write=True) as txn, txn.cursor(self.index_ip) as cursor:
# Retrieve and delete I-P index
if not cursor.set_range(first_key):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
while True:
# We don't use iterator here because the delete already advances to the next
# record
key, value = cursor.item()
if key[:len(first_key)] != first_key:
return
# Need to copy out with tobytes because we're deleting
pk_enc = s_compat.memToBytes(value)
# Delete the row and the other indices
if not self._delRowAndIndices(txn, pk_enc, i_enc=i_enc, p_enc=p_enc,
delete_ip=False, only_if_val=valu):
if not cursor.next():
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
else:
if not cursor.delete():
raise s_common.BadCoreStore(store='lmdb', mesg='Delete failure')
def _delRowsById(self, iden):
i_enc = _encIden(iden)
with self._getTxn(write=True) as txn, txn.cursor(self.index_ip) as cursor:
# Get the first record >= i_enc
if not cursor.set_range(i_enc):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
while True:
# We don't use iterator here because the delete already advances to the next
# record
key, value = cursor.item()
if key[:len(i_enc)] != i_enc:
return
p_enc = s_compat.memToBytes(key[len(i_enc):])
# Need to copy out with tobytes because we're deleting
pk_enc = s_compat.memToBytes(value)
if not cursor.delete():
raise s_common.BadCoreStore(store='lmdb', mesg='Delete failure')
self._delRowAndIndices(txn, pk_enc, i_enc=i_enc, p_enc=p_enc,
delete_ip=False)
def _subrangeRows(self, p_enc, first_val, last_val, limit, right_closed, do_count_only):
''' Performs part of a range query, either completely negative or non-negative '''
first_key = p_enc + _encValKey(first_val)
am_going_backwards = (first_val < 0)
last_key = p_enc + _encValKey(last_val)
ret = []
count = 0
# Figure out the terminating condition of the loop
if am_going_backwards:
term_cmp = bytes.__lt__ if right_closed else bytes.__le__
else:
term_cmp = bytes.__gt__ if right_closed else bytes.__ge__
with self._getTxn() as txn, txn.cursor(self.index_pvt) as cursor:
if not cursor.set_range(first_key):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
if am_going_backwards:
# set_range sets the cursor at the first key >= first_key, if we're going backwards
# we actually want the first key <= first_key
if s_compat.memToBytes(cursor.key()[:len(first_key)]) > first_key:
if not cursor.prev():
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
it = cursor.iterprev(keys=True, values=True)
else:
it = cursor.iternext(keys=True, values=True)
for key, value in it:
if term_cmp(s_compat.memToBytes(key[:len(last_key)]), last_key):
break
count += 1
if not do_count_only:
ret.append(self._getRowByPkValEnc(txn, value))
if limit is not None and count >= limit:
break
return count if do_count_only else ret
def getRowsByIdProp(self, iden, prop, valu=None):
# For now not making a ipv index because multiple v for a given i,p are probably rare
iden_enc = _encIden(iden)
prop_enc = _encProp(prop)
first_key = iden_enc + prop_enc
ret = []
with self._getTxn() as txn, txn.cursor(self.index_ip) as cursor:
if not cursor.set_range(first_key):
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')
for key, value in cursor:
if s_compat.memToBytes(key) != first_key:
return ret
row = self._getRowByPkValEnc(txn, value)
if valu is not None and row[2] != valu:
continue
ret.append(row)
raise s_common.BadCoreStore(store='lmdb', mesg='Missing sentinel')