def test_clean_old_index(self):
from rom import util
if not util.USE_LUA:
return
class RomTestCleanOld(Model):
col1 = Integer(index=True)
col2 = String(index=True) if six.PY2 else Text(index=True)
a = RomTestCleanOld(col1=6, col2="this is content that should be indexed")
a.save()
id = a.id
self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col2='content')), 1)
session.rollback()
del a
c = connect(None)
self.assertEqual(c.hlen('RomTestCleanOld::'), 1)
self.assertEqual(c.scard('RomTestCleanOld:col2:content:idx'), 1)
self.assertEqual(c.zcard('RomTestCleanOld:col1:idx'), 1)
self.assertEqual(c.delete('RomTestCleanOld:%s'%id), 1)
all(util.clean_old_index(RomTestCleanOld, force_hscan=None))
self.assertEqual(c.hlen('RomTestCleanOld::'), 0)
self.assertEqual(c.scard('RomTestCleanOld:col2:content:idx'), 0)
self.assertEqual(c.zcard('RomTestCleanOld:col1:idx'), 0)
# okay, now test for longer scan/clear.
minid = int(c.get('RomTestCleanOld:id:')) + 1
_count = 100
for i in range(_count):
RomTestCleanOld(col1=i).save()
session.rollback()
version = list(map(int, c.info('server')['redis_version'].split('.')[:2]))
has_hscan = version >= [2, 8]
to_delete = list(range(minid, minid + _count, 37))
c.delete(*['RomTestCleanOld:%i'%i for i in to_delete])
self.assertTrue(all(c.hexists('RomTestCleanOld::', i) for i in to_delete))
all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=has_hscan))
self.assertTrue(all(not c.hexists('RomTestCleanOld::', i) for i in to_delete))
to_delete = list(range(minid+29, minid + _count, 29))
c.delete(*['RomTestCleanOld:%i'%i for i in to_delete])
self.assertTrue(all(c.hexists('RomTestCleanOld::', i) for i in to_delete))
all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=None))
self.assertTrue(all(not c.hexists('RomTestCleanOld::', i) for i in to_delete))
def test_clean_old_index(self):
from rom import util
if not util.USE_LUA:
return
class RomTestCleanOld(Model):
col1 = Integer(index=True)
col2 = String(index=True) if six.PY2 else Text(index=True)
a = RomTestCleanOld(col1=6, col2="this is content that should be indexed")
a.save()
id = a.id
self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col2="content")), 1)
session.rollback()
del a
c = connect(None)
self.assertEqual(c.hlen("RomTestCleanOld::"), 1)
self.assertEqual(c.scard("RomTestCleanOld:col2:content:idx"), 1)
self.assertEqual(c.zcard("RomTestCleanOld:col1:idx"), 1)
self.assertEqual(c.delete("RomTestCleanOld:%s" % id), 1)
all(util.clean_old_index(RomTestCleanOld))
self.assertEqual(c.hlen("RomTestCleanOld::"), 0)
self.assertEqual(c.scard("RomTestCleanOld:col2:content:idx"), 0)
self.assertEqual(c.zcard("RomTestCleanOld:col1:idx"), 0)
def test_clean_old_index(self):
from rom import util
if not util.USE_LUA:
return
class RomTestCleanOld(Model):
_namespace = 'RomTestNamespacedCleanup'
col1 = Integer(index=True)
col2 = string(index=True, keygen=FULL_TEXT)
col3 = string(unique=True)
now = str(time.time())
a = RomTestCleanOld(col1=6, col2="this is content that should be indexed", col3=now)
a.save()
id = a.id
self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1)
self.assertEqual(len(RomTestCleanOld.get_by(col2='content')), 1)
self.assertTrue(RomTestCleanOld.get_by(col3=now))
session.rollback()
del a
c = connect(None)
self.assertEqual(c.hlen('RomTestNamespacedCleanup::'), 1)
self.assertEqual(RomTestCleanOld.query.count(), 1)
self.assertEqual(c.scard('RomTestNamespacedCleanup:col2:content:idx'), 1)
self.assertEqual(c.zcard('RomTestNamespacedCleanup:col1:idx'), 1)
self.assertEqual(c.hlen('RomTestNamespacedCleanup:col3:uidx'), 1)
self.assertEqual(c.delete('RomTestNamespacedCleanup:%s'%id), 1)
with warnings.catch_warnings(record=True) as w:
all(util.clean_old_index(RomTestCleanOld, force_hscan=None))
self.assertEqual(len(w), 1)
self.assertEqual(c.hlen('RomTestNamespacedCleanup::'), 0)
self.assertEqual(RomTestCleanOld.query.count(), 0)
self.assertEqual(c.scard('RomTestNamespacedCleanup:col2:content:idx'), 0)
self.assertEqual(c.zcard('RomTestNamespacedCleanup:col1:idx'), 0)
# can't clean out unique index when force_hscan is None - aka HSCAN disabled
self.assertEqual(c.hlen('RomTestNamespacedCleanup:col3:uidx'), 1)
c.delete('RomTestNamespacedCleanup:col3:uidx')
# okay, now test for longer scan/clear.
minid = int(c.get('RomTestNamespacedCleanup:id:')) + 1
_count = 200
for i in range(minid, minid+_count):
RomTestCleanOld(col1=i, col3=str(i)).save()
session.rollback()
version = list(map(int, c.info('server')['redis_version'].split('.')[:2]))
has_hscan = version >= [2, 8]
if has_hscan:
self.assertEqual(len(RomTestCleanOld.query.all()), _count)
to_delete = list(range(minid, minid + _count, 37))
c.delete(*['RomTestNamespacedCleanup:%i'%i for i in to_delete])
self.assertTrue(all(c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete))
all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=has_hscan))
self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete))
self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup:col3:uidx', i) for i in to_delete))
to_delete = list(range(minid+29, minid + _count, 29))
c.delete(*['RomTestNamespacedCleanup:%i'%i for i in to_delete])
self.assertTrue(all(c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete))
# should cause a warning
with warnings.catch_warnings(record=True) as w:
all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=None))
self.assertEqual(len(w), 1)
self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete))
# We can't really clean out unique indexes when hscan is disabled or not
# available. :/
self.assertTrue(all(c.hexists('RomTestNamespacedCleanup:col3:uidx', i) for i in to_delete))
