def test_clean_old_index(self): from rom import util if not util.USE_LUA: return class RomTestCleanOld(Model): col1 = Integer(index=True) col2 = String(index=True) if six.PY2 else Text(index=True) a = RomTestCleanOld(col1=6, col2="this is content that should be indexed") a.save() id = a.id self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1) self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1) self.assertEqual(len(RomTestCleanOld.get_by(col2='content')), 1) session.rollback() del a c = connect(None) self.assertEqual(c.hlen('RomTestCleanOld::'), 1) self.assertEqual(c.scard('RomTestCleanOld:col2:content:idx'), 1) self.assertEqual(c.zcard('RomTestCleanOld:col1:idx'), 1) self.assertEqual(c.delete('RomTestCleanOld:%s'%id), 1) all(util.clean_old_index(RomTestCleanOld, force_hscan=None)) self.assertEqual(c.hlen('RomTestCleanOld::'), 0) self.assertEqual(c.scard('RomTestCleanOld:col2:content:idx'), 0) self.assertEqual(c.zcard('RomTestCleanOld:col1:idx'), 0) # okay, now test for longer scan/clear. minid = int(c.get('RomTestCleanOld:id:')) + 1 _count = 100 for i in range(_count): RomTestCleanOld(col1=i).save() session.rollback() version = list(map(int, c.info('server')['redis_version'].split('.')[:2])) has_hscan = version >= [2, 8] to_delete = list(range(minid, minid + _count, 37)) c.delete(*['RomTestCleanOld:%i'%i for i in to_delete]) self.assertTrue(all(c.hexists('RomTestCleanOld::', i) for i in to_delete)) all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=has_hscan)) self.assertTrue(all(not c.hexists('RomTestCleanOld::', i) for i in to_delete)) to_delete = list(range(minid+29, minid + _count, 29)) c.delete(*['RomTestCleanOld:%i'%i for i in to_delete]) self.assertTrue(all(c.hexists('RomTestCleanOld::', i) for i in to_delete)) all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=None)) self.assertTrue(all(not c.hexists('RomTestCleanOld::', i) for i in to_delete))
def test_clean_old_index(self): from rom import util if not util.USE_LUA: return class RomTestCleanOld(Model): col1 = Integer(index=True) col2 = String(index=True) if six.PY2 else Text(index=True) a = RomTestCleanOld(col1=6, col2="this is content that should be indexed") a.save() id = a.id self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1) self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1) self.assertEqual(len(RomTestCleanOld.get_by(col2="content")), 1) session.rollback() del a c = connect(None) self.assertEqual(c.hlen("RomTestCleanOld::"), 1) self.assertEqual(c.scard("RomTestCleanOld:col2:content:idx"), 1) self.assertEqual(c.zcard("RomTestCleanOld:col1:idx"), 1) self.assertEqual(c.delete("RomTestCleanOld:%s" % id), 1) all(util.clean_old_index(RomTestCleanOld)) self.assertEqual(c.hlen("RomTestCleanOld::"), 0) self.assertEqual(c.scard("RomTestCleanOld:col2:content:idx"), 0) self.assertEqual(c.zcard("RomTestCleanOld:col1:idx"), 0)
def test_clean_old_index(self): from rom import util if not util.USE_LUA: return class RomTestCleanOld(Model): _namespace = 'RomTestNamespacedCleanup' col1 = Integer(index=True) col2 = string(index=True, keygen=FULL_TEXT) col3 = string(unique=True) now = str(time.time()) a = RomTestCleanOld(col1=6, col2="this is content that should be indexed", col3=now) a.save() id = a.id self.assertEqual(len(RomTestCleanOld.get_by(col1=6)), 1) self.assertEqual(len(RomTestCleanOld.get_by(col1=(5, 7))), 1) self.assertEqual(len(RomTestCleanOld.get_by(col2='content')), 1) self.assertTrue(RomTestCleanOld.get_by(col3=now)) session.rollback() del a c = connect(None) self.assertEqual(c.hlen('RomTestNamespacedCleanup::'), 1) self.assertEqual(RomTestCleanOld.query.count(), 1) self.assertEqual(c.scard('RomTestNamespacedCleanup:col2:content:idx'), 1) self.assertEqual(c.zcard('RomTestNamespacedCleanup:col1:idx'), 1) self.assertEqual(c.hlen('RomTestNamespacedCleanup:col3:uidx'), 1) self.assertEqual(c.delete('RomTestNamespacedCleanup:%s'%id), 1) with warnings.catch_warnings(record=True) as w: all(util.clean_old_index(RomTestCleanOld, force_hscan=None)) self.assertEqual(len(w), 1) self.assertEqual(c.hlen('RomTestNamespacedCleanup::'), 0) self.assertEqual(RomTestCleanOld.query.count(), 0) self.assertEqual(c.scard('RomTestNamespacedCleanup:col2:content:idx'), 0) self.assertEqual(c.zcard('RomTestNamespacedCleanup:col1:idx'), 0) # can't clean out unique index when force_hscan is None - aka HSCAN disabled self.assertEqual(c.hlen('RomTestNamespacedCleanup:col3:uidx'), 1) c.delete('RomTestNamespacedCleanup:col3:uidx') # okay, now test for longer scan/clear. minid = int(c.get('RomTestNamespacedCleanup:id:')) + 1 _count = 200 for i in range(minid, minid+_count): RomTestCleanOld(col1=i, col3=str(i)).save() session.rollback() version = list(map(int, c.info('server')['redis_version'].split('.')[:2])) has_hscan = version >= [2, 8] if has_hscan: self.assertEqual(len(RomTestCleanOld.query.all()), _count) to_delete = list(range(minid, minid + _count, 37)) c.delete(*['RomTestNamespacedCleanup:%i'%i for i in to_delete]) self.assertTrue(all(c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete)) all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=has_hscan)) self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete)) self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup:col3:uidx', i) for i in to_delete)) to_delete = list(range(minid+29, minid + _count, 29)) c.delete(*['RomTestNamespacedCleanup:%i'%i for i in to_delete]) self.assertTrue(all(c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete)) # should cause a warning with warnings.catch_warnings(record=True) as w: all(util.clean_old_index(RomTestCleanOld, 10, force_hscan=None)) self.assertEqual(len(w), 1) self.assertTrue(all(not c.hexists('RomTestNamespacedCleanup::', i) for i in to_delete)) # We can't really clean out unique indexes when hscan is disabled or not # available. :/ self.assertTrue(all(c.hexists('RomTestNamespacedCleanup:col3:uidx', i) for i in to_delete))