def test_fetch(self):
subject = RedisState(get_pool(), 1)
r1 = Request("7",
int(time()) - 10,
'https://www.knuthellan.com/',
domain='knuthellan.com')
r1.meta[b'state'] = b'g'
r2 = Request("8",
int(time()) - 10,
'https://www.khellan.com/',
domain='khellan.com')
r2.meta[b'state'] = b'h'
batch = [r1, r2]
subject.update_cache(batch)
subject.flush(True)
r3 = Request("9",
int(time()) - 10,
'https://www.hellan.me/',
domain='hellan.me')
r3.meta[b'state'] = b'i'
subject.update_cache(r3)
self.assertEqual(1, len(subject._cache))
to_fetch = ["7", "9"]
subject.fetch(to_fetch)
self.assertEqual(2, len(subject._cache))
self.assertEqual(b'g', subject._cache["7"])
self.assertEqual(b'i', subject._cache["9"])
def test_update_cache(self):
subject = RedisState(get_pool(), 10)
r1 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
self.assertEqual(3, len(subject._cache))
self.assertEqual(b'a', subject._cache["1"])
self.assertEqual(b'b', subject._cache["2"])
self.assertEqual(b'c', subject._cache["3"])
def test_update_cache(self):
subject = RedisState(get_pool(), 10)
r1 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
self.assertEqual(3, len(subject._cache))
self.assertEqual(b'a', subject._cache["1"])
self.assertEqual(b'b', subject._cache["2"])
self.assertEqual(b'c', subject._cache["3"])
def test_flush_cache_overflow(self):
pool = get_pool()
subject = RedisState(pool, 1)
r1 = Request("4", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'd'
r2 = Request("5", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'e'
r3 = Request("6", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'f'
batch = [r1, r2, r3]
subject.update_cache(batch)
subject.flush(False)
self.assertEqual(0, len(subject._cache))
connection = StrictRedis(connection_pool=pool)
self.assertEqual({FIELD_STATE: b'd'}, connection.hgetall("4"))
self.assertEqual({FIELD_STATE: b'e'}, connection.hgetall("5"))
self.assertEqual({FIELD_STATE: b'f'}, connection.hgetall("6"))
def test_flush_no_force(self):
pool = get_pool()
subject = RedisState(pool, 10)
r1 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
subject.flush(False)
self.assertEqual(3, len(subject._cache))
connection = StrictRedis(connection_pool=pool)
self.assertEqual({FIELD_STATE: b'a'}, connection.hgetall("1"))
self.assertEqual({FIELD_STATE: b'b'}, connection.hgetall("2"))
self.assertEqual({FIELD_STATE: b'c'}, connection.hgetall("3"))
def test_set_states(self):
subject = RedisState(get_pool(), 10)
r1 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
r4 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r5 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r6 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
batch2 = [r4, r5, r6]
subject.set_states(batch2)
self.assertEqual(b'a', r4.meta[b'state'])
self.assertEqual(b'b', r5.meta[b'state'])
self.assertEqual(b'c', r6.meta[b'state'])
def test_flush_cache_overflow(self):
pool = get_pool()
subject = RedisState(pool, 1)
r1 = Request("4",
int(time()) - 10,
'https://www.knuthellan.com/',
domain='knuthellan.com')
r1.meta[b'state'] = b'd'
r2 = Request("5",
int(time()) - 10,
'https://www.khellan.com/',
domain='khellan.com')
r2.meta[b'state'] = b'e'
r3 = Request("6",
int(time()) - 10,
'https://www.hellan.me/',
domain='hellan.me')
r3.meta[b'state'] = b'f'
batch = [r1, r2, r3]
subject.update_cache(batch)
subject.flush(False)
self.assertEqual(0, len(subject._cache))
connection = StrictRedis(connection_pool=pool)
self.assertEqual({FIELD_STATE: b'd'}, connection.hgetall("4"))
self.assertEqual({FIELD_STATE: b'e'}, connection.hgetall("5"))
self.assertEqual({FIELD_STATE: b'f'}, connection.hgetall("6"))
def test_flush_no_force(self):
pool = get_pool()
subject = RedisState(pool, 10)
r1 = Request("1",
int(time()) - 10,
'https://www.knuthellan.com/',
domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2",
int(time()) - 10,
'https://www.khellan.com/',
domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3",
int(time()) - 10,
'https://www.hellan.me/',
domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
subject.flush(False)
self.assertEqual(3, len(subject._cache))
connection = StrictRedis(connection_pool=pool)
self.assertEqual({FIELD_STATE: b'a'}, connection.hgetall("1"))
self.assertEqual({FIELD_STATE: b'b'}, connection.hgetall("2"))
self.assertEqual({FIELD_STATE: b'c'}, connection.hgetall("3"))
def test_fetch(self):
subject = RedisState(get_pool(), 1)
r1 = Request("7", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'g'
r2 = Request("8", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'h'
batch = [r1, r2]
subject.update_cache(batch)
subject.flush(True)
r3 = Request("9", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'i'
subject.update_cache(r3)
self.assertEqual(1, len(subject._cache))
to_fetch = ["7", "9"]
subject.fetch(to_fetch)
self.assertEqual(2, len(subject._cache))
self.assertEqual(b'g', subject._cache["7"])
self.assertEqual(b'i', subject._cache["9"])
def test_set_states(self):
subject = RedisState(get_pool(), 10)
r1 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r1.meta[b'state'] = b'a'
r2 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r2.meta[b'state'] = b'b'
r3 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
r3.meta[b'state'] = b'c'
batch = [r1, r2, r3]
subject.update_cache(batch)
r4 = Request("1", int(time()) - 10, 'https://www.knuthellan.com/', domain='knuthellan.com')
r5 = Request("2", int(time()) - 10, 'https://www.khellan.com/', domain='khellan.com')
r6 = Request("3", int(time()) - 10, 'https://www.hellan.me/', domain='hellan.me')
batch2 = [r4, r5, r6]
subject.set_states(batch2)
self.assertEqual(b'a', r4.meta[b'state'])
self.assertEqual(b'b', r5.meta[b'state'])
self.assertEqual(b'c', r6.meta[b'state'])