def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
class PostgresqlKeyStoreTestCase(unittest.TestCase):
def setUp(self):
self.kv = KeyStore(CharField(), ordered=True, database=db)
self.kv.clear()
def test_non_native_upsert(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv['a'], 'A')
self.kv['a'] = 'C'
self.assertEqual(self.kv['a'], 'C')
class PostgresqlKeyStoreTestCase(unittest.TestCase):
def setUp(self):
self.kv = KeyStore(CharField(), ordered=True, database=db)
self.kv.clear()
def test_non_native_upsert(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv['a'], 'A')
self.kv['a'] = 'C'
self.assertEqual(self.kv['a'], 'C')
class TestPostgresqlKeyStore(PeeweeTestCase):
def setUp(self):
self.db = PostgresqlDatabase('peewee_test')
self.kv = KeyStore(CharField(), ordered=True, database=self.db)
self.kv.clear()
def tearDown(self):
self.db.close()
def test_non_native_upsert(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv['a'], 'A')
self.kv['a'] = 'C'
self.assertEqual(self.kv['a'], 'C')
class PostgresqlKeyStoreTestCase(PeeweeTestCase):
def setUp(self):
self.db = PostgresqlDatabase('peewee_test')
self.kv = KeyStore(CharField(), ordered=True, database=self.db)
self.kv.clear()
def tearDown(self):
self.db.close()
def test_non_native_upsert(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv['a'], 'A')
self.kv['a'] = 'C'
self.assertEqual(self.kv['a'], 'C')
class KeyStoreTestCase(PeeweeTestCase):
def setUp(self):
super(KeyStoreTestCase, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del(self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
del(self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(
list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(
list(self.kv.get(self.kv.key << ('x', 'y'))),
[])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])
def setUp(self):
self.db = PostgresqlDatabase('peewee_test')
self.kv = KeyStore(CharField(), ordered=True, database=self.db)
self.kv.clear()
def setUp(self):
super(KeyStoreTestCase, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
def setUp(self):
self.kv = KeyStore(CharField(), ordered=True, database=db)
self.kv.clear()
def setUp(self):
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
class GreylistEntry(Model):
key = CharField(index=True)
timestamp = DateTimeFieldExtend(default=utils.utcnow)
expire_time = DateTimeFieldExtend(null=True)
rejects = IntegerField(default=0)
accepts = IntegerField(default=0)
last_state = DateTimeFieldExtend(null=True)
delay = FloatField(default=0.0)
protocol = KeyStore(TextField())
policy = CharField(max_length=255, default='policy')
def accept(self, now=None, expire=35 * 86400):
now = now or utils.utcnow()
self.accepts += 1
self.last_state = now
if self.accepts == 1:
value = now - self.timestamp
self.delay = round(value.total_seconds(), 2)
if not self.expire_time:
self.expire_time = now + datetime.timedelta(seconds=expire)
self.save()
def reject(self, now=None):
now = now or utils.utcnow()
self.rejects += 1
self.last_state = now
self.save()
def expire(self, delta=60, now=None):
now = now or utils.utcnow()
expire_date = self.timestamp + datetime.timedelta(seconds=delta)
value = expire_date - now
return round(value.total_seconds(), 2)
@classmethod
def create_entry(cls,
key=None,
protocol=None,
policy='default',
timestamp=None,
last_state=None,
now=None,
**kwargs):
now = now or utils.utcnow()
with database_proxy.transaction():
return cls.create(key=key,
rejects=1,
timestamp=timestamp or now,
last_state=last_state or now,
policy=policy,
protocol=protocol,
**kwargs)
@classmethod
def search_entry(cls, key=None, now=None):
"""
expire_time is None or greater than or equal to now AND key == key
"""
now = now or utils.utcnow()
try:
return cls.select().where((
(cls.expire_time == None) | (cls.expire_time > now))
& (cls.key == key)).get()
except:
pass
@classmethod
def last_metrics(cls):
last_24_hours = arrow.utcnow().replace(hours=-24).datetime
objects = cls.select().where(cls.timestamp >= last_24_hours)
count = objects.count()
if count == 0:
return
last_1_hour = arrow.utcnow().replace(hours=-1).datetime
accepted = cls.select(fn.Sum(
cls.accepts)).where(cls.timestamp >= last_24_hours)
rejected = cls.select(fn.Sum(
cls.rejects)).where(cls.timestamp >= last_24_hours)
delay = cls.select(fn.Avg(cls.delay)).where(
cls.timestamp >= last_24_hours, cls.accepts >= 0, cls.delay >= 0)
metrics = {
'count':
count,
'accepted':
accepted or 0,
'rejected':
rejected or 0,
'delay':
delay or 0.0,
'abandoned':
objects.filter(cls.accepts == 0,
cls.timestamp <= last_1_hour).count(),
#'count_accepts': objects.filter(accepts__gte=1).count(),
}
metrics['requests'] = metrics['accepted'] + metrics['rejected']
return metrics
def __unicode__(self):
return self.key
class Meta: # noqa
database = database_proxy
order_by = ('-timestamp', )
def setUp(self):
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
def setUp(self):
self.kv = KeyStore(CharField(), ordered=True, database=db)
self.kv.clear()
def setUp(self):
self.db = PostgresqlDatabase('peewee_test')
self.kv = KeyStore(CharField(), ordered=True, database=self.db)
self.kv.clear()
class KeyStoreTestCase(PeeweeTestCase):
def setUp(self):
super(KeyStoreTestCase, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del (self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
del (self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(list(self.kv.get(self.kv.key << ('x', 'y'))), [])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
class TestKeyStore(PeeweeTestCase):
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def test_json(self):
self.json_kv['foo'] = 'bar'
self.json_kv['baze'] = {'baze': [1, 2, 3]}
self.json_kv['nugget'] = None
self.assertEqual(self.json_kv['foo'], 'bar')
self.assertEqual(self.json_kv['baze'], {'baze': [1, 2, 3]})
self.assertIsNone(self.json_kv['nugget'])
self.assertRaises(KeyError, lambda: self.json_kv['missing'])
results = self.json_kv[self.json_kv.key << ['baze', 'bar', 'nugget']]
self.assertEqual(results, [
{
'baze': [1, 2, 3]
},
None,
])
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del (self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
key = self.kv.key
results = self.kv[key << ('a', 'b')]
self.assertEqual(results, ['X', '1'])
del (self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(list(self.kv.get(self.kv.key << ('x', 'y'))), [])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])
def setUp(self):
super(KeyStoreTestCase, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.kv.clear()
class TestKeyStore(PeeweeTestCase):
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def test_json(self):
self.json_kv['foo'] = 'bar'
self.json_kv['baze'] = {'baze': [1, 2, 3]}
self.json_kv['nugget'] = None
self.assertEqual(self.json_kv['foo'], 'bar')
self.assertEqual(self.json_kv['baze'], {'baze': [1, 2, 3]})
self.assertIsNone(self.json_kv['nugget'])
self.assertRaises(KeyError, lambda: self.json_kv['missing'])
results = self.json_kv[self.json_kv.key << ['baze', 'bar', 'nugget']]
self.assertEqual(results, [
{'baze': [1, 2, 3]},
None,
])
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del(self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
key = self.kv.key
results = self.kv[key << ('a', 'b')]
self.assertEqual(results, ['X', '1'])
del(self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(
list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(
list(self.kv.get(self.kv.key << ('x', 'y'))),
[])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])