def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def __init__(self, databaseName="databases/bot.db"):
self.connection = SqliteDatabase(databaseName)
self.connection.connect()
self.lock = threading.Lock()
self.keyStore = JSONKeyStore(database=self.connection)
tasks = []
async with ClientSession() as session:
for feed in db.model.select().where(db.model.key.startswith("feed:")):
task = ensure_future(throttle(sem, session, feed, db))
tasks.append(task)
responses = gather(*tasks)
await responses
def feeds_from_opml(filename):
tree = ElementTree.parse(filename)
for feed in tree.findall('.//outline'):
if feed.get('xmlUrl'):
yield {'title': feed.get('title'), 'url': feed.get('xmlUrl')}
def update_database(db, filename):
for feed in feeds_from_opml(filename):
if not "feed:" + feed['url'] in db:
feed['last_fetched'] = 0
db["feed:" + feed['url']] = feed
if __name__ == '__main__':
db = JSONKeyStore(database=SqliteExtDatabase('/tmp/feeds.db'))
update_database(db, 'feeds.opml')
loop = get_event_loop()
tasks = ensure_future(fetcher(db))
loop.run_until_complete(tasks)
class TestKeyStore(PeeweeTestCase):
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def test_json(self):
self.json_kv['foo'] = 'bar'
self.json_kv['baze'] = {'baze': [1, 2, 3]}
self.json_kv['nugget'] = None
self.assertEqual(self.json_kv['foo'], 'bar')
self.assertEqual(self.json_kv['baze'], {'baze': [1, 2, 3]})
self.assertIsNone(self.json_kv['nugget'])
self.assertRaises(KeyError, lambda: self.json_kv['missing'])
results = self.json_kv[self.json_kv.key << ['baze', 'bar', 'nugget']]
self.assertEqual(results, [
{
'baze': [1, 2, 3]
},
None,
])
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del (self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
key = self.kv.key
results = self.kv[key << ('a', 'b')]
self.assertEqual(results, ['X', '1'])
del (self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(list(self.kv.get(self.kv.key << ('x', 'y'))), [])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])
class TestKeyStore(PeeweeTestCase):
def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def test_json(self):
self.json_kv['foo'] = 'bar'
self.json_kv['baze'] = {'baze': [1, 2, 3]}
self.json_kv['nugget'] = None
self.assertEqual(self.json_kv['foo'], 'bar')
self.assertEqual(self.json_kv['baze'], {'baze': [1, 2, 3]})
self.assertIsNone(self.json_kv['nugget'])
self.assertRaises(KeyError, lambda: self.json_kv['missing'])
results = self.json_kv[self.json_kv.key << ['baze', 'bar', 'nugget']]
self.assertEqual(results, [
{'baze': [1, 2, 3]},
None,
])
def test_storage(self):
self.kv['a'] = 'A'
self.kv['b'] = 1
self.assertEqual(self.kv['a'], 'A')
self.assertEqual(self.kv['b'], '1')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
del(self.kv['a'])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.kv['a'] = 'A'
self.kv['c'] = 'C'
self.assertEqual(self.kv[self.kv.key << ('a', 'c')], ['A', 'C'])
self.kv[self.kv.key << ('a', 'c')] = 'X'
self.assertEqual(self.kv['a'], 'X')
self.assertEqual(self.kv['b'], '1')
self.assertEqual(self.kv['c'], 'X')
key = self.kv.key
results = self.kv[key << ('a', 'b')]
self.assertEqual(results, ['X', '1'])
del(self.kv[self.kv.key << ('a', 'c')])
self.assertRaises(KeyError, self.kv.__getitem__, 'a')
self.assertRaises(KeyError, self.kv.__getitem__, 'c')
self.assertEqual(self.kv['b'], '1')
self.pickled_kv['a'] = 'A'
self.pickled_kv['b'] = 1.1
self.assertEqual(self.pickled_kv['a'], 'A')
self.assertEqual(self.pickled_kv['b'], 1.1)
def test_container_properties(self):
self.kv['x'] = 'X'
self.kv['y'] = 'Y'
self.assertEqual(len(self.kv), 2)
self.assertTrue('x' in self.kv)
self.assertFalse('a' in self.kv)
def test_dict_methods(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
self.assertEqual(list(kv.keys()), ['a', 'b', 'c'])
self.assertEqual(list(kv.values()), ['A', 'B', 'C'])
self.assertEqual(list(kv.items()), [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_iteration(self):
for kv in (self.ordered_kv, self.pickled_kv):
kv['a'] = 'A'
kv['c'] = 'C'
kv['b'] = 'B'
items = list(kv)
self.assertEqual(items, [
('a', 'A'),
('b', 'B'),
('c', 'C'),
])
def test_shared_mem(self):
self.kv['a'] = 'xxx'
self.assertEqual(self.ordered_kv['a'], 'xxx')
def set_k():
kv_t = KeyStore(CharField())
kv_t['b'] = 'yyy'
t = threading.Thread(target=set_k)
t.start()
t.join()
self.assertEqual(self.kv['b'], 'yyy')
def test_get(self):
self.kv['a'] = 'A'
self.kv['b'] = 'B'
self.assertEqual(self.kv.get('a'), 'A')
self.assertEqual(self.kv.get('x'), None)
self.assertEqual(self.kv.get('x', 'y'), 'y')
self.assertEqual(
list(self.kv.get(self.kv.key << ('a', 'b'))),
['A', 'B'])
self.assertEqual(
list(self.kv.get(self.kv.key << ('x', 'y'))),
[])
def test_pop(self):
self.ordered_kv['a'] = 'A'
self.ordered_kv['b'] = 'B'
self.ordered_kv['c'] = 'C'
self.assertEqual(self.ordered_kv.pop('a'), 'A')
self.assertEqual(list(self.ordered_kv.keys()), ['b', 'c'])
self.assertRaises(KeyError, self.ordered_kv.pop, 'x')
self.assertEqual(self.ordered_kv.pop('x', 'y'), 'y')
self.assertEqual(
list(self.ordered_kv.pop(self.ordered_kv.key << ['b', 'c'])),
['B', 'C'])
self.assertEqual(list(self.ordered_kv.keys()), [])