def setUp(self):
super(TestKeyStore, self).setUp()
self.kv = KeyStore(CharField())
self.ordered_kv = KeyStore(CharField(), ordered=True)
self.pickled_kv = PickledKeyStore(ordered=True)
self.json_kv = JSONKeyStore(ordered=True)
self.kv.clear()
self.json_kv.clear()
def __init__(self, databaseName="databases/bot.db"):
self.connection = SqliteDatabase(databaseName)
self.connection.connect()
self.lock = threading.Lock()
self.keyStore = JSONKeyStore(database=self.connection)
tasks = []
async with ClientSession() as session:
for feed in db.model.select().where(db.model.key.startswith("feed:")):
task = ensure_future(throttle(sem, session, feed, db))
tasks.append(task)
responses = gather(*tasks)
await responses
def feeds_from_opml(filename):
tree = ElementTree.parse(filename)
for feed in tree.findall('.//outline'):
if feed.get('xmlUrl'):
yield {'title': feed.get('title'), 'url': feed.get('xmlUrl')}
def update_database(db, filename):
for feed in feeds_from_opml(filename):
if not "feed:" + feed['url'] in db:
feed['last_fetched'] = 0
db["feed:" + feed['url']] = feed
if __name__ == '__main__':
db = JSONKeyStore(database=SqliteExtDatabase('/tmp/feeds.db'))
update_database(db, 'feeds.opml')
loop = get_event_loop()
tasks = ensure_future(fetcher(db))
loop.run_until_complete(tasks)
