def read_data():
scanner = db.Scanner(client, 'Feeds', ['Meta:'])
for row in scanner:
feed, categs = row.row, row.columns['Meta:categs'].value
if allowed_categs and not any_in(split_csv(categs), allowed_categs):
continue
yield feed, categs
def refresh_feeds(client, allowed_categs):
"""
Refresh all feeds found in the database using a pool of threads.
"""
log.info('Starting to refresh all feeds')
allowed_categs = split_csv(allowed_categs)
def read_data():
scanner = db.Scanner(client, 'Feeds', ['Meta:'])
for row in scanner:
feed, categs = row.row, row.columns['Meta:categs'].value
if allowed_categs and not any_in(split_csv(categs), allowed_categs):
continue
yield feed, categs
feeds.aggregate_all(client, read_data(), get_hbase_client)
