def read_data(): scanner = db.Scanner(client, 'Feeds', ['Meta:']) for row in scanner: feed, categs = row.row, row.columns['Meta:categs'].value if allowed_categs and not any_in(split_csv(categs), allowed_categs): continue yield feed, categs
def refresh_feeds(client, allowed_categs): """ Refresh all feeds found in the database using a pool of threads. """ log.info('Starting to refresh all feeds') allowed_categs = split_csv(allowed_categs) def read_data(): scanner = db.Scanner(client, 'Feeds', ['Meta:']) for row in scanner: feed, categs = row.row, row.columns['Meta:categs'].value if allowed_categs and not any_in(split_csv(categs), allowed_categs): continue yield feed, categs feeds.aggregate_all(client, read_data(), get_hbase_client)