def __init__(self, worker_id, task_queue, completed_queue): ConsumerThread.__init__(self, worker_id, task_queue, completed_queue) self._storage = DatabaseStorage() self._crawler = GenericSourceCrawler(Scraper([GenericSourceParser]), Scraper([RSSParser])) topic_scraper = Scraper([RSSParser, GenericFrontPageParser]) article_scraper = Scraper([TheOnionParser, EngadgetParser, TechCrunchParser, TorontoStarParser, TheGlobeAndMailParser, GenericContentParser]) sub_task = ArticleScrapingTask(article_scraper) self._task = TopicScrapingTask(topic_scraper, sub_task)
def __init__(self, worker_id, task_queue, completed_queue): ConsumerThread.__init__(self, worker_id, task_queue, completed_queue) self._library = AlgorithmLibrary([DummyAlgorithm(), KeywordAlgorithm()])
def __init__(self, worker_id, task_queue, completed_queue, algorithm): ConsumerThread.__init__(self, worker_id, task_queue, completed_queue) self._algorithm = algorithm()