def main(): db = MongoDB() db.set_unique_key('WWA_app_vioation_content_info', 'url') db.set_ensure_index('WWA_app_vioation_content_info', 'read_status') def begin_callback(): log.info('\n********** WWA_APP begin **********') db.delete('WWA_app_urls', {}) def end_callback(): export_data.main() log.info('\n********** WWA_APP end **********') # 配置spider spider = Spider(tab_urls = 'WWA_app_urls', tab_site = 'WWA_app_site_info', tab_content = 'WWA_app_content_info', parser_count = 1, begin_callback = begin_callback, end_callback = end_callback, parser_params = {}) # 添加parser spider.add_parser(headline_parser) spider.add_parser(kuaibao_parser) spider.start()
class Spider(threading.Thread): def __init__(self, tab_urls, tab_site, tab_content, parser_count=None, parser_params={}, begin_callback=None, end_callback=None, content_unique_key='url'): ''' @summary: --------- @param tab_urls: url表名 @param tab_site: 网站表名 @param parser_count: parser 的线程数,为空时以配置文件为准 @param parser_params : 解析器所用的參數 @param begin_callback: 爬虫开始的回调 @param end_callback: 爬虫结束的回调 --------- @result: ''' super(Spider, self).__init__() self._tab_urls = tab_urls self._db = MongoDB() self._db.set_unique_key(tab_urls, 'url') self._db.set_unique_key(tab_site, 'site_id') self._db.set_unique_key(tab_content, content_unique_key) #设置索引 加快查询速度 self._db.set_ensure_index(tab_urls, 'depth') self._db.set_ensure_index(tab_urls, 'status') self._db.set_ensure_index(tab_site, 'read_status') self._db.set_ensure_index(tab_content, 'read_status') self._collector = Collector(tab_urls) self._parsers = [] self._parser_params = parser_params self._begin_callback = begin_callback self._end_callabck = end_callback self._parser_count = int( tools.get_conf_value( 'config.conf', 'parser', 'parser_count')) if not parser_count else parser_count self._spider_site_name = tools.get_conf_value( 'config.conf', "spider_site", "spider_site_name").split(',') self._except_site_name = tools.get_conf_value( 'config.conf', "spider_site", "except_site_name").split(',') def add_parser(self, parser): if self._spider_site_name[0] == 'all': for except_site_name in self._except_site_name: if parser.NAME != except_site_name.strip(): self._parsers.append(parser) else: for spider_site_name in self._spider_site_name: if parser.NAME == spider_site_name.strip(): self._parsers.append(parser) def run(self): self.__start() def __start(self): if self._begin_callback: self._begin_callback() if not self._parsers: if self._end_callabck: self._end_callabck() return # 启动collector self._collector.add_finished_callback(self._end_callabck) self._collector.start() # 启动parser 的add site 和 add root #print(self._parser_params) for parser in self._parsers: threading.Thread(target=parser.add_site_info).start() threading.Thread(target=parser.add_root_url, args=(self._parser_params, )).start() # 启动parser control while self._parser_count: parser_control = PaserControl(self._collector, self._tab_urls) for parser in self._parsers: parser_control.add_parser(parser) parser_control.start() self._parser_count -= 1
class Spider(threading.Thread): def __init__(self, tab_list, tab_unique_key_list, tab_ensure_index_list, parser_count=None, site_parsers=None, parser_params={}, begin_callback=None, end_callback=None): ''' @summary: --------- @param tab_urls: url表名 @param tab_site: 网站表名 @param parser_count: parser 的线程数,为空时以配置文件为准 @param parser_params : 解析器所用的参数 @param begin_callback: 爬虫开始的回调 @param end_callback: 爬虫结束的回调 --------- @result: ''' super(Spider, self).__init__() self._tab_urls = tab_list[0] self._site_parsers = site_parsers self._db = MongoDB() for tab_index in range(len(tab_list)): self._db.set_unique_key(tab_list[tab_index], tab_unique_key_list[tab_index]) # 设置索引 加快查询速度 for ensure_index in tab_ensure_index_list[tab_index]: self._db.set_ensure_index(tab_list[tab_index], ensure_index) self._collector = Collector(self._tab_urls, self._site_parsers) self._parsers = [] self._parser_params = parser_params self._begin_callback = begin_callback self._end_callabck = end_callback self._parser_count = int( tools.get_conf_value( 'config.conf', 'parser', 'parser_count')) if not parser_count else parser_count self._spider_site_name = tools.get_conf_value( 'config.conf', "spider_site", "spider_site_name").split(',') self._except_site_name = tools.get_conf_value( 'config.conf', "spider_site", "except_site_name").split(',') def add_parser(self, parser): if self._spider_site_name[0] == 'all': for except_site_name in self._except_site_name: if parser.NAME != except_site_name.strip(): self._parsers.append(parser) else: for spider_site_name in self._spider_site_name: if parser.NAME == spider_site_name.strip(): self._parsers.append(parser) def run(self): self.__start() def __start(self): if self._begin_callback: self._begin_callback() if not self._parsers: if self._end_callabck: self._end_callabck() return # 启动parser 的add site 和 add root # print(self._parser_params) for parser in self._parsers: parser.add_site_info() parser.add_root_url(self._parser_params) print('添加跟url完毕') # 启动collector self._collector.add_finished_callback(self._end_callabck) self._collector.start() # 启动parser control while self._parser_count: parser_control = PaserControl(self._collector, self._tab_urls) for parser in self._parsers: parser_control.add_parser(parser) parser_control.start() self._parser_count -= 1
class Spider(threading.Thread): def __init__(self, tab_urls, tab_site, tab_content, parser_count=None, search_keyword1=[], search_keyword2=[], search_keyword3=[], begin_callback=None, end_callback=None, content_unique_key=None): ''' @summary: --------- @param tab_urls: url表名 @param tab_site: 网站表名 @param parser_count: parser 的线程数,为空时以配置文件为准 @param search_keyword1: 搜索关键字(列表)全部包含 @param search_keyword2: 搜索关键字(列表)至少包含一个 @param search_keyword3: 搜索关键字(列表)一个都不能包含 @param begin_callback: 爬虫开始的回调 @param end_callback: 爬虫结束的回调 --------- @result: ''' super(Spider, self).__init__() self._tab_urls = tab_urls self._db = MongoDB() self._db.set_unique_key(tab_urls, 'url') self._db.set_unique_key(tab_site, 'site_id') self._db.set_unique_key( tab_content, 'url' if not content_unique_key else content_unique_key) self._collector = Collector(tab_urls) self._parsers = [] self._search_keyword1 = search_keyword1 self._search_keyword2 = search_keyword2 self._search_keyword3 = search_keyword3 self._begin_callback = begin_callback self._end_callabck = end_callback self._parser_count = int( tools.get_conf_value( 'config.conf', 'parser', 'parser_count')) if not parser_count else parser_count self._spider_site_name = tools.get_conf_value( 'config.conf', "spider_site", "spider_site_name").split(',') self._except_site_name = tools.get_conf_value( 'config.conf', "spider_site", "except_site_name").split(',') def add_parser(self, parser): if self._spider_site_name[0] == 'all': for except_site_name in self._except_site_name: if parser.NAME != except_site_name.strip(): self._parsers.append(parser) else: for spider_site_name in self._spider_site_name: if parser.NAME == spider_site_name.strip(): self._parsers.append(parser) def run(self): self.__start() def __start(self): if self._begin_callback: self._begin_callback() if not self._parsers: if self._end_callabck: self._end_callabck() return # 启动collector self._collector.add_finished_callback(self._end_callabck) self._collector.start() # 启动parser 的add site 和 add root for parser in self._parsers: threading.Thread(target=parser.add_site_info).start() threading.Thread(target=parser.add_root_url, args=(self._search_keyword1, self._search_keyword2, self._search_keyword3)).start() # 启动parser control while self._parser_count: parser_control = PaserControl(self._collector, self._tab_urls) for parser in self._parsers: parser_control.add_parser(parser) parser_control.start() self._parser_count -= 1