def test_comment_parser(): with open('./test_files/shop_reviews.html', 'r') as html_file: data = html_file.read() url = 'https://www.dianping.com/shop/90556783/review_all' task = Task(url, '', '') parser = get_parser(url) _parse(parser, task, data)
def _process_task(self, task: Task) -> None: content = self.downloader.download_task(task) try: parser = get_parser(task.url) for item in parser.parse(task, content): if isinstance(item, Task): logger.info(f'Append new task {item}') self.task_queue.push_task(item) elif isinstance(item, Document): logger.info(f'Save the parsed item {item}') item.__class__.store_item(item) else: raise Exception( f'Unsupported parse result: class={item.__class__}') except Exception as e: with open('exception.html', 'w') as ofile: ofile.write(content) logger.error(f'Parse failed with error:') logger.exception(e) raise self.task_queue.drop_top_task(task.type_) # 等待指定的秒数+-2s # delay = config.DOWNLOAD_DELAY + random.randint(20, 50) / 10 delay = config.DOWNLOAD_DELAY logger.info(f'Delay for {delay} seconds.') time.sleep(delay)
def test_xinhua_search_list_parser(): url = f'http://so.news.cn/getNews' params = { 'keyword': '疫情', 'curPage': 0, 'sortField': 0, 'searchFields': 1, 'lang': 'cn', } task = Task(url, '', '', params=params) data = _load_html_file('./test_files/xinhua_list.json') parser = get_parser(url) _parse(parser, task, data)
def test_china_cdc_search_list_parser(): url = 'http://www.chinacdc.cn/was5/web/search' params = { 'page': 2, 'channelid': 233877, 'searchword': '疫情', 'keyword': '疫情', 'orderby': '-日期', 'perpage': 10, 'outlinepage': 5, } task = Task(url, '', '', params=params) data = _load_html_file('./test_files/china_cdc_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_china_list_parser(): url = 'http://search1.china.com.cn/search/searchcn.jsp' body = { 'searchText': '新冠', 'submit': '%E6%90%9C%E7%B4%A2', 'nodeid': '', 'strKeyword': '', 'strUrl': '', 'strNodename': '', 'sourcename=': '', 'LateTag': 1, 'strFromdate': '', 'strTodate': '', 'strSortBy': 0, 'server': 1, } task = Task(url, '', '', method='POST', body=body) data = _load_html_file('./test_files/china_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_sina_news_search_list_parser(): url = 'https://search.sina.com.cn/?q=%E7%96%AB%E6%83%85&range=all&c=news&sort=time' task = Task(url, '', '') data = _load_html_file('./test_files/sina_news_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_china_news_search_list_parser(): url = 'http://sou.chinanews.com/search.do' task = Task(url, '', '', method='POST') data = _load_html_file('./test_files/china_news_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_people_search_list_parser(): url = 'http://search.people.com.cn/cnpeople/news/getNewsResult.jsp' task = Task(url, '', '') data = _load_html_file('./test_files/people_list.html', 'gbk') parser = get_parser(url) _parse(parser, task, data)
def test_cnr_search_list_parser(): url = 'http://was.cnr.cn/' task = Task(url, '', '') data = _load_html_file('./test_files/cnr_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_topic_list_parser(): data = _load_html_file('./test_files/topic_list.html') url = 'https://s.weibo.com/topic?q=%E7%96%AB%E6%83%85&pagetype=topic&topic=1&Refer=weibo_topic&page=2' task = Task(url, '', '', metadata={'keyword': '疫情'}) parser = get_parser(url) _parse(parser, task, data)
def test_detail_parser(): data = _load_html_file('./test_files/shop_detail.html') url = 'https://www.dianping.com/shop/90556783' task = Task(url, '', '') parser = get_parser(url) _parse(parser, task, data)
def test_weibo_topic_detail_parser(): url = 'https://m.s.weibo.com/ajax_topic/trend?q=%23疫情%23&time=30d' task = Task(url, '', '', metadata={'keyword': '疫情', 'title': '#疫情#'}) data = _load_html_file('./test_files/weibo_topic_info.json') parser = get_parser(url) _parse(parser, task, data)
def test_the_paper_list_parser(): url = 'https://www.thepaper.cn/searchResult.jsp?inpsearch=%E7%96%AB%E6%83%85' task = Task(url, '', '') data = _load_html_file('./test_files/the_paper_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_cctv_list_parser(): url = 'https://search.cctv.com/search.php?qtext=%E6%96%B0%E5%86%A0&sort=relevance&type=web' task = Task(url, '', '') data = _load_html_file('./test_files/cctv_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_gov_search_list_parser(): url = 'http://sousuo.gov.cn/s.htm?t=govall&q=%E7%96%AB%E6%83%85' task = Task(url, '', '') data = _load_html_file('./test_files/gov_list.html') parser = get_parser(url) _parse(parser, task, data)
def test_weibo_parser(): data = _load_html_file('./test_files/search_weibos.html') url = 'https://s.weibo.com/weibo?q=%23%E7%96%AB%E6%83%85%E5%8F%AF' task = Task(url, '', '') parser = get_parser(url) _parse(parser, task, data)
def test_list_parser(): data = _load_html_file('./test_files/shop_list.html') url = 'https://www.dianping.com/search/keyword/2/0_%E4%B9%A6%E5%BA%97%E9%9F%B3%E5%83%8F' task = Task(url, '', '') parser = get_parser(url) _parse(parser, task, data)