def test_count(self): self.assertEqual(pytime.count('2015517', '2015519'), datetime.timedelta(-2)) self.assertEqual(pytime.count('2015517', '2015519 23:23:23'), datetime.timedelta(-3, 2197)) self.assertEqual(pytime.count('2015517 23:23:23', '2015519 23:23:23'), datetime.timedelta(-2)) self.assertEqual(pytime.count('2015519 23:23:23', '2015-5-17'), datetime.timedelta(2, 84203))
def parse(self, response): # 解析列表页中的所有文章url并交给scrapy下载后并进行解析 while response: post_nodes = response.css( "#dnn_ctr59828_ArticleList__ctl0_ArtDataList a::attr(href)" ).extract() news_time = response.css( "#dnn_ctr59828_ArticleList__ctl0_ArtDataList__ctl0_Label6::text" ).extract_first() if pytime.count(pytime.today(), news_time) > datetime.timedelta(TIME_DELTA_DAYS): print(news_time + "\n") return for post_node in post_nodes: yield Request(url=parse.urljoin(response.url, post_node), callback=self.parse_detail) self.browser.find_element_by_css_selector( "#dnn_ctr59828_ArticleList__ctl0_lbtnNext").click() selector = Selector(text=self.browser.page_source) page_num = selector.css( "#dnn_ctr59828_ArticleList__ctl0_plPageNum::text" ).extract_first() print("page is " + page_num) response = HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8") return
def parse(self, response): # 解析列表页中的所有文章url并交给scrapy下载后并进行解析 if response.url == "http://www.jwc.shu.edu.cn/index/tzgg.htm": self.tag = "通知公告" elif response.url == "http://www.jwc.shu.edu.cn/index/xw.htm": self.tag = "新闻" post_nodes = response.css( "#dnn_ctr43516_ArticleList__ctl0_ArtDataList__ctl1_titleLink1::attr(href)" ).extract() news_time = response.css( "dnn_ctr43516_ArticleList__ctl0_ArtDataList__ctl1_Label6::text" ).extract_first() if pytime.count(pytime.today(), news_time) > datetime.timedelta(TIME_DELTA_DAYS): print(news_time) return for post_node in post_nodes: yield Request(url=parse.urljoin(response.url, post_node), callback=self.parse_detail) # 提取下一页并交给scrapy进行下载 next_url = response.css( "a.Next:nth-child(3)::attr(href)").extract_first("") if next_url: yield Request(url=parse.urljoin(response.url, next_url), meta={"tag": self.tag}, callback=self.parse)
def parse(self, response): # 解析列表页中的所有文章url并交给scrapy下载后并进行解析 post_nodes = response.css(".views-table > tbody:nth-child(1) tr") for post_node in post_nodes: create_date = post_node.css( ".views-field-created::text").extract_first().strip() create_date = datetime.datetime.strptime(create_date, "%y-%m-%d") post_node_url = post_node.css("a::attr(href)").extract_first() if pytime.count(pytime.today(), create_date) < datetime.timedelta(TIME_DELTA_DAYS): url = parse.urljoin(response.url, post_node_url) delta = pytime.count(pytime.today(), create_date) yield Request(url=parse.urljoin(response.url, post_node_url), meta={"create_date": create_date}, callback=self.parse_detail, dont_filter=True) else: break
def parse(self, response): # 解析列表页中的所有文章url并交给scrapy下载后并进行解析//*[@id="dnn_dnnBREADCRUMB_lblBreadCrumb"]/a[2] tag = response.css("#dnn_dnnBREADCRUMB_lblBreadCrumb > a:nth-child(2)::text").extract_first() post_nodes = response.css( "#dnn_ctr1053_ArticleList_ctl00_lstArticles > tbody:nth-child(1) a::attr(href)").extract() news_time = response.css( "#dnn_ctr1053_ArticleList_ctl00_lstArticles_ctl00_lblPublishDate::text").extract_first() if pytime.count(pytime.today(), news_time) > datetime.timedelta(TIME_DELTA_DAYS): print(news_time) return for post_node in post_nodes: yield Request(url=parse.urljoin(response.url, post_node), meta ={"tag": tag}, callback=self.parse_detail) # 提取下一页并交给scrapy进行下载 next_url = response.css("a.Next:nth-child(3)::attr(href)").extract_first("") if next_url: yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)
def parse(self, response): # 解析列表页中的所有文章url并交给scrapy下载后并进行解析 post_nodes = response.css("#dnn_ctr59825_ArticleList__ctl0_ArtDataList a::attr(href)").extract() news_time = response.css("#dnn_ctr59825_ArticleList__ctl0_ArtDataList__ctl0_Label6::text") .extract_first() if pytime.count(pytime.today(), news_time) > datetime.timedelta(TIME_DELTA_DAYS): print(news_time+"\n") return tag = response.css("#dnn_dnnBREADCRUMB_lblBreadCrumb > a::text") if "tabid=31641" in response.url: tag = '学工新闻' for post_node in post_nodes: yield Request(url=parse.urljoin(response.url, post_node), meta={"tag",tag}, callback=self.parse_detail) # 提取下一页并交给scrapy进行下载 next_url = response.css("#dnn_ctr59825_ArticleList__ctl0_lbtnNext::attr(href)").extract_first("") if next_url: yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)
def test_count(self): self.assertEqual(pytime.count('2015517', '2015519'), datetime.timedelta(-2)) self.assertEqual(pytime.count('2015517', '2015519 23:23:23'), datetime.timedelta(-3, 2197)) self.assertEqual(pytime.count('2015517 23:23:23', '2015519 23:23:23'), datetime.timedelta(-2)) self.assertEqual(pytime.count('2015519 23:23:23', '2015-5-17'), datetime.timedelta(2, 84203))
def test_count(self): self.assertTrue(pytime.count('2015517', '2015519'), datetime.timedelta(-2))
def process_item(self, item, spider): if pytime.count( pytime.today(), item['create_date']) < datetime.timedelta(TIME_DELTA_DAYS): return item