def parse(self, response): max_page_temp = response.xpath( '//div[@class="ewb-page"]//li[@class="ewb-page-li ewb-page-noborder ewb-page-num"]/span/text()' ).extract_first() max_page = max_page_temp.split('/')[1] contents = response.xpath('//ul[@class="wb-data-item"]/li') for content in contents: title = content.xpath( './/div[@class="wb-data-infor l"]/a/text()').extract_first() date = content.xpath( './/span[@class="wb-data-date"]/text()').extract_first() short_url = content.xpath( './/div[@class="wb-data-infor l"]/a/@href').extract_first() detail_url = "http://www.hngzzx.com" + short_url if "002002001001" in response.url: content_type = "02" elif "002002002001" in response.url: content_type = "01" elif "002002003001" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="湖南", content_type=content_type, publish_id="430000", thing_type_id="88") for page in range(2, int(max_page) + 1): next_url = re.sub(r"[1-9]\d*.html", str(page) + ".html", response.url) yield scrapy.Request(url=next_url)
def parse(self, response): max_page = response.xpath( '//div[@class="pagesite"]/div//option/text()')[-1].extract() contents = response.xpath('//table[@class="newtable"]//tr')[:-1] for content in contents: title = content.xpath('.//td[3]/a/text()').extract_first() date = content.xpath('.//td[4]/text()').extract_first() detail_url = content.xpath('.//td[3]/a/@href').extract_first() if "jgzbgg" in response.url: content_type = "02" elif "cggg" in response.url: content_type = "01" else: content_type = "03" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="海南", content_type=content_type, publish_id="150303", thing_type_id="88") for i in range(2, int(max_page) + 1): next_url = re.sub(r'index_[1-9]\d*', "index_" + str(i), response.url) yield scrapy.Request(next_url)
def parse(self, response): max_page_temp = response.xpath( '//ul[@class="wb-page-items clearfix"]/li[@class="wb-page-li"]/span/text()' ).extract_first() max_page = max_page_temp.split('/')[1] contents = response.xpath('div[@class="ewb-infolist"]/ul') for content in contents: title = content.xpath( './/a[@class="ewb-list-name"]/text()').extract_first() date = content.xpath( './/span[@class="ewb-list-date"]/text()').extract_first() short_url = content.xpath( './/a[@class="ewb-list-name"]/@href').extract_first() detail_url = response.urljoin(short_url) if "002001" in response.url: content_type = "02" elif "002006" in response.url: content_type = "01" elif "002007" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="江西", content_type=content_type, publish_id="360000", thing_type_id="88") for page in range(2, int(max_page) + 1): next_url = re.sub(r'[1-9]\d*.html', str(page) + ".html", response.url) yield scrapy.Request(url=next_url)
def parse(self, response): total_page = response.xpath('//div[@class="showpage"]//a').extract() page_str = total_page[len(total_page) - 1] total_re = re.match(".*?page=(\d+)", page_str) max_page_num = int(total_re.group(1)) all_tr = response.xpath('//td[@class="lmlb"]//tr') for tr in all_tr: title = tr.xpath('.//a/text()').extract_first() detail_url = tr.xpath('.//a/@href').extract_first() title_str = tr.xpath('.//a').extract_first() try: match_re = re.match( '.*?([0-9]{4}/[0-9]{0,2}/[0-9]{0,2} \d{1,2}:\d{1,2}:\d{1,2})', title_str, re.S) except Exception: print(title_str) if match_re: date_time = str(match_re.group(1)) else: date_time = '9999/99/99 00:00:00' content_type = "04" yield GovSpiderItem(title=title, date=date_time, detail_url=detail_url, area_code="GAOPING", content_type=content_type, publish_id="181818", thing_id="42") for page in range(2, int(max_page_num) + 1): next_url = self.base_url + '&page=' + str(page) yield scrapy.Request(url=next_url)
def parse(self, response): max_page_temp = response.xpath('//table[@algin="center"]/tr/td[@class="huifont"]/text()').extract_first() max_page = max_page_temp.split('/')[1] contents = response.xpath('//table[@width="98%"]//tr') for content in contents: title = content.xpath('.//td[2]/a/@title').extract_first() date = content.xpath('.//td[3]/text()').extract_first() if date: date=date[1:-1] else: date = '2018-04-22' short_url =content.xpath('.//td[2]/a/@href').extract_first() detail_url = response.urljoin(short_url) if "004001002" in response.url: content_type = "02" elif "004002006" in response.url: content_type = "01" elif "004004001" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title,notice_date=date, detail_url=detail_url,area_code="新疆", content_type=content_type, publish_id="650000", thing_type_id="88") for page in range(2, int(max_page)+1): next_url = response.urljoin('?Paging='+str(page)) yield scrapy.Request(url=next_url)
def parse(self, response): max_page_temp1 = response.xpath('//div[@id="pageZone"]/span/@title') if len(max_page_temp1) >4: max_page_temp = max_page_temp1[-2].extract() index = max_page_temp.find(u'页') max_page = max_page_temp[1:index] else: max_page = response.xpath('//div[@id="pageZone"]/span/text()').extract_first() contents = response.xpath('//div[@class="dq_nl"]/ul')[1:] for content in contents: title = content.xpath('.//li[@class="td_1"]/a/text()').extract_first() date = content.xpath('.//li[@class="td_2 dq_rq"]/text()').extract_first() short_url = content.xpath('.//li[@class="td_1"]/a/@href').extract_first() detail_url = "https://www.chinabidding.cn/"+short_url if "liaoning_gcjs" in response.url: content_type = "02" else: content_type = "01" yield GovSpiderItem(title=title,date=date, detail_url=detail_url,area_code="辽宁", content_type=content_type, publish_id= "181818", thing_id="42") for i in range(2, int(max_page)+1): next_url = re.sub(r'[1-9]\d*', str(i), response.url) yield scrapy.Request(next_url)
def parse(self, response): max_page_temp = response.xpath( '//ul[@class="pages-list"]/li[1]/a/text()').extract_first() max_page1 = max_page_temp.split('/')[1] index = max_page1.find(u'页') max_page = max_page1[:index] contents = response.xpath( '//div[@class="article_listbox"]/ul[@id="listbox"]/li') for content in contents: title = content.xpath('.//div[@class="content_left"]/a/text()' ).extract_first().strip() date = content.xpath( './/div[@class="content_right"]/span/text()').extract_first() detail_url = content.xpath( './/div[@class="content_left"]/a/@href').extract_first() if "jygkjsgc" in response.url: content_type = "02" elif "jygkzfcg" in response.url: content_type = "01" elif "jygkkyq" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="贵州", content_type=content_type, publish_id="520000", thing_type_id="88") for page in range(2, int(max_page) + 1): next_url = re.sub(r'index_[1-9]\d*', 'index_' + str(page), response.url) yield scrapy.Request(url=next_url)
def parse(self, response): contents = response.xpath('//table//tr')[1:] for content in contents: title = content.xpath('.//td[3]/a/text()').extract_first().strip() date = content.xpath('.//td[4]/text()').extract_first().strip() short_url = content.xpath('.//td[3]/a/@href').extract_first() detail_url = response.urljoin(short_url) if "jsgcZbgg" in response.url: content_type = "02" elif "zfcg" in response.url: content_type = "01" elif "tdAndKq" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="内蒙古", content_type=content_type, publish_id="150000", thing_type_id="88") max_page = response.xpath( '//div[@class="page"]/div[@class="mmggxlh"]/a/text()')[-2].extract( ) for i in range(2, int(max_page) + 1): next_url = re.sub(r'currentPage=[1-9]\d*', 'currentPage=' + str(i), response.url) yield scrapy.Request(next_url)
def parse(self, response): max_page = response.xpath( '//div[@class="mmggxlh"]/a/text()')[-2].extract() contents = response.xpath('//table[@id="data_tab"]/tbody/tr')[1:] for content in contents: title = content.xpath('.//td[3]/a/@title').extract_first() date = content.xpath('.//td[4]/text()').extract_first() short_url = content.xpath('.//td[3]/a/@href').extract_first() detail_url = "https://www.ynggzy.com/" + short_url if "zfcg" in response.url: content_type = "01" elif "jsgcZbgg" in response.url: content_type = "02" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="云南", content_type=content_type, publish_id="530000", thing_type_id="88") for i in range(2, int(max_page) + 1): next_url = re.sub(r'[1-9]\d*', str(i), response.url) yield scrapy.Request(next_url)
def parse_item(self,response): res = json.loads(response.text) base_url = 'http://www.hebpr.cn' for i in res['result']['records']: title = i['title'] detail_url = base_url + i['linkurl'] area_code = i['zhuanzai'] date = i['showdate'] content_type = '01' yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code=area_code, content_type=content_type, publish_id="130100", thing_type_id="88")
def parse_detail(self, response): data = json.loads(response.text)['data'] res = json.loads(data) base_url = 'http://58.87.81.13/Info/ProjectDetail' for i in res: title = i['Title'] notice_date = i['CreateDate'] detail_url = base_url + i['Link'] area_code = i['username'] businessType = i['businessType'] if businessType == '工程建设': content_type = '01' elif businessType == '工程建设': content_type = '03' elif businessType == '政府采购': content_type = '02' yield GovSpiderItem(notice_title=title, notice_date=notice_date, detail_url=detail_url, area_code=area_code, content_type=content_type, publish_id="530000", thing_type_id="88",source = 'sichuanspider')
def parse(self, response): max_page_temp = response.xpath('//div[@class="pagemargin"]//td[@class="huifont"]/text()').extract_first() max_page = max_page_temp.split('/')[1] contents = response.xpath('//table[@width="100%"]/tr[@height="27"]') for content in contents: title = content.xpath('.//td[2]/a/text()').extract_first() date = content.xpath('.//td[3]/font/text()').extract_first()[1:-1] short_url = content.xpath('.//td[2]/a/@href').extract_first() detail_url = response.urljoin(short_url) if "002001001" in response.url: content_type = "02" elif "002002001" in response.url: content_type = "01" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="河南", content_type=content_type, publish_id= "410000", thing_type_id="88") for page in range(2, int(max_page)+1): next_url = response.urljoin('?Page='+str(page)) yield scrapy.Request(url=next_url)
def parse(self, response): max_count_temp = response.xpath( '//div[@id="TestView_pageableDiv"]/@totalsize').extract_first() max_count = int(max_count_temp) if max_count <= 20: max_page = max_count elif max_count % 20 == 0: max_page = max_count / 20 else: max_page = max_count / 20 + 1 contents = response.xpath('//div[@class="list_box"]/ul') for content in contents: item = GovSpiderItem() item['title'] = content.xpath( './/li[@class="l"]/a/@title').extract_first() item['date'] = contents.xpath( './/li[@class="r"]/text()').extract_first() short_url = content.xpath( './/li[@class="l"]/a/@href').extract_first() detail_url = "http://www.gdggzy.org.cn" + short_url item['detail_url'] = detail_url item['area_code'] = "GUANGDONG" item['publish_id'] = "181818" item['thing_id'] = "42" if "30091" == response.meta["ss"]: content_type = "03" elif "30011" in response.meta["ss"]: content_type = "01" else: content_type = "02" item["content_type"] = content_type yield item for i in range(2, max_page + 1): for w in self.k: next_url = 'http://www.gdggzy.org.cn/prip-portal-web/main/viewList.do' formdata = {"currPage": str(i), "typeId": w, "pageSize": "20"} yield scrapy.FormRequest(url=next_url, formdata=formdata, dont_filter=False, meta={"ss": w})
def parse(self, response): max_page_temp = response.xpath('//div[@class="pagemargin"]//td[@class="huifont"]/text()').extract_first() max_page = max_page_temp.split('/')[1] trs = response.xpath('//table[@width="98%"]/tr[@height="30"]') for tr in trs: title = tr.xpath('.//td[2]/a/@title').extract_first() date = tr.xpath('.//td[3]/text()').extract_first()[1:-1] short_url = tr.xpath('.//td[2]/a/@href').extract_first() detail_url = response.urljoin(short_url) if "004001001" in response.url: # 工程03 content_type = "03" elif "004001002" in response.url: content_type = "01" else: content_type = "04" yield GovSpiderItem(title=title,date=date, detail_url=detail_url,area_code="浙江", content_type=content_type, publish_id= "330000", thing_type_id="88") for page in range(2, int(max_page)+1): next_url = response.urljoin('?Page='+str(page)) yield scrapy.Request(url=next_url)
def parse(self, response): max_page_temp = response.xpath( '//ul[@class="pages-list"]/li[1]/a/text()').extract_first() max_page1 = max_page_temp.split('/')[1] index = max_page1.find(u'页') max_page = max_page1[:index] contens = response.xpath('//ul[@class="article-list-a"]/li') for conten in contens: titleList = conten.xpath( './/div[@class="article-list3-t"]/a/text()').extract() title = "".join(titleList) date = conten.xpath( './/div[@class="article-list3-t"]/div[@class="list-times"]/text()' ).extract_first() detail_url = conten.xpath( './/div[@class="article-list3-t"]/a/@href').extract_first() if "channelId=78" in response.url: content_type = "02" elif "channelId=79" in response.url: content_type = "01" elif "channelId=80" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title, notice_date=date, detail_url=detail_url, area_code="山东", content_type=content_type, publish_id="370000", thing_type_id="88") for page in range(2, int(max_page) + 1): next_url = re.sub(r'queryContent_\d', "queryContent_" + str(page), response.url) yield scrapy.Request(url=next_url)
def parse(self, response): max_page_temp = response.xpath('//div[@class="pages"]/ul[@class="pages-list"]/li[1]/a/text()').extract_first() max_page1 = max_page_temp.split('/')[1] index = max_page1.find(u'页') max_page = max_page1[:index] contents = response.xpath('//ul[@class="article-list-old"]/li') for content in contents: title = content.xpath('.//a/@title').extract_first() date1 = content.xpath('.//div[@class="list-times-old"]/text()').extract_first() date2 = time.strptime(date1, "%Y-%m-%d %H:%M:%S") date = time.strftime("%Y-%m-%d", date2) detail_url = content.xpath('.//a/@href').extract_first() if "zbzsgg" in response.url: content_type = "02" elif "td" in response.url: content_type = "03" else: content_type = "04" yield GovSpiderItem(notice_title=title,notice_date=date, detail_url=detail_url,area_code="西藏", content_type=content_type, publish_id= "540000", thing_type_id="88") for i in range(2, int(max_page)+1): next_url = re.sub(r'index_[1-9]\d*', "index_"+str(i), response.url) yield scrapy.Request(next_url)
def parse(self, response): url = 'http://jsggzy.jszwfw.gov.cn/jyxx/tradeInfonew.html' max_counts = json.loads(response.text)["result"]["totalcount"] for i in range(15, max_counts, 15): formdata1 = { "token": "", "pn": i, "rn": "15", "sdt": "", "edt": "", "wd": "", "inc_wd": "", "exc_wd": "", "fields": "title", "cnum": "001", "sort": "{\"infodatepx\":\"0\"}", "ssort": "title", "cl": 200, "terminal": "", "condition": [{ "fieldName": "categorynum", "isLike": True, "likeType": 2, "equal": "%s" % response.meta.keys()[0] }, { "fieldName": "fieldvalue", "isLike": True, "likeType": 2, "equal": "省级" }], "time": None, "highlights": "title", "statistics": None, "unionCondition": None, "accuracy": "", "noParticiple": "0", "searchRange": None, "isBusiness": "1" } formdata = json.dumps(formdata1) yield scrapy.Request(url=url, method='POST', body=formdata, headers=self.headers, callback=self.parse, dont_filter=False, meta=response.meta) datas = json.loads(response.text)["result"]["records"] for data in datas: item = GovSpiderItem() item["notice_title"] = data["title"] item["notice_date"] = datas["infodateformat"] item["area_code"] = "江苏" item["publish_id"] = "320000" item["thing_type_id"] = "88" if response.meta.has_key("003001") == True: item["content_type"] = "02" elif response.meta.has_key("003004") == True: item["content_type"] = "01" elif response.meta.has_key("003005") == True: item["content_type"] = "03" else: item["content_type"] = "04" yield item
def parse(self, response): url = 'http://www.hebpr.cn/inteligentsearch/rest/inteligentSearch/getFullTextDataNew' max_counts = json.loads(response.text)["result"]["totalcount"] datas = json.loads(response.text)["result"]["records"] for data in datas: hbitem = GovSpiderItem() hbitem["notice_title"] = data["title"] hbitem["notice_date"] = data["showdate"] hbitem["detail_url"] = "http://www.hebpr.cn" + data["linkurl"] hbitem["area_code"] = "河北" hbitem["publish_id"] = "130000" hbitem["thing_type_id"] = "88" if "003005001" in hbitem["detail_url"]: hbitem["content_type"] = "01" elif "003005002" in hbitem["detail_url"]: hbitem["content_type"] = "02" elif "003005004" in hbitem["detail_url"]: hbitem["content_type"] = "03" else: hbitem["content_type"] = "04" yield hbitem for i in range(10, max_counts + 10, 10): for w in self.results: formdata2 = { "token": "", "pn": i, "rn": 10, "sdt": "", "edt": "", "wd": "", "inc_wd": "", "exc_wd": "", "fields": "title", "cnum": "001", "sort": "{\"showdate\":\"0\"}", "ssort": "title", "cl": 200, "terminal": "", "condition": [{ "fieldName": "categorynum", "isLike": True, "likeType": 2, "equal": w }], "time": None, "highlights": "title", "statistics": None, "unionCondition": None, "accuracy": "", "noParticiple": "0", "searchRange": None, "isBusiness": 1 } formdata = json.dumps(formdata2) yield scrapy.Request(url=url, method='POST', body=formdata, headers=self.headers, callback=self.parse, dont_filter=False)