def parse_page(self, response): if not response: return BiddinginfospiderItem() print('request_url= ', response.request.url) body = json.loads(str(response.body, "utf-8")) li = body.get("data") print("Num :", len(li)) for l in li: item = BiddinginfospiderItem() sheng = l.get('districtShow') shiQu = l.get('platformName') shi = self.getSHI(shiQu) href = l.get("url"), if isinstance(href, tuple): href = href[0] print("href is,", href) # href = href.replace("a", "b") item.update( city=sheng + "-" + shi if shi else sheng, title=l.get("title"), ctime=l.get("timeShow"), category=l.get("classifyShow"), href=href, industry=l.get("tradeShow"), ) print("ITEM IS") # print(item) yield item
def parse_page(self, response): rs = json.loads(response.body.decode('utf8')) rs = rs['data'].get("datalist", False) for a in rs: item = BiddinginfospiderItem() item[ 'href'] = 'http://www.wysggzy.cn:81/hyweb/wysebid/bidDetails.do?handle=1&tenderProjCode={0}¬iceType=1&flag=1&tenderProjId={1}&proj_id={2}&pre_evaId=&evaId={3}&signUpType={4}'.format( a['tenderProjCode'], a['tenderProjId'], a['proj_id'], a['evaId'], a['signUpType']) item['title'] = a['noticeTitle'] item['ctime'] = a['sendTime'] item['city'] = '武夷山' data = { "tenderProjCode": a['tenderProjCode'], "noticeType": "1", "noticeId": "" } # yield scrapy.Request(url='http://www.wysggzy.cn:81/hyweb/transInfo/getProjBuildNoticeById.do', # dont_filter=True, callback=self.parse_item, method="POST", body=json.dumps(data), # meta={'meta': item, }) yield item
def parse_shangdong_xml(self, response): # response已经完成返回xml的解析 # datastore = etree.XML(response.text) html = ''.join(response.xpath('//record/text()').extract()) # 产生内容页的请求 for li in etree.HTML(html).xpath('//li'): item = BiddinginfospiderItem() a = li.xpath(".//a")[0] item['href'] = a.xpath(".//@href")[0] item['title'] = a.xpath(".//text()")[0] item['ctime'] = li.xpath('.//span//text()')[0] # print(item) yield item # 如果是全部爬取则继续获取下一页 if not self.biddingInfo_update: selector = etree.XML(response.body) a = [0] next_select = selector.xpath('//nextgroup/text()') if next_select: a = next_select[0] href = etree.HTML(a).xpath('//a/@href')[0] req = self.make_requests_from_url(response.urljoin(href)) req.callback = self.parse_shangdong_xml yield req
def parse_start_url(self, response): print('request_url= ', response.request.url) li = response.xpath('//ul[@class="newslist"]//a') for a in li: item = BiddinginfospiderItem() title = a.xpath("..//h1").xpath('normalize-space(string(.))').get() href = a.xpath('.//@href').get() code = a.xpath( './/ul[@class="newsinfo"]//li[1]//span//text()').get() t = a.xpath('.//div[@class="newsDate"]').xpath( 'normalize-space(string(.))').get() if t: t = t.replace(" ", "").replace("/", "-") ctime = t[:3] + "-" + t[4:] item.update( code=code, industry=self.industry, category=self.category, title=title, ctime=ctime, href=href, ) # print(item) yield item
def parse(self, response): for i in range(1, self.endPageNum): form_data = { "page.currentPage": str(i), "page.perPageSize": "20", "noticeBean.companyName": "", "noticeBean.title": "", "noticeBean.startDate": "", "noticeBean.endDate": "", } response = requests.post(self.tmpl_url, headers=self.headers, data=form_data) res = scrapy.Selector(text=response.text) li = res.xpath('//table[@class="jtgs_table"]//tr') article_tmp_url = 'https://b2b.10086.cn/b2b/main/viewNoticeContent.html?noticeBean.id={0}' for l in li[1:]: item = BiddinginfospiderItem() a = l.xpath(".//a") id = l.xpath('@onclick').get()[14:-2] href = article_tmp_url.format(id) title = a.xpath('.//text()').get() item.update( title=title, href=href, ) yield item
def parse_start_url(self, response): print('request_url= ', response.request.url) body = json.loads(str(response.body, "utf-8")) li = body.get("obj") print(len(li)) for l in li: item = BiddinginfospiderItem() title = l.get("PROJECTNAME") ctime = l.get("RECEIVETIME") category = l.get("TABLENAME") code = l.get("PROJECTCODE") url = l.get("URL", "") + "&id=" id = l.get("ID", "") href = response.urljoin("?getNoticeDetail&url=" + url + id) print(href) item.update( category=self.category_dict[category], title=title, ctime=ctime, href=href, code=code ) # yield scrapy.Request(method="GET", url=href, dont_filter=True, callback=self.parse_item, # meta={'item': item}) yield item
def parse_start_url(self, response): a_list = response.xpath('//div[@id="listChangeDiv"]//td//a') for a in a_list: item = BiddinginfospiderItem() item['href'] = response.urljoin(a.xpath('@href').get()) item['ctime'] = a.xpath('../../td[3]//text()').get() item['title'] = a.xpath("@title").get() # print(item) yield item
def parse_start_url(self, response): print('reponse_request_url= ', response.request.url) li = response.xpath('//div[@id="listChangeDiv"]//li') for line in li: item = BiddinginfospiderItem() a = line.xpath('.//a') item['href'] = response.urljoin(a.xpath('@href').get()) item['title'] = a.xpath('@title').get() item['ctime'] = line.xpath(".//span/text()").get() yield item
def parse(self, response): # a标签 a = response.xpath('//div[@class="lb-link"]//ul//li//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin(a1.xpath('.//@href').get()) item['title'] = a1.xpath(".//@title").get().strip() item['ctime'] = a1.xpath('.//span[3]//text()').get() yield item
def parse(self, response): # a标签 a = response.xpath('//ul[@class="newslist"]//li//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin(a1.xpath('.//@href').get()) item['title'] = a1.xpath(".//h1//text()").getall()[3].strip() item['ctime'] = a1.xpath('normalize-space(string(.//div[@class="newsDate"]))').get().replace('/','-').replace(' ','-') yield item
def parse(self, response): rs = json.loads(response.body.decode('utf8')) for a1 in rs: item = BiddinginfospiderItem() item['href'] = a1['pdf_url'] item['title'] = a1['message_title'] time0 = time.localtime(int(str(a1['publish_time'])[0:10])) item['ctime'] = time.strftime("%Y-%m-%d", time0) yield item
def parse(self, response): rs = json.loads(response.body.decode('utf8')) rs = rs['_rows'] for a1 in rs: item = BiddinginfospiderItem() item[ 'href'] = 'http://ecm.crcc.cn/unlogin/queryPurchaseTenderDetailInit.jhtml?model=1&id=' + a1[ 'id'] item['title'] = a1['v_notice_title'] item['ctime'] = a1['d_createdate'] yield item
def parse(self, response): rs = json.loads(response.body.decode('utf8')) rs = rs['_rows'] for a1 in rs: item = BiddinginfospiderItem() item[ 'href'] = 'http://ece.crcc.cn/homepage/inviteInfo.jhtml?type=1&id=' + str( a1['inviteid']) item['title'] = a1['texttitle'] item['ctime'] = a1['recordtime'][0:10] yield item
def parse_start_url(self, response): res = Selector(text=response.text[11:-2]) a_list = res.xpath('//a') for a in a_list: item = BiddinginfospiderItem() item['href'] = urljoin("http://qdstc.qingdao.gov.cn", a.xpath('@href').get()) item['ctime'] = a.xpath('../../td[3]//text()').get() item['title'] = a.xpath("@title").get() # print(item) yield item
def parse(self, response): rs = json.loads(response.body.decode('utf8')) rs = rs['RESULT'] for a1 in rs: item = BiddinginfospiderItem() item[ 'href'] = 'http://scs.inspur.com/GGDetail.htm?dataType=CRFQ&id=' + a1[ 'NoticeID'] item['title'] = a1['NoticeTitle'] item['ctime'] = a1['PubTime'][0:10] yield item
def parse_page(self, response): rs = json.loads(response.body.decode('utf8')) rs = rs['result']['records'] for a in rs: item = BiddinginfospiderItem() item['href'] = response.urljoin(a['linkurl']) item['title'] = a['title'] item['ctime'] = a['showdate'][0:10] item['city'] = a['xiaquname'] # yield scrapy.Request(url=item['href'], dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse(self, response): rs = json.loads(response.body.decode('utf8')) rs = eval(rs['return'])['Table'] for a in rs: item = BiddinginfospiderItem() item['href'] = response.urljoin(a['href']) item['title'] = a['title'] item['ctime'] = a['infodate'] item['city'] = a['infoc'] # yield scrapy.Request(url=item['href'], dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse_page(self, response): # a标签 a = response.xpath('//table[@id="p2"]//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin(a1.xpath('.//@href').extract_first()) item['title'] = a1.xpath(".//@title").extract_first().strip() item['ctime'] = a1.xpath('..//..//td[4]//text()').extract_first() item['city'] = '安阳' # yield scrapy.Request(url=item['href'], dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse_page(self, response): # a标签 rs = response.body.decode('utf8') # selector=Selector(text=rs) a = response.xpath('//div[@class="zbgg_table"]//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin(a1.xpath('.//@href').get()) item['title'] = a1.xpath(".//@title").get().strip() item['ctime'] = a1.xpath('..//..//..//td[5]//text()').get().strip() yield item
def parse_page(self, response): # a标签 rs=eval(response.body.decode('utf8'))['list'] selector=Selector(text=rs) a = selector.xpath('//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin(a1.xpath('.//@href').get()) item['title'] = a1.xpath(".//@title").get().strip() item['ctime'] = a1.xpath('..//..//span//text()').get() yield item
def parse(self, response): # a标签 a = response.xpath('//div[@class="prcont TableListLine"]//li//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin( a1.xpath('.//@href').extract_first()) item['title'] = a1.xpath(".//@title").extract_first().strip() item['ctime'] = a1.xpath('.//span//text()').extract_first() item['city'] = '广州' # yield scrapy.Request(url=item['href'], dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse_page(self, response): a = response.xpath('//div[@class="publicont"]//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = response.urljoin( a1.xpath('.//@href').extract_first()) item['title'] = a1.xpath(".//text()").extract_first().strip() item['ctime'] = a1.xpath('..//..//span//text()').extract_first() item['city'] = a1.xpath( '..//..//..//p//span[2]//text()').extract_first() # yield scrapy.Request(url=item['href'], dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse_page(self, response): li_lst = response.xpath('//div[@class="filter-content"]/ul/li') for l in li_lst: item = BiddinginfospiderItem() a = l.xpath('./a') title = a.xpath('.//@title').extract_first() href = response.urljoin(a.xpath('.//@href').extract_first()) ctime = self.get_ctime(a.xpath('.//span[@class="time"]//text()')) item.update( title=title, ctime=ctime, href=href, ) yield item
def parse_page(self, response): rs = json.loads(response.body.decode('utf8')) for l in json.loads(rs['custom'])['Table']: title = l['title'] href = response.urljoin(l['href']) ctime = l['postdate'] city = l['city'] item = BiddinginfospiderItem( title=title, ctime=ctime, href=href, city=city, ) # yield scrapy.Request(url=href, dont_filter=True, callback=self.parse_item, meta={'meta': item, }) yield item
def parse_page(self, response): res = scrapy.Selector(response) li = res.xpath('//div[@class="titlecss"]') for l in li: item = BiddinginfospiderItem() a = l.xpath(".//a") title = a.xpath('.//@title').get() href = response.urljoin(a.xpath('.//@href').get()) ctime = self.get_ctime(l.xpath('../following-sibling::td[1]//text()')) item.update( title=title, href=href, ctime=ctime, ) # print(item) yield item
def parse(self, response): a = response.xpath('//div[@class="W750 Right"]//li//a') for a1 in a: item = BiddinginfospiderItem() href = response.urljoin(a1.xpath('.//@href').extract_first()) title = a1.xpath(".//text()").extract_first().strip() ctime = a1.xpath('..//..//span//text()').extract_first() city = '南方电网' item.update( href=href, title=title, ctime=ctime, city=city ) yield item
def parse_start_url(self, response): print('request_url= ', response.request.url) li = response.xpath('//li[@class="now-hd-items clearfix"]') for l in li: item = BiddinginfospiderItem() a = l.xpath('./a') title = a.xpath('.//@title').extract_first() href = response.urljoin(a.xpath('.//@href').extract_first()) ctime = self.get_ctime(l.xpath('.//span//text()')) item.update( title=title, ctime=ctime, href=href, ) # yield scrapy.Request(url=href, dont_filter=True, callback=self.parse_item, meta={'item': item}) yield item
def parse_page(self, response): # a标签 rs = response.body.decode('utf8') selector = Selector(text=rs) a = selector.xpath('//table[@class="wwFormTable"]//a') for a1 in a: item = BiddinginfospiderItem() item['href'] = 'https://www.chdtp.com/staticPage/' + a1.xpath( './/@href').get()[25:-2] item['title'] = a1.xpath(".//@title").get().strip() item['ctime'] = a1.xpath( '..//..//..//td[2]//span[@class="index_s12_c666"]//text()' ).get()[1:-1] yield item
def parse_start_url(self, response): print('request_url= ', response.request.url) li = response.xpath('//li[@name="li_name"]//a') for a in li: item = BiddinginfospiderItem() title = a.xpath('@title').get() href = response.urljoin(a.xpath('.//@href').get()) ctime = self.get_ctime(a.xpath('.//em[1]//text()')) item.update( ctime=ctime, industry=self.industry, category=self.category, title=title, href=href, ) # print(item) yield item
def parse_start_url(self, response): print('request_url= ', response.request.url) li = response.xpath('//ul[@class="ewb-news-items"]//li') for l in li: item = BiddinginfospiderItem() a = l.xpath('.//a') title = a.xpath('.//@title').extract_first() href = response.urljoin(a.xpath('.//@href').extract_first()) ctime = self.get_ctime(a.xpath('.//span//text()')) item.update( city="湖北", title=title, ctime=ctime, href=href, ) yield item