def parse_detail(self,response): item = response.meta['meta_1'] content = response.xpath('//*[@id="articleC"]') if content != []: content = content.extract_first() next_page = response.xpath('//div[@class="page"]') if next_page != []: next_url = response.xpath('//*[@id="nextPage"]/a/@href').extract_first() url = urljoin(response.url, next_url) data = requests.get(url) content_1= re.findall(r'<div id="articleC" class="article_con" >(.*?)</div>',data.text,re.S) content = content+content_1[0] author1 = response.xpath('//*[@id="laiyuan_mp"]/a/span/text()') author2 = response.xpath('//*[@id="laiyuan"]/span') author = author1.extract_first() if author is None: author = author2.xpath('string(.)').extract_first() author = author.split(':')[1].strip() item['Author'] = author str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" item['Content'] = content1 content2 = etree.HTML(content1) img_list = content2.xpath('//img/@src') get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Web_Id'] = '5-32' yield item
def parse_detail(self, response): content = response.xpath('//div[@class="newscontxt"]') item = response.meta['meta_1'] if content != []: content = content.extract_first() keywords = response.xpath('//div[@class="newscontxt"]//h4//a' ).xpath('string(.)').extract() item['Keywords'] = ','.join(keywords) author1 = 'EEWORLD' author = response.xpath( '//*[@id="newsptit"]/div[1]/div/h6/span[2]/text()') if author != []: author = author.extract_first() author_1 = author.split(':') if len(author_1) < 2: author_1 = author.split(':') author = author_1[1] if author != ' ': author1 = author item['Author'] = author1 str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" item['Content'] = content1 content2 = etree.HTML(content1) img_list = content2.xpath('//img/@src') get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Web_Id'] = '5-30' #print(item) yield item
def parse_detail(self, response): content = response.xpath('//div[@class="contentlist"]') if content != []: item = EleIndustryItem() content = content.extract_first() title = response.xpath('//h1/text()') if title != []: title = title.extract_first().strip() node1 = response.xpath('//div[@class="newstitle"]/span/text()') if node1 != []: node2 = node1.extract_first() time_1 = re.findall(r'\d+-\d+-\d+', node2) if time_1 != []: time1 = time_1[0] + ' ' + Get_Time() try: author = node2.split(',')[1].strip() except Exception as E: author = '' node3 = response.xpath( '//div[@class="newstitle"]/span/a/text()') keywords = '' if node3 != []: keywords = node3.extract_first() item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author item['Keywords'] = keywords str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split( ' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, 'http://www.ic37.com') item['Content'] = data[0] img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-37' #print(item) yield item
def parse_detail(self, response): content = response.xpath('//div[@class="article-body"]') if content != []: item = EleIndustryItem() content = content.extract_first() title = response.xpath('//h1[@class="headline"]/text()') if title != []: title = title.extract_first().strip() # print(title) node1 = response.xpath( '//div[@class="muted subline"]//span[@class="mr20"]/text()' ) if node1 != []: node2 = node1.extract() time1 = node2[0].strip().split(' ')[0] + ' ' + Get_Time() author = '' if len(node2) == 2: author = node2[1].strip() node3 = response.xpath( '//div[@class="overhide overhidden new-tags"]/a/text()' ) keywords = '' if node3 != []: keywords = ','.join(node3.extract()) item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author item['Keywords'] = keywords str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" item['Content'] = content1 content2 = etree.HTML(item['Content']) img_list = content2.xpath('//img/@src') get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-31' #print(item) yield item
def parse_item1(self, response): # print(response.url) # print(response.url) content = response.xpath('//*[@id="art_body"]') if content != []: content = content.extract_first() title = response.xpath('//h1') if title != []: item = EleIndustryItem() title = title.xpath('string(.)').extract_first().strip() time1 = response.xpath('//span[@class="time"]') if time1 != []: time1 = re.findall(r'\d+-\d+-\d+', time1.extract_first()) if time1 != []: time1 = time1[0] + ' ' + Get_Time() author = response.xpath( '//span[@class="zuozhe"]').xpath( 'string(.)').extract_first() if author == '作者:': author = '' Keywords = response.xpath( '//span[@class="mbx"]/text()').extract_first() item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author item['Keywords'] = Keywords str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split( ' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" item['Content'] = content1 content2 = etree.HTML(item['Content']) img_list = content2.xpath('//img/@src') get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-34' #print(item) yield item
def parse_item(self,response): content = response.xpath('//*[@id="articlebody"]') if content != []: item = EleIndustryItem() content = content.extract_first() node = re.findall(r'<center>.*?</center>', content, re.S) if node != []: content = content.replace(node[0],'') title = response.xpath('//*[@id="articledetail"]/h1/text()') if title != []: title = title.extract_first().strip() node1 = response.xpath('//*[@id="articledetail"]/p/span[2]/text()') if node1 != []: node2 = node1.extract_first().split('\xa0\xa0') time1 = re.findall(r'\d+-\d+-\d+',str(node1.extract_first)) if time1 != []: time1 = time1[0]+ ' ' + Get_Time() author = re.findall(r'作者:(.*)',node2[1]) author1 = '' if author != []: if author[0].strip() != 'n': author1 = author[0].strip() node3 = response.xpath('//*[@id="navigation"]/p/a[3]/text()') keywords = node3.extract_first() item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author1 item['Keywords'] = keywords str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" item['Content'] = content1 content2 = etree.HTML(item['Content']) img_list = content2.xpath('//img/@src') get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-33' #print(item) yield item
def parse_item1(self, response): content = response.xpath('//div[@class="content"]') if content != []: item = EleIndustryItem() content = content.extract_first() title = response.xpath('//h2/text()') if title != []: title = title.extract_first() time1 = response.xpath( '//div[@class="title-box"]/p/span[1]/text()') if time1 != []: time1 = time1.extract_first().split( ':')[1] + ' ' + Get_Time() author = response.xpath( '//div[@class="title-box"]/p/span[2]/text()') author1 = '' if author != []: author1 = author.extract_first().split(':')[1] if author1.endswith('.com'): author1 = '' item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author1 item['Keywords'] = '' str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, self.start_urls[0]) item['Content'] = data[0] # content2 = etree.HTML(item['Content']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-26' yield item
def parse_item(self, response): content = response.xpath('//*[@id="contentDiv"]') if content != []: content = content.extract_first() title = response.xpath('//h1/text()') if title != []: title = title.extract_first() time1 = response.xpath( '//div[@class="authorTimeSource"]/span[2]/text()') if time1 != []: item = EleIndustryItem() time1 = time1.extract_first().split(':')[1] time1 = time1 + ' ' + Get_Time() author = response.xpath( '//div[@class="authorTimeSource"]/span[3]/text()' ).extract_first() author = author.split(':')[1] tags = response.xpath('//div[@class="keyWord"]//em').xpath( 'string(.)').extract() key = ','.join(tags) item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author item['Keywords'] = key str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, self.start_urls[0]) item['Content'] = data[0] img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-27' yield item
def parse_detail(self,response): item = response.meta['meta_1'] content = response.xpath('//*[@id="newsInfo"]') if content != []: content = content.extract_first() item['Author'] = '华强资讯' item['URL'] = response.url str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + str( item['News_Dt'].split(' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1,'http://news.hqew.com') item['Content'] = data[0] if data[1]!=[]: data[1].pop() if data[1] == []: data[1].append(item['Image_URL']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Web_Id'] = '5-36' #print(item) yield item
def parse_item1(self, response): content = response.xpath('//div[@class="detailcon"]') if content != []: item = EleIndustryItem() content = content.extract_first() title = response.xpath('//div[@class=" detailtitle"]/text()') if title != []: title = title.extract_first().strip() node = response.xpath('//div[@class="detailintro"]').xpath( 'string(.)') if node != []: node = node.extract_first() node = node.split('|') author = node[0] time1 = re.compile(r'\d+-\d+-\d+', re.S).findall(node[1]) if time1 != []: time1 = time1[0] + ' ' + Get_Time() item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author item['Keywords'] = '' str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split( ' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, self.start_urls[0]) item['Content'] = data[0] img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-27' yield item
def parse_item(self, response): content = response.xpath('//div[@class="article-bd"]') if content != []: item = EleIndustryItem() content = content.extract_first() title = response.xpath('//h1[@class="title-h1"]/text()') if title != []: title = title.extract_first().strip() time1_node = response.xpath('//div[@class="article-attr"]') if time1_node != []: time1 = time1_node.xpath('string(.)').extract_first() time1 = time1.replace('/', '-') time1 = GetTime(time1) if time1 is not None: author = '' keywords = response.xpath( '//span[@class="article-keywords"]/a/text()' ).extract() keywords = ','.join(keywords) try: content = change_content( content, '//div[@class="article-bd"]') except Exception as E: logging.error('img有错' + response.url) item['newsTitle'] = title item['newsDt'] = time1 item['author'] = author item['keywords'] = keywords item['updateTm'] = get_time_stamp() item['abstract'] = '' item['url'] = response.url item['content'] = content item['aid'] = md5_url(item['url']) item['bid'] = '1' yield item
def parse_item(self, response): content = response.xpath('//div[@class="simditor-body clearfix"]') if content == []: content1 = response.xpath('//div[@class="pct"]') if content1 != []: content1 = content1.extract_first() time1 = response.xpath( '//div[@class="bar_tip float_l"]/em/span/@title') if time1 == []: time1 = response.xpath( '//div[@class="bar_tip float_l"]/em/text()') if time1 != []: time1 = time1.extract_first() time1 = re.findall(r'(\d+-\d+-\d+)', time1) if time1 != []: title = response.xpath( '//*[@id="thread_subject"]/text()') if title != []: item = EleIndustryItem() title = title.extract_first() item['News_Title'] = title time1 = time1[0] time1 = time1 + ' ' + Get_Time() item['News_Dt'] = time1 author = response.xpath( '//div[@class="bar_tip float_l"]/div/span/a/span/text()' ) author1 = '' if author != []: author1 = author.extract_first() item['Author'] = author1 str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split( ' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content1 + "</div>" data = change_content(content1, self.start_urls[0]) item['Content'] = data[0] # content2 = etree.HTML(item['Content']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['Keywords'] = '' item['URL'] = response.url item['Web_Id'] = '5-24' #print(item) yield item if content1 == []: content2 = response.xpath('//div[@class="article-content"]') content3 = response.xpath('//div[@class="author_des"]') if content2 != []: content2 = content2.extract_first() title = response.xpath('//h1/text()') if title != []: title = title.extract_first() time1 = response.xpath( '//div[@class="fl"]/em[3]/text()') if time1 != []: item = EleIndustryItem() time1 = time1.extract_first().replace( '年', '-').replace('月', '-').replace('日', '') time1 = time1.strip().split(' ')[0] time1 = time1 + ' ' + Get_Time() author = response.xpath( '//div[@class="fl"]/em[1]/text()') author1 = '' if author != []: author1 = author.extract_first() tag = response.xpath( '//div[@class="tag"]//span').xpath( 'string(.)').extract() tags = ','.join(tag) item['News_Title'] = title item['Author'] = author1 item['News_Dt'] = time1 item['Keywords'] = tags str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + \ item['News_Dt'].split(' ')[0] + '</time></div>' content_1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content2 + "</div>" data = change_content(content_1, self.start_urls[0]) item['Content'] = data[0] # content2 = etree.HTML(item['Content']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-24' #print(item) yield item if content3 != []: content3 = content3.extract_first() title = response.xpath('//h1/text()') if title != []: title = title.extract_first() span_node = response.xpath( '//span[@class="float_left font-small color_gray"]' ).xpath('string(.)').extract_first() span_text = span_node.split(' ') if len(span_text) == 3: item = EleIndustryItem() author = span_text[0].replace('\r', '').replace( '\n', '').replace('\t', '').replace('发表于', '') time1 = span_text[1] + ' ' + Get_Time() tag = response.xpath( '//ul[@class="article_tags clearfix"]/li/span' ).xpath('string(.)').extract() tags = ','.join(tag) item['News_Title'] = title item['Author'] = author item['News_Dt'] = time1 item['Keywords'] = tags str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + \ item['News_Dt'].split(' ')[0] + '</time></div>' content_1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content3 + "</div>" data = change_content(content_1, self.start_urls[0]) item['Content'] = data[0] # content2 = etree.HTML(item['Content']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-24' #print(item) yield item if content != []: content = content.extract_first() title = response.xpath('//h1/text()') if title != []: title = title.extract_first() time1 = response.xpath('//section//span[@class="time"]/text()') if time1 != []: item = EleIndustryItem() item['News_Title'] = title time1 = time1.extract_first().replace('年', '-').replace( '月', '-').replace('日', '') time1 = time1.strip().split(' ')[0] time1 = time1 + ' ' + Get_Time() item['News_Dt'] = time1 author2 = response.xpath( '//div[@class="article-info art-share-layout m-share-layout clearfix"]/a' ).xpath('string(.)').extract() author = '电子发烧友网' if author2 != []: author = author2[0] if author == '': author = '电子发烧友网' if author2 == ['']: uid = response.xpath('//input[@id="webMID"]/@value') if uid != []: uid = uid.extract_first() url = 'http://www.elecfans.com/webapi/member/getUserInfoNew/uid/{}'.format( str(uid)) data = requests.get(url).text try: data = json.loads(data)['data']['writer_uname'] author = data except Exception as E: pass item['Author'] = author tags = response.xpath( '//ul[@class="hot-main clearfix"]/li/text()').extract( ) tag = '' if tags != []: tag = ','.join(tags).replace('\n', '').replace( '\r', '').replace(' ', '').replace(',,', ',').strip(',') item['Keywords'] = tag str_time = '<div class="explain"><span>' + item['Author'] + '</span><time>' + \ item['News_Dt'].split(' ')[0] + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, self.start_urls[0]) item['Content'] = data[0] # content2 = etree.HTML(item['Content']) img_list = data[1] get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['Abstract'] = '' item['URL'] = response.url item['Web_Id'] = '5-24' #print(item) yield item
def parse_item2(self, response): content = response.xpath('//div[@class="art-con article_body"]') if content != []: content = content.extract_first() title = response.xpath('//h1') if title != []: item = EleIndustryItem() title = title.xpath('string(.)').extract_first().strip() node1 = response.xpath('//div[@class="detailwarn"]') if node1 != []: data = node1.xpath('string(.)').extract_first() time1 = re.findall(r'\d+-\d+-\d+', data) author = re.findall(r'作者:(.*)', data) if time1 != []: time1 = time1[0] + ' ' + Get_Time() author1 = '' if author != []: author1 = author[0].strip() abstract = response.xpath( '//span[@class="art-lead-text"]/text()' ).extract_first() Keywords = response.xpath( '//div[@class="art-relative-tags"]/a/text()' ).extract() Keywords = ','.join(Keywords) item['News_Title'] = title item['News_Dt'] = time1 item['Author'] = author1 item['Keywords'] = Keywords item['Abstract'] = abstract str_time = '<div class="explain"><span>' + item[ 'Author'] + '</span><time>' + str( item['News_Dt'].split( ' ')[0]) + '</time></div>' content1 = '<h1>' + item[ 'News_Title'] + '</h1>' + str_time + "<div class='content'>" + content + "</div>" data = change_content(content1, 'https://www.eet-china.com') item['Content'] = data[0] img_url2 = '' img = response.xpath('//div[@class="cover-img"]/@style' ).extract_first() if img is not None: img = img.replace('(', '').replace(')', '') img_url = re.search(r'url(.*)', img) if img_url: img_url1 = img_url.group(1) img_url2 = urljoin('https://www.eet-china.com', img_url1) img_list = data[1] if img_list == []: if img_url2 != '': img_list.append(img_url2) get_pic(item, img_list) item['Update_Tm'] = get_time_stamp() item['URL'] = response.url item['Web_Id'] = '5-38' #print(item) yield item