def parse_itme(self, response): article_p = response.xpath('//div[@class="content"]//p') title_index = [] for i, p in enumerate(article_p): if p.xpath("./strong").extract(): title_index.append(i) for s_index, i_item in enumerate(title_index): category = response.xpath("//h1/text()").extract()[0] title = article_p[i_item].xpath("./strong/text()").extract()[0] try: next_tile = title_index[s_index + 1] content_str = [ "".join(item.xpath("./text()").extract()) for item in article_p[i_item:next_tile] ] except Exception as e: content_str = [ "".join(item.xpath("./text()").extract()) for item in article_p[i_item:] ] item = TangspiderframeItem() item['url'] = response.url item['category'] = category item['content'] = "".join([item.strip() for item in content_str]) item['title'] = title yield item
def parse(self, response): category = response.xpath('//div[@id="place"]/a[2]/text()').extract()[0] title = response.xpath('//div[@id="info"]/dl/h1/a/text()').extract()[0] content = response.xpath('//dl[@id="zi"]//p//text()').extract() item = TangspiderframeItem() item['url'] = response.url item['category'] = category item['content'] = "".join([item.strip() for item in content]) item['title'] = title yield item
def parse(self, response): category = response.xpath( '//div[@class="position"]/a[2]/text()').extract()[0] title = response.xpath( '//div[@class="article"]/h1/text()').extract()[0] content = response.xpath('//div[@class="article"]/p//text()').extract() item = TangspiderframeItem() item['url'] = response.url item['category'] = category item['content'] = "".join([item.strip() for item in content]) item['title'] = title yield item
def parse(self, response): title = response.xpath('//h1//text()').extract() content = response.xpath('//p/text()').extract() last1_link = response.xpath('//p[last()]//@href').extract() last2_link = response.xpath('//p[last()-1]//@href').extract() content = ''.join(content) content = content.replace("\n", " ") content = content.replace("\t", " ") if last1_link and last2_link: last1_content = response.xpath('//p[last()]/text()').extract() last2_content = response.xpath('//p[last()-1]/text()').extract() last1_content = ''.join(last1_content) last2_content = ''.join(last2_content) content = content.replace(last1_content, "") content = content.replace(last2_content, "") elif last1_link and not last2_link: last1_content = response.xpath('//p[last()]/text()').extract() last1_content = ''.join(last1_content) content = content.replace(last1_content, "") elif last2_link and not last1_link: last2_content = response.xpath('//p[last()-1]/text()').extract() last2_content = ''.join(last2_content) content = content.replace(last2_content, "") else: content = content item = TangspiderframeItem() item['url'] = response.url item['category'] = response.url.split('/')[3] item['title'] = ''.join(title) item['content'] = content # print(item) yield item
def parse(self, response): hrefs = re.findall('<td ><a href=(.*?)>', response.text) for href in hrefs: url = "http://kids.hankooki.com/community/" + href item = TangspiderframeItem() item['url'] = url yield item
def parse(self, response): links = response.xpath('//article/a/@href').extract() for link in links: item = TangspiderframeItem() item['url'] = link # print(item) yield item
def parse(self, response): links = response.xpath('//div[@class="box"]//ul/li//a//@href').extract() for link in links: item = TangspiderframeItem() item['url'] = "http://www.enet.gr/"+link # print(item) yield item
def parse(self, response): links = response.xpath( '//div[@class="lx-stream__feed"]//a/@href').extract() links.extend( response.xpath('//div[@class="mpu-available"]//a/@href').extract()) links.extend( response.xpath('//div[@role="region"]//a/@href').extract()) urls = [ "https://www.bbc.com" + link for link in links if "http" not in link ] urls = list(set(urls)) for url in urls: item = TangspiderframeItem() item["url"] = url yield item type_id = self.type_id t = self.t base_url = "https://push.api.bbci.co.uk/p?" one_arg = "morph://data/bbc-morph-lx-commentary-latest-data/assetUri/news%2Flive%2F{}-{}/limit/31/version/4.1.27".format( t, type_id) one_url = "t={}&c=1".format(quote(one_arg, safe='')) frist_url = base_url + one_url two_arg = "morph://data/bbc-morph-lx-commentary-data/assetUri/news%2Flive%2F{}-{}/limit/31/version/5.0.24/withPayload/11".format( t, type_id) two_url = "t={}&c=1".format(quote(two_arg, safe='')) second_url = base_url + two_url three_arg_1 = "morph://data/bbc-morph-feature-toggle-manager/assetUri/news%2Flive%2F{}-{}/featureToggle/dot-com-ads-enabled/project/bbc-live/version/1.0.3".format( t, type_id) three_arg_2 = "morph://data/bbc-morph-feature-toggle-manager/assetUri/news%2Flive%2F{}-{}/featureToggle/lx-old-stream-map-rerender/project/bbc-live/version/1.0.3".format( t, type_id) three_arg_3 = "morph://data/bbc-morph-feature-toggle-manager/assetUri/news%2Flive%2F{}-{}/featureToggle/reactions-stream-v4/project/bbc-live/version/1.0.3".format( t, type_id) three_arg_4 = "morph://data/bbc-morph-lx-commentary-latest-data/assetUri/news%2Flive%2F{}-{}/limit/21/version/4.1.27".format( t, type_id) three_arg_5 = "morph://data/bbc-morph-lx-commentary-latest-data/assetUri/news%2Flive%2F{}-{}/limit/31/version/4.1.27".format( t, type_id) arg = [] for item in [ three_arg_1, three_arg_2, three_arg_3, three_arg_4, three_arg_5 ]: arg.append(quote(item, safe='')) three_url = "t={}&c=1&t={}&c=1&t={}&c=1&t={}&c=1&t={}&c=1".format(*arg) three_url = base_url + three_url yield scrapy.Request(url=frist_url, callback=self.parse_item, dont_filter=True) yield scrapy.Request(url=second_url, callback=self.parse_item, dont_filter=True) yield scrapy.Request(url=three_url, callback=self.parse_item, dont_filter=True, meta={"page": 31})
def parse_item(self, response): links = response.xpath('//li[@class="hh"]/a/@href').extract() for link in links: url = "https:" + link item = TangspiderframeItem() item['url'] = url yield item
def parse(self, response): # 结构化内容 content = response.xpath("//script/text()").extract() data_s = content[1].replace("window.$$data=", "") data = json.loads(data_s) # 抓取对话 questionDialog = data.get("questionDialog") def parse_dialog(dialog_data): # 对话解析 dialogs = [] for dialog in dialog_data: dialog_type = dialog.get("type") dialog_content = dialog.get("content") dialog_content = re.sub("<.*?>|\n|\t|\r", "", dialog_content) if not dialog_content: dialog_content = dialog.get("dialog_audio", {}).get("audio_text") if dialog_type: new_content = "doctor:" + dialog_content else: new_content = "patient:" + dialog_content dialogs.append(new_content) return "\t".join(dialogs) dialog = parse_dialog(questionDialog) dialog = self.delete_emoji(dialog) disease = data.get("disease", {}).get("title") item = TangspiderframeItem() item['url'] = response.url item['category'] = disease item['content'] = dialog yield item
def parse(self, response): link_arg = response.xpath('//a[@class=" ga-link"]/@href').extract() for link in link_arg: url = "https://www.ted.com" + link head_url, language = url.split("?") item = TangspiderframeItem() item['url'] = head_url + "/transcript" + "?" + language yield item
def parse(self, response): links = response.xpath( '//a[@class="iframe cboxElement"]/@href').extract() for link in links: item = TangspiderframeItem() item['url'] = link # print(item) yield item
def parse(self, response): products = response.xpath('//div[@class="tit-list"]/div') for product in products: product_url = ''.join(product.xpath('.//a/@href').extract()) item = TangspiderframeItem() item["url"] = product_url yield item
def parse(self, response): resp = json.loads(response.text) data = resp.get("data") links = [item.get("metadata", {}).get("url") for item in data] for link in links: item = TangspiderframeItem() item['url'] = link yield item
def parse(self, response): links = response.xpath('//h4/a//@href').extract() for pattern in links: link = "https://www.vientianemai.net" + pattern item = TangspiderframeItem() item['url'] = link # print(item) yield item
def parse(self, response): title = response.xpath("//h1/text()").extract() contents = response.xpath('//div[@id="Content"]/p//text()').extract() paragraph = [content for content in contents] item = TangspiderframeItem() item['category'] = "opinion" item['url'] = response.url item['title'] = title[0].strip() if title else "" item['content'] = " ".join(paragraph) yield item
def parse(self, response): print(response.text) data = json.loads(response.text) items = data.get("data").get("items") for item in items: q_id = item.get("question", {}).get("id") url = "https://ask.dxy.com/question/{}".format(q_id) item = TangspiderframeItem() item['url'] = url yield item
def parse(self, response): word = response.xpath("//h2/text()").extract()[0].replace( "的成语", "").replace("的组词", "") url = response.url if "com/zi" in url: item_li = response.xpath('//div[@class="mcon f14"]/ul/li') for li in item_li: phrase = li.xpath("./a[1]/text()").extract() pronunciation = li.xpath("./a[2]/text()").extract() item = TangspiderframeItem() item['url'] = response.url item['category'] = word item['title'] = pronunciation[0].strip() item['content'] = phrase[0] yield item next_page = response.xpath( '//a[contains(text(), "下一页")]/@href').extract() if next_page: next_url = "http://zuci.miaochaxun.com/" + next_page[0] yield scrapy.Request(url=next_url, meta={"word": response.meta.get("word")}, callback=self.parse, dont_filter=True) if "com/zuci" in url: item_li = response.xpath('//div[@class="mcon bt"]/ul/li') for li in item_li: phrase = li.xpath("./a[1]/text()").extract() item = TangspiderframeItem() item['url'] = response.url item['category'] = word item['content'] = phrase[0] yield item next_page = response.xpath( '//a[contains(text(), "下一页")]/@href').extract() if next_page: next_url = "http://chengyu.miaochaxun.com/" + next_page[0] yield scrapy.Request(url=next_url, meta={"word": response.meta.get("word")}, callback=self.parse, dont_filter=True)
def parse(self, response): res = response.xpath('//*[@id="pronWR"]/text()').extract() item = TangspiderframeItem() item['url'] = response.url item['title'] = response.meta.get("word") if res: item['content'] = res[0] else: item["content"] = "" return item
def parse(self, response): links = response.xpath( '//*[@id="listNewsLG"]/div[1]/div[3]/div[contains(@class, "col-xs-")]/a/@href' ).extract() for pattern in links: link = "http://www.kongthap.gov.la/index1.php" + pattern item = TangspiderframeItem() item['url'] = link # print(item) yield item
def parse_url(self, response): # next_links = response.xpath('//a[(contains(@class, "btn-bs-pagination"))]/@href').extract() # for next_link in next_links: # yield scrapy.Request(url=next_link, callback=self.parse_url, dont_filter=True) links = response.xpath( '//h2/a[@class="post-title post-url"]/@href').extract() for link in links: item = TangspiderframeItem() item['url'] = link yield item
def parse(self, response): content = re.findall( '<td colspan="12" style="padding:15px">(.*?)</td>', response.text, re.S) new_content = "<td>" + "".join(content) + "</td>" html = etree.HTML(new_content) res = html.xpath(".//text()") item = TangspiderframeItem() item['url'] = response.url item['content'] = "".join(res) return item
def parse(self, response): title = response.xpath( '//h1[@class="headline__title"]/text()').extract() content = response.xpath('//p/text()').extract() catagpory = response.url.split("/")[3] item = TangspiderframeItem() item['url'] = response.url item['title'] = " ".join(title) item['content'] = " ".join(content) item['category'] = catagpory yield item
def parse_item(self, response): next_links = response.xpath('//div[@id="div_currpage"]/a[@class="pagestyle"]/@href').extract() for next_link in next_links: next_link = "http:" + next_link yield scrapy.Request(url=next_link, callback=self.parse_item, dont_filter=True) links = response.xpath('//h4/a/@href').extract() for link in links: url = "http:" + link item = TangspiderframeItem() item["url"] = url yield item
def parse(self, response): resp = json.loads(response.text) data = resp.get("data") url_lists = data["response"]["videos"] for url_list in url_lists: url = url_list["play_url"] self.youtube_url = url self.download(self.youtube_url) item = TangspiderframeItem() item['url'] = url # print(item) yield item
def parse(self, response): lis = response.xpath('//div[@class="se_li"]') for li in lis: sen_en = li.xpath(".//div[@class='sen_en']//text()").extract() sentens = "".join(sen_en) md = md5(sentens) item = TangspiderframeItem() item['content'] = sentens item['title'] = response.meta.get("keyword") item['category'] = '' item['fingerprint'] = md yield item
def parse(self, response): category = response.xpath('//div[@class="dingtou"]/div[@class="da-bre"]/a[2]/text()').extract() title = response.xpath("//h1/text()").extract() # 文章标题 contents, para = [], [] contents.extend(response.xpath('//div[contains(@class, "article")]//p')) contents.extend(response.xpath('//div[contains(@class, "zhengwen")]//p')) contents.extend(response.xpath('//div[contains(@class, "c_body")]//p')) contents.extend(response.xpath('//div[contains(@class, "cnt_bd")]//p')) for item in contents: para.extend(item.xpath("./text()").extract()) sub_tag = item.xpath(".//*") for sub in sub_tag: if sub.root.tag != "script": para.extend(sub.xpath(".//text()").extract()) item = TangspiderframeItem() item['url'] = response.url item['category'] = "".join(category).replace(" > ", '').strip() item['content'] = "".join([item.strip() for item in para]) item['title'] = "".join(title).strip() yield item
def parse(self, response): pinyin = response.xpath( "//td[@class='resultswrap']//tbody/tr/td[@class='details'][4]//text()" ).extract() item = TangspiderframeItem() item['url'] = response.url item['title'] = response.meta.get("word") if pinyin: item['content'] = "|".join(pinyin) else: item["content"] = "" return item
def parse(self, response): zi_a = response.xpath('//p[@class="zi"]/a') for zi in zi_a: base_url = zi.xpath("./@href").extract()[0] word = zi.xpath("./text()").extract()[0] pron_span = zi.xpath("./span/text()").extract() item = TangspiderframeItem() item['url'] = response.url item['category'] = base_url if pron_span: item['title'] = pron_span[0] item['content'] = word yield item
def parse(self, response): ipa_content = response.xpath( """//*[@id="mw-content-text"]//span[@class="ipa"]/text()""" ).extract() keyword = response.meta.get("keyword") show_word = urllib.parse.unquote(response.url).split("/")[-1] item = TangspiderframeItem() if ipa_content: item['url'] = response.url item['title'] = keyword item["category"] = show_word item['content'] = ipa_content[0] yield item