def parse(self, response):
if self.mode == "c":
article_list = response.xpath(self.company_list_xpath)
for article in article_list:
time_str = get_time(article.xpath(self.company_time_xpath).extract_first().strip()[-10:])
if is_in_filtered_time(time_str):
article_detail = {
"title": article.xpath(self.company_title_xpath).extract_first().strip(),
"time": time_str,
"link": self.target_root + article.xpath(self.company_link_xpath).extract_first().strip()}
yield article_detail
else:
market_list = response.xpath(self.market_list_xpath)
if response.request.url.find("?page=1") > -1:
for article in market_list[0].xpath("./li"):
time_str = get_time(article.xpath(self.market_top_time_xpath).extract_first().strip())
if is_in_filtered_time(time_str):
article_detail = {
"title": article.xpath(self.market_top_title_xpath).extract_first().strip() + " " + time_str,
"time": time_str,
"link": self.target_root + article.xpath(self.market_top_link_xpath).extract_first().strip()}
yield article_detail
for article in market_list[1].xpath("./li"):
time_str = get_time(article.xpath(self.market_time_xpath).extract_first().strip())
if is_in_filtered_time(time_str):
article_detail = {
"title": article.xpath(self.market_title_xpath).extract_first().strip() + " " + time_str,
"time": time_str,
"link": self.target_root + article.xpath(self.market_link_xpath).extract_first().strip()}
yield article_detail
def parse(self, response):
# Get article in list
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
article_link = article.xpath(
self.link_xpath).extract_first().strip()
article_detail = {
"title":
article.xpath(self.title_xpath).extract_first().strip(),
"time":
get_time(
article.xpath(self.time_xpath).extract_first().strip()),
"init":
article.xpath(self.init_xpath).extract_first().strip(),
"link":
article_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=article_link,
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
def examine_article(response):
article_content_xpath = "//article[@id='detail-content']/div[@class='post-content ']/p"
# 2018/11/22-START-Myhq-REMOVED: Redundant scraped item
# article_init_xpath = "//article[@id='detail-content']/div[2]/h2/strong/text()"
# 2018/11/22-END-Myhq-REMOVED: Redundant scraped item
article_detail = response.meta.get("article_detail")
# 2018/11/22-START-Myhq-REMOVED: Redundant scraped item
# article_detail["init"] = response.selector.xpath(article_init_xpath).extract_first().strip()
# 2018/11/22-END-Myhq-REMOVED: Redundant scraped item
keyword_list = response.meta.get("keyword")
match_flg = True
article_content = response.selector.xpath(article_content_xpath)
for kw in keyword_list:
for paragraph in article_content:
paragraph_content = str(
paragraph.xpath(".//text()").extract_first())
if paragraph_content.lower().find(" " + kw + " ") != -1:
# Keyword found
match_flg = True
break
# Keyword not found
match_flg = False
# Article did not have a keyword
if not match_flg:
break
if match_flg:
time_xpath = "//div[@class='pull-right mt-5 mr-10']/text()"
time_str = response.selector.xpath(time_xpath)
formatted_time = get_time("".join(time_str.extract()).strip())
article_detail["time"] = formatted_time
if is_in_filtered_time(formatted_time):
yield article_detail
pass
def parse(self, response):
response.selector.remove_namespaces()
res_html = HtmlResponse(
url="my HTML string",
body=response.selector.xpath(
"//CallbackContent/text()").extract_first().strip(),
encoding='utf-8')
article_list = res_html.xpath(self.list_xpath)
for article in article_list:
time_str = get_time(
article.xpath(self.time_xpath).extract_first().strip())
if is_in_filtered_time(time_str):
article_link = article.xpath(
self.link_xpath).extract_first().strip()
article_detail = {
"title":
article.xpath(self.title_xpath).extract_first().strip(),
"time":
time_str,
"init":
"",
"link":
article_link
}
yield Request(url=article_link,
callback=self.get_summary,
meta={"article_detail": article_detail})
def parse(self, response):
# Get article in list
response_json = json.loads(response.text)
article_list = response_json["members"]["items"]
for article in article_list:
article_link = article["url"]
article_detail = {
"title": article["headline"],
"time": get_time(article["created"]),
"init": article["summary"],
"link": article_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=article_link,
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
def examine_article(response):
article_detail = response.meta.get("article_detail")
keyword_list = response.meta.get("keyword")
article_content_xpath = "//div[@class='news-body-content']/p"
article_content = response.selector.xpath(article_content_xpath)
time_str = "".join(
response.selector.xpath(
"//div[@class='news-author-info']//text()").extract()).strip()
time_regex = "((0?[1-9]|[12][0-9]|3[01])/(0?[1-9]|1[0-2])/\d\d\d\d)"
article_detail["time"] = get_time(
re.search(time_regex, time_str).group(1))
match_flg = False
if is_in_filtered_time(article_detail.get("time")):
if keyword_list:
for kw in keyword_list:
for paragraph in article_content:
paragraph_content = str(
paragraph.xpath(".//text()").extract_first())
if paragraph_content.lower().find(" " + kw +
" ") != -1:
# Keyword found
match_flg = True
break
# Keyword not found
match_flg = False
# Article did not have a keyword
if not match_flg:
break
if match_flg:
yield response.meta.get("article_detail")
pass
else:
yield response.meta.get("article_detail")
def parse(self, response):
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
if response.request.url.find("NextPageTinCPNY_CBTCPH") != -1:
title = article.xpath(self.stock_id_xpath).extract_first().strip() \
+ " - " + article.xpath(self.org_name_xpath).extract_first().strip()
init = article.xpath(
self.title_issuer_xpath).extract_first().strip()
else:
title = article.xpath(
self.stock_id_xpath).extract_first().strip()
init = article.xpath(
self.title_hnx_xpath).extract_first().strip()
article_detail = {
"title":
title,
"time":
get_time(
article.xpath(self.time_xpath).extract_first().strip()),
"init":
init
}
if is_in_filtered_time(article_detail.get("time")):
yield article_detail
def parse(self, response):
# Get top_focus
if str(response.request.url).endswith("p1.htm"):
top_focus = response.selector.xpath(self.top_focus_xpath)
top_link = top_focus.xpath(
self.top_focus_link_xpath).extract_first().strip()
article_detail = {
"title":
top_focus.xpath(
self.top_focus_title_xpath).extract_first().strip(),
"time":
get_time_from_link(top_link),
"intro":
"",
"link":
self.target_root + top_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=(self.target_root + top_link),
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
top_list = response.selector.xpath(self.top_list_xpath)
for item in top_list:
top_list_item_class = item.xpath(
self.top_list_item_class_xpath).extract_first().strip()
if top_list_item_class == "block-normal-item" or top_list_item_class == "block-normal-item last":
item_link = item.xpath(
self.top_list_item_link_xpath).extract_first().strip()
article_detail = {
"title":
item.xpath(self.top_list_item_title_xpath).
extract_first().strip(),
"time":
get_time_from_link(item_link),
"intro":
"",
"link":
self.target_root + item_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=(self.target_root + item_link),
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
# Get articles
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
article_link = article.xpath(
self.article_link_xpath).extract_first().strip()
article_detail = {
"title":
article.xpath(
self.article_title_xpath).extract_first().strip(),
"time":
get_time_from_link(article_link),
"intro":
article.xpath(
self.article_intro_xpath).extract_first().strip(),
"link":
self.target_root + article_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=(self.target_root + article_link),
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
def parse(self, response):
if str(response.request.url).endswith("page-1.html"):
# Get top item
top_item = response.selector.xpath(self.top_item_xpath)
top_item_link = top_item.xpath(
self.top_item_link_xpath).extract_first().strip()
article_detail = {
"title":
top_item.xpath(
self.top_item_title_xpath).extract_first().strip(),
"time":
date.today().strftime("%Y/%m/%d"),
# 2018/11/22-START-Myhq-MODIFIED: Redundant scraped item
# "init": "",
"init":
top_item.xpath(
self.top_item_init_xpath).extract_first().strip(),
# 2018/11/22-END-Myhq-MODIFIED: Redundant scraped item
"link":
top_item_link
}
if self.keyword:
yield Request(url=top_item_link,
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
# 2018/11/22-START-Myhq-REMOVED: Redundant scraped item
# # Get top list
# top_list = response.selector.xpath(self.top_list_xpath)
# top_list_item_link = ""
# for item in top_list:
# top_list_item_link = item.xpath(self.top_list_item_link_xpath).extract_first().strip()
# article_detail = {"title": item.xpath(self.top_list_item_title_xpath).extract_first().strip(),
# "time": date.today().strftime("%Y/%m/%d"),
# "init": "",
# "link": top_list_item_link}
# if self.keyword:
# yield Request(url=top_list_item_link, callback=self.examine_article,
# meta={"article_detail": article_detail,
# "keyword": self.keyword})
# else:
# yield article_detail
# 2018/11/22-END-Myhq-REMOVED: Redundant scraped item
# Get article in list
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
article_link = article.xpath(
self.list_item_link_xpath).extract_first().strip()
time_str = "".join(
article.xpath(self.list_item_time_xpath).extract()).strip()
time_regex = "((0?[1-9]|[12][0-9]|3[01])/(0?[1-9]|1[0-2])/\d\d\d\d)"
time_str = re.search(time_regex, time_str).group(1)
article_detail = {
"title":
article.xpath(
self.list_item_title_xpath).extract_first().strip(),
"time":
get_time(time_str),
"init":
article.xpath(
self.list_item_init_xpath).extract_first().strip(),
"link":
article_link
}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=article_link,
callback=self.examine_article,
meta={
"article_detail": article_detail,
"keyword": self.keyword
})
else:
yield article_detail
def parse(self, response):
# Not make-it sub
if str(response.request.url).find("make-it") == -1:
# Get headline article
if str(response.request.url).endswith("?page=1"):
headline = response.selector.xpath(self.headline_xpath)
if headline is not None:
headline_link = self.target_root + headline.xpath(self.link_xpath).extract_first().strip()
if headline_link.find("/video/") == -1:
article_detail = {"title": headline.xpath(self.title_xpath).extract_first().strip(),
"time": headline.xpath(self.time_xpath).extract_first().strip()[1:11],
"init": headline.xpath(self.init_xpath).extract_first().strip(),
"link": headline_link}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=headline_link, callback=self.examine_article,
meta={"article_detail": article_detail,
"keyword": self.keyword})
else:
yield article_detail
# Get headline article in list
headline_list = response.selector.xpath(self.headline_list_xpath)
for headline in headline_list:
if headline.xpath("./@id").extract_first() is None:
headline_link = self.target_root + headline.xpath(self.link_xpath).extract_first().strip()
if headline_link.find("/video/") == -1:
article_detail = {"title": headline.xpath(self.title_xpath).extract_first().strip(),
"time": headline.xpath(self.time_xpath).extract_first().strip()[1:11],
"init": headline.xpath(self.init_xpath).extract_first().strip(),
"link": headline_link}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=headline_link, callback=self.examine_article,
meta={"article_detail": article_detail,
"keyword": self.keyword})
else:
yield article_detail
# Get article in list
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
if article.xpath("./@id").extract_first() is None:
article_link = self.target_root + article.xpath(self.link_xpath).extract_first().strip()
if article_link.find("/video/") == -1:
article_detail = {"title": article.xpath(self.title_xpath).extract_first().strip(),
"time": article.xpath(self.time_xpath).extract_first().strip()[1:11],
"init": article.xpath(self.init_xpath).extract_first().strip(),
"link": article_link}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=article_link, callback=self.examine_article,
meta={"article_detail": article_detail,
"keyword": self.keyword})
else:
yield article_detail
# Make-it sub
else:
# Get headline article
if str(response.request.url).endswith("?page=1"):
makeit_headline_xpath = self.list_xpath + "/div/a"
headline = response.selector.xpath(makeit_headline_xpath)
if headline is not None:
headline_link = self.target_root + headline.xpath("./@href").extract_first().strip()
if headline_link.find("/video/") == -1:
article_detail = {
"title": headline.xpath("./div[@class='headline']/text()").extract_first().strip(),
"time": headline.xpath("./@href").extract_first().strip()[1:11],
"init": headline.xpath(self.init_xpath).extract_first().strip(),
"link": headline_link}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=headline_link, callback=self.examine_article,
meta={"article_detail": article_detail,
"keyword": self.keyword})
else:
yield article_detail
# Get article in list
article_list = response.selector.xpath(self.list_xpath)
for article in article_list:
if str(article.xpath("./@class").extract_first()).find(" card") != -1:
article_link = self.target_root + article.xpath(self.link_xpath).extract_first().strip()
if article_link.find("/video/") == -1:
article_detail = {"title": article.xpath(self.title_xpath).extract_first().strip(),
"time": article.xpath(self.time_xpath).extract_first().strip()[1:11],
"init": article.xpath(self.init_xpath).extract_first().strip(),
"link": article_link}
if is_in_filtered_time(article_detail.get("time")):
if self.keyword:
yield Request(url=article_link, callback=self.examine_article,
meta={"article_detail": article_detail,
"keyword": self.keyword})
else:
yield article_detail