def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
parsed_news = json.loads(str(response.body))[0]
# Initialize item loader
# extract news title, published_at, author, content, url
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', parsed_news['url'])
if not parsed_news['title']:
# Will be dropped on the item pipeline
return loader.load_item()
loader.add_value('title', parsed_news['title'])
# Convert HTML text to a scrapy response
html_response = HtmlResponse(url=parsed_news['url'],
body=parsed_news['content'].encode('utf-8', 'ignore'))
xpath_query = '''
//body/node()
[not(descendant-or-self::comment()|
descendant-or-self::style|
descendant-or-self::script|
descendant-or-self::div|
descendant-or-self::span|
descendant-or-self::image|
descendant-or-self::img|
descendant-or-self::iframe
)]
'''
raw_content_selectors = html_response.xpath(xpath_query)
if not raw_content_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
raw_content = raw_content_selectors.extract()
raw_content = ' '.join([w.strip() for w in raw_content])
loader.add_value('raw_content', raw_content)
if not parsed_news['published']:
# Will be dropped on the item pipeline
return loader.load_item()
# Parse date information
# Example: 12 Oct 2016 - 05:25
date_time_str = ' '.join([_(w) for w in parsed_news['published'].split(',')[1].strip()[:-4].split(' ')])
try:
published_at_wib = datetime.strptime(date_time_str,
'%d %b %Y - %H:%M')
except ValueError:
# Will be dropped on the item pipeline
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
if not parsed_news['author']:
loader.add_value('author_name', '')
else:
loader.add_value('author_name', parsed_news['author'])
# Move scraped news to pipeline
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
json_response = json.loads(response.body)
try:
url = json_response['NewsML']['NewsItem']['NewsComponent'][
'NewsComponent']['NewsComponent']['NewsLines']['MoreLink']
except KeyError:
return loader.load_item()
loader.add_value('url', url)
try:
title = json_response['NewsML']['NewsItem']['NewsComponent'][
'NewsComponent']['NewsComponent']['NewsLines']['HeadLine']
except KeyError:
return loader.load_item()
if not title:
return loader.load_item()
loader.add_value('title', title)
try:
raw_content = json_response['NewsML']['NewsItem']['NewsComponent'][
'NewsComponent']['NewsComponent']['ContentItem'][
'DataContent']['nitf']['body']['body.content']['p']
except KeyError:
return loader.load_item()
if not raw_content:
return loader.load_item()
loader.add_value('raw_content', raw_content)
try:
author_name = json_response['NewsML']['NewsItem']['NewsComponent'][
'NewsComponent']['Author']
except KeyError:
return loader.load_item()
if not author_name:
loader.add_value('author_name', '')
else:
loader.add_value('author_name', author_name)
try:
date_time_str = json_response['NewsML']['NewsItem'][
'NewsManagement']['FirstCreated']
except KeyError:
return loader.load_item()
if not date_time_str:
return loader.load_item()
date_time_str = date_time_str.split('T')
date_time_str[1] = '0' * (6 - len(date_time_str[1])) + date_time_str[1]
try:
published_at_wib = datetime.strptime(' '.join(date_time_str),
'%Y%m%d %H%M%S')
except Exception:
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
return loader.load_item()
def parse_news(self, article, sub_article):
if not (sub_article['news_url'] and article['news_title'] and
article['news_reporter'] and sub_article['news_description']
and article['news_date_publish']):
return
self.logger.info('parse_news: %s' % article)
# Example: https://m.merdeka.com/tag/p/pilgub-dki/politik/nachrowi-pastikan-agus-sylvi-tak-cuma-incar-suara-santri-ulama.html
url = 'https://www.merdeka.com' + sub_article['news_url']
# Initialize item loader
# extract news title, published_at, author, content, url
loader = ItemLoader(item=News())
loader.add_value('url', url)
loader.add_value('title', article['news_title'])
loader.add_value('author_name', article['news_reporter'])
loader.add_value('raw_content', sub_article['news_description'])
# Parse date information
try:
# Example: 2016-10-12 15:16:04
date_time_str = article['news_date_publish']
self.logger.info('parse_date: parse_news: date_str: %s',
date_time_str)
published_at = wib_to_utc(
datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S'))
loader.add_value('published_at', published_at)
except Exception as e:
raise CloseSpider('cannot_parse_date: %s' % e)
# Move scraped news to pipeline
return loader.load_item()
def parse_news(self, response):
title = response.css('h1[itemprop="headline"]::text').extract()[0]
author_name = response.css('a[rel="author"] > span::text').extract()[0]
raw_content = response.css('.content').extract()[0]
if not (title and author_name and raw_content):
return
self.logger.info('parse_news: %s' % response)
# Initialize item loader
# extract news title, published_at, author, content, url
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('title', title)
loader.add_value('author_name', author_name)
loader.add_value('raw_content', raw_content)
# Parse date information
try:
# Example: Selasa, 6 Oktober 2015 - 05:23 WIB
date_time_str = response.css('article > div.time::text').extract()[0]
date_time_str = date_time_str.split(',')[1].strip()[:-4]
date_time_str = ' '.join([_(w) for w in date_time_str.split(' ')])
self.logger.info('parse_date: parse_news: date_str: %s', date_time_str)
published_at = wib_to_utc(datetime.strptime(date_time_str, '%d %B %Y - %H:%M'))
loader.add_value('published_at', published_at)
except Exception as e:
raise CloseSpider('cannot_parse_date: %s' % e)
# Move scraped news to pipeline
return loader.load_item()
def parse(self, response):
self.logger.info('parse: {}'.format(response))
is_no_update = False
# Collect list of news from current page
articles = json.loads(response.body)['response']
for article in articles:
# Example: 2016-10-12 15:16:04
date_time_str = article['news_date_publish']
# Parse date information
try:
published_at_wib = datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
except Exception as e:
raise CloseSpider('cannot_parse_date: {}'.format(e))
published_at = wib_to_utc(published_at_wib)
if (self.media['last_scraped_at'] >= published_at):
is_no_update = True
break;
for sub_article in article['news_content']:
yield self.parse_news(article, sub_article)
if is_no_update:
self.logger.info('Media have no update')
return
# Collect news on next page
if len(articles) > 0:
# Example: 'http://api.merdeka.com/mobile/gettag/pilgub-dki/0/20/L9pTAoWB269T&-E/'
next_page_url = response.url.split('/')
next_page_url[-4] = str(int(next_page_url[-4]) + 20)
next_page_url = '/'.join(next_page_url)
yield Request(next_page_url, callback=self.parse)
def parse(self, response):
self.logger.info('parse: {}'.format(response))
is_no_update = False
# Collect list of news from current page
articles_grid = response.css('li:not(.last) > div.grid')
articles = zip(articles_grid, [NEWS_GRID] * len(articles_grid))
articles += zip(response.css('div.topic'), [NEWS_HEADLINE])
if not articles:
raise CloseSpider('article not found')
for article in articles:
# Close the spider if we don't find the list of urls
url_selectors = None
if article[1] == NEWS_GRID:
url_selectors = article[0].css('h2 > a::attr(href)')
elif article[1] == NEWS_HEADLINE:
url_selectors = article[0].css('h1 > a::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
self.logger.info('Url: {}'.format(url))
# Example: Minggu, 09 Oct 2016 15:14
info_selectors = article[0].css('div.reg::text')
if not info_selectors:
raise CloseSpider('info_selectors not found')
info = info_selectors.extract()[1]
# Example: 09 Oct 2016 15:14
info_time = info.split(',')[1].strip()
# Parse date information
try:
published_at_wib = datetime.strptime(info_time,
'%d %b %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: {}'.format(e))
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy request
yield Request(url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
# Collect news on next page
if response.css('div.bu.fr > a'):
next_page = response.css(
'div.bu.fr > a[rel="next"]::attr(href)').extract()[0]
next_page_url = response.urljoin(next_page)
yield Request(next_page_url, callback=self.parse)
def parse_news(self, response):
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css(
'div.detail > article > div.detail_area > h1::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
# parse date
date_selectors = response.css(
"div.detail > article > div.detail_area > div.date::text")
if not date_selectors:
return loader.load_item()
# Selasa 10 Oktober 2017, 13:40 WIB
date_str = date_selectors.extract()[0]
date_str = filter(None, re.split('[\s,]', date_str))[1:5]
info_time = ' '.join([_(s) for s in date_str if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#TODO check the published_at, if it is smaller than the last time
#we crawl, just drop the data.
#parse author name
author_name_selectors = response.css(
"div.detail > article > div.detail_area > div.author > strong::text"
)
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract_first()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css(
"div.detail > article > div.text_detail.detail_area")
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
# Initialize item loader
# extract news title, published_at, author, content, url
# Required: title, raw_content, published_at
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
title_selectors = response.css('div.detail_area > h1.jdl::text')
if not title_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
title = title_selectors.extract()[0]
loader.add_value('title', title)
# Extract the content using XPath instead of CSS selector
# We get the XPath from chrome developer tools (copy XPath)
# or equivalent tools from other browser
xpath_query = """
//div[@class="text_detail detail_area"]/node()
[not(self::comment()|self::script|self::div)]
"""
raw_content_selectors = response.xpath(xpath_query)
if not raw_content_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
raw_content = ' '.join(raw_content_selectors.extract())
raw_content = raw_content.strip()
loader.add_value('raw_content', raw_content)
# Parse date information
# Example: Kamis 15 Sep 2016, 18:33 WIB
date_selectors = response.css('div.detail_area > div.date::text')
if not date_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
date_str = date_selectors.extract()[0]
# Example: '15 Sep 2016, 18:33'
date_str = ' '.join(date_str.split(' ')[1:5])
try:
published_at_wib = datetime.strptime(date_str, '%d %b %Y, %H:%M')
except ValueError:
# Will be dropped on the item pipeline
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
author_name_selectors = response.css('div.author > strong::text')
if not author_name_selectors:
loader.add_value('author_name', '')
else:
author_name = author_name_selectors.extract()[0]
loader.add_value('author_name', author_name)
# Move scraped news to pipeline
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css('div.artikel > h1.artikel::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
#parse date
date_selectors = response.css('div.artikel > div.tanggal::text')
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract_first()
# eg: Tuesday, 12 September 2017 | 20:21 WIB
time_arr = filter(None, re.split('[\s,|]', date_str))[1:-1]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
return loader.load_item()
loader.add_value('published_at', published_at)
#parse author name
author_name_selectors = response.css(
'div.artikel > div > p > strong::text')
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract()[-1].strip()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css('div.artikel > div > p')
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
# Initialize item loader
# extract news title, published_at, author, content, url
# Required: title, raw_content, published_at
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
title_selectors = response.css('h1.detailtitle::text')
if not title_selectors:
# If error, drop from the item pipeline
return loader.load_item()
title = title_selectors.extract_first().strip()
loader.add_value('title', title)
# Parse date information
date_time = response.css(
'body > div > div.container > div.page-header > div::text'
).extract_first().strip()
date_time = date_time.split(',')[-1].strip()
date_time = ' '.join([_(w) for w in date_time.split(' ')
]) # October => Oktober
try:
published_at_wib = datetime.strptime(date_time, '%d %B %Y %H:%M')
except ValueError:
# If error, drop from the item pipeline
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
# If multipage
multipage_selectors = response.css('.newsPagingWrap > a')
if multipage_selectors:
return self.parse_indices(multipage_selectors, loader)
# Else if not multipage
author_name_selectors = response.css('.newsContent > p > strong::text')
if not author_name_selectors:
loader.add_value('author_name', '')
else:
author_name = author_name_selectors.extract()[-1].strip()
loader.add_value('author_name', author_name)
# Extract the news content
raw_content_selectors = response.css('.newsContent > p')
if not raw_content_selectors:
# Drop from the item pipeline
return loader.load_item()
raw_content = ' '.join(raw_content_selectors.extract())
raw_content = raw_content.strip()
loader.add_value('raw_content', raw_content)
# Move scraped news to pipeline
return loader.load_item()
def parse_news(self, response):
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css(
'div.main-container > div > section.main-content > h1.page-header::text'
)
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
#parse date
date_selectors = response.css(
'div.post-meta > div > div > div.submitted > span::text')
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract_first()
info_time = re.sub(r'[,-]', '', date_str)
info_time = re.sub(r'\s+', ' ', info_time)
time_arr = filter(None, re.split('[\s,|-]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
return loader.load_item()
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#parse author name
author_name_selectors = response.css(
'div.post-meta > div > div > div.items-penulis > span > a::text'
).extract_first()
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css(
'div.region.region-content > section > article > div.field.field-name-body.field-type-text-with-summary > div.field-items > div.field-item.even'
)
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css('div#mdk-news-title::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
#parse date
date_selectors = response.css("div.mdk-date-reporter > span::text")
# We need to do this because sometimes selector can contains 3 or 2 elements.
pos = len(date_selectors) - 2
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract()[pos]
# eg: 8 September 2017 21:02
date_str = date_str.split("|")[1].strip()
time_arr = filter(None, re.split('[\s,|]', date_str))
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#parse author name
author_name_selectors = response.css(
"div.mdk-date-reporter > span::text")
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract()[1]
author_name = author_name.split(":")[1].strip()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css("div.mdk-body-paragraph")
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css('h1::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
# parse date
date_selectors = response.css("div.date > span::text")
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract()[0]
# eg: Selasa, 12 Sep 2017 20:08
date_str = date_str.split(",")[1].strip()
time_arr = filter(None, re.split('[\s,|]', date_str))
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %b %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#TODO check the published_at, if it is smaller than the last time
#we crawl, just drop the data.
#parse author name
author_name_selectors = response.css("div.date > span")[1].css(
"span > span::text")
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract_first()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css("div.contentdetail")
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse(self, response):
is_no_update = False
articles = response.css("div.view-content > div > article")
if articles:
for article in articles:
url_selector = article.css(
"div.entry-main > h2.entry-title > a ::attr(href)")
if not url_selector:
continue
url = url_selector.extract_first()
info_selectors = article.css(
"div.entry-main > div.entry-meta > span.entry-date > span::text"
)
if not info_selectors:
continue
# 25 July, 2018 - 11:04
info = info_selectors.extract_first()
info_time = re.sub(r'[,-]', '', info)
info_time = re.sub(r'\s+', ' ', info_time)
time_arr = filter(None, re.split('[\s,|-]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(
info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
break
print url
print published_at_wib
full_url = '%s%s' % (self.base_url, url)
yield Request(url=full_url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
else:
next_selectors = response.css("ul.pagination > li.next")
if next_selectors:
next_url = next_selectors.css("a::attr(href)").extract_first()
next_url = '%s%s' % (self.base_url, next_url)
yield Request(next_url, callback=self.parse)
def parse_news(self, response):
# self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css('div.wrap-head > h2 > a::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
##parse date
date_selectors = response.css('div.wrap-head > span::text')
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract()[0]
# eg: Tuesday, 12 September 2017 | 20:21 WIB
time_arr = filter(None, re.split('[\s,|]', date_str))[1:-1]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#parse author name
author_name_selectors = response.css('div.red::text')
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract()[0].strip()
author_name = author_name.replace('Rep: ',
'').replace('Red: ', '').strip()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css('div.content-detail')
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def convert_date(self, idn_date):
# Example Rabu, 12 Oktober 2016 | 10:23 WIB
info_time = idn_date.split(',')[1].strip().split('|')
info_date = info_time[0].strip().split(' ')
info_hours = info_time[1].strip().split(' ')[0].strip()
day = info_date[0]
month = self.translator.translate('idn')[info_date[1]]
year = info_date[2]
formatted_date = day + ' ' + month + ' ' + year + ', ' + info_hours
return wib_to_utc(datetime.strptime(formatted_date, '%d %B %Y, %H:%M'))
def parse(self, response):
self.logger.info('parse: %s' % response)
is_no_update = False
# Get list of news from the current page
articles = response.css('article > div > div.post-content')
if not articles:
raise CloseSpider('article not found')
for article in articles:
# Close the spider if we don't find the list of urls
url_selectors = article.css('a.timestamp-link::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
# Example 'Sabtu, November 19, 2016'
date_selectors = article.css('a.timestamp-link > abbr::text')
if not date_selectors:
raise CloseSpider('date_selectors not found')
# Parse date information
try:
date = date_selectors.extract()[0].split(' ')
# Sanitize month - Indo month to Eng month
# Example: Nov 19 2016
date[1] = sanitize(date[1])
published_at_wib = datetime.strptime(' '.join(date[1:]),
'%b %d, %Y')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy request
yield Request(url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
# try getting next page
if len(articles) > 0:
try:
yield Request(
'http://www.nusanews.co/search/label/Pilkada?updated-max='
+ str(published_at_wib).replace(' ', 'T') +
'%2B07:00&max-results=20',
callback=self.parse)
except Exception as e:
pass
def parse(self, response):
is_no_update = False
articles = response.css("div.list.media_rows.list-berita > article")
if articles:
for article in articles:
# TODO handle for photo
url_selector = article.css("a::attr(href)")
if not url_selector:
continue
url = url_selector.extract_first()
self.logger.info(url)
# parse date
date_selectors = article.css("a > span.box_text > span.date::text")
if not date_selectors:
continue
date_str = date_selectors.extract_first()
# date_str = Senin, 09 Okt 2017 16:12 WIB
info_time = date_str.split(',')[1].strip()
time_arr = filter(None, re.split('[\s,|]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s in %s' % (e, url))
# convert to utc+0
published_at = wib_to_utc(published_at_wib)
# TODO
# eg: 6 Hours ago
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
self.logger.info('Done sending requests. No more')
break
yield Request(url=url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
else:
if response.css("div.paging.text_center > a.last"):
navs = response.css("div.paging.text_center > a.last")
for nav in navs:
direction = nav.css("img::attr(alt)").extract_first()
if direction.lower() == 'kanan':
next_page = nav.css("a::attr(href)").extract_first()
yield Request(next_page, callback=self.parse)
def parse_news_metro(self, response):
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
date_selector = response.css('.artikel > div.block-tanggal::text')
if not date_selector:
return self.parse_news_pilkada(loader, response)
try:
date_time_str = date_selector.extract()[0].split(',')[1].strip()[:-4]
date_time_str = ' '.join([_(x) for x in date_time_str.split(' ')])
published_at_wib = datetime.strptime(date_time_str, '%d %B %Y | %H:%M')
except Exception:
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
if (self.media['last_scraped_at'] >= published_at):
is_no_update = True
self.logger.info('Media have no update')
raise CloseSpider('finished')
loader.add_value('published_at', published_at)
title_selector = response.css('.artikel > h1::text')
if not title_selector:
return loader.load_item()
loader.add_value('title', title_selector.extract()[0])
# Select all p which don't have iframe inside it
raw_content_selector = response.xpath('//div[@class="artikel"]//p[not(iframe)]')
if not raw_content_selector:
return loader.load_item()
raw_content = ''
for raw_content_selector_one in raw_content_selector:
raw_content = raw_content + raw_content_selector_one.extract()
# Go to next page while there is next page button
next_page_selector = response.css('.pagination-nb').xpath('//a[text()="next"]/@href')
if next_page_selector:
return Request(next_page_selector.extract()[0], callback=lambda x, loader=loader, raw_content=raw_content: self.parse_next_page_metro(x, loader, raw_content))
loader.add_value('raw_content', raw_content)
# The author usually put inside <strong> tag, however, some news is not using <strong> tag.
# NOTE: this block of code may need revision in the future
author_name = ''
for author_name_selector in reversed(raw_content_selector):
author_name_selector = author_name_selector.css('strong::text')
for tmp in reversed(author_name_selector.extract()):
tmp = tmp.strip()
if tmp and all((x.isalpha() and x.isupper()) or x.isspace() or x == '.' or x == '|' for x in tmp):
author_name = tmp
break
if author_name:
break
author_name = ','.join(author_name.split(' | '))
loader.add_value('author_name', author_name)
return loader.load_item()
def parse(self, response):
self.logger.info('parse: {}'.format(response))
is_no_update = False
# Collect list of news from current page
article_selectors = response.css('ul.indexlist > li')
if not article_selectors:
raise CloseSpider('article_selectors not found')
for article in article_selectors:
url_selectors = article.css('a::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
# Example: 7 Oktober 2016 19:37
info_selectors = article.css('div.upperdeck::text')
if not info_selectors:
raise CloseSpider('info_selectors not found')
info = info_selectors.extract()[1]
info = info.split(',')[1].replace('\t', '').strip()
# Example: 7 October 2016 19:37
info_time = info.split(' ')
info_time = ' '.join([_(s) for s in info_time])
# Parse date information
try:
published_at_wib = datetime.strptime(info_time,
'%d %B %Y %H:%M')
except ValueError as err:
raise CloseSpider('cannot_parse_date: {}'.format(err))
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy Request
yield Request(url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
# Collect news on next page
tag_selectors = response.css('div.pagination > a')
if not tag_selectors:
raise CloseSpider('tag_selectors not found')
for tag in tag_selectors:
more_selectors = tag.css('a::text')
if not more_selectors:
raise CloseSpider('more_selectors not found')
more = more_selectors.extract()[0]
if more == 'NEXT':
next_page = tag.css('a::attr(href)').extract()[0]
next_page_url = response.urljoin(next_page)
yield Request(next_page_url, callback=self.parse)
def parse(self, response):
is_no_update = False
articles = response.css(
"div.left_ib_rightx > div.set_subkanal > div.txt_subkanal")
if articles:
for article in articles:
# TODO handle for photo
url_selector = article.css("h2 > a::attr(href)")
if not url_selector:
continue
url = url_selector.extract_first()
# parse date
date_selectors = article.css("h6::text")
if not date_selectors:
continue
date_str = date_selectors.extract_first()
# Tuesday, 14 Aug 2018 21:37 WIB
info_time = date_str.split(',')[1].strip()
time_arr = filter(None, re.split('[\s,|]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
try:
published_at_wib = datetime.strptime(
info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s in %s' % (e, url))
# convert to utc+0
published_at = wib_to_utc(published_at_wib)
# TODO
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
break
yield Request(url=url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
else:
if response.css("div.pagination > section > nav > a"):
links = response.css("div.pagination > section > nav > a")
for link in links:
l = link.css("a::text").extract_first()
if l.lower() == 'next':
next_page = link.css("a::attr(href)").extract_first()
yield Request(next_page, callback=self.parse)
else:
continue
def parse_news(self, response):
self.logger.info('parse_news: {}'.format(response))
# Init item loader
# extract news title, published_at, author, content, url
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
title_selectors = response.css('div.content-detail > h4::text')
if not title_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
title = title_selectors.extract()[0]
loader.add_value('title', title)
# Extract raw html, not the text
raw_content_selectors = response.css('div.content-body')
if not raw_content_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
raw_content = raw_content_selectors.extract()
raw_content = ' '.join([w.strip() for w in raw_content])
raw_content = raw_content.strip()
loader.add_value('raw_content', raw_content)
# Example: Selasa, 11 Oktober 2016 | 10:48
date_selectors = response.css('div.date::text')
if not date_selectors:
# Will be dropped on the item pipeline
return loader.load_item()
date_str = date_selectors.extract()[0]
# Example: 11 October 2016 10:48
date_str = re.split('[\s,|-]', date_str)
date_str = ' '.join([_(s) for s in date_str[1:] if s])
# Parse date information
try:
published_at_wib = datetime.strptime(date_str, '%d %B %Y %H:%M')
except ValueError:
# Will be dropped on the item pipeline
return loader.load_item()
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
author_selectors = response.css('div.content-detail > p::text')
if not author_selectors:
loader.add_value('author_name', '')
else:
author_name = author_selectors.extract()[0]
author_name = author_name.split('/')[0]
loader.add_value('author_name', author_name)
# Move scraped news to pipeline
return loader.load_item()
def extract_date_from_url(self, url):
# date equals to '2017 01 31'
date = ' '.join(url.split('/')[-5:-2])
try:
published_at_wib = datetime.strptime(date, '%Y %m %d')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
return published_at
def parse(self, response):
self.logger.info('parse: %s' % response)
is_no_update = False
# Get list of news from the current page
articles = response.css('article')
if not articles:
raise CloseSpider('article not found')
for article in articles:
# Close the spider if we don't find the list of urls
url_selectors = article.css('a::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
# Example 'detikNews | Sabtu 08 Oct 2016, 14:54 WIB'
info_selectors = article.css('a > .text > span.info::text')
if not info_selectors:
raise CloseSpider('info_selectors not found')
info = info_selectors.extract()[0]
# Example 'Sabtu 08 Oct 2016, 14:54 WIB'
info_time = info.split('|')[1].strip()
# Example '08 Oct 2016, 14:54'
info_time = ' '.join(info_time.split(' ')[1:5])
# Parse date information
try:
published_at_wib = datetime.strptime(info_time,
'%d %b %Y, %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy request
yield Request(url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
if response.css('a.btn_more'):
next_page = response.css('a.btn_more::attr(href)')[0].extract()
next_page_url = response.urljoin(next_page)
yield Request(next_page_url, callback=self.parse)
elif response.css('div.pag-nextprev > a'):
next_page = response.css(
'div.pag-nextprev > a::attr(href)')[1].extract()
next_page_url = response.urljoin(next_page)
yield scrapy.Request(next_page_url, callback=self.parse)
def parse(self, response):
self.logger.info('parse: %s' % response)
is_no_update = False
# Get list of news from the current page
articles = response.css('li.media')
if not articles:
raise CloseSpider('article not found')
for article in articles:
# Close the spider if we don't find the list of urls
url_selectors = article.css('a::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
# Example '02 November 2016'
date_selectors = article.css('time::text')
if not date_selectors:
raise CloseSpider('date_selectors not found')
# Parse date information
try:
date = date_selectors.extract()[0].split(' ')
# Sanitize month - Indo month to Eng month
# Example: 02 Nov 2016
date[1] = sanitize(date[1])
published_at_wib = datetime.strptime(' '.join(date),
'%d %b %Y')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy request
yield Request('http:' + url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
# try getting next page
try:
next_page_url = response.xpath(
'//section[@class="pagination-numeric"]/span/a/@href'
)[-1].extract()
if next_page_url and next_page_url != response.url:
yield Request(next_page_url, callback=self.parse)
except:
pass
def parse_news(self, response):
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css(
'div.pa15.bgwhite > h1.f32.fno.crimson::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
#parse date
date_selectors = response.css(
'div.pa15.bgwhite > div.mt10.mb10 > time.grey.f13.dip::text')
if not date_selectors:
return loader.load_item()
date_str = date_selectors.extract_first()
# eg: Kompas.com - 10/10/2017, 13:37 WIB
info_time = date_str.split(',')[1].strip()
time_arr = filter(None, re.split('[\s,|]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
return loader.load_item()
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#parse author name
# author_name_selectors = response.css('div.read__author > a::text').extract_first()
# if not author_name_selectors:
loader.add_value('author_name', 'N/A')
# else:
# author_name = author_name_selectors
# loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css(
'div.ptb15 > div.txt-article.mb20')
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract_first()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse(self, response):
is_no_update = False
articles = response.css("div.article__list.clearfix")
if articles:
for article in articles:
url_selector = article.css(
"div.article__list__title > h3 > a::attr(href)")
if not url_selector:
continue
url = url_selector.extract_first()
info_selectors = article.css(
"div.article__list__info > div.article__date::text")
if not info_selectors:
continue
#info = 10/10/2017, 13:37 WIB
info = info_selectors.extract_first()
time_arr = filter(None, re.split('[\s,]', info))[:2]
info_time = ' '.join([s for s in time_arr if s])
# print info_time
#parse date information
try:
published_at_wib = datetime.strptime(
info_time, '%d/%m/%Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
break
yield Request(url=url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
else:
next_selectors = response.css(
"div.paging__wrap > div.paging__item > a.paging__link.paging__link--next"
)
for num in range(0, len(next_selectors)):
if next_selectors[num].css(
"a::attr(rel)").extract_first() == 'next':
next_url = next_selectors[num].css(
"a::attr(href)").extract_first()
yield Request(next_url, callback=self.parse)
break
def parse_news(self, response):
self.logger.info('parse_news: %s' % response)
loader = ItemLoader(item=News(), response=response)
loader.add_value('url', response.url)
loader.add_value('media_id', self.media_id)
loader.add_value('election_id', self.election_id)
#parse title
title_selectors = response.css('section.main-content > h1::text')
if not title_selectors:
return loader.load_item()
title = title_selectors.extract_first()
loader.add_value('title', title)
#parse date
date_selectors = response.css("div.submitted > span::text")
if not date_selectors:
return loader.load_item()
# eg: 5 September, 2017 - 18:54
date_str = date_selectors.extract_first()
time_arr = filter(None, re.split('[\s,-]', date_str))
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
loader.add_value('published_at', published_at)
#parse author name
author_name_selectors = response.css(
"div.items-penulis > span > a::text")
if not author_name_selectors:
loader.add_value('author_name', 'N/A')
else:
author_name = author_name_selectors.extract()[0].strip()
loader.add_value('author_name', author_name)
#parse raw content
raw_content_selectors = response.css("div.field-item.even")
if not raw_content_selectors:
return loader.load_item()
raw_content = raw_content_selectors.extract()
loader.add_value('raw_content', raw_content)
return loader.load_item()
def parse(self, response):
is_no_update = False
articles = response.css("div.lsi > ul#latestul > li > div")
if articles:
for article in articles:
url_selector = article.css(
"div.m-btsquare > h3 > a::attr(href)")
if not url_selector:
continue
url = url_selector.extract_first()
info_selectors = article.css(
" div.m-btsquare > div > time::text")
if not info_selectors:
continue
#info = Jumat, 17 Agustus 2018 04:43 WIB
info = info_selectors.extract_first()
info_time = info.split(',')[1].strip()
time_arr = filter(None, re.split('[\s,|]', info_time))[:4]
info_time = ' '.join([_(s) for s in time_arr if s])
#parse date information
try:
published_at_wib = datetime.strptime(
info_time, '%d %B %Y %H:%M')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
break
yield Request(url=url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
else:
next_selectors = response.css("div.ma10.paging#paginga > a")
for num in range(0, len(next_selectors)):
if next_selectors[num].css(
"a::text").extract_first().lower() == 'next':
next_url = next_selectors[num].css(
"a::attr(href)").extract_first()
yield Request(next_url, callback=self.parse)
break
def parse(self, response):
base_url = "https://m.merdeka.com"
self.logger.info('parse: %s' % response)
print self.media['last_crawl_at']
is_no_update = False
articles = response.css("div#mdk-tag-news-list_mobile > ul > li")
if not articles:
raise CloseSpider('articles not found')
for article in articles:
url_selector = article.css("div > a::attr(href)")
if not url_selector:
raise CloseSpider('url_selectors not found')
url = base_url + url_selector.extract()[0]
print url
info_selectors = article.css("div > b.mdk-time::text")
if not info_selectors:
raise CloseSpider('info_selectors not found')
#info = Jumat, 8 September 2017 16:45:19
info = info_selectors.extract_first().replace(u'\xa0', u'')
#info_time = 8 September 2017 16:45:19
info_time = info.split(',')[1].strip()
time_arr = filter(None, re.split('[\s,|]', info_time))
info_time = ' '.join([_(s) for s in time_arr if s])
print info_time
#parse date information
try:
published_at_wib = datetime.strptime(info_time,
'%d %B %Y %H:%M:%S')
except ValueError as e:
raise CloseSpider('cannot_parse_date: %s' % e)
#convert to utc+0
published_at = wib_to_utc(published_at_wib)
if self.media['last_crawl_at'] >= published_at:
is_no_update = True
break
yield Request(url=url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
if response.css('div.paging-box'):
next_page = response.css('a.link_next::attr(href)')[0].extract()
yield Request(next_page, callback=self.parse)