def create_suggestion_item(self, response):
"""
Create a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`, from given response.
:param response: scrapy response
:return: scrapy item
"""
suggestion_item = items.SuggestionItem()
suggestion_item['suggestion_id'] = self.get_suggestion_id(response)
suggestion_item['title'] = self.get_suggestion_title(response)
suggestion_item['date_time'] = self.get_suggestion_datetime(response)
suggestion_item['category'] = self.get_suggestion_category(response)
suggestion_item['author'] = self.get_suggestion_author(response)
suggestion_item['approval'] = self.get_suggestion_approval(response)
suggestion_item['address'] = self.get_suggestion_address(response)
suggestion_item['content'] = self.get_suggestion_content(response)
suggestion_item['comment_count'] = self.get_suggestion_comment_count(response)
return suggestion_item
def parse_post(response: HtmlResponse) -> SuggestionItem:
"""
Parse thread and yield a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`.
"""
suggestion_item = items.SuggestionItem()
suggestion_item['suggestion_id'] = response.url.split("/")[-1]
suggestion_item['title'] = response.css(
"h2.node-title::text").extract_first()
suggestion_item['date_time'] = datetime.strptime(
response.css("p.user-and-date:first-child::text").extract()
[1].strip(), "am %d.%m.%Y")
suggestion_item['category'] = response.css(
".category_button span::text").extract_first().strip()
suggestion_item['author'] = response.css(
"span.username::text").extract_first()
suggestion_item['address'] = response.css(
"p.user-and-date::text")[-1].extract().strip()
suggestion_item['content'] = response.css(
".field p::text").extract_first()
return suggestion_item
def create_suggestion_item(self, response):
'''
Creates a suggestion item based on information in response
:param response: scrapy response
:return: suggestion item
'''
self.suggestions_counter += 1
sug_item = items.SuggestionItem()
#sug_item['id'] = response.css('.ecm_draftBillParagraphTabs>div>div>div::attr(id)').extract_first()
sug_item['id'] = next(self.id_counter)
sug_item['title'] = ' '.join(
response.css(
'.ecm_draftBillParagraphContent.push-top>h1::text').extract())
sug_item['content'] = ' '.join(
response.css(
'.ecm_draftBillParagraphContent.push-top>div>h3::text,.ecm_draftBillParagraphContent.push-top>div>p>strong::text'
).extract())
sug_item['comment_count'] = int(self.extract_num_comments(response))
sug_item['category'] = self.get_category(response)
sug_item['parent'] = 'None'
sug_item['children'] = []
return sug_item