def create_suggestion_item(self, response): """ Create a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`, from given response. :param response: scrapy response :return: scrapy item """ suggestion_item = items.SuggestionItem() suggestion_item['suggestion_id'] = self.get_suggestion_id(response) suggestion_item['title'] = self.get_suggestion_title(response) suggestion_item['date_time'] = self.get_suggestion_datetime(response) suggestion_item['category'] = self.get_suggestion_category(response) suggestion_item['author'] = self.get_suggestion_author(response) suggestion_item['approval'] = self.get_suggestion_approval(response) suggestion_item['address'] = self.get_suggestion_address(response) suggestion_item['content'] = self.get_suggestion_content(response) suggestion_item['comment_count'] = self.get_suggestion_comment_count(response) return suggestion_item
def parse_post(response: HtmlResponse) -> SuggestionItem: """ Parse thread and yield a SuggestionItem, see :class:`~OnlineParticipationDataset.items.SuggestionItem`. """ suggestion_item = items.SuggestionItem() suggestion_item['suggestion_id'] = response.url.split("/")[-1] suggestion_item['title'] = response.css( "h2.node-title::text").extract_first() suggestion_item['date_time'] = datetime.strptime( response.css("p.user-and-date:first-child::text").extract() [1].strip(), "am %d.%m.%Y") suggestion_item['category'] = response.css( ".category_button span::text").extract_first().strip() suggestion_item['author'] = response.css( "span.username::text").extract_first() suggestion_item['address'] = response.css( "p.user-and-date::text")[-1].extract().strip() suggestion_item['content'] = response.css( ".field p::text").extract_first() return suggestion_item
def create_suggestion_item(self, response): ''' Creates a suggestion item based on information in response :param response: scrapy response :return: suggestion item ''' self.suggestions_counter += 1 sug_item = items.SuggestionItem() #sug_item['id'] = response.css('.ecm_draftBillParagraphTabs>div>div>div::attr(id)').extract_first() sug_item['id'] = next(self.id_counter) sug_item['title'] = ' '.join( response.css( '.ecm_draftBillParagraphContent.push-top>h1::text').extract()) sug_item['content'] = ' '.join( response.css( '.ecm_draftBillParagraphContent.push-top>div>h3::text,.ecm_draftBillParagraphContent.push-top>div>p>strong::text' ).extract()) sug_item['comment_count'] = int(self.extract_num_comments(response)) sug_item['category'] = self.get_category(response) sug_item['parent'] = 'None' sug_item['children'] = [] return sug_item