def create_suggestion_item(self, response):
"""
Create a Suggestion, see :class:`~OnlineParticipationDatasetsitems.Suggestion`, from given response.
:param response: scrapy response
:return: scrapy item
"""
suggestion_item = items.Suggestion()
suggestion_item['suggestion_id'] = self.get_suggestion_id(response)
suggestion_item['title'] = self.get_suggestion_title(response)
suggestion_item['date_time'] = self.get_suggestion_datetime(response)
suggestion_item['category'] = self.get_suggestion_category(response)
suggestion_item['author'] = self.get_suggestion_author(response)
suggestion_item['approval'] = self.get_suggestion_approval(response)
suggestion_item['address'] = self.get_suggestion_address(response)
suggestion_item['content'] = self.get_suggestion_content(response)
suggestion_item['comment_count'] = self.get_suggestion_comment_count(
response)
return suggestion_item
def parse_post(response: HtmlResponse) -> Suggestion:
"""
Parse thread and yield a Suggestion, see :class:`~OnlineParticipationDataset.items.Suggestion`.
"""
suggestion_item = items.Suggestion()
suggestion_item['suggestion_id'] = response.url.split("/")[-1]
suggestion_item['title'] = response.css(
"h2.node-title::text").extract_first()
suggestion_item['date_time'] = datetime.strptime(
response.css("p.user-and-date:first-child::text").extract()
[1].strip(), "am %d.%m.%Y")
suggestion_item['category'] = response.css(
".category_button span::text").extract_first().strip()
suggestion_item['author'] = response.css(
"span.username::text").extract_first()
suggestion_item['address'] = response.css(
"p.user-and-date::text")[-1].extract().strip()
suggestion_item['content'] = response.css(
".field p::text").extract_first()
return suggestion_item