def create_comment_item(self, response, suggestion_id):
"""
Create a Comment, see :class:`~OnlineParticipationDatasetsitems.Comment`, from given response.
:param response: scrapy response
:return: scrapy item
"""
comment_item = items.Comment()
comment_item['comment_id'] = self.get_comment_id(response)
comment_item['suggestion_id'] = suggestion_id
comment_item['date_time'] = self.get_comment_datetime(response)
comment_item['author'] = self.get_comment_author(response)
comment_item['title'] = self.get_comment_title(response)
comment_item['content'] = self.get_comment_content(response)
vote = self.get_comment_vote(response)
if vote == "sonstiges":
comment_item['vote'] = 'misc'
elif vote == 'unterstützt den Vorschlag':
comment_item['vote'] = 'approval'
elif vote == 'lehnt Vorschlag ab':
comment_item['vote'] = 'refusal'
elif response.xpath('.//div[@class="field-label"]/text()'
).extract_first() == 'Kommentarlabel:\xa0':
comment_item['vote'] = 'offical'
else:
comment_item['vote'] = 'answer'
return comment_item
def create_comment_item(self, response, suggestion_id, level=1):
"""
Create a Comment, see :class:`~OnlineParticipationDatasetsitems.Comment`, from given response.
:param response: scrapy response
:return: scrapy item
"""
comment_item = items.Comment()
comment_item['level'] = level
comment_item['comment_id'] = self.get_comment_id(response)
comment_item['suggestion_id'] = suggestion_id
if level > 1:
comment_item['parent_id'] = 0
comment_item['date_time'] = self.get_comment_datetime(response)
comment_item['author'] = self.get_comment_author(response)
comment_item['title'] = self.get_comment_title(response)
comment_item['content'] = self.get_comment_content(response)
return comment_item
def create_comment_item(self,
response,
suggestion_id,
parent_id=None,
level=1):
"""
Create a Comment, see :class:`~OnlineParticipationDatasetsitems.Comment`, from given response.
:param response: scrapy response
:return: scrapy item
"""
comment_item = items.Comment()
comment_item['level'] = level
comment_item['comment_id'] = self.get_comment_id(response)
comment_item['suggestion_id'] = suggestion_id
if parent_id:
comment_item['parent_id'] = parent_id
comment_item['date_time'] = self.get_comment_datetime(response)
comment_item['author'] = self.get_comment_author(response)
comment_item['title'] = self.get_comment_title(response)
comment_item['content'] = self.get_comment_content(response)
children_list = []
if level == 1:
children_xpath = 'following-sibling::div/div[@class="indented"]/article'
else:
children_xpath = 'following-sibling::div[@class="indented"]/article'
for child in response.xpath(children_xpath):
children_list.append(
self.create_comment_item(child, suggestion_id,
comment_item['comment_id'],
level + 1))
comment_item['children'] = children_list
return comment_item