def test_normalize_text(self):
text_origin = """<div class="myclass">\r\n<h1>My title</h1>\r\n <h1>My title 2</h1>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text_origin = html_diff_wrapper.normalize_text(text_origin)
self.assertEqual(normalized_text_origin, """<div class="myclass"><h1>My title</h1><h1>My title 2</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p></div>""")
text1 = """<div class="myclass">\r\n<h1>My title</h1>\r\n <h1>My title 2 autre</h1><p></p>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>propositions</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text1 = html_diff_wrapper.normalize_text(text1)
self.assertEqual(normalized_text1, """<div class="myclass"><h1>My title</h1><h1>My title 2 autre</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>propositions</strong> lors de la Fete.</p></div>""")
text2 = """<div class="myclass">\r\n<h1>My title</h1><p> </p>\r\n<p>Mon texte ici</p>\r\n<h1>My title 2</h1>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text2 = html_diff_wrapper.normalize_text(text2)
self.assertEqual(normalized_text2, """<div class="myclass"><h1>My title</h1><p>Mon texte ici</p><h1>My title 2</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p></div>""")
def start(self, context, request, appstruct, **kw):
root = getSite()
user = get_current()
proposal = context.proposal
working_group = proposal.working_group
author = working_group.get_member(user)
copy_of_amendment = copy(context,
(proposal, 'amendments'),
omit=('created_at',
'modified_at',
'explanations'))
copy_of_amendment.set_data(appstruct)
copy_of_amendment.text = html_diff_wrapper.normalize_text(
copy_of_amendment.text)
copy_of_amendment.setproperty('originalentity', context)
copy_of_amendment.state = PersistentList(['draft'])
copy_of_amendment.setproperty('author', author)
localizer = request.localizer
copy_of_amendment.title = localizer.translate(
_('Amended version ')) + str(getattr(proposal,
'_amendments_counter', 1))
grant_roles(user=author, roles=(('Owner', copy_of_amendment), ))
copy_of_amendment.text_diff = get_text_amendment_diff(
proposal, copy_of_amendment)
copy_of_amendment.reindex()
proposal._amendments_counter = getattr(
proposal, '_amendments_counter', 1) + 1
context.reindex()
request.registry.notify(ActivityExecuted(self, [context], author))
return {'newcontext': copy_of_amendment}
def start(self, context, request, appstruct, **kw):
user = get_current(request)
user = context.working_group.get_member(user)
related_ideas = appstruct.pop('related_ideas')
add_files = appstruct.pop('add_files')
copy_of_proposal = context.get_version(user, (context, 'version'))
context.working_group.init_nonproductive_cycle()
context.state = PersistentList(['amendable', 'published'])
context.set_data(appstruct)
context.text = html_diff_wrapper.normalize_text(
context.text)
context.modified_at = datetime.datetime.now(tz=pytz.UTC)
context.set_related_ideas(
related_ideas, user)
# Add Nia comment
alert_comment_nia(
context, request, getSite(),
internal_kind=InternalAlertKind.working_group_alert,
subject_type='proposal',
alert_kind='new_version',
diff=diff_analytics(
copy_of_proposal, context, ['title', 'text', 'description'])
)
add_attached_files({'add_files': add_files}, context)
context.reindex()
return {'newcontext': context}
def start(self, context, request, appstruct, **kw):
root = getSite()
user = get_current()
proposal = context.proposal
working_group = proposal.working_group
author = working_group.get_member(user)
copy_of_amendment = copy(context, (proposal, 'amendments'),
omit=('created_at', 'modified_at',
'explanations'))
copy_of_amendment.set_data(appstruct)
copy_of_amendment.text = html_diff_wrapper.normalize_text(
copy_of_amendment.text)
copy_of_amendment.setproperty('originalentity', context)
copy_of_amendment.state = PersistentList(['draft'])
copy_of_amendment.setproperty('author', author)
localizer = request.localizer
copy_of_amendment.title = localizer.translate(
_('Amended version ')) + str(
getattr(proposal, '_amendments_counter', 1))
grant_roles(user=author, roles=(('Owner', copy_of_amendment), ))
copy_of_amendment.text_diff = get_text_amendment_diff(
proposal, copy_of_amendment)
copy_of_amendment.reindex()
proposal._amendments_counter = getattr(proposal, '_amendments_counter',
1) + 1
context.reindex()
request.registry.notify(ActivityExecuted(self, [context], author))
return {'newcontext': copy_of_amendment}
def start(self, context, request, appstruct, **kw):
user = get_current(request)
user = context.working_group.get_member(user)
related_ideas = appstruct.pop('related_ideas')
add_files = appstruct.pop('add_files')
copy_of_proposal = context.get_version(user, (context, 'version'))
context.working_group.init_nonproductive_cycle()
context.state = PersistentList(['amendable', 'published'])
context.set_data(appstruct)
context.text = html_diff_wrapper.normalize_text(context.text)
context.modified_at = datetime.datetime.now(tz=pytz.UTC)
context.set_related_ideas(related_ideas, user)
# Add Nia comment
alert_comment_nia(
context,
request,
getSite(),
internal_kind=InternalAlertKind.working_group_alert,
subject_type='proposal',
alert_kind='new_version',
diff=diff_analytics(copy_of_proposal, context,
['title', 'text', 'description']))
add_attached_files({'add_files': add_files}, context)
context.reindex()
return {'newcontext': context}
def start(self, context, request, appstruct, **kw):
result = set()
working_group = context.working_group
for ballot in working_group.amendments_ballots:
electeds = ballot.report.get_electeds()
if electeds is not None:
result.update(electeds)
amendments = [a for a in result if isinstance(a, Amendment)]
members = working_group.members
root = getSite()
user = get_current()
if amendments:
merged_text = html_diff_wrapper.merge(context.text,
[a.text for a in amendments])
merged_text = html_diff_wrapper.normalize_text(merged_text)
#TODO merged_keywords + merged_description
version = context.get_version(user, (context, 'version'))
for amendment in version.amendments:
amendment.state = PersistentList(['archived'])
amendment.reindex()
self._send_ballot_result(context, request, result, members)
context.text = merged_text
related_ideas = [a.related_ideas for a in amendments]
related_ideas = [
item for sublist in related_ideas for item in sublist
]
related_ideas.extend(context.related_ideas)
related_ideas = list(set(related_ideas))
context.set_related_ideas(related_ideas, user)
context.working_group.init_nonproductive_cycle()
context.reindex()
alert('internal', [root],
members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context],
alert_kind='amendments_result')
# Add Nia comment
alert_comment_nia(
context,
request,
root,
internal_kind=InternalAlertKind.working_group_alert,
subject_type='proposal',
alert_kind='new_version',
diff=diff_analytics(version, context,
['title', 'text', 'description']))
else:
context.state = PersistentList(['amendable', 'published'])
alert('internal', [root],
members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context])
for amendment in context.amendments:
amendment.state = PersistentList(['archived'])
amendment.reindex()
context.reindex()
return {'newcontext': context}
def start(self, context, request, appstruct, **kw):
result = set()
working_group = context.working_group
for ballot in working_group.amendments_ballots:
electeds = ballot.report.get_electeds()
if electeds is not None:
result.update(electeds)
amendments = [a for a in result if isinstance(a, Amendment)]
members = working_group.members
root = getSite()
user = get_current()
if amendments:
merged_text = html_diff_wrapper.merge(
context.text, [a.text for a in amendments])
merged_text = html_diff_wrapper.normalize_text(merged_text)
#TODO merged_keywords + merged_description
version = context.get_version(
user, (context, 'version'))
for amendment in version.amendments:
amendment.state = PersistentList(['archived'])
amendment.reindex()
self._send_ballot_result(context, request,
result, members)
context.text = merged_text
related_ideas = [a.related_ideas for a in amendments]
related_ideas = [item for sublist in related_ideas
for item in sublist]
related_ideas.extend(context.related_ideas)
related_ideas = list(set(related_ideas))
context.set_related_ideas(related_ideas, user)
context.working_group.init_nonproductive_cycle()
context.reindex()
alert('internal', [root], members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context], alert_kind='amendments_result')
# Add Nia comment
alert_comment_nia(
context, request, root,
internal_kind=InternalAlertKind.working_group_alert,
subject_type='proposal',
alert_kind='new_version',
diff=diff_analytics(
version, context, ['title', 'text', 'description'])
)
else:
context.state = PersistentList(['amendable', 'published'])
alert('internal', [root], members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context])
for amendment in context.amendments:
amendment.state = PersistentList(['archived'])
amendment.reindex()
context.reindex()
return {'newcontext': context}
def start(self, context, request, appstruct, **kw):
user = get_current(request)
user = context.working_group.get_member(user)
correction = appstruct['_object_data']
correction.text = html_diff_wrapper.normalize_text(correction.text)
old_text = correction.text
correction.setproperty('author', user)
version = context.get_version(user, (context, 'version'))
context.addtoproperty('corrections', correction)
correction.setproperty('current_version', version)
context.setproperty('version', version.version)
souptextdiff, textdiff = html_diff_wrapper.render_html_diff(
getattr(context, 'text', ''),
getattr(correction, 'text', ''),
"correction")
soupdescriptiondiff, descriptiondiff = html_diff_wrapper.render_html_diff(
getattr(context, 'description', ''),
getattr(correction, 'description', ''),
"correction")
souptitlediff, titlediff = html_diff_wrapper.render_html_diff(
getattr(context, 'title', ''),
getattr(correction, 'title', ''),
"correction")
descriminator = 0
descriminator = self._identify_corrections(souptitlediff,
correction,
descriminator,
'title',
user)
self._add_actions(correction, request, souptitlediff)
descriminator = self._identify_corrections(soupdescriptiondiff,
correction,
descriminator,
'description',
user)
self._add_actions(correction, request, soupdescriptiondiff)
self._identify_corrections(souptextdiff, correction,
descriminator, 'text',
user)
self._add_actions(correction, request, souptextdiff)
correction.text = html_diff_wrapper.soup_to_text(souptextdiff)
correction.description = html_diff_wrapper.soup_to_text(soupdescriptiondiff)
correction.title = html_diff_wrapper.soup_to_text(souptitlediff)
if souptextdiff.find_all("span", id="correction") or \
soupdescriptiondiff.find_all("span", id="correction") or\
souptitlediff.find_all("span", id="correction"):
correction.state.append('in process')
alert('internal', [request.root], context.working_group.members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context], alert_kind='correction_added')
else:
context.text = old_text
return {}
def test_normalize_text(self):
text_origin = """<div class="myclass">\r\n<h1>My title</h1>\r\n <h1>My title 2</h1>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text_origin = html_diff_wrapper.normalize_text(text_origin)
self.assertEqual(
normalized_text_origin,
"""<div class="myclass"><h1>My title</h1><h1>My title 2</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p></div>"""
)
text1 = """<div class="myclass">\r\n<h1>My title</h1>\r\n <h1>My title 2 autre</h1><p></p>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>propositions</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text1 = html_diff_wrapper.normalize_text(text1)
self.assertEqual(
normalized_text1,
"""<div class="myclass"><h1>My title</h1><h1>My title 2 autre</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>propositions</strong> lors de la Fete.</p></div>"""
)
text2 = """<div class="myclass">\r\n<h1>My title</h1><p> </p>\r\n<p>Mon texte ici</p>\r\n<h1>My title 2</h1>\r\n<p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p>\r\n<h1>My title 3</h1>\r\n<p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p>\r\n</div>"""
normalized_text2 = html_diff_wrapper.normalize_text(text2)
self.assertEqual(
normalized_text2,
"""<div class="myclass"><h1>My title</h1><p>Mon texte ici</p><h1>My title 2</h1><p><span style="font-family: arial black,avant garde;">Organiser</span> des <strong>animation</strong> lors de la Fete <em>de la</em> science.</p><h1>My title 3</h1><p><span style="font-family: arial black,avant garde;">Publier</span> les <strong>idées</strong> lors de la Fete.</p></div>"""
)
def start(self, context, request, appstruct, **kw):
root = getSite()
user = get_current()
proposal = context.proposal
working_group = proposal.working_group
author = working_group.get_member(user)
context.set_data(appstruct)
context.text = html_diff_wrapper.normalize_text(context.text)
context.text_diff = get_text_amendment_diff(proposal, context)
context.modified_at = datetime.datetime.now(tz=pytz.UTC)
context.reindex()
request.registry.notify(ActivityExecuted(self, [context], author))
return {}
def start(self, context, request, appstruct, **kw):
user = get_current(request)
user = context.working_group.get_member(user)
correction = appstruct['_object_data']
correction.text = html_diff_wrapper.normalize_text(correction.text)
old_text = correction.text
correction.setproperty('author', user)
version = context.get_version(user, (context, 'version'))
context.addtoproperty('corrections', correction)
correction.setproperty('current_version', version)
context.setproperty('version', version.version)
souptextdiff, textdiff = html_diff_wrapper.render_html_diff(
getattr(context, 'text', ''), getattr(correction, 'text', ''),
"correction")
soupdescriptiondiff, descriptiondiff = html_diff_wrapper.render_html_diff(
getattr(context, 'description', ''),
getattr(correction, 'description', ''), "correction")
souptitlediff, titlediff = html_diff_wrapper.render_html_diff(
getattr(context, 'title', ''), getattr(correction, 'title', ''),
"correction")
descriminator = 0
descriminator = self._identify_corrections(souptitlediff, correction,
descriminator, 'title',
user)
self._add_actions(correction, request, souptitlediff)
descriminator = self._identify_corrections(soupdescriptiondiff,
correction, descriminator,
'description', user)
self._add_actions(correction, request, soupdescriptiondiff)
self._identify_corrections(souptextdiff, correction, descriminator,
'text', user)
self._add_actions(correction, request, souptextdiff)
correction.text = html_diff_wrapper.soup_to_text(souptextdiff)
correction.description = html_diff_wrapper.soup_to_text(
soupdescriptiondiff)
correction.title = html_diff_wrapper.soup_to_text(souptitlediff)
if souptextdiff.find_all("span", id="correction") or \
soupdescriptiondiff.find_all("span", id="correction") or\
souptitlediff.find_all("span", id="correction"):
correction.state.append('in process')
alert('internal', [request.root],
context.working_group.members,
internal_kind=InternalAlertKind.working_group_alert,
subjects=[context],
alert_kind='correction_added')
else:
context.text = old_text
return {}
def start(self, context, request, appstruct, **kw):
root = getSite()
user = get_current()
proposal = context.proposal
working_group = proposal.working_group
author = working_group.get_member(user)
context.set_data(appstruct)
context.text = html_diff_wrapper.normalize_text(context.text)
context.text_diff = get_text_amendment_diff(
proposal, context)
context.modified_at = datetime.datetime.now(tz=pytz.UTC)
context.reindex()
request.registry.notify(ActivityExecuted(
self, [context], author))
return {}
def deserialize(self, field, pstruct):
article = super(ArticleRichTextWidget, self).deserialize(field, pstruct)
if article is colander.null:
return colander.null
soup = BeautifulSoup(article, 'lxml')
clear_style_btns = soup.find_all('button', 'clear-style')
newparagraph_style_btns = soup.find_all('button', 'newparagraph-style')
for tag in clear_style_btns:
tag.extract()
for tag in newparagraph_style_btns:
tag.extract()
text = ''.join([str(t) for t in soup.body.contents])
return html_diff_wrapper.normalize_text(text, {redirect_links})
def deserialize(self, field, pstruct):
article = super(ArticleRichTextWidget,
self).deserialize(field, pstruct)
if article is colander.null:
return colander.null
soup = BeautifulSoup(article, 'lxml')
clear_style_btns = soup.find_all('button', 'clear-style')
newparagraph_style_btns = soup.find_all('button', 'newparagraph-style')
for tag in clear_style_btns:
tag.extract()
for tag in newparagraph_style_btns:
tag.extract()
text = ''.join([str(t) for t in soup.body.contents])
return html_diff_wrapper.normalize_text(text, {redirect_links})
def start(self, context, request, appstruct, **kw):
data = {}
user = get_current(request)
user = context.working_group.get_member(user)
localizer = request.localizer
data['title'] = localizer.translate(_('Amended version ')) + \
str(getattr(context, '_amendments_counter', 1))
data['text'] = html_diff_wrapper.normalize_text(appstruct['text'])
# data['description'] = appstruct['description']
# data['keywords'] = appstruct['keywords']
amendment = Amendment()
amendment.set_data(data)
context.addtoproperty('amendments', amendment)
amendment.state.append('draft')
grant_roles(user=user, roles=(('Owner', amendment), ))
amendment.setproperty('author', user)
amendment.text_diff = get_text_amendment_diff(context, amendment)
amendment.reindex()
context._amendments_counter = getattr(context, '_amendments_counter',
1) + 1
return {'newcontext': amendment}
def reviw_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IBaseReview
import html_diff_wrapper
contents = find_entities(interfaces=[IBaseReview])
len_entities = str(len(contents))
for index, review in enumerate(contents):
if getattr(review, 'article', None):
review.article = html_diff_wrapper.normalize_text(
review.article, {redirect_links})
review.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Review text evolved.')
def reviw_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IBaseReview
import html_diff_wrapper
contents = find_entities(interfaces=[IBaseReview])
len_entities = str(len(contents))
for index, review in enumerate(contents):
if getattr(review, 'article', None):
review.article = html_diff_wrapper.normalize_text(
review.article, {redirect_links})
review.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Review text evolved.')
def artist_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IArtistInformationSheet
import html_diff_wrapper
contents = find_entities(interfaces=[IArtistInformationSheet])
len_entities = str(len(contents))
for index, artist in enumerate(contents):
if getattr(artist, 'biography', None):
artist.biography = html_diff_wrapper.normalize_text(
artist.biography, {redirect_links})
artist.hash_artist_data()
artist.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Artists text evolved.')
def artist_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IArtistInformationSheet
import html_diff_wrapper
contents = find_entities(interfaces=[IArtistInformationSheet])
len_entities = str(len(contents))
for index, artist in enumerate(contents):
if getattr(artist, 'biography', None):
artist.biography = html_diff_wrapper.normalize_text(
artist.biography, {redirect_links})
artist.hash_artist_data()
artist.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Artists text evolved.')
def venue_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IVenue
import html_diff_wrapper
contents = find_entities(interfaces=[IVenue])
len_entities = str(len(contents))
for index, venue in enumerate(contents):
if getattr(venue, 'description', None):
venue.description = html_diff_wrapper.normalize_text(
venue.description, {redirect_links})
venue.hash_venue_data()
venue.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Venues text evolved.')
def start(self, context, request, appstruct, **kw):
data = {}
user = get_current(request)
user = context.working_group.get_member(user)
localizer = request.localizer
data['title'] = localizer.translate(_('Amended version ')) + \
str(getattr(context, '_amendments_counter', 1))
data['text'] = html_diff_wrapper.normalize_text(appstruct['text'])
# data['description'] = appstruct['description']
# data['keywords'] = appstruct['keywords']
amendment = Amendment()
amendment.set_data(data)
context.addtoproperty('amendments', amendment)
amendment.state.append('draft')
grant_roles(user=user, roles=(('Owner', amendment), ))
amendment.setproperty('author', user)
amendment.text_diff = get_text_amendment_diff(
context, amendment)
amendment.reindex()
context._amendments_counter = getattr(
context, '_amendments_counter', 1) + 1
return {'newcontext': amendment}
def _get_amendment_text(self, context, group):
items = [str(e['oid']) for e in group]
soup = BeautifulSoup(context.text_diff)
allexplanations = soup.find_all('span', {'id': 'explanation'})
explanations = [tag for tag in allexplanations
if tag['data-item'] not in items]
blocstodel = ('span', {'id': 'explanation_action'})
soup = html_diff_wrapper.include_diffs(
soup, explanations,
"ins", "del", blocstodel)
explanations = [tag for tag in allexplanations
if tag['data-item'] in items]
soup = html_diff_wrapper.include_diffs(
soup, explanations,
"del", "ins", blocstodel)
allmodal = [m for m in soup.find_all('div', {'class': 'modal'})
if m['id'].endswith("explanation_modal")]
for modal in allmodal:
modal.extract()
text = html_diff_wrapper.soup_to_text(soup)
return html_diff_wrapper.normalize_text(text)
def venue_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import IVenue
import html_diff_wrapper
contents = find_entities(interfaces=[IVenue])
len_entities = str(len(contents))
for index, venue in enumerate(contents):
if getattr(venue, 'description', None):
venue.description = html_diff_wrapper.normalize_text(
venue.description, {redirect_links})
venue.hash_venue_data()
venue.reindex()
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Venues text evolved.')
def event_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import ICulturalEvent
import html_diff_wrapper
contents = find_entities(interfaces=[ICulturalEvent])
len_entities = str(len(contents))
for index, event in enumerate(contents):
if getattr(event, 'details', None):
try:
event.details = html_diff_wrapper.normalize_text(
event.details, {redirect_links})
event.reindex()
except Exception as error:
log.warning(error)
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Event text evolved.')
def _get_amendment_text(self, context, group):
items = [str(e['oid']) for e in group]
soup = BeautifulSoup(context.text_diff)
allexplanations = soup.find_all('span', {'id': 'explanation'})
explanations = [
tag for tag in allexplanations if tag['data-item'] not in items
]
blocstodel = ('span', {'id': 'explanation_action'})
soup = html_diff_wrapper.include_diffs(soup, explanations, "ins",
"del", blocstodel)
explanations = [
tag for tag in allexplanations if tag['data-item'] in items
]
soup = html_diff_wrapper.include_diffs(soup, explanations, "del",
"ins", blocstodel)
allmodal = [
m for m in soup.find_all('div', {'class': 'modal'})
if m['id'].endswith("explanation_modal")
]
for modal in allmodal:
modal.extract()
text = html_diff_wrapper.soup_to_text(soup)
return html_diff_wrapper.normalize_text(text)
def event_normalize_text(root, registry):
from lac.views.filter import find_entities
from lac.views.widget import redirect_links
from lac.content.interface import ICulturalEvent
import html_diff_wrapper
contents = find_entities(interfaces=[ICulturalEvent])
len_entities = str(len(contents))
for index, event in enumerate(contents):
if getattr(event, 'details', None):
try:
event.details = html_diff_wrapper.normalize_text(
event.details, {redirect_links})
event.reindex()
except Exception as error:
log.warning(error)
if index % 1000 == 0:
log.info("**** Commit ****")
transaction.commit()
log.info(str(index) + "/" + len_entities)
log.info('Event text evolved.')
def deserialize(self, field, pstruct):
text = super(RichTextWidget, self).deserialize(field, pstruct)
if text is colander.null:
return colander.null
return html_diff_wrapper.normalize_text(text, {redirect_links})
def start(self, context, request, appstruct, **kw):
result = super(CrateAndPublishAsProposal, self).start(
context, request, appstruct, **kw)
root = getSite()
state = result.get('state', False)
if state:
idea = result.get('newcontext', None)
if idea:
user = get_current(request)
mask = user.get_mask(root) if hasattr(user, 'get_mask') else user
author = mask if appstruct.get('anonymous', False) and mask else user
idea.subscribe_to_channel(user)
related_ideas = [idea]
localizer = request.localizer
title = idea.title + \
localizer.translate(_(" (the proposal)"))
template = getattr(root, 'proposal_template', None)
text = '<p>{idea_text}</p>'
if template:
try:
text = template.fp.readall().decode()
except Exception as error:
log.warning(error)
proposal = Proposal(
title=title,
description=idea.text[:600],
text=text.format(
idea_text=idea.text.replace('\n', '<br/>')),
keywords=list(idea.keywords)
)
proposal.text = html_diff_wrapper.normalize_text(proposal.text)
root.addtoproperty('proposals', proposal)
proposal.state.append('draft')
grant_roles(user=author, roles=(('Owner', proposal), ))
grant_roles(user=author, roles=(('Participant', proposal), ))
proposal.setproperty('author', author)
challenge = idea.challenge
if challenge:
proposal.setproperty('challenge', challenge)
wg = WorkingGroup()
root.addtoproperty('working_groups', wg)
wg.init_workspace()
wg.setproperty('proposal', proposal)
wg.addtoproperty('members', author)
wg.state.append('deactivated')
if related_ideas:
connect(proposal,
related_ideas,
{'comment': _('Add related ideas'),
'type': _('Creation')},
author,
['related_proposals', 'related_ideas'],
CorrelationType.solid)
try:
files = {
'add_files': {
'attached_files': [{'_object_data': f.copy()} for
f in idea.attached_files]
}
}
add_attached_files(files, proposal)
except Exception:
pass
proposal.subscribe_to_channel(user)
proposal.reindex()
init_proposal_ballots(proposal)
wg.reindex()
request.registry.notify(
ActivityExecuted(self, [idea, proposal, wg], author))
return {'newcontext': proposal}
return {'newcontext': root}
def deserialize(self, field, pstruct):
text = super(RichTextWidget, self).deserialize(field, pstruct)
if text is colander.null:
return colander.null
return html_diff_wrapper.normalize_text(text, {redirect_links})
