def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('announce', sa.Column('title_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
op.add_column('announce', sa.Column('body_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {d.id: LanguageIdentificationService(d) for d in ds
if len(locales_of_discussion[d.id]) > 1}
announcement_strings = db.execute(
"SELECT id, title, body FROM announce")
announcement_strings = {id: (title, body)
for (id, title, body) in announcement_strings}
for announcement in db.query(m.Announcement):
candidate_langs = locales_of_discussion[announcement.discussion_id]
(title, body) = announcement_strings[announcement.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(filter(None, (
title or '',
sanitize_text(body or ''))))
lang = None
if text:
# Use idea language for priors?
lang, data = langid_services[announcement.discussion_id].identify(text)
if not lang:
print "***** Could not identify for announcement %d: %s" % (announcement.id, text)
lang = candidate_langs[0]
def as_lang_string(text):
ls = m.LangString.create(text, lang)
return ls
if title:
announcement.title = as_lang_string(title)
if body:
announcement.body = as_lang_string(body)
with context.begin_transaction():
op.drop_column('announce', 'title')
op.drop_column('announce', 'body')
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('announce', sa.Column('title_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
op.add_column('announce', sa.Column('body_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {d.id: LanguageIdentificationService(d) for d in ds
if len(locales_of_discussion[d.id]) > 1}
announcement_strings = db.execute(
"SELECT id, title, body FROM announce")
announcement_strings = {id: (title, body)
for (id, title, body) in announcement_strings}
for announcement in db.query(m.Announcement):
candidate_langs = locales_of_discussion[announcement.discussion_id]
(title, body) = announcement_strings[announcement.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(filter(None, (
title or '',
sanitize_text(body or ''))))
lang = None
if text:
# Use idea language for priors?
lang, data = langid_services[announcement.discussion_id].identify(text)
if not lang:
print("***** Could not identify for announcement %d: %s" % (announcement.id, text))
lang = candidate_langs[0]
def as_lang_string(text):
ls = m.LangString.create(text, lang)
return ls
if title:
announcement.title = as_lang_string(title)
if body:
announcement.body = as_lang_string(body)
with context.begin_transaction():
op.drop_column('announce', 'title')
op.drop_column('announce', 'body')
def upgrade(pyramid_env):
# Do stuff with the app's models here.
from assembl import models as m
from assembl.lib.clean_input import sanitize_html, sanitize_text
db = m.get_session_maker()()
with transaction.manager:
# sanitize body of assemblposts (not imported posts)
for lse in db.query(m.LangStringEntry
).join(m.AssemblPost,
m.Content.body_id == m.LangStringEntry.langstring_id
).filter(m.LangStringEntry.value.like('%<%')):
lse.value = sanitize_html(lse.value)
# sanitize subject of all posts
for lse in db.query(m.LangStringEntry
).join(m.Content,
m.Content.subject_id == m.LangStringEntry.langstring_id
).filter(m.LangStringEntry.value.like('%<%')):
lse.value = sanitize_text(lse.value)
def upgrade(pyramid_env):
# Do stuff with the app's models here.
from assembl import models as m
from assembl.lib.clean_input import sanitize_html, sanitize_text
db = m.get_session_maker()()
with transaction.manager:
# sanitize body of assemblposts (not imported posts)
for lse in db.query(m.LangStringEntry
).join(m.AssemblPost,
m.Content.body_id == m.LangStringEntry.langstring_id
).filter(m.LangStringEntry.value.like('%<%')):
lse.value = sanitize_html(lse.value)
# sanitize subject of all posts
for lse in db.query(m.LangStringEntry
).join(m.Content,
m.Content.subject_id == m.LangStringEntry.langstring_id
).filter(m.LangStringEntry.value.like('%<%')):
lse.value = sanitize_text(lse.value)
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
'idea',
sa.Column('synthesis_title_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
op.add_column(
'idea',
sa.Column('title_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
op.add_column(
'idea',
sa.Column('description_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {
d.id: LanguageIdentificationService(d)
for d in ds if len(locales_of_discussion[d.id]) > 1
}
idea_strings = db.execute(
"SELECT id, short_title, long_title, definition FROM idea")
idea_strings = {
id: (short_title, long_title, definition)
for (id, short_title, long_title, definition) in idea_strings
}
languages = {}
parents = dict(
list(
db.execute("SELECT target_id, source_id FROM idea_idea_link")))
for idea in db.query(m.Idea).order_by(
m.Idea.base_id, m.Idea.tombstone_date.nullsfirst()):
candidate_langs = locales_of_discussion[idea.discussion_id]
(short_title, long_title, definition) = idea_strings[idea.id]
if idea.tombstone_date is None:
live_idea = idea
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(
filter(None,
(short_title or '', sanitize_text(long_title or ''),
sanitize_text(definition or ''))))
lang = None
if text:
parent_lang = languages.get(parents.get(idea.id, None),
None)
if parent_lang:
priors = {
locale: 1 if locale == parent_lang else .1
for locale in candidate_langs
}
else:
priors = {locale: 1 for locale in candidate_langs}
lang, data = langid_services[idea.discussion_id].identify(
text, expected_locales=priors)
if not lang:
print("***** Could not identify for idea %d: %s" %
(idea.id, text))
lang = candidate_langs[0]
languages[idea.id] = lang
def as_lang_string(text, based_on):
if idea.tombstone_date is None or based_on is None:
ls = m.LangString.create(text, lang)
ls.tombstone_date = idea.tombstone_date
return ls
else:
db.add(
m.LangStringEntry(langstring=based_on,
value=text,
locale=lang,
tombstone_date=idea.tombstone_date))
return based_on
if idea.title_id:
if short_title:
assert short_title in [
lse.value for lse in idea.title.entries
]
elif short_title:
idea.title = as_lang_string(short_title, live_idea.title)
if idea.description_id:
if idea.definition:
assert idea.definition in [
lse.value for lse in idea.description.entries
]
elif definition:
idea.description = as_lang_string(definition,
live_idea.description)
if long_title:
idea.synthesis_title = as_lang_string(long_title,
None) # do not reuse
with context.begin_transaction():
op.drop_column('idea', 'long_title')
op.drop_column('idea', 'short_title')
op.drop_column('idea', 'definition')
def create_post(request):
"""
Create a new post in this discussion.
We use post, not put, because we don't know the id of the post
"""
localizer = request.localizer
request_body = json.loads(request.body)
user_id = request.authenticated_userid
if not user_id:
raise HTTPUnauthorized()
user = Post.default_db.query(User).filter_by(id=user_id).one()
body = request_body.get('body', None)
html = request_body.get('html',
None) # BG: Is this used now? I cannot see it.
reply_id = request_body.get('reply_id', None)
idea_id = request_body.get('idea_id', None)
subject = request_body.get('subject', None)
publishes_synthesis_id = request_body.get('publishes_synthesis_id', None)
message_classifier = request_body.get('message_classifier', None)
if not body and not publishes_synthesis_id:
# Should we allow empty messages otherwise?
raise HTTPBadRequest(localizer.translate(_("Your message is empty")))
if reply_id:
in_reply_to_post = Post.get_instance(reply_id)
else:
in_reply_to_post = None
if idea_id:
in_reply_to_idea = Idea.get_instance(idea_id)
else:
in_reply_to_idea = None
discussion_id = int(request.matchdict['discussion_id'])
discussion = Discussion.get_instance(discussion_id)
if not discussion:
raise HTTPNotFound(
localizer.translate(_("No discussion found with id=%s")) %
(discussion_id, ))
ctx = DummyContext({Discussion: discussion})
if html:
log.warning("Still using html")
# how to guess locale in this case?
body = LangString.create(sanitize_html(html))
# TODO: AssemblPosts are pure text right now.
# Allowing HTML requires changes to the model.
elif body:
# TODO: Accept HTML body.
for e in body['entries']:
e['value'] = sanitize_text(e['value'])
body = LangString.create_from_json(body, context=ctx, user_id=user_id)
else:
body = LangString.EMPTY(discussion.db)
if subject:
for e in subject['entries']:
e['value'] = sanitize_text(e['value'])
subject = LangString.create_from_json(subject,
context=ctx,
user_id=user_id)
else:
from assembl.models import Locale
locale = Locale.UNDEFINED
# print(in_reply_to_post.subject, discussion.topic)
if in_reply_to_post and in_reply_to_post.get_title():
original_subject = in_reply_to_post.get_title().first_original()
if original_subject:
locale = original_subject.locale_code
subject = (original_subject.value or ''
if in_reply_to_post.get_title() else '')
elif in_reply_to_idea:
# TODO: THis should use a cascade like the frontend
# also, some ideas have extra langstring titles
subject = (in_reply_to_idea.short_title
if in_reply_to_idea.short_title else '')
locale = discussion.main_locale
else:
subject = discussion.topic if discussion.topic else ''
locale = discussion.main_locale
# print subject
if subject is not None and len(subject):
new_subject = "Re: " + restrip_pat.sub('', subject).strip()
if (in_reply_to_post and new_subject == subject
and in_reply_to_post.get_title()):
# reuse subject and translations
subject = in_reply_to_post.get_title().clone(discussion.db)
else:
# how to guess locale in this case?
subject = LangString.create(new_subject, locale)
else:
capture_message(
"A message is about to be written to the database with an "
"empty subject. This is not supposed to happen.")
subject = LangString.EMPTY(discussion.db)
post_constructor_args = {
'discussion': discussion,
'creator_id': user_id,
'message_classifier': message_classifier,
'subject': subject,
'body': body
}
if publishes_synthesis_id:
published_synthesis = Synthesis.get_instance(publishes_synthesis_id)
post_constructor_args['publishes_synthesis'] = published_synthesis
new_post = SynthesisPost(**post_constructor_args)
new_post.finalize_publish()
else:
new_post = AssemblPost(**post_constructor_args)
new_post.guess_languages()
discussion.db.add(new_post)
discussion.db.flush()
if in_reply_to_post:
new_post.set_parent(in_reply_to_post)
if in_reply_to_idea:
idea_post_link = IdeaRelatedPostLink(creator_id=user_id,
content=new_post,
idea=in_reply_to_idea)
discussion.db.add(idea_post_link)
idea = in_reply_to_idea
while idea:
idea.send_to_changes()
parents = idea.get_parents()
idea = next(iter(parents)) if parents else None
else:
discussion.root_idea.send_to_changes()
for source in discussion.sources:
if 'send_post' in dir(source):
source.send_post(new_post)
permissions = get_permissions(user_id, discussion_id)
return new_post.generic_json('default', user_id, permissions)
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column(
'synthesis',
sa.Column('subject_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
op.add_column(
'synthesis',
sa.Column('introduction_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
op.add_column(
'synthesis',
sa.Column('conclusion_id', sa.Integer(),
sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {
d.id: LanguageIdentificationService(d)
for d in ds if len(locales_of_discussion[d.id]) > 1
}
synthesis_strings = db.execute(
"SELECT id, subject, introduction, conclusion FROM synthesis")
synthesis_strings = {
id: (subject, introduction, conclusion)
for (id, subject, introduction, conclusion) in synthesis_strings
}
for synthesis in db.query(m.Synthesis):
candidate_langs = locales_of_discussion[synthesis.discussion_id]
(subject, introduction,
conclusion) = synthesis_strings[synthesis.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(
filter(None, (
sanitize_text(subject or ''),
sanitize_text(introduction or ''),
sanitize_text(conclusion or ''),
)))
lang = None
if text:
lang, data = langid_services[
synthesis.discussion_id].identify(text)
if not lang:
print("***** Could not identify for synthesis %d: %s" %
(synthesis.id, text))
lang = candidate_langs[0]
def as_lang_string(text):
ls = m.LangString.create(text, lang)
return ls
if subject:
synthesis.subject = as_lang_string(subject)
if introduction:
synthesis.introduction = as_lang_string(introduction)
if conclusion:
synthesis.conclusion = as_lang_string(conclusion)
with context.begin_transaction():
op.drop_column('synthesis', 'subject')
op.drop_column('synthesis', 'introduction')
op.drop_column('synthesis', 'conclusion')
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('idea', sa.Column('synthesis_title_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {d.id: LanguageIdentificationService(d) for d in ds
if len(locales_of_discussion[d.id]) > 1}
idea_strings = db.execute(
"SELECT id, short_title, long_title, definition FROM idea")
idea_strings = {id: (short_title, long_title, definition)
for (id, short_title, long_title, definition) in idea_strings}
languages = {}
parents = dict(list(db.execute(
"SELECT target_id, source_id FROM idea_idea_link")))
for idea in db.query(m.Idea):
candidate_langs = locales_of_discussion[idea.discussion_id]
(short_title, long_title, definition) = idea_strings[idea.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(filter(None, (
short_title or '',
sanitize_text(long_title or ''),
sanitize_text(definition or ''))))
lang = None
if text:
parent_lang = languages.get(parents.get(idea.id, None), None)
if parent_lang:
priors = {locale: 1 if locale == parent_lang else .1
for locale in candidate_langs}
else:
priors = {locale: 1 for locale in candidate_langs}
lang, data = langid_services[idea.discussion_id].identify(text, expected_locales=priors)
if not lang:
print "***** Could not identify for idea %d: %s" % (idea.id, text)
lang = candidate_langs[0]
languages[idea.id] = lang
def as_lang_string(text):
ls = m.LangString.create(text, lang)
ls.tombstone_date = idea.tombstone_date
return ls
if idea.title_id:
if short_title:
assert short_title in [lse.value for lse in idea.title.entries]
elif short_title:
idea.title = as_lang_string(short_title)
if idea.description_id:
if idea.definition:
assert idea.definition in [lse.value for lse in idea.description.entries]
elif definition:
idea.description = as_lang_string(definition)
if long_title:
idea.synthesis_title = as_lang_string(long_title)
with context.begin_transaction():
op.drop_column('idea', 'long_title')
op.drop_column('idea', 'short_title')
op.drop_column('idea', 'definition')
def mutate(root, args, context, info):
EMBED_ATTACHMENT = models.AttachmentPurpose.EMBED_ATTACHMENT.value
discussion_id = context.matchdict['discussion_id']
user_id = context.authenticated_userid or Everyone
discussion = models.Discussion.get(discussion_id)
post_id = args.get('post_id')
post_id = int(Node.from_global_id(post_id)[1])
post = models.Post.get(post_id)
cls = models.Post
permissions = get_permissions(user_id, discussion_id)
allowed = post.user_can(user_id, CrudPermissions.UPDATE, permissions)
if not allowed:
raise HTTPUnauthorized()
changed = False
subject = args.get('subject')
if subject:
subject = sanitize_text(subject)
body = args.get('body')
if body:
body = sanitize_html(body)
# TODO: Here, an assumption that the modification uses the same
# language as the original. May need revisiting.
original_subject_entry = post.subject.first_original()
# subject is not required, be careful to not remove it if not specified
if subject and subject != original_subject_entry.value:
changed = True
post.subject.add_value(subject, original_subject_entry.locale_code)
# Edit subject for all descendants
children = post.children[:]
new_subject = u'Re: ' + restrip_pat.sub('', subject).strip()
while children:
child = children.pop()
children.extend(child.children)
child.subject.add_value(
new_subject,
child.subject.first_original().locale_code)
original_body_entry = post.body.first_original()
if body != original_body_entry.value:
post.body.add_value(body, original_body_entry.locale_code)
changed = True
original_attachments = post.attachments
original_attachments_doc_ids = []
if original_attachments:
original_attachments_doc_ids = [
str(a.document_id) for a in original_attachments
]
attachments = args.get('attachments', [])
for document_id in attachments:
if document_id not in original_attachments_doc_ids:
document = models.Document.get(document_id)
models.PostAttachment(
document=document,
discussion=discussion,
creator_id=context.authenticated_userid,
post=post,
title=document.title,
attachmentPurpose=EMBED_ATTACHMENT)
# delete attachments that has been removed
documents_to_delete = set(original_attachments_doc_ids) - set(
attachments) # noqa: E501
for document_id in documents_to_delete:
with cls.default_db.no_autoflush:
document = models.Document.get(document_id)
post_attachment = post.db.query(
models.PostAttachment).filter_by(
discussion_id=discussion_id,
post_id=post_id,
document_id=document_id).first()
document.delete_file()
post.db.delete(document)
post.attachments.remove(post_attachment)
post.db.flush()
publication_state = models.PublicationStates.from_string(
args.get('publication_state')) if args.get(
'publication_state') in models.PublicationStates.values(
) else None
if publication_state and publication_state != post.publication_state:
post.publication_state = publication_state
changed = True
# Update the creation date when switching from draft to published
if post.publication_state == models.PublicationStates.DRAFT and publication_state == models.PublicationStates.PUBLISHED:
post.creation_date = datetime.utcnow()
if changed:
post.modification_date = datetime.utcnow()
post.body_mime_type = u'text/html'
post.db.flush()
post.db.expire(post.subject, ["entries"])
post.db.expire(post.body, ["entries"])
return UpdatePost(post=post)
def mutate(root, args, context, info):
EMBED_ATTACHMENT = models.AttachmentPurpose.EMBED_ATTACHMENT.value
discussion_id = context.matchdict['discussion_id']
user_id = context.authenticated_userid or Everyone
discussion = models.Discussion.get(discussion_id)
idea_id = args.get('idea_id')
idea_id = int(Node.from_global_id(idea_id)[1])
in_reply_to_idea = models.Idea.get(idea_id)
if isinstance(in_reply_to_idea, models.Question):
cls = models.PropositionPost
else:
cls = models.AssemblPost
extract_id = args.get('extract_id')
if extract_id:
extract_id_global = int(Node.from_global_id(extract_id)[1])
extract = models.Extract.get(extract_id_global)
cls = models.ExtractComment
in_reply_to_post = None
if (cls == models.AssemblPost) or (cls == models.ExtractComment):
in_reply_to_post_id = args.get('parent_id')
if in_reply_to_post_id:
in_reply_to_post_id = int(
Node.from_global_id(in_reply_to_post_id)[1])
if in_reply_to_post_id:
in_reply_to_post = models.Post.get(in_reply_to_post_id)
permissions = get_permissions(user_id, discussion_id)
allowed = cls.user_can_cls(user_id, CrudPermissions.CREATE,
permissions)
if not allowed:
raise HTTPUnauthorized()
with cls.default_db.no_autoflush:
subject = args.get('subject')
body = args.get('body')
classifier = args.get('message_classifier', None)
body = sanitize_html(body)
body_langstring = models.LangString.create(body)
publication_state = models.PublicationStates.from_string(
args.get('publication_state')) if args.get(
'publication_state') in models.PublicationStates.values(
) else models.PublicationStates.PUBLISHED
if subject:
subject = sanitize_text(subject)
subject_langstring = models.LangString.create(subject)
elif issubclass(cls, models.PropositionPost):
# Specific case first. Respect inheritance. Since we are using
# a specific value, construct it with localization machinery.
subject_langstring = models.LangString.create_localized_langstring( # noqa: E501
_('Proposal'), discussion.discussion_locales,
{'fr': 'Proposition'})
else:
# We apply the same logic than in views/api/post.py::create_post # noqa: E501
locale = models.Locale.UNDEFINED
if in_reply_to_post and in_reply_to_post.get_title():
original_subject = in_reply_to_post.get_title(
).first_original()
locale = original_subject.locale_code
subject = original_subject.value
elif in_reply_to_idea:
# TODO: some ideas have extra langstring titles
# we try to guess the locale of the body to use the same locale for post's subject
body_lang, data = discussion.translation_service(
).identify(body_langstring.entries[0].value,
discussion.discussion_locales)
closest_subject = in_reply_to_idea.title.closest_entry(
body_lang)
if closest_subject:
subject = closest_subject.value
locale = closest_subject.locale.code
else:
# rather no subject than one in a random locale
subject = u''
locale = discussion.main_locale
else:
subject = discussion.topic if discussion.topic else ''
locale = discussion.main_locale
if subject is not None:
if in_reply_to_idea and in_reply_to_idea.message_view_override == u'messageColumns':
new_subject = subject
else:
new_subject = u'Re: ' + restrip_pat.sub(
'', subject).strip() # noqa: E501
if (in_reply_to_post and new_subject == subject
and in_reply_to_post.get_title()):
# reuse subject and translations
subject_langstring = in_reply_to_post.get_title(
).clone(discussion.db)
else:
subject_langstring = models.LangString.create(
new_subject, locale)
if cls == models.ExtractComment:
new_post = cls(discussion=discussion,
subject=subject_langstring,
body=body_langstring,
creator_id=user_id,
body_mime_type=u'text/html',
message_classifier=classifier,
creation_date=datetime.utcnow(),
publication_state=publication_state,
parent_extract_id=extract.id)
else:
new_post = cls(discussion=discussion,
subject=subject_langstring,
body=body_langstring,
creator_id=user_id,
body_mime_type=u'text/html',
message_classifier=classifier,
creation_date=datetime.utcnow(),
publication_state=publication_state)
new_post.guess_languages()
db = new_post.db
db.add(new_post)
db.flush()
if in_reply_to_post:
new_post.set_parent(in_reply_to_post)
elif in_reply_to_idea and cls != models.ExtractComment:
# don't create IdeaRelatedPostLink when we have both
# in_reply_to_post and in_reply_to_idea or if it's a comment
# for an extract
idea_post_link = models.IdeaRelatedPostLink(
creator_id=user_id,
content=new_post,
idea=in_reply_to_idea)
db.add(idea_post_link)
db.flush()
attachments = args.get('attachments', [])
for document_id in attachments:
document = models.Document.get(document_id)
models.PostAttachment(document=document,
discussion=discussion,
creator_id=context.authenticated_userid,
post=new_post,
title=document.title,
attachmentPurpose=EMBED_ATTACHMENT)
db.flush()
return CreatePost(post=new_post)
def create_post(request):
"""
Create a new post in this discussion.
We use post, not put, because we don't know the id of the post
"""
localizer = request.localizer
request_body = json.loads(request.body)
user_id = request.authenticated_userid
if not user_id:
raise HTTPUnauthorized()
user = Post.default_db.query(User).filter_by(id=user_id).one()
body = request_body.get('body', None)
html = request_body.get('html', None) # BG: Is this used now? I cannot see it.
reply_id = request_body.get('reply_id', None)
idea_id = request_body.get('idea_id', None)
subject = request_body.get('subject', None)
publishes_synthesis_id = request_body.get('publishes_synthesis_id', None)
message_classifier = request_body.get('message_classifier', None)
if not body and not publishes_synthesis_id:
# Should we allow empty messages otherwise?
raise HTTPBadRequest(localizer.translate(
_("Your message is empty")))
if reply_id:
in_reply_to_post = Post.get_instance(reply_id)
else:
in_reply_to_post = None
if idea_id:
in_reply_to_idea = Idea.get_instance(idea_id)
else:
in_reply_to_idea = None
discussion_id = int(request.matchdict['discussion_id'])
discussion = Discussion.get_instance(discussion_id)
if not discussion:
raise HTTPNotFound(localizer.translate(_(
"No discussion found with id=%s")) % (discussion_id,)
)
ctx = DummyContext({Discussion: discussion})
if html:
log.warning("Still using html")
# how to guess locale in this case?
body = LangString.create(sanitize_html(html))
# TODO: AssemblPosts are pure text right now.
# Allowing HTML requires changes to the model.
elif body:
# TODO: Accept HTML body.
for e in body['entries']:
e['value'] = sanitize_text(e['value'])
body = LangString.create_from_json(
body, context=ctx, user_id=user_id)
else:
body = LangString.EMPTY(discussion.db)
if subject:
for e in subject['entries']:
e['value'] = sanitize_text(e['value'])
subject = LangString.create_from_json(
subject, context=ctx, user_id=user_id)
else:
from assembl.models import Locale
locale = Locale.UNDEFINED
# print(in_reply_to_post.subject, discussion.topic)
if in_reply_to_post and in_reply_to_post.get_title():
original_subject = in_reply_to_post.get_title().first_original()
if original_subject:
locale = original_subject.locale_code
subject = (
original_subject.value or ''
if in_reply_to_post.get_title() else '')
elif in_reply_to_idea:
# TODO: THis should use a cascade like the frontend
# also, some ideas have extra langstring titles
subject = (in_reply_to_idea.short_title
if in_reply_to_idea.short_title else '')
locale = discussion.main_locale
else:
subject = discussion.topic if discussion.topic else ''
locale = discussion.main_locale
# print subject
if subject is not None and len(subject):
new_subject = "Re: " + restrip_pat.sub('', subject).strip()
if (in_reply_to_post and new_subject == subject and
in_reply_to_post.get_title()):
# reuse subject and translations
subject = in_reply_to_post.get_title().clone(discussion.db)
else:
# how to guess locale in this case?
subject = LangString.create(new_subject, locale)
else:
capture_message(
"A message is about to be written to the database with an "
"empty subject. This is not supposed to happen.")
subject = LangString.EMPTY(discussion.db)
post_constructor_args = {
'discussion': discussion,
'creator_id': user_id,
'message_classifier': message_classifier,
'subject': subject,
'body': body
}
if publishes_synthesis_id:
published_synthesis = Synthesis.get_instance(publishes_synthesis_id)
post_constructor_args['publishes_synthesis'] = published_synthesis
new_post = SynthesisPost(**post_constructor_args)
new_post.finalize_publish()
else:
new_post = AssemblPost(**post_constructor_args)
new_post.guess_languages()
discussion.db.add(new_post)
discussion.db.flush()
if in_reply_to_post:
new_post.set_parent(in_reply_to_post)
if in_reply_to_idea:
idea_post_link = IdeaRelatedPostLink(
creator_id=user_id,
content=new_post,
idea=in_reply_to_idea
)
discussion.db.add(idea_post_link)
idea = in_reply_to_idea
while idea:
idea.send_to_changes()
parents = idea.get_parents()
idea = next(iter(parents)) if parents else None
else:
discussion.root_idea.send_to_changes()
for source in discussion.sources:
if 'send_post' in dir(source):
source.send_post(new_post)
permissions = get_permissions(user_id, discussion_id)
return new_post.generic_json('default', user_id, permissions)
def mutate(root, args, context, info):
EMBED_ATTACHMENT = models.AttachmentPurpose.EMBED_ATTACHMENT.value
user_id = context.authenticated_userid or Everyone
discussion_id = context.matchdict['discussion_id']
discussion = models.Discussion.get(discussion_id)
post_id = args.get('post_id')
post_id = int(Node.from_global_id(post_id)[1])
post = models.Post.get(post_id)
cls = models.Post
permissions = get_permissions(user_id, discussion_id)
allowed = post.user_can(user_id, CrudPermissions.UPDATE, permissions)
if not allowed:
raise HTTPUnauthorized()
if (post.publication_state == models.PublicationStates.PUBLISHED and
P_MODERATE not in permissions and
discussion.preferences['with_moderation']):
raise HTTPUnauthorized()
changed = False
subject = args.get('subject')
if subject:
subject = sanitize_text(subject)
body = args.get('body')
if body:
body = sanitize_html(body)
# TODO: Here, an assumption that the modification uses the same
# language as the original. May need revisiting.
original_subject_entry = post.subject.first_original()
# subject is not required, be careful to not remove it if not specified
if subject and subject != original_subject_entry.value:
changed = True
post.subject.add_value(subject, original_subject_entry.locale_code)
# Edit subject for all descendants
children = post.children[:]
new_subject = u'Re: ' + restrip_pat.sub('', subject).strip()
while children:
child = children.pop()
children.extend(child.children)
child.subject.add_value(
new_subject, child.subject.first_original().locale_code)
original_body_entry = post.body.first_original()
if body != original_body_entry.value:
post.body.add_value(body, original_body_entry.locale_code)
changed = True
original_attachments = post.attachments
original_attachments_doc_ids = []
if original_attachments:
original_attachments_doc_ids = [
str(a.document_id) for a in original_attachments]
attachments = args.get('attachments', [])
for document_id in attachments:
if document_id not in original_attachments_doc_ids:
document = models.Document.get(document_id)
models.PostAttachment(
document=document,
discussion=discussion,
creator_id=context.authenticated_userid,
post=post,
title=document.title,
attachmentPurpose=EMBED_ATTACHMENT
)
# delete attachments that has been removed
documents_to_delete = set(original_attachments_doc_ids) - set(attachments) # noqa: E501
for document_id in documents_to_delete:
with cls.default_db.no_autoflush:
document = models.Document.get(document_id)
post_attachment = post.db.query(
models.PostAttachment
).filter_by(
discussion_id=discussion_id, post_id=post_id,
document_id=document_id
).first()
document.delete_file()
post.db.delete(document)
post.attachments.remove(post_attachment)
post.db.flush()
publication_state = models.PublicationStates.from_string(args.get('publication_state')) if args.get('publication_state') in models.PublicationStates.values() else None
if publication_state and publication_state != post.publication_state:
post.publication_state = publication_state
changed = True
# Update the creation date when switching from draft to published
if post.publication_state == models.PublicationStates.DRAFT and publication_state == models.PublicationStates.PUBLISHED:
post.creation_date = datetime.utcnow()
if changed:
post.modification_date = datetime.utcnow()
post.body_mime_type = u'text/html'
post.db.flush()
post.db.expire(post.subject, ["entries"])
post.db.expire(post.body, ["entries"])
return UpdatePost(post=post)
def mutate(root, args, context, info):
EMBED_ATTACHMENT = models.AttachmentPurpose.EMBED_ATTACHMENT.value
discussion_id = context.matchdict['discussion_id']
user_id = context.authenticated_userid or Everyone
discussion = models.Discussion.get(discussion_id)
idea_id = args.get('idea_id')
idea_id = int(Node.from_global_id(idea_id)[1])
in_reply_to_idea = models.Idea.get(idea_id)
if isinstance(in_reply_to_idea, models.Question):
cls = models.PropositionPost
else:
cls = models.AssemblPost
extract_id = args.get('extract_id')
if extract_id:
extract_id_global = int(Node.from_global_id(extract_id)[1])
extract = models.Extract.get(extract_id_global)
cls = models.ExtractComment
in_reply_to_post = None
if (cls == models.AssemblPost) or (cls == models.ExtractComment):
in_reply_to_post_id = args.get('parent_id')
if in_reply_to_post_id:
in_reply_to_post_id = int(
Node.from_global_id(in_reply_to_post_id)[1])
if in_reply_to_post_id:
in_reply_to_post = models.Post.get(in_reply_to_post_id)
require_cls_permission(CrudPermissions.CREATE, cls, context)
with cls.default_db.no_autoflush:
subject = args.get('subject')
body = args.get('body')
classifier = args.get('message_classifier', None)
body = sanitize_html(body)
body_langstring = models.LangString.create(body)
publication_state = models.PublicationStates.from_string(args.get('publication_state')) if args.get('publication_state') in models.PublicationStates.values() else models.PublicationStates.PUBLISHED
if subject:
subject = sanitize_text(subject)
subject_langstring = models.LangString.create(subject)
elif issubclass(cls, models.PropositionPost):
# Specific case first. Respect inheritance. Since we are using
# a specific value, construct it with localization machinery.
subject_langstring = models.LangString.create_localized_langstring( # noqa: E501
_('Proposal'),
discussion.discussion_locales, {'fr': 'Proposition'})
else:
# We apply the same logic than in views/api/post.py::create_post # noqa: E501
locale = models.Locale.UNDEFINED
if in_reply_to_post and in_reply_to_post.get_title():
original_subject = in_reply_to_post.get_title().first_original()
locale = original_subject.locale_code
subject = original_subject.value
elif in_reply_to_idea:
# TODO: some ideas have extra langstring titles
# we try to guess the locale of the body to use the same locale for post's subject
body_lang, data = discussion.translation_service().identify(
body_langstring.entries[0].value,
discussion.discussion_locales)
closest_subject = in_reply_to_idea.title.closest_entry(body_lang)
if closest_subject:
subject = closest_subject.value
locale = closest_subject.locale.code
else:
# rather no subject than one in a random locale
subject = u''
locale = discussion.main_locale
else:
subject = discussion.topic if discussion.topic else ''
locale = discussion.main_locale
if subject is not None:
if in_reply_to_idea and in_reply_to_idea.message_view_override == u'messageColumns':
new_subject = subject
else:
new_subject = u'Re: ' + restrip_pat.sub('', subject).strip() # noqa: E501
if (in_reply_to_post and new_subject == subject and
in_reply_to_post.get_title()):
# reuse subject and translations
subject_langstring = in_reply_to_post.get_title().clone(discussion.db)
else:
subject_langstring = models.LangString.create(
new_subject, locale)
if cls == models.ExtractComment:
new_post = cls(
discussion=discussion,
subject=subject_langstring,
body=body_langstring,
creator_id=user_id,
body_mime_type=u'text/html',
message_classifier=classifier,
creation_date=datetime.utcnow(),
publication_state=publication_state,
parent_extract_id=extract.id
)
else:
new_post = cls(
discussion=discussion,
subject=subject_langstring,
body=body_langstring,
creator_id=user_id,
body_mime_type=u'text/html',
message_classifier=classifier,
creation_date=datetime.utcnow(),
publication_state=publication_state
)
new_post.guess_languages()
db = new_post.db
db.add(new_post)
db.flush()
if in_reply_to_post:
new_post.set_parent(in_reply_to_post)
elif in_reply_to_idea and cls != models.ExtractComment:
# don't create IdeaRelatedPostLink when we have both
# in_reply_to_post and in_reply_to_idea or if it's a comment
# for an extract
idea_post_link = models.IdeaRelatedPostLink(
creator_id=user_id,
content=new_post,
idea=in_reply_to_idea
)
db.add(idea_post_link)
db.flush()
attachments = args.get('attachments', [])
for document_id in attachments:
document = models.Document.get(document_id)
models.PostAttachment(
document=document,
discussion=discussion,
creator_id=context.authenticated_userid,
post=new_post,
title=document.title,
attachmentPurpose=EMBED_ATTACHMENT
)
db.flush()
return CreatePost(post=new_post)
def upgrade(pyramid_env):
with context.begin_transaction():
op.add_column('synthesis', sa.Column('subject_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
op.add_column('synthesis', sa.Column('introduction_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
op.add_column('synthesis', sa.Column('conclusion_id',
sa.Integer(), sa.ForeignKey('langstring.id')))
# Do stuff with the app's models here.
from assembl import models as m
from assembl.nlp.translation_service import LanguageIdentificationService
db = m.get_session_maker()()
# Disable idea reindexation
from assembl.lib.sqla import BaseOps, orm_update_listener
if sa.event.contains(BaseOps, 'after_update', orm_update_listener):
sa.event.remove(BaseOps, 'after_update', orm_update_listener)
with transaction.manager:
ds = db.query(m.Discussion).all()
locales_of_discussion = {d.id: d.discussion_locales for d in ds}
langid_services = {d.id: LanguageIdentificationService(d) for d in ds
if len(locales_of_discussion[d.id]) > 1}
synthesis_strings = db.execute(
"SELECT id, subject, introduction, conclusion FROM synthesis")
synthesis_strings = {id: (subject, introduction, conclusion)
for (id, subject, introduction, conclusion) in synthesis_strings}
for synthesis in db.query(m.Synthesis):
candidate_langs = locales_of_discussion[synthesis.discussion_id]
(subject, introduction, conclusion) = synthesis_strings[synthesis.id]
if len(candidate_langs) == 1:
lang = candidate_langs[0]
else:
text = ' '.join(filter(None, (
sanitize_text(subject or ''),
sanitize_text(introduction or ''),
sanitize_text(conclusion or ''),
)))
lang = None
if text:
lang, data = langid_services[synthesis.discussion_id].identify(text)
if not lang:
print "***** Could not identify for synthesis %d: %s" % (synthesis.id, text)
lang = candidate_langs[0]
def as_lang_string(text):
ls = m.LangString.create(text, lang)
return ls
if subject:
synthesis.subject = as_lang_string(subject)
if introduction:
synthesis.introduction = as_lang_string(introduction)
if conclusion:
synthesis.conclusion = as_lang_string(conclusion)
with context.begin_transaction():
op.drop_column('synthesis', 'subject')
op.drop_column('synthesis', 'introduction')
op.drop_column('synthesis', 'conclusion')
