def process_entity_comment(entity, profile, receiving_profile=None):
"""Process an entity of type Comment."""
fid = safe_text(entity.id)
if not validate_against_old_content(fid, entity, profile):
return
try:
parent = Content.objects.fed(entity.target_id).get()
except Content.DoesNotExist:
logger.warning("No target found for comment: %s", entity)
return
values = {
"text": safe_text_for_markdown(entity.raw_content),
"author": profile,
"visibility": parent.visibility,
"remote_created": safe_make_aware(entity.created_at, "UTC"),
"parent": parent,
}
values["text"] = _embed_entity_images_to_post(entity._children, values["text"])
if getattr(entity, "guid", None):
values["guid"] = safe_text(entity.guid)
content, created = Content.objects.fed_update_or_create(fid, values)
_process_mentions(content, entity)
if created:
logger.info("Saved Content from comment entity: %s", content)
else:
logger.info("Updated Content from comment entity: %s", content)
if parent.visibility != Visibility.PUBLIC and receiving_profile:
content.limited_visibilities.add(receiving_profile)
logger.info("Added visibility to Comment %s to %s", content.uuid, receiving_profile.uuid)
if parent.local:
# We should relay this to participants we know of
from socialhome.federate.tasks import forward_entity
django_rq.enqueue(forward_entity, entity, parent.id)
def process_entity_post(entity, profile, receiving_profile=None):
"""Process an entity of type Post."""
fid = safe_text(entity.id)
if not validate_against_old_content(fid, entity, profile):
return
values = {
"fid": fid,
"text": safe_text_for_markdown(entity.raw_content),
"author": profile,
"visibility": Visibility.PUBLIC if entity.public else Visibility.LIMITED,
"remote_created": safe_make_aware(entity.created_at, "UTC"),
"service_label": safe_text(entity.provider_display_name) or "",
}
values["text"] = _embed_entity_images_to_post(entity._children, values["text"])
if getattr(entity, "guid", None):
values["guid"] = safe_text(entity.guid)
content, created = Content.objects.fed_update_or_create(fid, values)
_process_mentions(content, entity)
if created:
logger.info("Saved Content: %s", content)
else:
logger.info("Updated Content: %s", content)
if content.visibility != Visibility.PUBLIC and receiving_profile:
content.limited_visibilities.add(receiving_profile)
logger.info("Added visibility to Post %s to %s", content.fid, receiving_profile.fid)
def process_entity_retraction(entity, profile):
"""Process an entity of type Retraction."""
entity_type = safe_text(entity.entity_type)
if entity_type in ("Post", "Comment", "Share"):
target_fid = safe_text(entity.target_id)
_retract_content(target_fid, profile)
else:
logger.debug("Ignoring retraction of entity_type %s", entity_type)
def process_entity_share(entity, profile):
"""Process an entity of type Share."""
if not entity.entity_type == "Post":
# TODO: enable shares of replies too
logger.warning("Ignoring share entity type that is not of type Post")
return
try:
target_content = Content.objects.fed(entity.target_id, share_of__isnull=True).get()
except Content.DoesNotExist:
# Try fetching. If found, process and then try again
remote_target = retrieve_remote_content(
entity.target_id,
guid=entity.target_guid,
handle=entity.target_handle,
entity_type=entity.entity_type,
sender_key_fetcher=sender_key_fetcher,
)
if remote_target:
process_entities([remote_target])
try:
target_content = Content.objects.fed(entity.target_id, share_of__isnull=True).get()
except Content.DoesNotExist:
logger.warning("Share target was fetched from remote, but it is still missing locally! Share: %s",
entity)
return
else:
logger.warning("No target found for share even after fetching from remote: %s", entity)
return
values = {
"text": safe_text_for_markdown(entity.raw_content),
"author": profile,
# TODO: ensure visibility constraints depending on shared content?
"visibility": Visibility.PUBLIC if entity.public else Visibility.LIMITED,
"remote_created": safe_make_aware(entity.created_at, "UTC"),
"service_label": safe_text(entity.provider_display_name) or "",
}
values["text"] = _embed_entity_images_to_post(entity._children, values["text"])
fid = safe_text(entity.id)
if getattr(entity, "guid", None):
values["guid"] = safe_text(entity.guid)
content, created = Content.objects.fed_update_or_create(fid, values, extra_lookups={'share_of': target_content})
_process_mentions(content, entity)
if created:
logger.info("Saved share: %s", content)
else:
logger.info("Updated share: %s", content)
# TODO: send participation to the share from the author, if local
# We probably want that to happen even though our shares are not separate in the stream?
if target_content.local:
# We should relay this share entity to participants we know of
from socialhome.federate.tasks import forward_entity
django_rq.enqueue(forward_entity, entity, target_content.id)
def from_remote_profile(remote_profile):
"""Create a Profile from a remote Profile entity."""
logger.info("from_remote_profile - Create or updating %s", remote_profile)
values = {
"name": safe_text(remote_profile.name),
"visibility": Visibility.PUBLIC, # Any profile that has been federated has to be public
"image_url_large": Profile.absolute_image_url(remote_profile, "large"),
"image_url_medium": Profile.absolute_image_url(remote_profile, "medium"),
"image_url_small": Profile.absolute_image_url(remote_profile, "small"),
"location": safe_text(remote_profile.location),
"email": safe_text(remote_profile.email),
}
public_key = safe_text(remote_profile.public_key)
if public_key:
# Only update public key if it has a value
values["rsa_public_key"] = public_key
for img_size in ["small", "medium", "large"]:
# Possibly fix some broken by bleach urls
values["image_url_%s" % img_size] = values["image_url_%s" % img_size].replace("&", "&")
fid = safe_text(remote_profile.id)
if hasattr(remote_profile, "handle"):
values['handle'] = safe_text(remote_profile.handle)
if hasattr(remote_profile, "guid"):
values['guid'] = safe_text(remote_profile.guid)
logger.debug("from_remote_profile - values %s", values)
profile, created = Profile.objects.fed_update_or_create(fid, values)
logger.info("from_remote_profile - created %s, profile %s", created, profile)
return profile
def absolute_image_url(profile, image_name):
"""Returns absolute version of image URL of given size if they wasn't absolute"""
url = safe_text(profile.image_urls[image_name])
if url.startswith("/") and profile.handle:
return "https://%s%s" % (
profile.handle.split("@")[1], url,
)
return url
def _embed_entity_images_to_post(children, text):
"""Embed any entity `_children` of base.Image type to the text content as markdown.
Images are prefixed on top of the normal text content.
:param children: List of child entities
:param values: Text for creating the Post
:return: New text value to create the Post with
"""
images = []
for child in children:
if isinstance(child, base.Image):
image_url = "%s%s" % (
safe_text(child.remote_path), safe_text(child.remote_name)
)
images.append(" " % image_url)
if images:
return "%s\n\n%s" % (
"".join(images), text
)
return text
def fetch_og_preview(content, urls):
"""Fetch first opengraph entry for a list of urls."""
for url in urls:
# See first if recently cached already
if OpenGraphCache.objects.filter(url=url, modified__gte=now() - datetime.timedelta(days=7)).exists():
opengraph = OpenGraphCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(opengraph=opengraph)
return opengraph
try:
og = OpenGraph(url=url, parser="lxml")
except AttributeError:
continue
if not og or ("title" not in og and "site_name" not in og and "description" not in og and "image" not in og):
continue
try:
title = og.title if "title" in og else og.site_name if "site_name" in og else ""
description = og.description if "description" in og else ""
image = og.image if "image" in og and not content.is_nsfw else ""
try:
with transaction.atomic():
opengraph = OpenGraphCache.objects.create(
url=url,
title=truncate_letters(safe_text(title), 250),
description=safe_text(description),
image=safe_text(image),
)
except DataError:
continue
except IntegrityError:
# Some other process got ahead of us
opengraph = OpenGraphCache.objects.get(url=url)
Content.objects.filter(id=content.id).update(opengraph=opengraph)
return opengraph
Content.objects.filter(id=content.id).update(opengraph=opengraph)
return opengraph
return False
def get(self, request, *args, **kwargs):
"""See if we have a direct match. If so redirect, if not, search.
Try fetching a remote profile if the search term is a handle or fid.
"""
q = safe_text(request.GET.get("q"))
if q:
q = q.strip().lower()
self.q = q
# Check if direct tag matches
if q.startswith('#'):
try:
tag = Tag.objects.filter(
name=q[1:]
).annotate(
content_count=Count('contents')
).filter(
content_count__gt=0
).get()
except Tag.DoesNotExist:
pass
else:
return redirect(tag.get_absolute_url())
# Check if profile matches
profile = None
try:
profile = Profile.objects.visible_for_user(request.user).fed(q).get()
except Profile.DoesNotExist:
# Try a remote search
# TODO currently only if diaspora handle
if validate_handle(q):
try:
remote_profile = retrieve_remote_profile(q)
except (AttributeError, ValueError, xml.parsers.expat.ExpatError):
# Catch various errors parsing the remote profile
return super().get(request, *args, **kwargs)
if remote_profile:
profile = Profile.from_remote_profile(remote_profile)
if profile:
return redirect(reverse("users:profile-detail", kwargs={"uuid": profile.uuid}))
try:
return super().get(request, *args, **kwargs)
except QueryError:
# Re-render the form
messages.warning(self.request, _("Search string is invalid, please try another one."))
return HttpResponseRedirect(self.get_success_url())
def process_entity_comment(entity: Any, profile: Profile):
"""Process an entity of type Comment."""
fid = safe_text(entity.id)
if not validate_against_old_content(fid, entity, profile):
return
try:
parent = Content.objects.fed(entity.target_id).get()
except Content.DoesNotExist:
logger.warning("No target found for comment: %s", entity)
return
root_parent = parent
if entity.root_target_id:
try:
root_parent = Content.objects.fed(entity.root_target_id).get()
except Content.DoesNotExist:
pass
visibility = None
if getattr(entity, "public", None) is not None:
visibility = Visibility.PUBLIC if entity.public else Visibility.LIMITED
values = {
"text":
_embed_entity_images_to_post(
entity._children, safe_text_for_markdown(entity.raw_content)),
"author":
profile,
"visibility":
visibility if visibility is not None else parent.visibility,
"remote_created":
safe_make_aware(entity.created_at, "UTC"),
"parent":
parent,
"root_parent":
root_parent,
}
if getattr(entity, "guid", None):
values["guid"] = safe_text(entity.guid)
content, created = Content.objects.fed_update_or_create(fid, values)
_process_mentions(content, entity)
if created:
logger.info("Saved Content from comment entity: %s", content)
else:
logger.info("Updated Content from comment entity: %s", content)
if visibility == Visibility.LIMITED or (
visibility is None and parent.visibility == Visibility.LIMITED):
if entity._receivers:
receivers = get_profiles_from_receivers(entity._receivers)
if len(receivers):
content.limited_visibilities.add(*receivers)
logger.info("Added visibility to Comment %s to %s",
content.fid, receivers)
else:
logger.warning(
"No local receivers found for limited Comment %s",
content.fid)
else:
logger.warning("No receivers for limited Comment %s", content.fid)
if parent.local:
# We should relay this to participants we know of
from socialhome.federate.tasks import forward_entity
django_rq.enqueue(forward_entity, entity, root_parent.id)
def process_entity_share(entity, profile):
"""Process an entity of type Share."""
if not entity.entity_type == "Post":
# TODO: enable shares of replies too
logger.warning("Ignoring share entity type that is not of type Post")
return
try:
target_content = Content.objects.fed(entity.target_id,
share_of__isnull=True).get()
except Content.DoesNotExist:
# Try fetching. If found, process and then try again
logger.debug(
"process_entity_share - trying to fetch %s, %s, %s, %s, %s",
entity.target_id,
entity.target_guid,
entity.target_handle,
entity.entity_type,
sender_key_fetcher,
)
remote_target = retrieve_remote_content(
entity.target_id,
guid=entity.target_guid,
handle=entity.target_handle,
entity_type=entity.entity_type,
sender_key_fetcher=sender_key_fetcher,
)
if remote_target:
process_entities([remote_target])
try:
target_content = Content.objects.fed(
entity.target_id, share_of__isnull=True).get()
except Content.DoesNotExist:
logger.warning(
"Share target was fetched from remote, but it is still missing locally! Share: %s",
entity)
return
else:
logger.warning(
"No target found for share even after fetching from remote: %s",
entity)
return
if target_content.visibility != Visibility.PUBLIC:
# Don't process a share for non-public target content
logger.warning("Share '%s' target '%s' is not public - aborting",
entity, target_content)
return
values = {
"text": safe_text_for_markdown(entity.raw_content),
"author": profile,
"visibility": Visibility.PUBLIC,
"remote_created": safe_make_aware(entity.created_at, "UTC"),
"service_label": safe_text(entity.provider_display_name) or "",
}
# noinspection PyProtectedMember
values["text"] = _embed_entity_images_to_post(entity._children,
values["text"])
fid = safe_text(entity.id)
if getattr(entity, "guid", None):
values["guid"] = safe_text(entity.guid)
content, created = Content.objects.fed_update_or_create(
fid, values, extra_lookups={'share_of': target_content})
_process_mentions(content, entity)
if created:
logger.info("Saved share: %s", content)
else:
logger.info("Updated share: %s", content)
# TODO: send participation to the share from the author, if local
# We probably want that to happen even though our shares are not separate in the stream?
if target_content.local:
# We should relay this share entity to participants we know of
from socialhome.federate.tasks import forward_entity
django_rq.enqueue(forward_entity, entity, target_content.id)
def test_text_with_html_is_cleaned(self):
assert safe_text(HTML_TEXT) == "barceedaaafaa"
def test_text_with_script_is_cleaned(self):
assert safe_text(SCRIPT_TEXT) == "console.log"
def test_text_with_markdown_code_is_cleaned(self):
assert safe_text(MARKDOWN_CODE_TEXT) == "`\nalert('yup');\n`\n\n`alert('yup');`\n\n```\n" \
"alert('yap');\n```"
def test_text_with_markdown_survives(self):
assert safe_text(MARKDOWN_TEXT) == MARKDOWN_TEXT
def test_text_with_script_is_cleaned(self):
assert safe_text(SCRIPT_TEXT) == "console.log"
def clean_name(self):
return safe_text(self.cleaned_data["name"])
def test_text_with_html_is_cleaned__mention_link_removed(self):
assert safe_text(HTML_TEXT_WITH_MENTION_LINK) == '@jaywink boom'
def test_plain_text_survives(self):
assert safe_text(PLAIN_TEXT) == PLAIN_TEXT
def test_text_with_markdown_survives(self):
assert safe_text(MARKDOWN_TEXT) == MARKDOWN_TEXT
def test_text_with_html_is_cleaned(self):
assert safe_text(HTML_TEXT) == "barceedaaafaa"
def test_plain_text_survives(self):
assert safe_text(PLAIN_TEXT) == PLAIN_TEXT
def get(self, request, *args, **kwargs):
"""See if we have a direct match. If so redirect, if not, search.
Try fetching a remote profile if the search term is a handle or fid.
"""
q = safe_text(request.GET.get("q"))
if q:
q = q.strip().lower().strip("@")
self.q = q
# Check if direct tag matches
if q.startswith('#'):
try:
tag = Tag.objects.filter(
name=q[1:]
).annotate(
content_count=Count('contents')
).filter(
content_count__gt=0
).get()
except Tag.DoesNotExist:
pass
else:
return redirect(tag.get_absolute_url())
# Check if profile matches
profile = None
try:
profile = Profile.objects.visible_for_user(request.user).fed(q).get()
except Profile.DoesNotExist:
# Try a remote search
if is_url(q) or validate_handle(q):
try:
remote_profile = retrieve_remote_profile(q)
except (AttributeError, ValueError, xml.parsers.expat.ExpatError):
# Catch various errors parsing the remote profile
return super().get(request, *args, **kwargs)
if remote_profile and isinstance(remote_profile, base.Profile):
profile = Profile.from_remote_profile(remote_profile)
if profile:
return redirect(reverse("users:profile-detail", kwargs={"uuid": profile.uuid}))
# Check if content matches
content = None
try:
content = Content.objects.visible_for_user(request.user).fed(q).get()
except Content.DoesNotExist:
# Try a remote search
if is_url(q):
try:
remote_content = retrieve_remote_content(q)
except (AttributeError, ValueError):
# Catch various errors parsing the remote content
return super().get(request, *args, **kwargs)
if remote_content:
process_entities([remote_content])
# Try again
try:
content = Content.objects.visible_for_user(request.user).fed(remote_content.id).get()
except Content.DoesNotExist:
return super().get(request, *args, **kwargs)
if content:
return redirect(reverse("content:view", kwargs={"pk": content.id}))
try:
return super().get(request, *args, **kwargs)
except QueryError:
# Re-render the form
messages.warning(self.request, _("Search string is invalid, please try another one."))
return HttpResponseRedirect(self.get_success_url())
def test_text_with_markdown_code_is_cleaned(self):
assert safe_text(MARKDOWN_CODE_TEXT) == "`\nalert('yup');\n`\n\n`alert('yup');`\n\n```\n" \
"alert('yap');\n```"
