def process_entity_comment(entity, profile, receiving_profile=None): """Process an entity of type Comment.""" fid = safe_text(entity.id) if not validate_against_old_content(fid, entity, profile): return try: parent = Content.objects.fed(entity.target_id).get() except Content.DoesNotExist: logger.warning("No target found for comment: %s", entity) return values = { "text": safe_text_for_markdown(entity.raw_content), "author": profile, "visibility": parent.visibility, "remote_created": safe_make_aware(entity.created_at, "UTC"), "parent": parent, } values["text"] = _embed_entity_images_to_post(entity._children, values["text"]) if getattr(entity, "guid", None): values["guid"] = safe_text(entity.guid) content, created = Content.objects.fed_update_or_create(fid, values) _process_mentions(content, entity) if created: logger.info("Saved Content from comment entity: %s", content) else: logger.info("Updated Content from comment entity: %s", content) if parent.visibility != Visibility.PUBLIC and receiving_profile: content.limited_visibilities.add(receiving_profile) logger.info("Added visibility to Comment %s to %s", content.uuid, receiving_profile.uuid) if parent.local: # We should relay this to participants we know of from socialhome.federate.tasks import forward_entity django_rq.enqueue(forward_entity, entity, parent.id)
def process_entity_post(entity, profile, receiving_profile=None): """Process an entity of type Post.""" fid = safe_text(entity.id) if not validate_against_old_content(fid, entity, profile): return values = { "fid": fid, "text": safe_text_for_markdown(entity.raw_content), "author": profile, "visibility": Visibility.PUBLIC if entity.public else Visibility.LIMITED, "remote_created": safe_make_aware(entity.created_at, "UTC"), "service_label": safe_text(entity.provider_display_name) or "", } values["text"] = _embed_entity_images_to_post(entity._children, values["text"]) if getattr(entity, "guid", None): values["guid"] = safe_text(entity.guid) content, created = Content.objects.fed_update_or_create(fid, values) _process_mentions(content, entity) if created: logger.info("Saved Content: %s", content) else: logger.info("Updated Content: %s", content) if content.visibility != Visibility.PUBLIC and receiving_profile: content.limited_visibilities.add(receiving_profile) logger.info("Added visibility to Post %s to %s", content.fid, receiving_profile.fid)
def process_entity_retraction(entity, profile): """Process an entity of type Retraction.""" entity_type = safe_text(entity.entity_type) if entity_type in ("Post", "Comment", "Share"): target_fid = safe_text(entity.target_id) _retract_content(target_fid, profile) else: logger.debug("Ignoring retraction of entity_type %s", entity_type)
def process_entity_share(entity, profile): """Process an entity of type Share.""" if not entity.entity_type == "Post": # TODO: enable shares of replies too logger.warning("Ignoring share entity type that is not of type Post") return try: target_content = Content.objects.fed(entity.target_id, share_of__isnull=True).get() except Content.DoesNotExist: # Try fetching. If found, process and then try again remote_target = retrieve_remote_content( entity.target_id, guid=entity.target_guid, handle=entity.target_handle, entity_type=entity.entity_type, sender_key_fetcher=sender_key_fetcher, ) if remote_target: process_entities([remote_target]) try: target_content = Content.objects.fed(entity.target_id, share_of__isnull=True).get() except Content.DoesNotExist: logger.warning("Share target was fetched from remote, but it is still missing locally! Share: %s", entity) return else: logger.warning("No target found for share even after fetching from remote: %s", entity) return values = { "text": safe_text_for_markdown(entity.raw_content), "author": profile, # TODO: ensure visibility constraints depending on shared content? "visibility": Visibility.PUBLIC if entity.public else Visibility.LIMITED, "remote_created": safe_make_aware(entity.created_at, "UTC"), "service_label": safe_text(entity.provider_display_name) or "", } values["text"] = _embed_entity_images_to_post(entity._children, values["text"]) fid = safe_text(entity.id) if getattr(entity, "guid", None): values["guid"] = safe_text(entity.guid) content, created = Content.objects.fed_update_or_create(fid, values, extra_lookups={'share_of': target_content}) _process_mentions(content, entity) if created: logger.info("Saved share: %s", content) else: logger.info("Updated share: %s", content) # TODO: send participation to the share from the author, if local # We probably want that to happen even though our shares are not separate in the stream? if target_content.local: # We should relay this share entity to participants we know of from socialhome.federate.tasks import forward_entity django_rq.enqueue(forward_entity, entity, target_content.id)
def from_remote_profile(remote_profile): """Create a Profile from a remote Profile entity.""" logger.info("from_remote_profile - Create or updating %s", remote_profile) values = { "name": safe_text(remote_profile.name), "visibility": Visibility.PUBLIC, # Any profile that has been federated has to be public "image_url_large": Profile.absolute_image_url(remote_profile, "large"), "image_url_medium": Profile.absolute_image_url(remote_profile, "medium"), "image_url_small": Profile.absolute_image_url(remote_profile, "small"), "location": safe_text(remote_profile.location), "email": safe_text(remote_profile.email), } public_key = safe_text(remote_profile.public_key) if public_key: # Only update public key if it has a value values["rsa_public_key"] = public_key for img_size in ["small", "medium", "large"]: # Possibly fix some broken by bleach urls values["image_url_%s" % img_size] = values["image_url_%s" % img_size].replace("&", "&") fid = safe_text(remote_profile.id) if hasattr(remote_profile, "handle"): values['handle'] = safe_text(remote_profile.handle) if hasattr(remote_profile, "guid"): values['guid'] = safe_text(remote_profile.guid) logger.debug("from_remote_profile - values %s", values) profile, created = Profile.objects.fed_update_or_create(fid, values) logger.info("from_remote_profile - created %s, profile %s", created, profile) return profile
def absolute_image_url(profile, image_name): """Returns absolute version of image URL of given size if they wasn't absolute""" url = safe_text(profile.image_urls[image_name]) if url.startswith("/") and profile.handle: return "https://%s%s" % ( profile.handle.split("@")[1], url, ) return url
def _embed_entity_images_to_post(children, text): """Embed any entity `_children` of base.Image type to the text content as markdown. Images are prefixed on top of the normal text content. :param children: List of child entities :param values: Text for creating the Post :return: New text value to create the Post with """ images = [] for child in children: if isinstance(child, base.Image): image_url = "%s%s" % ( safe_text(child.remote_path), safe_text(child.remote_name) ) images.append("![](%s) " % image_url) if images: return "%s\n\n%s" % ( "".join(images), text ) return text
def fetch_og_preview(content, urls): """Fetch first opengraph entry for a list of urls.""" for url in urls: # See first if recently cached already if OpenGraphCache.objects.filter(url=url, modified__gte=now() - datetime.timedelta(days=7)).exists(): opengraph = OpenGraphCache.objects.get(url=url) Content.objects.filter(id=content.id).update(opengraph=opengraph) return opengraph try: og = OpenGraph(url=url, parser="lxml") except AttributeError: continue if not og or ("title" not in og and "site_name" not in og and "description" not in og and "image" not in og): continue try: title = og.title if "title" in og else og.site_name if "site_name" in og else "" description = og.description if "description" in og else "" image = og.image if "image" in og and not content.is_nsfw else "" try: with transaction.atomic(): opengraph = OpenGraphCache.objects.create( url=url, title=truncate_letters(safe_text(title), 250), description=safe_text(description), image=safe_text(image), ) except DataError: continue except IntegrityError: # Some other process got ahead of us opengraph = OpenGraphCache.objects.get(url=url) Content.objects.filter(id=content.id).update(opengraph=opengraph) return opengraph Content.objects.filter(id=content.id).update(opengraph=opengraph) return opengraph return False
def get(self, request, *args, **kwargs): """See if we have a direct match. If so redirect, if not, search. Try fetching a remote profile if the search term is a handle or fid. """ q = safe_text(request.GET.get("q")) if q: q = q.strip().lower() self.q = q # Check if direct tag matches if q.startswith('#'): try: tag = Tag.objects.filter( name=q[1:] ).annotate( content_count=Count('contents') ).filter( content_count__gt=0 ).get() except Tag.DoesNotExist: pass else: return redirect(tag.get_absolute_url()) # Check if profile matches profile = None try: profile = Profile.objects.visible_for_user(request.user).fed(q).get() except Profile.DoesNotExist: # Try a remote search # TODO currently only if diaspora handle if validate_handle(q): try: remote_profile = retrieve_remote_profile(q) except (AttributeError, ValueError, xml.parsers.expat.ExpatError): # Catch various errors parsing the remote profile return super().get(request, *args, **kwargs) if remote_profile: profile = Profile.from_remote_profile(remote_profile) if profile: return redirect(reverse("users:profile-detail", kwargs={"uuid": profile.uuid})) try: return super().get(request, *args, **kwargs) except QueryError: # Re-render the form messages.warning(self.request, _("Search string is invalid, please try another one.")) return HttpResponseRedirect(self.get_success_url())
def process_entity_comment(entity: Any, profile: Profile): """Process an entity of type Comment.""" fid = safe_text(entity.id) if not validate_against_old_content(fid, entity, profile): return try: parent = Content.objects.fed(entity.target_id).get() except Content.DoesNotExist: logger.warning("No target found for comment: %s", entity) return root_parent = parent if entity.root_target_id: try: root_parent = Content.objects.fed(entity.root_target_id).get() except Content.DoesNotExist: pass visibility = None if getattr(entity, "public", None) is not None: visibility = Visibility.PUBLIC if entity.public else Visibility.LIMITED values = { "text": _embed_entity_images_to_post( entity._children, safe_text_for_markdown(entity.raw_content)), "author": profile, "visibility": visibility if visibility is not None else parent.visibility, "remote_created": safe_make_aware(entity.created_at, "UTC"), "parent": parent, "root_parent": root_parent, } if getattr(entity, "guid", None): values["guid"] = safe_text(entity.guid) content, created = Content.objects.fed_update_or_create(fid, values) _process_mentions(content, entity) if created: logger.info("Saved Content from comment entity: %s", content) else: logger.info("Updated Content from comment entity: %s", content) if visibility == Visibility.LIMITED or ( visibility is None and parent.visibility == Visibility.LIMITED): if entity._receivers: receivers = get_profiles_from_receivers(entity._receivers) if len(receivers): content.limited_visibilities.add(*receivers) logger.info("Added visibility to Comment %s to %s", content.fid, receivers) else: logger.warning( "No local receivers found for limited Comment %s", content.fid) else: logger.warning("No receivers for limited Comment %s", content.fid) if parent.local: # We should relay this to participants we know of from socialhome.federate.tasks import forward_entity django_rq.enqueue(forward_entity, entity, root_parent.id)
def process_entity_share(entity, profile): """Process an entity of type Share.""" if not entity.entity_type == "Post": # TODO: enable shares of replies too logger.warning("Ignoring share entity type that is not of type Post") return try: target_content = Content.objects.fed(entity.target_id, share_of__isnull=True).get() except Content.DoesNotExist: # Try fetching. If found, process and then try again logger.debug( "process_entity_share - trying to fetch %s, %s, %s, %s, %s", entity.target_id, entity.target_guid, entity.target_handle, entity.entity_type, sender_key_fetcher, ) remote_target = retrieve_remote_content( entity.target_id, guid=entity.target_guid, handle=entity.target_handle, entity_type=entity.entity_type, sender_key_fetcher=sender_key_fetcher, ) if remote_target: process_entities([remote_target]) try: target_content = Content.objects.fed( entity.target_id, share_of__isnull=True).get() except Content.DoesNotExist: logger.warning( "Share target was fetched from remote, but it is still missing locally! Share: %s", entity) return else: logger.warning( "No target found for share even after fetching from remote: %s", entity) return if target_content.visibility != Visibility.PUBLIC: # Don't process a share for non-public target content logger.warning("Share '%s' target '%s' is not public - aborting", entity, target_content) return values = { "text": safe_text_for_markdown(entity.raw_content), "author": profile, "visibility": Visibility.PUBLIC, "remote_created": safe_make_aware(entity.created_at, "UTC"), "service_label": safe_text(entity.provider_display_name) or "", } # noinspection PyProtectedMember values["text"] = _embed_entity_images_to_post(entity._children, values["text"]) fid = safe_text(entity.id) if getattr(entity, "guid", None): values["guid"] = safe_text(entity.guid) content, created = Content.objects.fed_update_or_create( fid, values, extra_lookups={'share_of': target_content}) _process_mentions(content, entity) if created: logger.info("Saved share: %s", content) else: logger.info("Updated share: %s", content) # TODO: send participation to the share from the author, if local # We probably want that to happen even though our shares are not separate in the stream? if target_content.local: # We should relay this share entity to participants we know of from socialhome.federate.tasks import forward_entity django_rq.enqueue(forward_entity, entity, target_content.id)
def test_text_with_html_is_cleaned(self): assert safe_text(HTML_TEXT) == "barceedaaafaa"
def test_text_with_script_is_cleaned(self): assert safe_text(SCRIPT_TEXT) == "console.log"
def test_text_with_markdown_code_is_cleaned(self): assert safe_text(MARKDOWN_CODE_TEXT) == "`\nalert('yup');\n`\n\n`alert('yup');`\n\n```\n" \ "alert('yap');\n```"
def test_text_with_markdown_survives(self): assert safe_text(MARKDOWN_TEXT) == MARKDOWN_TEXT
def test_text_with_script_is_cleaned(self): assert safe_text(SCRIPT_TEXT) == "console.log"
def clean_name(self): return safe_text(self.cleaned_data["name"])
def test_text_with_html_is_cleaned__mention_link_removed(self): assert safe_text(HTML_TEXT_WITH_MENTION_LINK) == '@jaywink boom'
def test_plain_text_survives(self): assert safe_text(PLAIN_TEXT) == PLAIN_TEXT
def test_text_with_markdown_survives(self): assert safe_text(MARKDOWN_TEXT) == MARKDOWN_TEXT
def test_text_with_html_is_cleaned(self): assert safe_text(HTML_TEXT) == "barceedaaafaa"
def test_plain_text_survives(self): assert safe_text(PLAIN_TEXT) == PLAIN_TEXT
def get(self, request, *args, **kwargs): """See if we have a direct match. If so redirect, if not, search. Try fetching a remote profile if the search term is a handle or fid. """ q = safe_text(request.GET.get("q")) if q: q = q.strip().lower().strip("@") self.q = q # Check if direct tag matches if q.startswith('#'): try: tag = Tag.objects.filter( name=q[1:] ).annotate( content_count=Count('contents') ).filter( content_count__gt=0 ).get() except Tag.DoesNotExist: pass else: return redirect(tag.get_absolute_url()) # Check if profile matches profile = None try: profile = Profile.objects.visible_for_user(request.user).fed(q).get() except Profile.DoesNotExist: # Try a remote search if is_url(q) or validate_handle(q): try: remote_profile = retrieve_remote_profile(q) except (AttributeError, ValueError, xml.parsers.expat.ExpatError): # Catch various errors parsing the remote profile return super().get(request, *args, **kwargs) if remote_profile and isinstance(remote_profile, base.Profile): profile = Profile.from_remote_profile(remote_profile) if profile: return redirect(reverse("users:profile-detail", kwargs={"uuid": profile.uuid})) # Check if content matches content = None try: content = Content.objects.visible_for_user(request.user).fed(q).get() except Content.DoesNotExist: # Try a remote search if is_url(q): try: remote_content = retrieve_remote_content(q) except (AttributeError, ValueError): # Catch various errors parsing the remote content return super().get(request, *args, **kwargs) if remote_content: process_entities([remote_content]) # Try again try: content = Content.objects.visible_for_user(request.user).fed(remote_content.id).get() except Content.DoesNotExist: return super().get(request, *args, **kwargs) if content: return redirect(reverse("content:view", kwargs={"pk": content.id})) try: return super().get(request, *args, **kwargs) except QueryError: # Re-render the form messages.warning(self.request, _("Search string is invalid, please try another one.")) return HttpResponseRedirect(self.get_success_url())
def test_text_with_markdown_code_is_cleaned(self): assert safe_text(MARKDOWN_CODE_TEXT) == "`\nalert('yup');\n`\n\n`alert('yup');`\n\n```\n" \ "alert('yap');\n```"