Exemple #1
0
 def test_post_called_with_only_one_headers_kwarg(self, mock_post):
     # A failure might raise:
     # TypeError: MagicMock object got multiple values for keyword argument 'headers'
     send_document("http://localhost", {"foo": "bar"}, **self.call_args)
     mock_post.assert_called_once_with(
         "http://localhost", data={"foo": "bar"}, **self.call_args
     )
Exemple #2
0
def send_content(content_id):
    """Handle sending a Content object out via the federation layer.

    Currently we only deliver public content.
    """
    try:
        content = Content.objects.get(id=content_id)
    except Content.DoesNotExist:
        logger.warning("No content found with id %s", content_id)
        return
    if not content.visibility == Visibility.PUBLIC:
        return
    entity = make_federable_entity(content)
    if entity:
        if settings.DEBUG:
            # Don't send in development mode
            return
        # TODO: federation should provide one method to send,
        # which handles also payload creation and url calculation
        payload = handle_create_payload(entity, content.author)
        # Just dump to the relay system for now
        url = "https://%s/receive/public" % settings.SOCIALHOME_RELAY_DOMAIN
        send_document(url, payload)
    else:
        logger.warning("No entity for %s", content)
Exemple #3
0
def send_follow_change(profile_id, followed_id, follow):
    """Handle sending of a local follow of a remote profile."""
    try:
        profile = Profile.objects.get(id=profile_id, user__isnull=False)
    except Profile.DoesNotExist:
        logger.warning(
            "send_follow_change - No local profile %s found to send follow with",
            profile_id)
        return
    try:
        remote_profile = Profile.objects.get(id=followed_id, user__isnull=True)
    except Profile.DoesNotExist:
        logger.warning(
            "send_follow_change - No remote profile %s found to send follow for",
            followed_id)
        return
    if settings.DEBUG:
        # Don't send in development mode
        return
    entity = base.Follow(handle=profile.handle,
                         target_handle=remote_profile.handle,
                         following=follow)
    # TODO: add high level method support to federation for private payload delivery
    payload = handle_create_payload(entity, profile, to_user=remote_profile)
    url = "https://%s/receive/users/%s" % (
        remote_profile.handle.split("@")[1],
        remote_profile.guid,
    )
    send_document(url, payload)
    # Also trigger a profile send
    send_profile(profile_id, recipients=[(remote_profile.handle, None)])
Exemple #4
0
 def test_headers_in_either_case_are_handled_without_exception(self, mock_post):
     send_document("http://localhost", {"foo": "bar"}, **self.call_args)
     mock_post.assert_called_once_with(
         "http://localhost", data={"foo": "bar"}, headers={'user-agent': USER_AGENT}, timeout=10
     )
     mock_post.reset_mock()
     send_document("http://localhost", {"foo": "bar"}, headers={'User-Agent': USER_AGENT})
     mock_post.assert_called_once_with(
         "http://localhost", data={"foo": "bar"}, headers={'User-Agent': USER_AGENT}, timeout=10
     )
Exemple #5
0
 def test_post_is_called(self, mock_post):
     code, exc = send_document("http://localhost", {"foo": "bar"})
     mock_post.assert_called_once_with(
         "http://localhost", data={"foo": "bar"}, **self.call_args
     )
     assert code == 200
     assert exc == None
Exemple #6
0
def send_content_retraction(content, author_id):
    """Handle sending of retractions.

    Currently only for public content.
    """
    if not content.visibility == Visibility.PUBLIC:
        return
    author = Profile.objects.get(id=author_id)
    entity = make_federable_retraction(content, author)
    if entity:
        if settings.DEBUG:
            # Don't send in development mode
            return
        payload = handle_create_payload(entity, author)
        # Just dump to the relay system for now
        url = "https://%s/receive/public" % settings.SOCIALHOME_RELAY_DOMAIN
        send_document(url, payload)
    else:
        logger.warning("No retraction entity for %s", content)
Exemple #7
0
def send_profile(profile_id, recipients=None):
    """Handle sending a Profile object out via the federation layer.

    :param profile_id: Profile.id of profile to send
    :param recipients: Optional list of recipient tuples, in form tuple(handle, network), for example
        ("*****@*****.**", "diaspora"). Network can be None.
    """
    try:
        profile = Profile.objects.get(id=profile_id, user__isnull=False)
    except Profile.DoesNotExist:
        logger.warning("send_profile - No local profile found with id %s",
                       profile_id)
        return
    entity = make_federable_profile(profile)
    if not entity:
        logger.warning("send_profile - No entity for %s", profile)
        return
    if settings.DEBUG:
        # Don't send in development mode
        return
    # From diaspora devs: "if the profile is private it needs to be encrypted, so to the private endpoint,
    # starting with 0.7.0.0 diaspora starts sending public profiles to the public endpoint only once per pod".
    # Let's just send everything to private endpoints as 0.7 isn't out yet.
    # TODO: once 0.7 is out for longer, start sending public profiles to public endpoints
    # TODO: add high level method support to federation for private payload delivery
    if not recipients:
        recipients = _get_remote_followers(profile)
    for handle, _network in recipients:
        try:
            remote_profile = Profile.objects.get(handle=handle)
        except Profile.DoesNotExist:
            continue
        payload = handle_create_payload(entity,
                                        profile,
                                        to_user=remote_profile)
        url = "https://%s/receive/users/%s" % (handle.split("@")[1],
                                               remote_profile.guid)
        send_document(url, payload)
Exemple #8
0
def handle_send(entity, author_user, recipients=None, parent_user=None):
    """Send an entity to remote servers.

    Using this we will build a list of payloads per protocol, after resolving any that need to be guessed or
    looked up over the network. After that, each recipient will get the generated protocol payload delivered.

    NOTE! This will not (yet) support Diaspora limited messages - `handle_create_payload` above should be directly
    called instead and payload sent with `federation.utils.network.send_document`.

    Any given user arguments must have ``private_key`` and ``handle`` attributes.

    :arg entity: Entity object to send. Can be a base entity or a protocol specific one.
    :arg author_user: User authoring the object.
    :arg recipients: A list of tuples to delivery to. Tuple contains (recipient handle or domain, protocol or None).
                     For example ``[("*****@*****.**", "diaspora"), ("*****@*****.**", None)]``.
    :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
                      Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
                      be generated. If given, the payload will be sent as this user.
    """
    payloads = {"diaspora": {"payload": None, "recipients": set()}}
    # Generate payload per protocol and split recipients to protocols
    for recipient, protocol in recipients:
        # TODO currently we only support Diaspora protocol, so no need to guess, just generate the payload
        if not payloads["diaspora"]["payload"]:
            payloads["diaspora"]["payload"] = handle_create_payload(
                entity, author_user, parent_user=parent_user)
        if "@" in recipient:
            payloads["diaspora"]["recipients"].add(recipient.split("@")[1])
        else:
            payloads["diaspora"]["recipients"].add(recipient)
    # Do actual sending
    for protocol, data in payloads.items():
        for recipient in data.get("recipients"):
            # TODO protocol independant url generation by importing named helper under protocol
            url = get_public_endpoint(recipient)
            send_document(url, data.get("payload"))
Exemple #9
0
def process(payload):
    """Open payload and route it to any pods that might be interested in it."""
    try:
        sender, protocol_name, entities = handle_receive(
            payload, sender_key_fetcher=Profile.get_public_key)
        logging.debug("sender=%s, protocol_name=%s, entities=%s" %
                      (sender, protocol_name, entities))
    except NoSuitableProtocolFoundError:
        logging.warning("No suitable protocol found for payload")
        return
    if protocol_name != "diaspora":
        logging.warning("Unsupported protocol: %s, sender: %s" %
                        (protocol_name, sender))
        return
    if not entities:
        logging.warning("No entities in payload")
        return
    sent_amount = 0
    sent_success = 0
    nodes = set()
    sent_to_nodes = []
    entity = None
    try:
        for entity in entities:
            # Diaspora payloads should only have one top level entity. Once we find a suitable one, just start sending
            if isinstance(entity, SUPPORTED_ENTITIES):
                nodes = get_send_to_nodes(sender, entity)
                break
        # Send out
        if nodes:
            logging.info("Sending %s to %s nodes", entity, len(nodes))
        for node in nodes:
            status, error = send_document(
                url="https://%s/receive/public" % node,
                data=payload,
                headers=HEADERS,
            )
            is_success = status in [200, 202]
            if is_success:
                sent_success += 1
                sent_to_nodes.append(node)
            sent_amount += 1
            update_node(node, is_success)
        logging.info("Successfully sent to %s nodes", len(sent_to_nodes))
    finally:
        log_worker_receive_statistics(protocol_name, len(entities),
                                      sent_amount, sent_success)
Exemple #10
0
def parse_matrix_document(doc: Dict, host: str) -> Dict:
    result = deepcopy(defaults)
    result['host'] = host
    result['name'] = host
    result['protocols'] = ['matrix']
    result['platform'] = f'matrix|{doc["server"]["name"].lower()}'
    result['version'] = doc["server"]["version"]

    # Get signups status by posting to register endpoint and analyzing the status code coming back
    status_code, _error = send_document(
        f'https://{host}/_matrix/client/r0/register',
        data=json.dumps({'auth': {}}),
    )
    if status_code == 401:
        result['open_signups'] = True
    elif status_code == 403:
        result['open_signups'] = False

    return result
Exemple #11
0
def process(payload):
    """Open payload and route it to any pods that might be interested in it."""
    try:
        sender, protocol_name, entities = handle_receive(payload, skip_author_verification=True)
        logging.debug("sender=%s, protocol_name=%s, entities=%s" % (sender, protocol_name, entities))
    except NoSuitableProtocolFoundError:
        logging.warning("No suitable protocol found for payload")
        return
    if protocol_name != "diaspora":
        logging.warning("Unsupported protocol: %s, sender: %s" % (protocol_name, sender))
        return
    if not entities:
        logging.warning("No entities in payload")
        return
    sent_amount = 0
    sent_success = 0
    try:
        for entity in entities:
            logging.info("Entity: %s" % entity)
            # We only care about posts atm
            if isinstance(entity, SUPPORTED_ENTITIES):
                sent_to_nodes = []
                nodes = get_send_to_nodes(sender, entity)
                # Send out
                for node in nodes:
                    status, error = send_document(
                        url="https://%s/receive/public" % node,
                        data={"xml": payload},
                        headers={"User-Agent": config.USER_AGENT},
                    )
                    is_success = status in [200, 202]
                    if is_success:
                        sent_success += 1
                        sent_to_nodes.append(node)
                    sent_amount += 1
                    update_node(node, is_success)
                if sent_to_nodes and isinstance(entity, (DiasporaPost, Image)):
                    save_post_metadata(entity=entity, protocol=protocol_name, hosts=sent_to_nodes)
    finally:
        log_worker_receive_statistics(
            protocol_name, len(entities), sent_amount, sent_success
        )
Exemple #12
0
def handle_send(entity, author_user, recipients=None, parent_user=None):
    """Send an entity to remote servers.

    Using this we will build a list of payloads per protocol, after resolving any that need to be guessed or
    looked up over the network. After that, each recipient will get the generated protocol payload delivered.

    Any given user arguments must have ``private_key`` and ``handle`` attributes.

    :arg entity: Entity object to send. Can be a base entity or a protocol specific one.
    :arg author_user: User authoring the object.
    :arg recipients: A list of recipients to delivery to. Each recipient is a tuple
                     containing at minimum the "id", optionally "public key" for private deliveries.
                     Instead of a tuple, for public deliveries the "id" as str is also ok.
                     If public key is provided, Diaspora protocol delivery will be made as an encrypted
                     private delivery.
                     For example
                     [
                         ("diaspora://[email protected]/profile/zyx", <RSAPublicKey object>),
                         ("diaspora://[email protected]/profile/xyz", None),
                         "diaspora://[email protected]/profile/xyz",
                     ]
    :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
                      Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
                      be generated. If given, the payload will be sent as this user.
    """
    payloads = []
    public_payloads = {
        "diaspora": {
            "payload": None,
            "urls": set(),
        },
    }

    # Generate payloads and collect urls
    for recipient in recipients:
        id = recipient[0] if isinstance(recipient, tuple) else recipient
        public_key = recipient[1] if isinstance(
            recipient, tuple) and len(recipient) > 1 else None
        if public_key:
            # Private payload
            try:
                payload = handle_create_payload(entity,
                                                author_user,
                                                to_user_key=public_key,
                                                parent_user=parent_user)
                payload = json.dumps(payload)
            except Exception as ex:
                logger.error(
                    "handle_send - failed to generate private payload for %s: %s",
                    id, ex)
                continue
            # TODO get_private_endpoint should be imported per protocol
            url = get_private_endpoint(id)
            payloads.append({
                "urls": {url},
                "payload": payload,
                "content_type": "application/json",
            })
        else:
            if not public_payloads["diaspora"]["payload"]:
                public_payloads["diaspora"]["payload"] = handle_create_payload(
                    entity,
                    author_user,
                    parent_user=parent_user,
                )
            # TODO get_public_endpoint should be imported per protocol
            url = get_public_endpoint(id)
            public_payloads["diaspora"]["urls"].add(url)

    # Add public payload
    if public_payloads["diaspora"]["payload"]:
        payloads.append({
            "urls": public_payloads["diaspora"]["urls"],
            "payload": public_payloads["diaspora"]["payload"],
            "content_type": "application/magic-envelope+xml",
        })

    logger.debug("handle_send - %s", payloads)

    # Do actual sending
    for payload in payloads:
        for url in payload["urls"]:
            try:
                send_document(
                    url,
                    payload["payload"],
                    headers={"Content-Type": payload["content_type"]})
            except Exception as ex:
                logger.error(
                    "handle_send - failed to send payload to %s: %s, payload: %s",
                    url, ex, payload["payload"])
Exemple #13
0
 def test_post_raises_and_returns_exception(self, mock_post):
     code, exc = send_document("http://localhost", {"foo": "bar"})
     assert code == None
     assert exc.__class__ == RequestException
Exemple #14
0
def handle_send(
    entity: BaseEntity,
    author_user: UserType,
    recipients: List[Dict],
    parent_user: UserType = None,
    payload_logger: callable = None,
) -> None:
    """Send an entity to remote servers.

    Using this we will build a list of payloads per protocol. After that, each recipient will get the generated
    protocol payload delivered. Delivery to the same endpoint will only be done once so it's ok to include
    the same endpoint as a receiver multiple times.

    Any given user arguments must have ``private_key`` and ``fid`` attributes.

    :arg entity: Entity object to send. Can be a base entity or a protocol specific one.
    :arg author_user: User authoring the object.
    :arg recipients: A list of recipients to delivery to. Each recipient is a dict
                     containing at minimum the "endpoint", "fid", "public" and "protocol" keys.

                     For ActivityPub and Diaspora payloads, "endpoint" should be an URL of the endpoint to deliver to.

                     The "fid" can be empty for Diaspora payloads. For ActivityPub it should be the recipient
                     federation ID should the delivery be non-private.

                     The "protocol" should be a protocol name that is known for this recipient.

                     The "public" value should be a boolean to indicate whether the payload should be flagged as a
                     public payload.

                     TODO: support guessing the protocol over networks? Would need caching of results

                     For private deliveries to Diaspora protocol recipients, "public_key" is also required.

                     For example
                     [
                        {
                            "endpoint": "https://domain.tld/receive/users/1234-5678-0123-4567",
                            "fid": "",
                            "protocol": "diaspora",
                            "public": False,
                            "public_key": <RSAPublicKey object> | str,
                        },
                        {
                            "endpoint": "https://domain2.tld/receive/public",
                            "fid": "",
                            "protocol": "diaspora",
                            "public": True,
                        },
                        {
                            "endpoint": "https://domain4.tld/sharedinbox/",
                            "fid": "https://domain4.tld/profiles/jack/",
                            "protocol": "activitypub",
                            "public": True,
                        },
                        {
                            "endpoint": "https://domain4.tld/profiles/jill/inbox",
                            "fid": "https://domain4.tld/profiles/jill",
                            "protocol": "activitypub",
                            "public": False,
                        },
                        {
                            "endpoint": "https://matrix.domain.tld",
                            "fid": "#@user:domain.tld",
                            "protocol": "matrix",
                            "public": True,
                        }
                     ]
    :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
                      Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
                      be generated. If given, the payload will be sent as this user.
    :arg payload_logger: (Optional) Function to log the payloads with.
    """
    payloads = []
    ready_payloads = {
        "activitypub": {
            "auth": None,
            "headers": {},
            "payload": None,
            "urls": set(),
        },
        "diaspora": {
            "auth": None,
            "headers": {},
            "payload": None,
            "urls": set(),
        },
        "matrix": {
            "auth": None,
            "headers": {},
            "payload": None,
            "urls": set(),
        },
    }
    skip_ready_payload = {
        "activitypub": False,
        "diaspora": False,
        "matrix": False,
    }

    # Flatten to unique recipients
    # TODO supply a callable that empties "fid" in the case that public=True
    unique_recipients = unique_everseen(recipients)

    matrix_config = None

    # Generate payloads and collect urls
    for recipient in unique_recipients:
        payload = None
        endpoint = recipient["endpoint"]
        fid = recipient["fid"]
        public_key = recipient.get("public_key")
        if isinstance(public_key, str):
            public_key = RSA.importKey(public_key)
        protocol = recipient["protocol"]
        public = recipient["public"]

        if protocol == "activitypub":
            if skip_ready_payload["activitypub"]:
                continue
            if entity.__class__.__name__.startswith(
                    "Diaspora") or entity.__class__.__name__.startswith(
                        "Matrix"):
                # Don't try to do anything with these entities currently
                skip_ready_payload["activitypub"] = True
                continue
            # noinspection PyBroadException
            try:
                if not ready_payloads[protocol]["payload"]:
                    try:
                        # noinspection PyTypeChecker
                        ready_payloads[protocol][
                            "payload"] = handle_create_payload(
                                entity,
                                author_user,
                                protocol,
                                parent_user=parent_user,
                                payload_logger=payload_logger,
                            )
                    except ValueError as ex:
                        # No point continuing for this protocol
                        skip_ready_payload["activitypub"] = True
                        logger.warning(
                            "handle_send - skipping activitypub due to failure to generate payload: %s",
                            ex)
                        continue
                payload = copy.copy(ready_payloads[protocol]["payload"])
                if public:
                    payload["to"] = [NAMESPACE_PUBLIC]
                    payload["cc"] = [fid]
                    if isinstance(payload.get("object"), dict):
                        payload["object"]["to"] = [NAMESPACE_PUBLIC]
                        payload["object"]["cc"] = [fid]
                else:
                    payload["to"] = [fid]
                    if isinstance(payload.get("object"), dict):
                        payload["object"]["to"] = [fid]
                rendered_payload = json.dumps(payload).encode("utf-8")
            except Exception:
                logger.error(
                    "handle_send - failed to generate activitypub payload for %s, %s: %s",
                    fid,
                    endpoint,
                    traceback.format_exc(),
                    extra={
                        "recipient": recipient,
                        "unique_recipients": list(unique_recipients),
                        "payload": payload,
                        "payloads": payloads,
                        "ready_payloads": ready_payloads,
                        "entity": entity,
                        "author": author_user.id,
                        "parent_user": parent_user.id,
                    })
                continue
            payloads.append({
                "auth":
                get_http_authentication(author_user.rsa_private_key,
                                        f"{author_user.id}#main-key"),
                "headers": {
                    "Content-Type":
                    'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
                },
                "payload":
                rendered_payload,
                "urls": {endpoint},
            })
        elif protocol == "diaspora":
            if entity.__class__.__name__.startswith(
                    "Activitypub") or entity.__class__.__name__.startswith(
                        "Matrix"):
                # Don't try to do anything with these entities currently
                skip_ready_payload["diaspora"] = True
                continue
            if public:
                if skip_ready_payload["diaspora"]:
                    continue
                if public_key:
                    logger.warning(
                        "handle_send - Diaspora recipient cannot be public and use encrypted delivery"
                    )
                    continue
                if not ready_payloads[protocol]["payload"]:
                    try:
                        # noinspection PyTypeChecker
                        ready_payloads[protocol][
                            "payload"] = handle_create_payload(
                                entity,
                                author_user,
                                protocol,
                                parent_user=parent_user,
                                payload_logger=payload_logger,
                            )
                    except Exception as ex:
                        # No point continuing for this protocol
                        skip_ready_payload["diaspora"] = True
                        logger.warning(
                            "handle_send - skipping diaspora due to failure to generate payload: %s",
                            ex)
                        continue
                ready_payloads["diaspora"]["urls"].add(endpoint)
            else:
                if not public_key:
                    logger.warning(
                        "handle_send - Diaspora recipient cannot be private without a public key for "
                        "encrypted delivery")
                    continue
                # Private payload
                try:
                    payload = handle_create_payload(
                        entity,
                        author_user,
                        "diaspora",
                        to_user_key=public_key,
                        parent_user=parent_user,
                        payload_logger=payload_logger,
                    )
                    payload = json.dumps(payload)
                except Exception as ex:
                    logger.error(
                        "handle_send - failed to generate private payload for %s: %s",
                        endpoint, ex)
                    continue
                payloads.append({
                    "auth": None,
                    "headers": {
                        "Content-Type": "application/json",
                    },
                    "payload": payload,
                    "urls": {endpoint},
                })
        elif protocol == "matrix":
            if skip_ready_payload["matrix"]:
                continue
            if entity.__class__.__name__.startswith(
                    "Activitypub") or entity.__class__.__name__.startswith(
                        "Diaspora"):
                # Don't try to do anything with these entities currently
                skip_ready_payload["matrix"] = True
                continue
            payload_info = []
            # noinspection PyBroadException
            try:
                try:
                    # For matrix we actually might get multiple payloads and endpoints
                    payload_info = handle_create_payload(
                        entity,
                        author_user,
                        protocol,
                        parent_user=parent_user,
                        payload_logger=payload_logger,
                    )
                except ValueError as ex:
                    # No point continuing for this protocol
                    skip_ready_payload["matrix"] = True
                    logger.warning(
                        "handle_send - skipping matrix due to failure to generate payload: %s",
                        ex)
                    continue
                if not matrix_config:
                    matrix_config = get_matrix_configuration()
                for payload in payload_info:
                    rendered_payload = json.dumps(
                        payload["payload"]).encode("utf-8")
                    payloads.append({
                        "auth": None,
                        "headers": {
                            "Authorization":
                            f"Bearer {matrix_config['appservice']['token']}",
                            "Content-Type": "application/json",
                        },
                        "payload": rendered_payload,
                        "urls": {payload["endpoint"]},
                        "method": payload.get("method"),
                    })
            except Exception:
                logger.error(
                    "handle_send - failed to generate matrix payload for %s, %s: %s",
                    fid,
                    endpoint,
                    traceback.format_exc(),
                    extra={
                        "recipient": recipient,
                        "unique_recipients": list(unique_recipients),
                        "payload_info": payload_info,
                        "payloads": payloads,
                        "ready_payloads": ready_payloads,
                        "entity": entity,
                        "author": author_user.id,
                        "parent_user": parent_user.id,
                    })
                continue

    # Add public diaspora payload
    if ready_payloads["diaspora"]["payload"]:
        payloads.append({
            "auth": None,
            "headers": {
                "Content-Type": "application/magic-envelope+xml",
            },
            "payload": ready_payloads["diaspora"]["payload"],
            "urls": ready_payloads["diaspora"]["urls"],
        })

    logger.debug("handle_send - %s", payloads)

    # Do actual sending
    for payload in payloads:
        for url in payload["urls"]:
            try:
                # TODO send_document and fetch_document need to handle rate limits
                send_document(
                    url,
                    payload["payload"],
                    auth=payload.get("auth"),
                    headers=payload.get("headers"),
                    method=payload.get("method"),
                )
            except Exception as ex:
                logger.error(
                    "handle_send - failed to send payload to %s: %s, payload: %s",
                    url, ex, payload["payload"])
Exemple #15
0
def handle_send(
        entity: BaseEntity,
        author_user: UserType,
        recipients: List[Dict],
        parent_user: UserType = None,
) -> None:
    """Send an entity to remote servers.

    Using this we will build a list of payloads per protocol. After that, each recipient will get the generated
    protocol payload delivered. Delivery to the same endpoint will only be done once so it's ok to include
    the same endpoint as a receiver multiple times.

    Any given user arguments must have ``private_key`` and ``fid`` attributes.

    :arg entity: Entity object to send. Can be a base entity or a protocol specific one.
    :arg author_user: User authoring the object.
    :arg recipients: A list of recipients to delivery to. Each recipient is a dict
                     containing at minimum the "endpoint", "fid", "public" and "protocol" keys.

                     For ActivityPub and Diaspora payloads, "endpoint" should be an URL of the endpoint to deliver to.

                     The "fid" can be empty for Diaspora payloads. For ActivityPub it should be the recipient
                     federation ID should the delivery be non-private.

                     The "protocol" should be a protocol name that is known for this recipient.

                     The "public" value should be a boolean to indicate whether the payload should be flagged as a
                     public payload.

                     TODO: support guessing the protocol over networks? Would need caching of results

                     For private deliveries to Diaspora protocol recipients, "public_key" is also required.

                     For example
                     [
                        {
                            "endpoint": "https://domain.tld/receive/users/1234-5678-0123-4567",
                            "fid": "",
                            "protocol": "diaspora",
                            "public": False,
                            "public_key": <RSAPublicKey object> | str,
                        },
                        {
                            "endpoint": "https://domain2.tld/receive/public",
                            "fid": "",
                            "protocol": "diaspora",
                            "public": True,
                        },
                        {
                            "endpoint": "https://domain4.tld/sharedinbox/",
                            "fid": "https://domain4.tld/profiles/jack/",
                            "protocol": "activitypub",
                            "public": True,
                        },
                        {
                            "endpoint": "https://domain4.tld/profiles/jill/inbox",
                            "fid": "https://domain4.tld/profiles/jill",
                            "protocol": "activitypub",
                            "public": False,
                        },
                     ]
    :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
                      Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
                      be generated. If given, the payload will be sent as this user.
    """
    payloads = []
    public_payloads = {
        "activitypub": {
            "auth": None,
            "payload": None,
            "urls": set(),
        },
        "diaspora": {
            "auth": None,
            "payload": None,
            "urls": set(),
        },
    }

    # Flatten to unique recipients
    # TODO supply a callable that empties "fid" in the case that public=True
    unique_recipients = unique_everseen(recipients)

    # Generate payloads and collect urls
    for recipient in unique_recipients:
        endpoint = recipient["endpoint"]
        fid = recipient["fid"]
        public_key = recipient.get("public_key")
        if isinstance(public_key, str):
            public_key = RSA.importKey(public_key)
        protocol = recipient["protocol"]
        public = recipient["public"]

        if protocol == "activitypub":
            try:
                payload = handle_create_payload(entity, author_user, protocol, parent_user=parent_user)
                if public:
                    payload["to"] = [NAMESPACE_PUBLIC]
                    payload["cc"] = [fid]
                    if isinstance(payload.get("object"), dict):
                        payload["object"]["to"] = [NAMESPACE_PUBLIC]
                        payload["object"]["cc"] = [fid]
                else:
                    payload["to"] = [fid]
                    if isinstance(payload.get("object"), dict):
                        payload["object"]["to"] = [fid]
                payload = json.dumps(payload).encode("utf-8")
            except Exception as ex:
                logger.error("handle_send - failed to generate payload for %s, %s: %s", fid, endpoint, ex)
                continue
            payloads.append({
                "auth": get_http_authentication(author_user.rsa_private_key, f"{author_user.id}#main-key"),
                "payload": payload,
                "content_type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
                "urls": {endpoint},
            })
        elif protocol == "diaspora":
            if public:
                if public_key:
                    raise ValueError("handle_send - Diaspora recipient cannot be public and use encrypted delivery")
                if not public_payloads[protocol]["payload"]:
                    try:
                        # noinspection PyTypeChecker
                        public_payloads[protocol]["payload"] = handle_create_payload(
                            entity, author_user, protocol, parent_user=parent_user,
                        )
                    except Exception as ex:
                        logger.error("handle_send - failed to generate public payload for %s: %s", endpoint, ex)
                public_payloads["diaspora"]["urls"].add(endpoint)
            else:
                if not public_key:
                    raise ValueError("handle_send - Diaspora recipient cannot be private without a public key for "
                                     "encrypted delivery")
                # Private payload
                try:
                    payload = handle_create_payload(
                        entity, author_user, "diaspora", to_user_key=public_key, parent_user=parent_user,
                    )
                    payload = json.dumps(payload)
                except Exception as ex:
                    logger.error("handle_send - failed to generate private payload for %s: %s", endpoint, ex)
                    continue
                payloads.append({
                    "urls": {endpoint}, "payload": payload, "content_type": "application/json", "auth": None,
                })

    # Add public diaspora payload
    if public_payloads["diaspora"]["payload"]:
        payloads.append({
            "urls": public_payloads["diaspora"]["urls"], "payload": public_payloads["diaspora"]["payload"],
            "content_type": "application/magic-envelope+xml", "auth": None,
        })

    logger.debug("handle_send - %s", payloads)

    # Do actual sending
    for payload in payloads:
        for url in payload["urls"]:
            try:
                send_document(
                    url,
                    payload["payload"],
                    auth=payload["auth"],
                    headers={"Content-Type": payload["content_type"]},
                )
            except Exception as ex:
                logger.error("handle_send - failed to send payload to %s: %s, payload: %s", url, ex, payload["payload"])