def migrate(self) -> None: for data in find_activities({ **by_type(ap.ActivityType.CREATE), **not_deleted() }): try: activity = ap.parse_activity(data["activity"]) mentions = [] obj = activity.get_object() for m in obj.get_mentions(): mentions.append(m.href) hashtags = [] for h in obj.get_hashtags(): hashtags.append(h.name[1:]) # Strip the # update_one_activity( by_remote_id(data["remote_id"]), upsert({ MetaKey.MENTIONS: mentions, MetaKey.HASHTAGS: hashtags }), ) except Exception: logger.exception(f"failed to process activity {data!r}")
def _emoji_reaction_process_inbox(emoji_reaction: ap.EmojiReaction, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={emoji_reaction!r}") obj = emoji_reaction.get_object() # Try to update an existing emoji reaction counter entry for the activity emoji if not update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id), "meta.emoji_reactions.emoji": emoji_reaction.content, }, {"$inc": { "meta.emoji_reactions.$.count": 1 }}, ): # Bootstrap the current emoji counter update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, { "$push": { "meta.emoji_reactions": { "emoji": emoji_reaction.content, "count": 1, } } }, )
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={update!r}") obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: update_one_activity({"activity.object.id": obj.id}, {"$set": { "activity.object": obj.to_dict() }}) elif obj.has_type(ap.ActivityType.QUESTION): choices = obj._data.get("oneOf", obj.anyOf) total_replies = 0 _set = {} for choice in choices: answer_key = _answer_key(choice["name"]) cnt = choice["replies"]["totalItems"] total_replies += cnt _set[f"meta.question_answers.{answer_key}"] = cnt _set["meta.question_replies"] = total_replies update_one_activity({ **in_inbox(), **by_object_id(obj.id) }, {"$set": _set}) # Also update the cached copies of the question (like Announce and Like) DB.activities.update_many(by_object_id(obj.id), upsert({MetaKey.OBJECT: obj.to_dict()})) elif obj.has_type(ap.ACTOR_TYPES): actor = ap.fetch_remote_activity(obj.id, no_cache=True) update_cached_actor(actor) else: raise ValueError(f"don't know how to update {obj!r}")
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={announce!r}") # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else # or remove it? try: obj = announce.get_object() except NotAnActivityError: _logger.exception( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) update_one_activity( by_remote_id(announce.id), upsert({ MetaKey.OBJECT: obj.to_dict(embed=True), MetaKey.OBJECT_ACTOR: obj.get_actor().to_dict(embed=True), }), ) update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, inc(MetaKey.COUNT_BOOST, 1), )
def _delete_process_inbox(delete: ap.Delete, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={delete!r}") obj_id = delete.get_object_id() _logger.debug(f"delete object={obj_id}") try: # FIXME(tsileo): call the DB here instead? like for the oubox obj = ap.fetch_remote_activity(obj_id) _logger.info(f"inbox_delete handle_replies obj={obj!r}") in_reply_to = obj.get_in_reply_to() if obj.inReplyTo else None if obj.has_type(ap.CREATE_TYPES): in_reply_to = ap._get_id( DB.activities.find_one({ "meta.object_id": obj_id, "type": ap.ActivityType.CREATE.value })["activity"]["object"].get("inReplyTo")) if in_reply_to: back._handle_replies_delete(MY_PERSON, in_reply_to) except Exception: _logger.exception(f"failed to handle delete replies for {obj_id}") update_one_activity( { **by_object_id(obj_id), **by_type(ap.ActivityType.CREATE) }, upsert({MetaKey.DELETED: True}), ) # Foce undo other related activities DB.activities.update(by_object_id(obj_id), upsert({MetaKey.UNDO: True}))
def api_ack_reply() -> _Response: reply_iri = _user_api_arg("reply_iri") obj = ap.fetch_remote_activity(reply_iri) if obj.has_type(ap.ActivityType.CREATE): obj = obj.get_object() # TODO(tsileo): tweak the adressing? update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, {"$set": { "meta.reply_acked": True }}, ) read = ap.Read( actor=MY_PERSON.id, object=obj.id, to=[MY_PERSON.followers], cc=[ap.AS_PUBLIC, obj.get_actor().id], published=now(), context=new_context(obj), ) read_id = post_to_outbox(read) return _user_api_response(activity=read_id)
def migrate(self) -> None: for data in find_activities({ **by_type(ap.ActivityType.CREATE), **not_deleted() }): try: in_reply_to = data["activity"]["object"].get("inReplyTo") if in_reply_to: update_one_activity( by_remote_id(data["remote_id"]), upsert({MetaKey.IN_REPLY_TO: in_reply_to}), ) except Exception: logger.exception(f"failed to process activity {data!r}") for data in DB.replies.find({**not_deleted()}): try: in_reply_to = data["activity"].get("inReplyTo") if in_reply_to: DB.replies.update_one( by_remote_id(data["remote_id"]), upsert({MetaKey.IN_REPLY_TO: in_reply_to}), ) except Exception: logger.exception(f"failed to process activity {data!r}")
def task_cache_object() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") obj = activity.get_object() # Refetch the object actor (without cache) with no_cache(): obj_actor = ap.fetch_remote_activity(obj.get_actor().id) cache = {MetaKey.OBJECT: obj.to_dict(embed=True)} if activity.get_actor().id != obj_actor.id: # Cache the object actor obj_actor_hash = _actor_hash(obj_actor) cache[MetaKey.OBJECT_ACTOR] = obj_actor.to_dict(embed=True) cache[MetaKey.OBJECT_ACTOR_ID] = obj_actor.id cache[MetaKey.OBJECT_ACTOR_HASH] = obj_actor_hash # Update the actor cache for the other activities update_cached_actor(obj_actor) update_one_activity(by_remote_id(activity.id), upsert(cache)) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no object caching") except Exception as err: app.logger.exception(f"failed to cache object for {iri}") raise TaskError() from err return ""
def _follow_process_inbox(activity: ap.Follow, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={activity!r}") # Reply to a Follow with an Accept if we're not manully approving them if not config.MANUALLY_APPROVES_FOLLOWERS: accept_follow(activity) else: update_one_activity( by_remote_id(activity.id), upsert({MetaKey.FOLLOW_STATUS: FollowStatus.WAITING.value}), )
def _like_process_inbox(like: ap.Like, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={like!r}") obj = like.get_object() # Update the meta counter if the object is published by the server update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, inc(MetaKey.COUNT_LIKE, 1), )
def _announce_process_outbox(announce: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={announce!r}") obj = announce.get_object() if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) Tasks.cache_object(announce.id) update_one_activity( {**by_object_id(obj.id), **by_type(ap.ActivityType.CREATE)}, upsert({MetaKey.BOOSTED: announce.id}), )
def _like_process_outbox(like: ap.Like, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={like!r}") obj = like.get_object() if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) # Cache the object for display on the "Liked" public page Tasks.cache_object(like.id) update_one_activity( {**by_object_id(obj.id), **by_type(ap.ActivityType.CREATE)}, {**inc(MetaKey.COUNT_LIKE, 1), **upsert({MetaKey.LIKED: like.id})}, )
def migrate(self) -> None: for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): try: update_one_activity( { **by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"]), }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), ) # Check if we are following this actor follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_actor_id(data["meta"]["actor_id"]), **not_undo(), } raw_follow = DB.activities.find_one(follow_query) if raw_follow: DB.activities.update_many( follow_query, {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, ) except Exception: logger.exception(f"failed to process activity {data!r}") for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}): try: update_one_activity( { **by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"]), }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}), ) except Exception: logger.exception(f"failed to process activity {data!r}") DB.activities.update_many( { **by_type(ap.ActivityType.FOLLOW), **in_inbox(), "meta.follow_status": {"$exists": False}, }, {"$set": {"meta.follow_status": "waiting"}}, )
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={announce!r}") # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else # or remove it? try: obj = announce.get_object() except NotAnActivityError: _logger.exception( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) # Cache the announced object Tasks.cache_object(announce.id) # Process the reply of the announced object if any in_reply_to = obj.get_in_reply_to() if in_reply_to: reply = ap.fetch_remote_activity(in_reply_to) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to} # Update the activity to save some data about the reply if reply.get_actor().id == obj.get_actor().id: in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True}) else: in_reply_to_data.update({ MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True) }) update_one_activity(by_remote_id(announce.id), upsert(in_reply_to_data)) # Spawn a task to process it (and determine if it needs to be saved) Tasks.process_reply(reply.id) update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, inc(MetaKey.COUNT_BOOST, 1), )
def migrate(self) -> None: for data in find_activities({"meta.published": {"$exists": False}}): try: raw = data["activity"] # If the activity has its own `published` field, we'll use it if "published" in raw: published = raw["published"] else: # Otherwise, we take the date we received the activity as the published time published = ap.format_datetime(data["_id"].generation_time) # Set the field in the DB update_one_activity( {"_id": data["_id"]}, {"$set": {_meta(MetaKey.PUBLISHED): published}}, ) except Exception: logger.exception(f"failed to process activity {data!r}")
def accept_follow(activity: ap.BaseActivity) -> str: actor_id = activity.get_actor().id accept = ap.Accept( actor=ID, context=new_context(activity), object={ "type": "Follow", "id": activity.id, "object": activity.get_object_id(), "actor": actor_id, }, to=[actor_id], published=now(), ) update_one_activity( by_remote_id(activity.id), upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), ) return post_to_outbox(accept)
def _delete_process_outbox(delete: ap.Delete, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={delete!r}") obj_id = delete.get_object_id() # Flag everything referencing the deleted object as deleted (except the Delete activity itself) update_many_activities( { **by_object_id(obj_id), "remote_id": { "$ne": delete.id } }, upsert({ MetaKey.DELETED: True, MetaKey.UNDO: True }), ) # If the deleted activity was in DB, decrease some threads-related counter data = find_one_activity({ **by_object_id(obj_id), **by_type(ap.ActivityType.CREATE) }) _logger.info(f"found local copy of deleted activity: {data}") if data: obj = ap.parse_activity(data["activity"]).get_object() _logger.info(f"obj={obj!r}") in_reply_to = obj.get_in_reply_to() if in_reply_to: update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(in_reply_to) }, { "$inc": { "meta.count_reply": -1, "meta.count_direct_reply": -1 } }, )
def _undo_process_outbox(undo: ap.Undo, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={undo!r}") obj = undo.get_object() update_one_activity({"remote_id": obj.id}, {"$set": {"meta.undo": True}}) # Undo Like if obj.has_type(ap.ActivityType.LIKE): liked = obj.get_object_id() update_one_activity( { **by_object_id(liked), **by_type(ap.ActivityType.CREATE) }, { **inc(MetaKey.COUNT_LIKE, -1), **upsert({MetaKey.LIKED: False}) }, ) elif obj.has_type(ap.ActivityType.ANNOUNCE): announced = obj.get_object_id() update_one_activity( { **by_object_id(announced), **by_type(ap.ActivityType.CREATE) }, upsert({MetaKey.BOOSTED: False}), ) # Undo Follow (undo new following) elif obj.has_type(ap.ActivityType.FOLLOW): pass
def _undo_process_inbox(activity: ap.Undo, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={activity!r}") # Fetch the object that's been undo'ed obj = activity.get_object() # Set the undo flag on the mentionned activity update_one_activity(by_remote_id(obj.id), upsert({MetaKey.UNDO: True})) # Handle cached counters if obj.has_type(ap.ActivityType.LIKE): # Update the meta counter if the object is published by the server update_one_activity( { **by_object_id(obj.get_object_id()), **by_type(ap.ActivityType.CREATE) }, inc(MetaKey.COUNT_LIKE, -1), ) elif obj.has_type(ap.ActivityType.ANNOUNCE): announced = obj.get_object() # Update the meta counter if the object is published by the server update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(announced.id) }, inc(MetaKey.COUNT_BOOST, -1), )
def _update_process_outbox(update: ap.Update, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={update!r}") obj = update._data["object"] update_prefix = "activity.object." to_update: Dict[str, Any] = {"$set": dict(), "$unset": dict()} to_update["$set"][f"{update_prefix}updated"] = ( datetime.utcnow().replace(microsecond=0).isoformat() + "Z") for k, v in obj.items(): if k in ["id", "type"]: continue if v is None: to_update["$unset"][f"{update_prefix}{k}"] = "" else: to_update["$set"][f"{update_prefix}{k}"] = v if len(to_update["$unset"]) == 0: del to_update["$unset"] _logger.info(f"updating note from outbox {obj!r} {to_update}") update_one_activity({"activity.object.id": obj["id"]}, to_update)
def _announce_process_outbox(announce: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"process_outbox activity={announce!r}") obj = announce.get_object() if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) update_one_activity( by_remote_id(announce.id), upsert({ MetaKey.OBJECT: obj.to_dict(embed=True), MetaKey.OBJECT_ACTOR: obj.get_actor().to_dict(embed=True), }), ) update_one_activity( { **by_object_id(obj.id), **by_type(ap.ActivityType.CREATE) }, upsert({MetaKey.BOOSTED: announce.id}), )
def task_process_reply() -> _Response: """Process `Announce`d posts from Pleroma relays in order to process replies of activities that are in the inbox.""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"checking for reply activity={activity!r}") # Some AP server always return Create when requesting an object if activity.has_type(ap.ActivityType.CREATE): activity = activity.get_object() in_reply_to = activity.get_in_reply_to() if not in_reply_to: # If it's not reply, we can drop it app.logger.info( f"activity={activity!r} is not a reply, dropping it") return "" root_reply = in_reply_to # Fetch the activity reply reply = ap.fetch_remote_activity(in_reply_to) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() new_replies = [activity, reply] while 1: in_reply_to = reply.get_in_reply_to() if not in_reply_to: break root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() new_replies.append(reply) app.logger.info(f"root_reply={reply!r} for activity={activity!r}") # In case the activity was from the inbox update_one_activity( { **by_object_id(activity.id), **by_type(ap.ActivityType.CREATE) }, upsert({MetaKey.THREAD_ROOT_PARENT: root_reply}), ) for (new_reply_idx, new_reply) in enumerate(new_replies): if find_one_activity({ **by_object_id(new_reply.id), **by_type(ap.ActivityType.CREATE) }) or DB.replies.find_one(by_remote_id(new_reply.id)): continue actor = new_reply.get_actor() is_root_reply = new_reply_idx == len(new_replies) - 1 if is_root_reply: reply_flags: Dict[str, Any] = {} else: reply_actor = new_replies[new_reply_idx + 1].get_actor() is_in_reply_to_self = actor.id == reply_actor.id reply_flags = { MetaKey.IN_REPLY_TO_SELF.value: is_in_reply_to_self, MetaKey.IN_REPLY_TO.value: new_reply.get_in_reply_to(), } if not is_in_reply_to_self: reply_flags[MetaKey.IN_REPLY_TO_ACTOR. value] = reply_actor.to_dict(embed=True) # Save the reply with the cached actor and the thread flag/ID save_reply( new_reply, { **reply_flags, MetaKey.THREAD_ROOT_PARENT.value: root_reply, MetaKey.ACTOR.value: actor.to_dict(embed=True), MetaKey.ACTOR_HASH.value: _actor_hash(actor), }, ) # Update the reply counters if new_reply.get_in_reply_to(): update_one_activity( { **by_object_id(new_reply.get_in_reply_to()), **by_type(ap.ActivityType.CREATE), }, inc(MetaKey.COUNT_REPLY, 1), ) DB.replies.update_one( by_remote_id(new_reply.get_in_reply_to()), inc(MetaKey.COUNT_REPLY, 1), ) # Cache the actor icon _cache_actor_icon(actor) # And cache the attachments Tasks.cache_attachments(new_reply.id) except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"dropping activity {iri}, skip processing") return "" except Exception as err: app.logger.exception(f"failed to process new activity {iri}") raise TaskError() from err return ""
def _update_follow_status(follow_id: str, status: FollowStatus) -> None: _logger.info(f"{follow_id} is {status}") update_one_activity(by_remote_id(follow_id), upsert({MetaKey.FOLLOW_STATUS: status.value}))
def handle_replies(create: ap.Create) -> None: """Go up to the root reply, store unknown replies in the `threads` DB and set the "meta.thread_root_parent" key to make it easy to query a whole thread.""" in_reply_to = create.get_object().get_in_reply_to() if not in_reply_to: return reply = ap.fetch_remote_activity(in_reply_to) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() # FIXME(tsileo): can be a 403 too, in this case what to do? not error at least # Ensure the this is a local reply, of a question, with a direct "to" addressing if ( reply.id.startswith(BASE_URL) and reply.has_type(ap.ActivityType.QUESTION.value) and _is_local_reply(create) and not create.is_public() ): return handle_question_reply(create, reply) elif ( create.id.startswith(BASE_URL) and reply.has_type(ap.ActivityType.QUESTION.value) and not create.is_public() ): # Keep track of our own votes DB.activities.update_one( {"activity.object.id": reply.id, "box": "inbox"}, { "$set": { f"meta.poll_answers_sent.{_answer_key(create.get_object().name)}": True } }, ) # Mark our reply as a poll answers, to "hide" it from the UI update_one_activity( by_remote_id(create.id), upsert({MetaKey.POLL_ANSWER: True, MetaKey.POLL_ANSWER_TO: reply.id}), ) return None in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to} # Update the activity to save some data about the reply if reply.get_actor().id == create.get_actor().id: in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True}) else: in_reply_to_data.update( {MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True)} ) update_one_activity(by_remote_id(create.id), upsert(in_reply_to_data)) # It's a regular reply, try to increment the reply counter creply = DB.activities.find_one_and_update( {**by_object_id(in_reply_to), **by_type(ap.ActivityType.CREATE)}, inc(MetaKey.COUNT_REPLY, 1), ) if not creply: # Maybe it's the reply of a reply? DB.replies.find_one_and_update( by_remote_id(in_reply_to), inc(MetaKey.COUNT_REPLY, 1) ) # Spawn a task to process it (and determine if it needs to be saved) Tasks.process_reply(create.get_object_id())