def migrate(self) -> None:
for data in find_activities({
**by_type(ap.ActivityType.CREATE),
**not_deleted()
}):
try:
activity = ap.parse_activity(data["activity"])
mentions = []
obj = activity.get_object()
for m in obj.get_mentions():
mentions.append(m.href)
hashtags = []
for h in obj.get_hashtags():
hashtags.append(h.name[1:]) # Strip the #
update_one_activity(
by_remote_id(data["remote_id"]),
upsert({
MetaKey.MENTIONS: mentions,
MetaKey.HASHTAGS: hashtags
}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
def _emoji_reaction_process_inbox(emoji_reaction: ap.EmojiReaction,
new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={emoji_reaction!r}")
obj = emoji_reaction.get_object()
# Try to update an existing emoji reaction counter entry for the activity emoji
if not update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id),
"meta.emoji_reactions.emoji":
emoji_reaction.content,
},
{"$inc": {
"meta.emoji_reactions.$.count": 1
}},
):
# Bootstrap the current emoji counter
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
{
"$push": {
"meta.emoji_reactions": {
"emoji": emoji_reaction.content,
"count": 1,
}
}
},
)
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={update!r}")
obj = update.get_object()
if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE:
update_one_activity({"activity.object.id": obj.id},
{"$set": {
"activity.object": obj.to_dict()
}})
elif obj.has_type(ap.ActivityType.QUESTION):
choices = obj._data.get("oneOf", obj.anyOf)
total_replies = 0
_set = {}
for choice in choices:
answer_key = _answer_key(choice["name"])
cnt = choice["replies"]["totalItems"]
total_replies += cnt
_set[f"meta.question_answers.{answer_key}"] = cnt
_set["meta.question_replies"] = total_replies
update_one_activity({
**in_inbox(),
**by_object_id(obj.id)
}, {"$set": _set})
# Also update the cached copies of the question (like Announce and Like)
DB.activities.update_many(by_object_id(obj.id),
upsert({MetaKey.OBJECT: obj.to_dict()}))
elif obj.has_type(ap.ACTOR_TYPES):
actor = ap.fetch_remote_activity(obj.id, no_cache=True)
update_cached_actor(actor)
else:
raise ValueError(f"don't know how to update {obj!r}")
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={announce!r}")
# TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else
# or remove it?
try:
obj = announce.get_object()
except NotAnActivityError:
_logger.exception(
f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message'
)
return
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
update_one_activity(
by_remote_id(announce.id),
upsert({
MetaKey.OBJECT: obj.to_dict(embed=True),
MetaKey.OBJECT_ACTOR: obj.get_actor().to_dict(embed=True),
}),
)
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
inc(MetaKey.COUNT_BOOST, 1),
)
def _delete_process_inbox(delete: ap.Delete, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={delete!r}")
obj_id = delete.get_object_id()
_logger.debug(f"delete object={obj_id}")
try:
# FIXME(tsileo): call the DB here instead? like for the oubox
obj = ap.fetch_remote_activity(obj_id)
_logger.info(f"inbox_delete handle_replies obj={obj!r}")
in_reply_to = obj.get_in_reply_to() if obj.inReplyTo else None
if obj.has_type(ap.CREATE_TYPES):
in_reply_to = ap._get_id(
DB.activities.find_one({
"meta.object_id": obj_id,
"type": ap.ActivityType.CREATE.value
})["activity"]["object"].get("inReplyTo"))
if in_reply_to:
back._handle_replies_delete(MY_PERSON, in_reply_to)
except Exception:
_logger.exception(f"failed to handle delete replies for {obj_id}")
update_one_activity(
{
**by_object_id(obj_id),
**by_type(ap.ActivityType.CREATE)
},
upsert({MetaKey.DELETED: True}),
)
# Foce undo other related activities
DB.activities.update(by_object_id(obj_id), upsert({MetaKey.UNDO: True}))
def api_ack_reply() -> _Response:
reply_iri = _user_api_arg("reply_iri")
obj = ap.fetch_remote_activity(reply_iri)
if obj.has_type(ap.ActivityType.CREATE):
obj = obj.get_object()
# TODO(tsileo): tweak the adressing?
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
{"$set": {
"meta.reply_acked": True
}},
)
read = ap.Read(
actor=MY_PERSON.id,
object=obj.id,
to=[MY_PERSON.followers],
cc=[ap.AS_PUBLIC, obj.get_actor().id],
published=now(),
context=new_context(obj),
)
read_id = post_to_outbox(read)
return _user_api_response(activity=read_id)
def migrate(self) -> None:
for data in find_activities({
**by_type(ap.ActivityType.CREATE),
**not_deleted()
}):
try:
in_reply_to = data["activity"]["object"].get("inReplyTo")
if in_reply_to:
update_one_activity(
by_remote_id(data["remote_id"]),
upsert({MetaKey.IN_REPLY_TO: in_reply_to}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
for data in DB.replies.find({**not_deleted()}):
try:
in_reply_to = data["activity"].get("inReplyTo")
if in_reply_to:
DB.replies.update_one(
by_remote_id(data["remote_id"]),
upsert({MetaKey.IN_REPLY_TO: in_reply_to}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
def task_cache_object() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
obj = activity.get_object()
# Refetch the object actor (without cache)
with no_cache():
obj_actor = ap.fetch_remote_activity(obj.get_actor().id)
cache = {MetaKey.OBJECT: obj.to_dict(embed=True)}
if activity.get_actor().id != obj_actor.id:
# Cache the object actor
obj_actor_hash = _actor_hash(obj_actor)
cache[MetaKey.OBJECT_ACTOR] = obj_actor.to_dict(embed=True)
cache[MetaKey.OBJECT_ACTOR_ID] = obj_actor.id
cache[MetaKey.OBJECT_ACTOR_HASH] = obj_actor_hash
# Update the actor cache for the other activities
update_cached_actor(obj_actor)
update_one_activity(by_remote_id(activity.id), upsert(cache))
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}})
app.logger.exception(f"flagging activity {iri} as deleted, no object caching")
except Exception as err:
app.logger.exception(f"failed to cache object for {iri}")
raise TaskError() from err
return ""
def _follow_process_inbox(activity: ap.Follow, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={activity!r}")
# Reply to a Follow with an Accept if we're not manully approving them
if not config.MANUALLY_APPROVES_FOLLOWERS:
accept_follow(activity)
else:
update_one_activity(
by_remote_id(activity.id),
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.WAITING.value}),
)
def _like_process_inbox(like: ap.Like, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={like!r}")
obj = like.get_object()
# Update the meta counter if the object is published by the server
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
inc(MetaKey.COUNT_LIKE, 1),
)
def _announce_process_outbox(announce: ap.Announce, new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={announce!r}")
obj = announce.get_object()
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
Tasks.cache_object(announce.id)
update_one_activity(
{**by_object_id(obj.id), **by_type(ap.ActivityType.CREATE)},
upsert({MetaKey.BOOSTED: announce.id}),
)
def _like_process_outbox(like: ap.Like, new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={like!r}")
obj = like.get_object()
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
# Cache the object for display on the "Liked" public page
Tasks.cache_object(like.id)
update_one_activity(
{**by_object_id(obj.id), **by_type(ap.ActivityType.CREATE)},
{**inc(MetaKey.COUNT_LIKE, 1), **upsert({MetaKey.LIKED: like.id})},
)
def migrate(self) -> None:
for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}):
try:
update_one_activity(
{
**by_type(ap.ActivityType.FOLLOW),
**by_remote_id(data["meta"]["object_id"]),
},
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}),
)
# Check if we are following this actor
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_actor_id(data["meta"]["actor_id"]),
**not_undo(),
}
raw_follow = DB.activities.find_one(follow_query)
if raw_follow:
DB.activities.update_many(
follow_query,
{"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}):
try:
update_one_activity(
{
**by_type(ap.ActivityType.FOLLOW),
**by_remote_id(data["meta"]["object_id"]),
},
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
DB.activities.update_many(
{
**by_type(ap.ActivityType.FOLLOW),
**in_inbox(),
"meta.follow_status": {"$exists": False},
},
{"$set": {"meta.follow_status": "waiting"}},
)
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={announce!r}")
# TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else
# or remove it?
try:
obj = announce.get_object()
except NotAnActivityError:
_logger.exception(
f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message'
)
return
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
# Cache the announced object
Tasks.cache_object(announce.id)
# Process the reply of the announced object if any
in_reply_to = obj.get_in_reply_to()
if in_reply_to:
reply = ap.fetch_remote_activity(in_reply_to)
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to}
# Update the activity to save some data about the reply
if reply.get_actor().id == obj.get_actor().id:
in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True})
else:
in_reply_to_data.update({
MetaKey.IN_REPLY_TO_ACTOR:
reply.get_actor().to_dict(embed=True)
})
update_one_activity(by_remote_id(announce.id),
upsert(in_reply_to_data))
# Spawn a task to process it (and determine if it needs to be saved)
Tasks.process_reply(reply.id)
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
inc(MetaKey.COUNT_BOOST, 1),
)
def migrate(self) -> None:
for data in find_activities({"meta.published": {"$exists": False}}):
try:
raw = data["activity"]
# If the activity has its own `published` field, we'll use it
if "published" in raw:
published = raw["published"]
else:
# Otherwise, we take the date we received the activity as the published time
published = ap.format_datetime(data["_id"].generation_time)
# Set the field in the DB
update_one_activity(
{"_id": data["_id"]},
{"$set": {_meta(MetaKey.PUBLISHED): published}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
def accept_follow(activity: ap.BaseActivity) -> str:
actor_id = activity.get_actor().id
accept = ap.Accept(
actor=ID,
context=new_context(activity),
object={
"type": "Follow",
"id": activity.id,
"object": activity.get_object_id(),
"actor": actor_id,
},
to=[actor_id],
published=now(),
)
update_one_activity(
by_remote_id(activity.id),
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}),
)
return post_to_outbox(accept)
def _delete_process_outbox(delete: ap.Delete, new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={delete!r}")
obj_id = delete.get_object_id()
# Flag everything referencing the deleted object as deleted (except the Delete activity itself)
update_many_activities(
{
**by_object_id(obj_id), "remote_id": {
"$ne": delete.id
}
},
upsert({
MetaKey.DELETED: True,
MetaKey.UNDO: True
}),
)
# If the deleted activity was in DB, decrease some threads-related counter
data = find_one_activity({
**by_object_id(obj_id),
**by_type(ap.ActivityType.CREATE)
})
_logger.info(f"found local copy of deleted activity: {data}")
if data:
obj = ap.parse_activity(data["activity"]).get_object()
_logger.info(f"obj={obj!r}")
in_reply_to = obj.get_in_reply_to()
if in_reply_to:
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(in_reply_to)
},
{
"$inc": {
"meta.count_reply": -1,
"meta.count_direct_reply": -1
}
},
)
def _undo_process_outbox(undo: ap.Undo, new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={undo!r}")
obj = undo.get_object()
update_one_activity({"remote_id": obj.id}, {"$set": {"meta.undo": True}})
# Undo Like
if obj.has_type(ap.ActivityType.LIKE):
liked = obj.get_object_id()
update_one_activity(
{
**by_object_id(liked),
**by_type(ap.ActivityType.CREATE)
},
{
**inc(MetaKey.COUNT_LIKE, -1),
**upsert({MetaKey.LIKED: False})
},
)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
announced = obj.get_object_id()
update_one_activity(
{
**by_object_id(announced),
**by_type(ap.ActivityType.CREATE)
},
upsert({MetaKey.BOOSTED: False}),
)
# Undo Follow (undo new following)
elif obj.has_type(ap.ActivityType.FOLLOW):
pass
def _undo_process_inbox(activity: ap.Undo, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={activity!r}")
# Fetch the object that's been undo'ed
obj = activity.get_object()
# Set the undo flag on the mentionned activity
update_one_activity(by_remote_id(obj.id), upsert({MetaKey.UNDO: True}))
# Handle cached counters
if obj.has_type(ap.ActivityType.LIKE):
# Update the meta counter if the object is published by the server
update_one_activity(
{
**by_object_id(obj.get_object_id()),
**by_type(ap.ActivityType.CREATE)
},
inc(MetaKey.COUNT_LIKE, -1),
)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
announced = obj.get_object()
# Update the meta counter if the object is published by the server
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(announced.id)
},
inc(MetaKey.COUNT_BOOST, -1),
)
def _update_process_outbox(update: ap.Update, new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={update!r}")
obj = update._data["object"]
update_prefix = "activity.object."
to_update: Dict[str, Any] = {"$set": dict(), "$unset": dict()}
to_update["$set"][f"{update_prefix}updated"] = (
datetime.utcnow().replace(microsecond=0).isoformat() + "Z")
for k, v in obj.items():
if k in ["id", "type"]:
continue
if v is None:
to_update["$unset"][f"{update_prefix}{k}"] = ""
else:
to_update["$set"][f"{update_prefix}{k}"] = v
if len(to_update["$unset"]) == 0:
del to_update["$unset"]
_logger.info(f"updating note from outbox {obj!r} {to_update}")
update_one_activity({"activity.object.id": obj["id"]}, to_update)
def _announce_process_outbox(announce: ap.Announce,
new_meta: _NewMeta) -> None:
_logger.info(f"process_outbox activity={announce!r}")
obj = announce.get_object()
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
update_one_activity(
by_remote_id(announce.id),
upsert({
MetaKey.OBJECT: obj.to_dict(embed=True),
MetaKey.OBJECT_ACTOR: obj.get_actor().to_dict(embed=True),
}),
)
update_one_activity(
{
**by_object_id(obj.id),
**by_type(ap.ActivityType.CREATE)
},
upsert({MetaKey.BOOSTED: announce.id}),
)
def task_process_reply() -> _Response:
"""Process `Announce`d posts from Pleroma relays in order to process replies of activities that are in the inbox."""
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"checking for reply activity={activity!r}")
# Some AP server always return Create when requesting an object
if activity.has_type(ap.ActivityType.CREATE):
activity = activity.get_object()
in_reply_to = activity.get_in_reply_to()
if not in_reply_to:
# If it's not reply, we can drop it
app.logger.info(
f"activity={activity!r} is not a reply, dropping it")
return ""
root_reply = in_reply_to
# Fetch the activity reply
reply = ap.fetch_remote_activity(in_reply_to)
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
new_replies = [activity, reply]
while 1:
in_reply_to = reply.get_in_reply_to()
if not in_reply_to:
break
root_reply = in_reply_to
reply = ap.fetch_remote_activity(root_reply)
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
new_replies.append(reply)
app.logger.info(f"root_reply={reply!r} for activity={activity!r}")
# In case the activity was from the inbox
update_one_activity(
{
**by_object_id(activity.id),
**by_type(ap.ActivityType.CREATE)
},
upsert({MetaKey.THREAD_ROOT_PARENT: root_reply}),
)
for (new_reply_idx, new_reply) in enumerate(new_replies):
if find_one_activity({
**by_object_id(new_reply.id),
**by_type(ap.ActivityType.CREATE)
}) or DB.replies.find_one(by_remote_id(new_reply.id)):
continue
actor = new_reply.get_actor()
is_root_reply = new_reply_idx == len(new_replies) - 1
if is_root_reply:
reply_flags: Dict[str, Any] = {}
else:
reply_actor = new_replies[new_reply_idx + 1].get_actor()
is_in_reply_to_self = actor.id == reply_actor.id
reply_flags = {
MetaKey.IN_REPLY_TO_SELF.value: is_in_reply_to_self,
MetaKey.IN_REPLY_TO.value: new_reply.get_in_reply_to(),
}
if not is_in_reply_to_self:
reply_flags[MetaKey.IN_REPLY_TO_ACTOR.
value] = reply_actor.to_dict(embed=True)
# Save the reply with the cached actor and the thread flag/ID
save_reply(
new_reply,
{
**reply_flags,
MetaKey.THREAD_ROOT_PARENT.value: root_reply,
MetaKey.ACTOR.value: actor.to_dict(embed=True),
MetaKey.ACTOR_HASH.value: _actor_hash(actor),
},
)
# Update the reply counters
if new_reply.get_in_reply_to():
update_one_activity(
{
**by_object_id(new_reply.get_in_reply_to()),
**by_type(ap.ActivityType.CREATE),
},
inc(MetaKey.COUNT_REPLY, 1),
)
DB.replies.update_one(
by_remote_id(new_reply.get_in_reply_to()),
inc(MetaKey.COUNT_REPLY, 1),
)
# Cache the actor icon
_cache_actor_icon(actor)
# And cache the attachments
Tasks.cache_attachments(new_reply.id)
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"dropping activity {iri}, skip processing")
return ""
except Exception as err:
app.logger.exception(f"failed to process new activity {iri}")
raise TaskError() from err
return ""
def _update_follow_status(follow_id: str, status: FollowStatus) -> None:
_logger.info(f"{follow_id} is {status}")
update_one_activity(by_remote_id(follow_id),
upsert({MetaKey.FOLLOW_STATUS: status.value}))
def handle_replies(create: ap.Create) -> None:
"""Go up to the root reply, store unknown replies in the `threads` DB and set the "meta.thread_root_parent"
key to make it easy to query a whole thread."""
in_reply_to = create.get_object().get_in_reply_to()
if not in_reply_to:
return
reply = ap.fetch_remote_activity(in_reply_to)
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
# FIXME(tsileo): can be a 403 too, in this case what to do? not error at least
# Ensure the this is a local reply, of a question, with a direct "to" addressing
if (
reply.id.startswith(BASE_URL)
and reply.has_type(ap.ActivityType.QUESTION.value)
and _is_local_reply(create)
and not create.is_public()
):
return handle_question_reply(create, reply)
elif (
create.id.startswith(BASE_URL)
and reply.has_type(ap.ActivityType.QUESTION.value)
and not create.is_public()
):
# Keep track of our own votes
DB.activities.update_one(
{"activity.object.id": reply.id, "box": "inbox"},
{
"$set": {
f"meta.poll_answers_sent.{_answer_key(create.get_object().name)}": True
}
},
)
# Mark our reply as a poll answers, to "hide" it from the UI
update_one_activity(
by_remote_id(create.id),
upsert({MetaKey.POLL_ANSWER: True, MetaKey.POLL_ANSWER_TO: reply.id}),
)
return None
in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to}
# Update the activity to save some data about the reply
if reply.get_actor().id == create.get_actor().id:
in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True})
else:
in_reply_to_data.update(
{MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True)}
)
update_one_activity(by_remote_id(create.id), upsert(in_reply_to_data))
# It's a regular reply, try to increment the reply counter
creply = DB.activities.find_one_and_update(
{**by_object_id(in_reply_to), **by_type(ap.ActivityType.CREATE)},
inc(MetaKey.COUNT_REPLY, 1),
)
if not creply:
# Maybe it's the reply of a reply?
DB.replies.find_one_and_update(
by_remote_id(in_reply_to), inc(MetaKey.COUNT_REPLY, 1)
)
# Spawn a task to process it (and determine if it needs to be saved)
Tasks.process_reply(create.get_object_id())
