def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None:
"""Handle notification for new followers."""
_logger.info(f"set_inbox_flags activity={activity!r}")
# Check if we're already following this actor
follows_back = False
accept_query = {
**in_inbox(),
**by_type(ap.ActivityType.ACCEPT),
**by_actor(activity.get_actor()),
**not_undo(),
}
raw_accept = DB.activities.find_one(accept_query)
if raw_accept:
follows_back = True
DB.activities.update_many(
accept_query,
{"$set": {
_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True
}})
# This Follow will be a "$actor started following you" notification
_flag_as_notification(activity, new_meta)
_set_flag(new_meta, MetaKey.GC_KEEP)
_set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back)
return None
def _announce_set_inbox_flags(activity: ap.Announce,
new_meta: _NewMeta) -> None:
_logger.info(f"set_inbox_flags activity={activity!r}")
obj = activity.get_object()
# Is it a Annnounce/boost of local acitivty/from the outbox
if is_from_outbox(obj):
# Flag it as a notification
_flag_as_notification(activity, new_meta)
# Also set the "keep mark" for the GC (as we want to keep it forever)
_set_flag(new_meta, MetaKey.GC_KEEP)
# Dedup boosts (it's annoying to see the same note multipe times on the same page)
if not find_one_activity({
**in_inbox(),
**by_type([ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]),
**by_object_id(obj.id),
**flag(MetaKey.STREAM, True),
**published_after(
datetime.now(timezone.utc) - timedelta(hours=12)),
}):
# Display it in the stream only it not there already (only looking at the last 12 hours)
_set_flag(new_meta, MetaKey.STREAM)
return None
def _accept_set_inbox_flags(activity: ap.Accept, new_meta: _NewMeta) -> None:
"""Handle notifications for "accepted" following requests."""
_logger.info(f"set_inbox_flags activity={activity!r}")
# Check if this actor already follow us back
follows_back = False
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_actor(activity.get_actor()),
**not_undo(),
}
raw_follow = DB.activities.find_one(follow_query)
if raw_follow:
follows_back = True
DB.activities.update_many(
follow_query,
{"$set": {
_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True
}})
# This Accept will be a "You started following $actor" notification
_flag_as_notification(activity, new_meta)
_set_flag(new_meta, MetaKey.GC_KEEP)
_set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back)
return None
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={update!r}")
obj = update.get_object()
if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE:
update_one_activity({"activity.object.id": obj.id},
{"$set": {
"activity.object": obj.to_dict()
}})
elif obj.has_type(ap.ActivityType.QUESTION):
choices = obj._data.get("oneOf", obj.anyOf)
total_replies = 0
_set = {}
for choice in choices:
answer_key = _answer_key(choice["name"])
cnt = choice["replies"]["totalItems"]
total_replies += cnt
_set[f"meta.question_answers.{answer_key}"] = cnt
_set["meta.question_replies"] = total_replies
update_one_activity({
**in_inbox(),
**by_object_id(obj.id)
}, {"$set": _set})
# Also update the cached copies of the question (like Announce and Like)
DB.activities.update_many(by_object_id(obj.id),
upsert({MetaKey.OBJECT: obj.to_dict()}))
elif obj.has_type(ap.ACTOR_TYPES):
actor = ap.fetch_remote_activity(obj.id, no_cache=True)
update_cached_actor(actor)
else:
raise ValueError(f"don't know how to update {obj!r}")
def migrate(self) -> None:
for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}):
try:
update_one_activity(
{
**by_type(ap.ActivityType.FOLLOW),
**by_remote_id(data["meta"]["object_id"]),
},
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}),
)
# Check if we are following this actor
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_actor_id(data["meta"]["actor_id"]),
**not_undo(),
}
raw_follow = DB.activities.find_one(follow_query)
if raw_follow:
DB.activities.update_many(
follow_query,
{"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}):
try:
update_one_activity(
{
**by_type(ap.ActivityType.FOLLOW),
**by_remote_id(data["meta"]["object_id"]),
},
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}),
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
DB.activities.update_many(
{
**by_type(ap.ActivityType.FOLLOW),
**in_inbox(),
"meta.follow_status": {"$exists": False},
},
{"$set": {"meta.follow_status": "waiting"}},
)
def migrate(self) -> None:
DB.activities.update_many(
{
**by_type(ap.ActivityType.FOLLOW),
**in_inbox(),
"meta.follow_status": {"$exists": False},
},
{"$set": {"meta.follow_status": "accepted"}},
)
def perform() -> None: # noqa: C901
start = perf_counter()
d = (datetime.utcnow() - timedelta(days=DAYS_TO_KEEP)).strftime("%Y-%m-%d")
toi = threads_of_interest()
logger.info(f"thread_of_interest={toi!r}")
delete_deleted = DB.activities.delete_many({
**in_inbox(),
**by_type(ap.ActivityType.DELETE),
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
}).deleted_count
logger.info(f"{delete_deleted} Delete deleted")
create_deleted = 0
create_count = 0
# Go over the old Create activities
for data in DB.activities.find({
"box": Box.INBOX.value,
"type": ap.ActivityType.CREATE.value,
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
},
}).limit(500):
try:
logger.info(f"data={data!r}")
create_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
obj = None
if not meta.get("deleted"):
try:
activity = ap.parse_activity(data["activity"])
logger.info(f"activity={activity!r}")
obj = activity.get_object()
except (RemoteServerUnavailableError, ActivityGoneError):
logger.exception(
f"failed to load {remote_id}, this activity will be deleted"
)
# This activity mentions the server actor, keep it
if obj and obj.has_mention(ID):
_keep(data)
continue
# This activity is a direct reply of one the server actor activity, keep it
if obj:
in_reply_to = obj.get_in_reply_to()
if in_reply_to and in_reply_to.startswith(ID):
_keep(data)
continue
# This activity is part of a thread we want to keep, keep it
if obj and in_reply_to and meta.get("thread_root_parent"):
thread_root_parent = meta["thread_root_parent"]
if thread_root_parent.startswith(
ID) or thread_root_parent in toi:
_keep(data)
continue
# This activity was boosted or liked, keep it
if meta.get("boosted") or meta.get("liked"):
_keep(data)
continue
# TODO(tsileo): remove after tests
if meta.get("keep"):
logger.warning(
f"{activity!r} would not have been deleted, skipping for now"
)
_keep(data)
continue
# Delete the cached attachment
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.activities.delete_one({"_id": data["_id"]})
create_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
for data in DB.replies.find({
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
}
}).limit(500):
try:
logger.info(f"data={data!r}")
create_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
obj = ap.parse_activity(data["activity"])
# This activity is a direct reply of one the server actor activity, keep it
in_reply_to = obj.get_in_reply_to()
# This activity is part of a thread we want to keep, keep it
if in_reply_to and meta.get("thread_root_parent"):
thread_root_parent = meta["thread_root_parent"]
if thread_root_parent.startswith(
ID) or thread_root_parent in toi:
_keep(data)
continue
# This activity was boosted or liked, keep it
if meta.get("boosted") or meta.get("liked"):
_keep(data)
continue
# Delete the cached attachment
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.replies.delete_one({"_id": data["_id"]})
create_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
after_gc_create = perf_counter()
time_to_gc_create = after_gc_create - start
logger.info(
f"{time_to_gc_create:.2f} seconds to analyze {create_count} Create, {create_deleted} deleted"
)
announce_count = 0
announce_deleted = 0
# Go over the old Create activities
for data in DB.activities.find({
"box": Box.INBOX.value,
"type": ap.ActivityType.ANNOUNCE.value,
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
},
}).limit(500):
try:
announce_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
activity = ap.parse_activity(data["activity"])
logger.info(f"activity={activity!r}")
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
object_id = activity.get_object_id()
# This announce is for a local activity (i.e. from the outbox), keep it
if object_id.startswith(ID):
_keep(data)
continue
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# TODO(tsileo): here for legacy reason, this needs to be removed at some point
for grid_item in MEDIA_CACHE.fs.find({"remote_id": object_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.activities.delete_one({"_id": data["_id"]})
announce_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
after_gc_announce = perf_counter()
time_to_gc_announce = after_gc_announce - after_gc_create
logger.info(
f"{time_to_gc_announce:.2f} seconds to analyze {announce_count} Announce, {announce_deleted} deleted"
)
def inject_config():
q = {
**in_outbox(),
"$or": [
{
**by_type(ActivityType.CREATE),
**not_deleted(),
**by_visibility(ap.Visibility.PUBLIC),
},
{
**by_type(ActivityType.ANNOUNCE),
**not_undo()
},
],
}
notes_count = DB.activities.count(q)
# FIXME(tsileo): rename to all_count, and remove poll answers from it
all_q = {
**in_outbox(),
**by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]),
**not_deleted(),
**not_undo(),
**not_poll_answer(),
}
liked_q = {
**in_outbox(),
**by_type(ActivityType.LIKE),
**not_undo(),
**not_deleted(),
}
followers_q = {
**in_inbox(),
**by_type(ActivityType.FOLLOW),
**not_undo(),
**not_deleted(),
}
following_q = {
**in_outbox(),
**by_type(ActivityType.FOLLOW),
**follow_request_accepted(),
**not_undo(),
**not_deleted(),
}
unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True}
logged_in = session.get("logged_in", False)
return dict(
microblogpub_version=VERSION,
config=config,
logged_in=logged_in,
followers_count=DB.activities.count(followers_q),
following_count=DB.activities.count(following_q)
if logged_in or not config.HIDE_FOLLOWING else 0,
notes_count=notes_count,
liked_count=DB.activities.count(liked_q) if logged_in else 0,
with_replies_count=DB.activities.count(all_q) if logged_in else 0,
unread_notifications_count=DB.activities.count(unread_notifications_q)
if logged_in else 0,
me=ME,
base_url=config.BASE_URL,
highlight_css=HIGHLIGHT_CSS,
)
