def _flag_as_notification(activity: ap.BaseActivity,
new_meta: _NewMeta) -> None:
new_meta.update({
_meta(MetaKey.NOTIFICATION): True,
_meta(MetaKey.NOTIFICATION_UNREAD): True
})
return None
def task_process_new_activity() -> _Response:
"""Process an activity received in the inbox"""
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
flags = {}
if not activity.published:
flags[_meta(MetaKey.PUBLISHED)] = now()
else:
flags[_meta(MetaKey.PUBLISHED)] = activity.published
set_inbox_flags(activity, flags)
app.logger.info(f"a={activity}, flags={flags!r}")
if flags:
DB.activities.update_one({"remote_id": activity.id}, {"$set": flags})
app.logger.info(f"new activity {iri} processed")
if not activity.has_type(ap.ActivityType.DELETE):
Tasks.cache_actor(iri)
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"dropping activity {iri}, skip processing")
return ""
except Exception as err:
app.logger.exception(f"failed to process new activity {iri}")
raise TaskError() from err
return ""
def api_mark_notification_as_read() -> _Response:
nid = ObjectId(_user_api_arg("nid"))
DB.activities.update_many(
{_meta(MetaKey.NOTIFICATION_UNREAD): True, "_id": {"$lte": nid}},
{"$set": {_meta(MetaKey.NOTIFICATION_UNREAD): False}},
)
return _user_api_response()
def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None:
"""Handle notification for new followers."""
_logger.info(f"set_inbox_flags activity={activity!r}")
# Check if we're already following this actor
follows_back = False
accept_query = {
**in_inbox(),
**by_type(ap.ActivityType.ACCEPT),
**by_actor(activity.get_actor()),
**not_undo(),
}
raw_accept = DB.activities.find_one(accept_query)
if raw_accept:
follows_back = True
DB.activities.update_many(
accept_query,
{"$set": {
_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True
}})
# This Follow will be a "$actor started following you" notification
_flag_as_notification(activity, new_meta)
_set_flag(new_meta, MetaKey.GC_KEEP)
_set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back)
return None
def _accept_set_inbox_flags(activity: ap.Accept, new_meta: _NewMeta) -> None:
"""Handle notifications for "accepted" following requests."""
_logger.info(f"set_inbox_flags activity={activity!r}")
# Check if this actor already follow us back
follows_back = False
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_actor(activity.get_actor()),
**not_undo(),
}
raw_follow = DB.activities.find_one(follow_query)
if raw_follow:
follows_back = True
DB.activities.update_many(
follow_query,
{"$set": {
_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True
}})
# This Accept will be a "You started following $actor" notification
_flag_as_notification(activity, new_meta)
_set_flag(new_meta, MetaKey.GC_KEEP)
_set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back)
return None
def inject_config():
q = {
"type": "Create",
"activity.object.inReplyTo": None,
"meta.deleted": False,
"meta.public": True,
}
notes_count = DB.activities.find({
"box":
Box.OUTBOX.value,
"$or": [q, {
"type": "Announce",
"meta.undo": False
}]
}).count()
# FIXME(tsileo): rename to all_count, and remove poll answers from it
all_q = {
"box": Box.OUTBOX.value,
"type": {
"$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value]
},
"meta.undo": False,
"meta.deleted": False,
"meta.poll_answer": False,
}
liked_count = DB.activities.count({
"box": Box.OUTBOX.value,
"meta.deleted": False,
"meta.undo": False,
"type": ActivityType.LIKE.value,
})
followers_q = {
"box": Box.INBOX.value,
"type": ActivityType.FOLLOW.value,
"meta.undo": False,
}
following_q = {
"box": Box.OUTBOX.value,
"type": ActivityType.FOLLOW.value,
"meta.undo": False,
}
unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True}
logged_in = session.get("logged_in", False)
return dict(
microblogpub_version=VERSION,
config=config,
logged_in=logged_in,
followers_count=DB.activities.count(followers_q),
following_count=DB.activities.count(following_q) if logged_in else 0,
notes_count=notes_count,
liked_count=liked_count,
with_replies_count=DB.activities.count(all_q) if logged_in else 0,
unread_notifications_count=DB.activities.count(unread_notifications_q)
if logged_in else 0,
me=ME,
base_url=config.BASE_URL,
)
def migrate(self) -> None:
for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}):
try:
update_one_activity(
{
**by_type(ap.ActivityType.FOLLOW),
**by_remote_id(data["meta"]["object_id"]),
},
upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}),
)
# Check if we are following this actor
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_object_id(data["meta"]["actor_id"]),
**not_undo(),
}
raw_follow = DB.activities.find_one(follow_query)
if raw_follow:
DB.activities.update_many(
follow_query,
{"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
for data in find_activities({**by_type(ap.ActivityType.FOLLOW), **in_outbox()}):
try:
print(data)
follow_query = {
**in_inbox(),
**by_type(ap.ActivityType.FOLLOW),
**by_actor_id(data["meta"]["object_id"]),
**not_undo(),
}
raw_accept = DB.activities.find_one(follow_query)
print(raw_accept)
if raw_accept:
DB.activities.update_many(
by_remote_id(data["remote_id"]),
{"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
def migrate(self) -> None:
for data in find_activities({"meta.published": {"$exists": False}}):
try:
raw = data["activity"]
# If the activity has its own `published` field, we'll use it
if "published" in raw:
published = raw["published"]
else:
# Otherwise, we take the date we received the activity as the published time
published = ap.format_datetime(data["_id"].generation_time)
# Set the field in the DB
update_one_activity(
{"_id": data["_id"]},
{"$set": {_meta(MetaKey.PUBLISHED): published}},
)
except Exception:
logger.exception(f"failed to process activity {data!r}")
def create_indexes():
if "trash" not in DB.collection_names():
DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB
if "activities" in DB.collection_names():
DB.command("compact", "activities")
try:
MEDIA_CACHE.fs._GridFS__database.command("compact", "fs.files")
MEDIA_CACHE.fs._GridFS__database.command("compact", "fs.chunks")
except Exception:
pass
DB.activities.create_index([(_meta(MetaKey.NOTIFICATION),
pymongo.ASCENDING)])
DB.activities.create_index([(_meta(MetaKey.NOTIFICATION_UNREAD),
pymongo.ASCENDING)])
DB.activities.create_index([("remote_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)
])
DB.activities.create_index([
("meta.thread_root_parent", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
])
DB.activities.create_index([("activity.object.id", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING)])
DB.activities.create_index([("meta.object_id", pymongo.ASCENDING),
("type", pymongo.ASCENDING)])
# Index for the block query
DB.activities.create_index([
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
])
# Index for count queries
DB.activities.create_index([
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
])
DB.activities.create_index([("box", pymongo.ASCENDING)])
# Outbox query
DB.activities.create_index([
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
("meta.public", pymongo.ASCENDING),
])
DB.activities.create_index([
("type", pymongo.ASCENDING),
("activity.object.type", pymongo.ASCENDING),
("activity.object.inReplyTo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
])
# For the is_actor_icon_cached query
MEDIA_CACHE.fs._GridFS__files.create_index([("url", 1), ("kind", 1)])
def _set_flag(meta: _NewMeta, meta_key: MetaKey, value: Any = True) -> None:
meta.update({_meta(meta_key): value})
return None
def perform() -> None: # noqa: C901
start = perf_counter()
d = (datetime.utcnow() - timedelta(days=DAYS_TO_KEEP)).strftime("%Y-%m-%d")
toi = threads_of_interest()
logger.info(f"thread_of_interest={toi!r}")
delete_deleted = DB.activities.delete_many({
**in_inbox(),
**by_type(ap.ActivityType.DELETE),
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
}).deleted_count
logger.info(f"{delete_deleted} Delete deleted")
create_deleted = 0
create_count = 0
# Go over the old Create activities
for data in DB.activities.find({
"box": Box.INBOX.value,
"type": ap.ActivityType.CREATE.value,
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
},
}).limit(500):
try:
logger.info(f"data={data!r}")
create_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
obj = None
if not meta.get("deleted"):
try:
activity = ap.parse_activity(data["activity"])
logger.info(f"activity={activity!r}")
obj = activity.get_object()
except (RemoteServerUnavailableError, ActivityGoneError):
logger.exception(
f"failed to load {remote_id}, this activity will be deleted"
)
# This activity mentions the server actor, keep it
if obj and obj.has_mention(ID):
_keep(data)
continue
# This activity is a direct reply of one the server actor activity, keep it
if obj:
in_reply_to = obj.get_in_reply_to()
if in_reply_to and in_reply_to.startswith(ID):
_keep(data)
continue
# This activity is part of a thread we want to keep, keep it
if obj and in_reply_to and meta.get("thread_root_parent"):
thread_root_parent = meta["thread_root_parent"]
if thread_root_parent.startswith(
ID) or thread_root_parent in toi:
_keep(data)
continue
# This activity was boosted or liked, keep it
if meta.get("boosted") or meta.get("liked"):
_keep(data)
continue
# TODO(tsileo): remove after tests
if meta.get("keep"):
logger.warning(
f"{activity!r} would not have been deleted, skipping for now"
)
_keep(data)
continue
# Delete the cached attachment
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.activities.delete_one({"_id": data["_id"]})
create_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
for data in DB.replies.find({
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
}
}).limit(500):
try:
logger.info(f"data={data!r}")
create_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
obj = ap.parse_activity(data["activity"])
# This activity is a direct reply of one the server actor activity, keep it
in_reply_to = obj.get_in_reply_to()
# This activity is part of a thread we want to keep, keep it
if in_reply_to and meta.get("thread_root_parent"):
thread_root_parent = meta["thread_root_parent"]
if thread_root_parent.startswith(
ID) or thread_root_parent in toi:
_keep(data)
continue
# This activity was boosted or liked, keep it
if meta.get("boosted") or meta.get("liked"):
_keep(data)
continue
# Delete the cached attachment
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.replies.delete_one({"_id": data["_id"]})
create_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
after_gc_create = perf_counter()
time_to_gc_create = after_gc_create - start
logger.info(
f"{time_to_gc_create:.2f} seconds to analyze {create_count} Create, {create_deleted} deleted"
)
announce_count = 0
announce_deleted = 0
# Go over the old Create activities
for data in DB.activities.find({
"box": Box.INBOX.value,
"type": ap.ActivityType.ANNOUNCE.value,
_meta(MetaKey.PUBLISHED): {
"$lt": d
},
"meta.gc_keep": {
"$exists": False
},
}).limit(500):
try:
announce_count += 1
remote_id = data["remote_id"]
meta = data["meta"]
activity = ap.parse_activity(data["activity"])
logger.info(f"activity={activity!r}")
# This activity has been bookmarked, keep it
if meta.get("bookmarked"):
_keep(data)
continue
object_id = activity.get_object_id()
# This announce is for a local activity (i.e. from the outbox), keep it
if object_id.startswith(ID):
_keep(data)
continue
for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# TODO(tsileo): here for legacy reason, this needs to be removed at some point
for grid_item in MEDIA_CACHE.fs.find({"remote_id": object_id}):
MEDIA_CACHE.fs.delete(grid_item._id)
# Delete the activity
DB.activities.delete_one({"_id": data["_id"]})
announce_deleted += 1
except Exception:
logger.exception(f"failed to process {data!r}")
after_gc_announce = perf_counter()
time_to_gc_announce = after_gc_announce - after_gc_create
logger.info(
f"{time_to_gc_announce:.2f} seconds to analyze {announce_count} Announce, {announce_deleted} deleted"
)
def inject_config():
q = {
**in_outbox(),
"$or": [
{
**by_type(ActivityType.CREATE),
**not_deleted(),
**by_visibility(ap.Visibility.PUBLIC),
},
{
**by_type(ActivityType.ANNOUNCE),
**not_undo()
},
],
}
notes_count = DB.activities.count(q)
# FIXME(tsileo): rename to all_count, and remove poll answers from it
all_q = {
**in_outbox(),
**by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]),
**not_deleted(),
**not_undo(),
**not_poll_answer(),
}
liked_q = {
**in_outbox(),
**by_type(ActivityType.LIKE),
**not_undo(),
**not_deleted(),
}
followers_q = {
**in_inbox(),
**by_type(ActivityType.FOLLOW),
**not_undo(),
**not_deleted(),
}
following_q = {
**in_outbox(),
**by_type(ActivityType.FOLLOW),
**follow_request_accepted(),
**not_undo(),
**not_deleted(),
}
unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True}
logged_in = session.get("logged_in", False)
return dict(
microblogpub_version=VERSION,
config=config,
logged_in=logged_in,
followers_count=DB.activities.count(followers_q),
following_count=DB.activities.count(following_q)
if logged_in or not config.HIDE_FOLLOWING else 0,
notes_count=notes_count,
liked_count=DB.activities.count(liked_q) if logged_in else 0,
with_replies_count=DB.activities.count(all_q) if logged_in else 0,
unread_notifications_count=DB.activities.count(unread_notifications_q)
if logged_in else 0,
me=ME,
base_url=config.BASE_URL,
highlight_css=HIGHLIGHT_CSS,
)
def create_indexes():
if "trash" not in DB.collection_names():
DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB
if "activities" in DB.collection_names():
DB.command("compact", "activities")
DB.activities.create_index([(_meta(MetaKey.NOTIFICATION), pymongo.ASCENDING)])
DB.activities.create_index(
[(_meta(MetaKey.NOTIFICATION_UNREAD), pymongo.ASCENDING)]
)
DB.activities.create_index([("remote_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)])
DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)])
DB.activities.create_index(
[
("meta.thread_root_parent", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
]
)
DB.activities.create_index(
[("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)]
)
DB.activities.create_index(
[("meta.object_id", pymongo.ASCENDING), ("type", pymongo.ASCENDING)]
)
# Index for the block query
DB.activities.create_index(
[
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
]
)
# Index for count queries
DB.activities.create_index(
[
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
]
)
DB.activities.create_index([("box", pymongo.ASCENDING)])
# Outbox query
DB.activities.create_index(
[
("box", pymongo.ASCENDING),
("type", pymongo.ASCENDING),
("meta.undo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
("meta.public", pymongo.ASCENDING),
]
)
DB.activities.create_index(
[
("type", pymongo.ASCENDING),
("activity.object.type", pymongo.ASCENDING),
("activity.object.inReplyTo", pymongo.ASCENDING),
("meta.deleted", pymongo.ASCENDING),
]
)
