def _flag_as_notification(activity: ap.BaseActivity, new_meta: _NewMeta) -> None: new_meta.update({ _meta(MetaKey.NOTIFICATION): True, _meta(MetaKey.NOTIFICATION_UNREAD): True }) return None
def task_process_new_activity() -> _Response: """Process an activity received in the inbox""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") flags = {} if not activity.published: flags[_meta(MetaKey.PUBLISHED)] = now() else: flags[_meta(MetaKey.PUBLISHED)] = activity.published set_inbox_flags(activity, flags) app.logger.info(f"a={activity}, flags={flags!r}") if flags: DB.activities.update_one({"remote_id": activity.id}, {"$set": flags}) app.logger.info(f"new activity {iri} processed") if not activity.has_type(ap.ActivityType.DELETE): Tasks.cache_actor(iri) except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"dropping activity {iri}, skip processing") return "" except Exception as err: app.logger.exception(f"failed to process new activity {iri}") raise TaskError() from err return ""
def api_mark_notification_as_read() -> _Response: nid = ObjectId(_user_api_arg("nid")) DB.activities.update_many( {_meta(MetaKey.NOTIFICATION_UNREAD): True, "_id": {"$lte": nid}}, {"$set": {_meta(MetaKey.NOTIFICATION_UNREAD): False}}, ) return _user_api_response()
def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None: """Handle notification for new followers.""" _logger.info(f"set_inbox_flags activity={activity!r}") # Check if we're already following this actor follows_back = False accept_query = { **in_inbox(), **by_type(ap.ActivityType.ACCEPT), **by_actor(activity.get_actor()), **not_undo(), } raw_accept = DB.activities.find_one(accept_query) if raw_accept: follows_back = True DB.activities.update_many( accept_query, {"$set": { _meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True }}) # This Follow will be a "$actor started following you" notification _flag_as_notification(activity, new_meta) _set_flag(new_meta, MetaKey.GC_KEEP) _set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back) return None
def _accept_set_inbox_flags(activity: ap.Accept, new_meta: _NewMeta) -> None: """Handle notifications for "accepted" following requests.""" _logger.info(f"set_inbox_flags activity={activity!r}") # Check if this actor already follow us back follows_back = False follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_actor(activity.get_actor()), **not_undo(), } raw_follow = DB.activities.find_one(follow_query) if raw_follow: follows_back = True DB.activities.update_many( follow_query, {"$set": { _meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True }}) # This Accept will be a "You started following $actor" notification _flag_as_notification(activity, new_meta) _set_flag(new_meta, MetaKey.GC_KEEP) _set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back) return None
def inject_config(): q = { "type": "Create", "activity.object.inReplyTo": None, "meta.deleted": False, "meta.public": True, } notes_count = DB.activities.find({ "box": Box.OUTBOX.value, "$or": [q, { "type": "Announce", "meta.undo": False }] }).count() # FIXME(tsileo): rename to all_count, and remove poll answers from it all_q = { "box": Box.OUTBOX.value, "type": { "$in": [ActivityType.CREATE.value, ActivityType.ANNOUNCE.value] }, "meta.undo": False, "meta.deleted": False, "meta.poll_answer": False, } liked_count = DB.activities.count({ "box": Box.OUTBOX.value, "meta.deleted": False, "meta.undo": False, "type": ActivityType.LIKE.value, }) followers_q = { "box": Box.INBOX.value, "type": ActivityType.FOLLOW.value, "meta.undo": False, } following_q = { "box": Box.OUTBOX.value, "type": ActivityType.FOLLOW.value, "meta.undo": False, } unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True} logged_in = session.get("logged_in", False) return dict( microblogpub_version=VERSION, config=config, logged_in=logged_in, followers_count=DB.activities.count(followers_q), following_count=DB.activities.count(following_q) if logged_in else 0, notes_count=notes_count, liked_count=liked_count, with_replies_count=DB.activities.count(all_q) if logged_in else 0, unread_notifications_count=DB.activities.count(unread_notifications_q) if logged_in else 0, me=ME, base_url=config.BASE_URL, )
def migrate(self) -> None: for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): try: update_one_activity( { **by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"]), }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), ) # Check if we are following this actor follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_object_id(data["meta"]["actor_id"]), **not_undo(), } raw_follow = DB.activities.find_one(follow_query) if raw_follow: DB.activities.update_many( follow_query, {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, ) except Exception: logger.exception(f"failed to process activity {data!r}") for data in find_activities({**by_type(ap.ActivityType.FOLLOW), **in_outbox()}): try: print(data) follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_actor_id(data["meta"]["object_id"]), **not_undo(), } raw_accept = DB.activities.find_one(follow_query) print(raw_accept) if raw_accept: DB.activities.update_many( by_remote_id(data["remote_id"]), {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, ) except Exception: logger.exception(f"failed to process activity {data!r}")
def migrate(self) -> None: for data in find_activities({"meta.published": {"$exists": False}}): try: raw = data["activity"] # If the activity has its own `published` field, we'll use it if "published" in raw: published = raw["published"] else: # Otherwise, we take the date we received the activity as the published time published = ap.format_datetime(data["_id"].generation_time) # Set the field in the DB update_one_activity( {"_id": data["_id"]}, {"$set": {_meta(MetaKey.PUBLISHED): published}}, ) except Exception: logger.exception(f"failed to process activity {data!r}")
def create_indexes(): if "trash" not in DB.collection_names(): DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB if "activities" in DB.collection_names(): DB.command("compact", "activities") try: MEDIA_CACHE.fs._GridFS__database.command("compact", "fs.files") MEDIA_CACHE.fs._GridFS__database.command("compact", "fs.chunks") except Exception: pass DB.activities.create_index([(_meta(MetaKey.NOTIFICATION), pymongo.ASCENDING)]) DB.activities.create_index([(_meta(MetaKey.NOTIFICATION_UNREAD), pymongo.ASCENDING)]) DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING) ]) DB.activities.create_index([ ("meta.thread_root_parent", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ]) DB.activities.create_index([("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)]) DB.activities.create_index([("meta.object_id", pymongo.ASCENDING), ("type", pymongo.ASCENDING)]) # Index for the block query DB.activities.create_index([ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ]) # Index for count queries DB.activities.create_index([ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ]) DB.activities.create_index([("box", pymongo.ASCENDING)]) # Outbox query DB.activities.create_index([ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ("meta.public", pymongo.ASCENDING), ]) DB.activities.create_index([ ("type", pymongo.ASCENDING), ("activity.object.type", pymongo.ASCENDING), ("activity.object.inReplyTo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ]) # For the is_actor_icon_cached query MEDIA_CACHE.fs._GridFS__files.create_index([("url", 1), ("kind", 1)])
def _set_flag(meta: _NewMeta, meta_key: MetaKey, value: Any = True) -> None: meta.update({_meta(meta_key): value}) return None
def perform() -> None: # noqa: C901 start = perf_counter() d = (datetime.utcnow() - timedelta(days=DAYS_TO_KEEP)).strftime("%Y-%m-%d") toi = threads_of_interest() logger.info(f"thread_of_interest={toi!r}") delete_deleted = DB.activities.delete_many({ **in_inbox(), **by_type(ap.ActivityType.DELETE), _meta(MetaKey.PUBLISHED): { "$lt": d }, }).deleted_count logger.info(f"{delete_deleted} Delete deleted") create_deleted = 0 create_count = 0 # Go over the old Create activities for data in DB.activities.find({ "box": Box.INBOX.value, "type": ap.ActivityType.CREATE.value, _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False }, }).limit(500): try: logger.info(f"data={data!r}") create_count += 1 remote_id = data["remote_id"] meta = data["meta"] # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue obj = None if not meta.get("deleted"): try: activity = ap.parse_activity(data["activity"]) logger.info(f"activity={activity!r}") obj = activity.get_object() except (RemoteServerUnavailableError, ActivityGoneError): logger.exception( f"failed to load {remote_id}, this activity will be deleted" ) # This activity mentions the server actor, keep it if obj and obj.has_mention(ID): _keep(data) continue # This activity is a direct reply of one the server actor activity, keep it if obj: in_reply_to = obj.get_in_reply_to() if in_reply_to and in_reply_to.startswith(ID): _keep(data) continue # This activity is part of a thread we want to keep, keep it if obj and in_reply_to and meta.get("thread_root_parent"): thread_root_parent = meta["thread_root_parent"] if thread_root_parent.startswith( ID) or thread_root_parent in toi: _keep(data) continue # This activity was boosted or liked, keep it if meta.get("boosted") or meta.get("liked"): _keep(data) continue # TODO(tsileo): remove after tests if meta.get("keep"): logger.warning( f"{activity!r} would not have been deleted, skipping for now" ) _keep(data) continue # Delete the cached attachment for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.activities.delete_one({"_id": data["_id"]}) create_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") for data in DB.replies.find({ _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False } }).limit(500): try: logger.info(f"data={data!r}") create_count += 1 remote_id = data["remote_id"] meta = data["meta"] # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue obj = ap.parse_activity(data["activity"]) # This activity is a direct reply of one the server actor activity, keep it in_reply_to = obj.get_in_reply_to() # This activity is part of a thread we want to keep, keep it if in_reply_to and meta.get("thread_root_parent"): thread_root_parent = meta["thread_root_parent"] if thread_root_parent.startswith( ID) or thread_root_parent in toi: _keep(data) continue # This activity was boosted or liked, keep it if meta.get("boosted") or meta.get("liked"): _keep(data) continue # Delete the cached attachment for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.replies.delete_one({"_id": data["_id"]}) create_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") after_gc_create = perf_counter() time_to_gc_create = after_gc_create - start logger.info( f"{time_to_gc_create:.2f} seconds to analyze {create_count} Create, {create_deleted} deleted" ) announce_count = 0 announce_deleted = 0 # Go over the old Create activities for data in DB.activities.find({ "box": Box.INBOX.value, "type": ap.ActivityType.ANNOUNCE.value, _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False }, }).limit(500): try: announce_count += 1 remote_id = data["remote_id"] meta = data["meta"] activity = ap.parse_activity(data["activity"]) logger.info(f"activity={activity!r}") # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue object_id = activity.get_object_id() # This announce is for a local activity (i.e. from the outbox), keep it if object_id.startswith(ID): _keep(data) continue for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # TODO(tsileo): here for legacy reason, this needs to be removed at some point for grid_item in MEDIA_CACHE.fs.find({"remote_id": object_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.activities.delete_one({"_id": data["_id"]}) announce_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") after_gc_announce = perf_counter() time_to_gc_announce = after_gc_announce - after_gc_create logger.info( f"{time_to_gc_announce:.2f} seconds to analyze {announce_count} Announce, {announce_deleted} deleted" )
def inject_config(): q = { **in_outbox(), "$or": [ { **by_type(ActivityType.CREATE), **not_deleted(), **by_visibility(ap.Visibility.PUBLIC), }, { **by_type(ActivityType.ANNOUNCE), **not_undo() }, ], } notes_count = DB.activities.count(q) # FIXME(tsileo): rename to all_count, and remove poll answers from it all_q = { **in_outbox(), **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), **not_deleted(), **not_undo(), **not_poll_answer(), } liked_q = { **in_outbox(), **by_type(ActivityType.LIKE), **not_undo(), **not_deleted(), } followers_q = { **in_inbox(), **by_type(ActivityType.FOLLOW), **not_undo(), **not_deleted(), } following_q = { **in_outbox(), **by_type(ActivityType.FOLLOW), **follow_request_accepted(), **not_undo(), **not_deleted(), } unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True} logged_in = session.get("logged_in", False) return dict( microblogpub_version=VERSION, config=config, logged_in=logged_in, followers_count=DB.activities.count(followers_q), following_count=DB.activities.count(following_q) if logged_in or not config.HIDE_FOLLOWING else 0, notes_count=notes_count, liked_count=DB.activities.count(liked_q) if logged_in else 0, with_replies_count=DB.activities.count(all_q) if logged_in else 0, unread_notifications_count=DB.activities.count(unread_notifications_q) if logged_in else 0, me=ME, base_url=config.BASE_URL, highlight_css=HIGHLIGHT_CSS, )
def create_indexes(): if "trash" not in DB.collection_names(): DB.create_collection("trash", capped=True, size=50 << 20) # 50 MB if "activities" in DB.collection_names(): DB.command("compact", "activities") DB.activities.create_index([(_meta(MetaKey.NOTIFICATION), pymongo.ASCENDING)]) DB.activities.create_index( [(_meta(MetaKey.NOTIFICATION_UNREAD), pymongo.ASCENDING)] ) DB.activities.create_index([("remote_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.actor_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.object_id", pymongo.ASCENDING)]) DB.activities.create_index([("meta.thread_root_parent", pymongo.ASCENDING)]) DB.activities.create_index( [ ("meta.thread_root_parent", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ] ) DB.activities.create_index( [("activity.object.id", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING)] ) DB.activities.create_index( [("meta.object_id", pymongo.ASCENDING), ("type", pymongo.ASCENDING)] ) # Index for the block query DB.activities.create_index( [ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ] ) # Index for count queries DB.activities.create_index( [ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ] ) DB.activities.create_index([("box", pymongo.ASCENDING)]) # Outbox query DB.activities.create_index( [ ("box", pymongo.ASCENDING), ("type", pymongo.ASCENDING), ("meta.undo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ("meta.public", pymongo.ASCENDING), ] ) DB.activities.create_index( [ ("type", pymongo.ASCENDING), ("activity.object.type", pymongo.ASCENDING), ("activity.object.inReplyTo", pymongo.ASCENDING), ("meta.deleted", pymongo.ASCENDING), ] )