def _follow_set_inbox_flags(activity: ap.Follow, new_meta: _NewMeta) -> None: """Handle notification for new followers.""" _logger.info(f"set_inbox_flags activity={activity!r}") # Check if we're already following this actor follows_back = False accept_query = { **in_inbox(), **by_type(ap.ActivityType.ACCEPT), **by_actor(activity.get_actor()), **not_undo(), } raw_accept = DB.activities.find_one(accept_query) if raw_accept: follows_back = True DB.activities.update_many( accept_query, {"$set": { _meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True }}) # This Follow will be a "$actor started following you" notification _flag_as_notification(activity, new_meta) _set_flag(new_meta, MetaKey.GC_KEEP) _set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back) return None
def _announce_set_inbox_flags(activity: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"set_inbox_flags activity={activity!r}") obj = activity.get_object() # Is it a Annnounce/boost of local acitivty/from the outbox if is_from_outbox(obj): # Flag it as a notification _flag_as_notification(activity, new_meta) # Also set the "keep mark" for the GC (as we want to keep it forever) _set_flag(new_meta, MetaKey.GC_KEEP) # Dedup boosts (it's annoying to see the same note multipe times on the same page) if not find_one_activity({ **in_inbox(), **by_type([ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]), **by_object_id(obj.id), **flag(MetaKey.STREAM, True), **published_after( datetime.now(timezone.utc) - timedelta(hours=12)), }): # Display it in the stream only it not there already (only looking at the last 12 hours) _set_flag(new_meta, MetaKey.STREAM) return None
def _accept_set_inbox_flags(activity: ap.Accept, new_meta: _NewMeta) -> None: """Handle notifications for "accepted" following requests.""" _logger.info(f"set_inbox_flags activity={activity!r}") # Check if this actor already follow us back follows_back = False follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_actor(activity.get_actor()), **not_undo(), } raw_follow = DB.activities.find_one(follow_query) if raw_follow: follows_back = True DB.activities.update_many( follow_query, {"$set": { _meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True }}) # This Accept will be a "You started following $actor" notification _flag_as_notification(activity, new_meta) _set_flag(new_meta, MetaKey.GC_KEEP) _set_flag(new_meta, MetaKey.NOTIFICATION_FOLLOWS_BACK, follows_back) return None
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={update!r}") obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: update_one_activity({"activity.object.id": obj.id}, {"$set": { "activity.object": obj.to_dict() }}) elif obj.has_type(ap.ActivityType.QUESTION): choices = obj._data.get("oneOf", obj.anyOf) total_replies = 0 _set = {} for choice in choices: answer_key = _answer_key(choice["name"]) cnt = choice["replies"]["totalItems"] total_replies += cnt _set[f"meta.question_answers.{answer_key}"] = cnt _set["meta.question_replies"] = total_replies update_one_activity({ **in_inbox(), **by_object_id(obj.id) }, {"$set": _set}) # Also update the cached copies of the question (like Announce and Like) DB.activities.update_many(by_object_id(obj.id), upsert({MetaKey.OBJECT: obj.to_dict()})) elif obj.has_type(ap.ACTOR_TYPES): actor = ap.fetch_remote_activity(obj.id, no_cache=True) update_cached_actor(actor) else: raise ValueError(f"don't know how to update {obj!r}")
def migrate(self) -> None: for data in find_activities({**by_type(ap.ActivityType.ACCEPT), **in_inbox()}): try: update_one_activity( { **by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"]), }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.ACCEPTED.value}), ) # Check if we are following this actor follow_query = { **in_inbox(), **by_type(ap.ActivityType.FOLLOW), **by_actor_id(data["meta"]["actor_id"]), **not_undo(), } raw_follow = DB.activities.find_one(follow_query) if raw_follow: DB.activities.update_many( follow_query, {"$set": {_meta(MetaKey.NOTIFICATION_FOLLOWS_BACK): True}}, ) except Exception: logger.exception(f"failed to process activity {data!r}") for data in find_activities({**by_type(ap.ActivityType.REJECT), **in_inbox()}): try: update_one_activity( { **by_type(ap.ActivityType.FOLLOW), **by_remote_id(data["meta"]["object_id"]), }, upsert({MetaKey.FOLLOW_STATUS: FollowStatus.REJECTED.value}), ) except Exception: logger.exception(f"failed to process activity {data!r}") DB.activities.update_many( { **by_type(ap.ActivityType.FOLLOW), **in_inbox(), "meta.follow_status": {"$exists": False}, }, {"$set": {"meta.follow_status": "waiting"}}, )
def migrate(self) -> None: DB.activities.update_many( { **by_type(ap.ActivityType.FOLLOW), **in_inbox(), "meta.follow_status": {"$exists": False}, }, {"$set": {"meta.follow_status": "accepted"}}, )
def perform() -> None: # noqa: C901 start = perf_counter() d = (datetime.utcnow() - timedelta(days=DAYS_TO_KEEP)).strftime("%Y-%m-%d") toi = threads_of_interest() logger.info(f"thread_of_interest={toi!r}") delete_deleted = DB.activities.delete_many({ **in_inbox(), **by_type(ap.ActivityType.DELETE), _meta(MetaKey.PUBLISHED): { "$lt": d }, }).deleted_count logger.info(f"{delete_deleted} Delete deleted") create_deleted = 0 create_count = 0 # Go over the old Create activities for data in DB.activities.find({ "box": Box.INBOX.value, "type": ap.ActivityType.CREATE.value, _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False }, }).limit(500): try: logger.info(f"data={data!r}") create_count += 1 remote_id = data["remote_id"] meta = data["meta"] # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue obj = None if not meta.get("deleted"): try: activity = ap.parse_activity(data["activity"]) logger.info(f"activity={activity!r}") obj = activity.get_object() except (RemoteServerUnavailableError, ActivityGoneError): logger.exception( f"failed to load {remote_id}, this activity will be deleted" ) # This activity mentions the server actor, keep it if obj and obj.has_mention(ID): _keep(data) continue # This activity is a direct reply of one the server actor activity, keep it if obj: in_reply_to = obj.get_in_reply_to() if in_reply_to and in_reply_to.startswith(ID): _keep(data) continue # This activity is part of a thread we want to keep, keep it if obj and in_reply_to and meta.get("thread_root_parent"): thread_root_parent = meta["thread_root_parent"] if thread_root_parent.startswith( ID) or thread_root_parent in toi: _keep(data) continue # This activity was boosted or liked, keep it if meta.get("boosted") or meta.get("liked"): _keep(data) continue # TODO(tsileo): remove after tests if meta.get("keep"): logger.warning( f"{activity!r} would not have been deleted, skipping for now" ) _keep(data) continue # Delete the cached attachment for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.activities.delete_one({"_id": data["_id"]}) create_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") for data in DB.replies.find({ _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False } }).limit(500): try: logger.info(f"data={data!r}") create_count += 1 remote_id = data["remote_id"] meta = data["meta"] # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue obj = ap.parse_activity(data["activity"]) # This activity is a direct reply of one the server actor activity, keep it in_reply_to = obj.get_in_reply_to() # This activity is part of a thread we want to keep, keep it if in_reply_to and meta.get("thread_root_parent"): thread_root_parent = meta["thread_root_parent"] if thread_root_parent.startswith( ID) or thread_root_parent in toi: _keep(data) continue # This activity was boosted or liked, keep it if meta.get("boosted") or meta.get("liked"): _keep(data) continue # Delete the cached attachment for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.replies.delete_one({"_id": data["_id"]}) create_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") after_gc_create = perf_counter() time_to_gc_create = after_gc_create - start logger.info( f"{time_to_gc_create:.2f} seconds to analyze {create_count} Create, {create_deleted} deleted" ) announce_count = 0 announce_deleted = 0 # Go over the old Create activities for data in DB.activities.find({ "box": Box.INBOX.value, "type": ap.ActivityType.ANNOUNCE.value, _meta(MetaKey.PUBLISHED): { "$lt": d }, "meta.gc_keep": { "$exists": False }, }).limit(500): try: announce_count += 1 remote_id = data["remote_id"] meta = data["meta"] activity = ap.parse_activity(data["activity"]) logger.info(f"activity={activity!r}") # This activity has been bookmarked, keep it if meta.get("bookmarked"): _keep(data) continue object_id = activity.get_object_id() # This announce is for a local activity (i.e. from the outbox), keep it if object_id.startswith(ID): _keep(data) continue for grid_item in MEDIA_CACHE.fs.find({"remote_id": remote_id}): MEDIA_CACHE.fs.delete(grid_item._id) # TODO(tsileo): here for legacy reason, this needs to be removed at some point for grid_item in MEDIA_CACHE.fs.find({"remote_id": object_id}): MEDIA_CACHE.fs.delete(grid_item._id) # Delete the activity DB.activities.delete_one({"_id": data["_id"]}) announce_deleted += 1 except Exception: logger.exception(f"failed to process {data!r}") after_gc_announce = perf_counter() time_to_gc_announce = after_gc_announce - after_gc_create logger.info( f"{time_to_gc_announce:.2f} seconds to analyze {announce_count} Announce, {announce_deleted} deleted" )
def inject_config(): q = { **in_outbox(), "$or": [ { **by_type(ActivityType.CREATE), **not_deleted(), **by_visibility(ap.Visibility.PUBLIC), }, { **by_type(ActivityType.ANNOUNCE), **not_undo() }, ], } notes_count = DB.activities.count(q) # FIXME(tsileo): rename to all_count, and remove poll answers from it all_q = { **in_outbox(), **by_type([ActivityType.CREATE, ActivityType.ANNOUNCE]), **not_deleted(), **not_undo(), **not_poll_answer(), } liked_q = { **in_outbox(), **by_type(ActivityType.LIKE), **not_undo(), **not_deleted(), } followers_q = { **in_inbox(), **by_type(ActivityType.FOLLOW), **not_undo(), **not_deleted(), } following_q = { **in_outbox(), **by_type(ActivityType.FOLLOW), **follow_request_accepted(), **not_undo(), **not_deleted(), } unread_notifications_q = {_meta(MetaKey.NOTIFICATION_UNREAD): True} logged_in = session.get("logged_in", False) return dict( microblogpub_version=VERSION, config=config, logged_in=logged_in, followers_count=DB.activities.count(followers_q), following_count=DB.activities.count(following_q) if logged_in or not config.HIDE_FOLLOWING else 0, notes_count=notes_count, liked_count=DB.activities.count(liked_q) if logged_in else 0, with_replies_count=DB.activities.count(all_q) if logged_in else 0, unread_notifications_count=DB.activities.count(unread_notifications_q) if logged_in else 0, me=ME, base_url=config.BASE_URL, highlight_css=HIGHLIGHT_CSS, )