def task_cache_actor() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload["iri"] try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") # Reload the actor without caching (in case it got upated) actor = ap.fetch_remote_activity(activity.get_actor().id, no_cache=True) # Fetch the Open Grah metadata if it's a `Create` if activity.has_type(ap.ActivityType.CREATE): obj = activity.get_object() links = opengraph.links_from_note(obj.to_dict()) if links: Tasks.fetch_og_meta(iri) # Send Webmentions only if it's from the outbox, and public if (is_from_outbox(obj) and ap.get_visibility(obj) == ap.Visibility.PUBLIC): Tasks.send_webmentions(activity, links) if activity.has_type(ap.ActivityType.FOLLOW): if actor.id == config.ID: # It's a new following, cache the "object" (which is the actor we follow) DB.activities.update_one( by_remote_id(iri), upsert({ MetaKey.OBJECT: activity.get_object().to_dict(embed=True) }), ) # Cache the actor info update_cached_actor(actor) app.logger.info(f"actor cached for {iri}") if not activity.has_type( [ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]): return "" if activity.get_object()._data.get( "attachment", []) or activity.get_object().has_type( ap.ActivityType.VIDEO): Tasks.cache_attachments(iri) except (ActivityGoneError, ActivityNotFoundError): DB.activities.update_one({"remote_id": iri}, {"$set": { "meta.deleted": True }}) app.logger.exception( f"flagging activity {iri} as deleted, no actor caching") except Exception as err: app.logger.exception(f"failed to cache actor for {iri}") raise TaskError() from err return ""
def task_cache_object() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") obj = activity.get_object() # Refetch the object actor (without cache) with no_cache(): obj_actor = ap.fetch_remote_activity(obj.get_actor().id) cache = {MetaKey.OBJECT: obj.to_dict(embed=True)} if activity.get_actor().id != obj_actor.id: # Cache the object actor obj_actor_hash = _actor_hash(obj_actor) cache[MetaKey.OBJECT_ACTOR] = obj_actor.to_dict(embed=True) cache[MetaKey.OBJECT_ACTOR_ID] = obj_actor.id cache[MetaKey.OBJECT_ACTOR_HASH] = obj_actor_hash # Update the actor cache for the other activities update_cached_actor(obj_actor) update_one_activity(by_remote_id(activity.id), upsert(cache)) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no object caching") except Exception as err: app.logger.exception(f"failed to cache object for {iri}") raise TaskError() from err return ""
def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None: """Go up to the root reply, store unknown replies in the `threads` DB and set the "meta.thread_root_parent" key to make it easy to query a whole thread.""" in_reply_to = create.get_object().inReplyTo if not in_reply_to: return new_threads = [] root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply) creply = DB.activities.find_one_and_update( {"activity.object.id": in_reply_to}, {"$inc": { "meta.count_reply": 1, "meta.count_direct_reply": 1 }}, ) if not creply: # It means the activity is not in the inbox, and not in the outbox, we want to save it self.save(Box.REPLIES, reply) new_threads.append(reply.id) while reply is not None: in_reply_to = reply.inReplyTo if not in_reply_to: break root_reply = in_reply_to reply = ap.fetch_remote_activity(root_reply) q = {"activity.object.id": root_reply} if not DB.activities.count(q): self.save(Box.REPLIES, reply) new_threads.append(reply.id) DB.activities.update_one( {"remote_id": create.id}, {"$set": { "meta.thread_root_parent": root_reply }}) DB.activities.update( { "box": Box.REPLIES.value, "remote_id": { "$in": new_threads } }, {"$set": { "meta.thread_root_parent": root_reply }}, )
def task_process_new_activity() -> _Response: """Process an activity received in the inbox""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") flags = {} if not activity.published: flags[_meta(MetaKey.PUBLISHED)] = now() else: flags[_meta(MetaKey.PUBLISHED)] = activity.published set_inbox_flags(activity, flags) app.logger.info(f"a={activity}, flags={flags!r}") if flags: DB.activities.update_one({"remote_id": activity.id}, {"$set": flags}) app.logger.info(f"new activity {iri} processed") if not activity.has_type(ap.ActivityType.DELETE): Tasks.cache_actor(iri) except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"dropping activity {iri}, skip processing") return "" except Exception as err: app.logger.exception(f"failed to process new activity {iri}") raise TaskError() from err return ""
def task_cache_attachments() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") # Generates thumbnails for the actor's icon and the attachments if any obj = activity.get_object() # Iter the attachments for attachment in obj._data.get("attachment", []): try: config.MEDIA_CACHE.cache_attachment(attachment, iri) except ValueError: app.logger.exception(f"failed to cache {attachment}") app.logger.info(f"attachments cached for {iri}") except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"dropping activity {iri}, no attachment caching") except Exception as err: app.logger.exception(f"failed to cache attachments for {iri}") raise TaskError() from err return ""
def api_ack_reply() -> _Response: reply_iri = _user_api_arg("reply_iri") obj = ap.fetch_remote_activity(reply_iri) if obj.has_type(ap.ActivityType.CREATE): obj = obj.get_object() # TODO(tsileo): tweak the adressing? update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, {"$set": { "meta.reply_acked": True }}, ) read = ap.Read( actor=MY_PERSON.id, object=obj.id, to=[MY_PERSON.followers], cc=[ap.AS_PUBLIC, obj.get_actor().id], published=now(), context=new_context(obj), ) read_id = post_to_outbox(read) return _user_api_response(activity=read_id)
def _delete_process_inbox(delete: ap.Delete, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={delete!r}") obj_id = delete.get_object_id() _logger.debug(f"delete object={obj_id}") try: # FIXME(tsileo): call the DB here instead? like for the oubox obj = ap.fetch_remote_activity(obj_id) _logger.info(f"inbox_delete handle_replies obj={obj!r}") in_reply_to = obj.get_in_reply_to() if obj.inReplyTo else None if obj.has_type(ap.CREATE_TYPES): in_reply_to = ap._get_id( DB.activities.find_one({ "meta.object_id": obj_id, "type": ap.ActivityType.CREATE.value })["activity"]["object"].get("inReplyTo")) if in_reply_to: back._handle_replies_delete(MY_PERSON, in_reply_to) except Exception: _logger.exception(f"failed to handle delete replies for {obj_id}") update_one_activity( { **by_object_id(obj_id), **by_type(ap.ActivityType.CREATE) }, upsert({MetaKey.DELETED: True}), ) # Foce undo other related activities DB.activities.update(by_object_id(obj_id), upsert({MetaKey.UNDO: True}))
def task_process_new_activity() -> _Response: """Process an activity received in the inbox""" task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") flags: _NewMeta = {} set_inbox_flags(activity, flags) app.logger.info(f"a={activity}, flags={flags!r}") if flags: DB.activities.update_one({"remote_id": activity.id}, {"$set": flags}) app.logger.info(f"new activity {iri} processed") except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"dropping activity {iri}, skip processing") return "" except Exception as err: app.logger.exception(f"failed to process new activity {iri}") raise TaskError() from err return ""
def fetch_og_metadata(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") if activity.has_type(ap.ActivityType.CREATE): note = activity.get_object() links = opengraph.links_from_note(note.to_dict()) og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links) for og in og_metadata: if not og.get("image"): continue MEDIA_CACHE.cache_og_image(og["image"]) log.debug(f"OG metadata {og_metadata!r}") DB.activities.update_one( {"remote_id": iri}, {"$set": { "meta.og_metadata": og_metadata }}) log.info(f"OG metadata fetched for {iri}") except (ActivityGoneError, ActivityNotFoundError): log.exception(f"dropping activity {iri}, skip OG metedata") except requests.exceptions.HTTPError as http_err: if 400 <= http_err.response.status_code < 500: log.exception("bad request, no retry") return log.exception("failed to fetch OG metadata") self.retry(exc=http_err, countdown=int(random.uniform(2, 4)**self.request.retries)) except Exception as err: log.exception(f"failed to fetch OG metadata for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def task_finish_post_to_outbox() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") recipients = activity.recipients() process_outbox(activity, {}) app.logger.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) for recp in recipients: app.logger.debug(f"posting to {recp}") Tasks.post_to_remote_inbox(payload, recp) except (ActivityGoneError, ActivityNotFoundError): app.logger.exception(f"no retry") except Exception as err: app.logger.exception(f"failed to post to remote inbox for {iri}") raise TaskError() from err return ""
def cache_object(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") obj = activity.get_object() DB.activities.update_one( {"remote_id": activity.id}, { "$set": { "meta.object": obj.to_dict(embed=True), "meta.object_actor": activitypub._actor_to_meta(obj.get_actor()), } }, ) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): DB.activities.update_one({"remote_id": iri}, {"$set": { "meta.deleted": True }}) log.exception(f"flagging activity {iri} as deleted, no object caching") except Exception as err: log.exception(f"failed to cache object for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def finish_post_to_inbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") if activity.has_type(ap.ActivityType.DELETE): back.inbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): back.inbox_update(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.CREATE): back.inbox_create(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.ANNOUNCE): back.inbox_announce(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.LIKE): back.inbox_like(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.FOLLOW): # Reply to a Follow with an Accept accept = ap.Accept(actor=ID, object=activity.to_dict(embed=True)) post_to_outbox(accept) elif activity.has_type(ap.ActivityType.UNDO): obj = activity.get_object() if obj.has_type(ap.ActivityType.LIKE): back.inbox_undo_like(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.ANNOUNCE): back.inbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_follower(MY_PERSON, obj) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): log.exception(f"no retry") except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def task_cache_object() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") obj = activity.get_object() DB.activities.update_one( {"remote_id": activity.id}, { "$set": { "meta.object": obj.to_dict(embed=True), # FIXME(tsileo): set object actor only if different from actor? "meta.object_actor": obj.get_actor().to_dict(embed=True), } }, ) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no object caching") except Exception as err: app.logger.exception(f"failed to cache object for {iri}") raise TaskError() from err return ""
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={update!r}") obj = update.get_object() if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE: update_one_activity({"activity.object.id": obj.id}, {"$set": { "activity.object": obj.to_dict() }}) elif obj.has_type(ap.ActivityType.QUESTION): choices = obj._data.get("oneOf", obj.anyOf) total_replies = 0 _set = {} for choice in choices: answer_key = _answer_key(choice["name"]) cnt = choice["replies"]["totalItems"] total_replies += cnt _set[f"meta.question_answers.{answer_key}"] = cnt _set["meta.question_replies"] = total_replies update_one_activity({ **in_inbox(), **by_object_id(obj.id) }, {"$set": _set}) # Also update the cached copies of the question (like Announce and Like) DB.activities.update_many(by_object_id(obj.id), upsert({MetaKey.OBJECT: obj.to_dict()})) elif obj.has_type(ap.ACTOR_TYPES): actor = ap.fetch_remote_activity(obj.id, no_cache=True) update_cached_actor(actor) else: raise ValueError(f"don't know how to update {obj!r}")
def lookup(url: str) -> ap.BaseActivity: """Try to find an AP object related to the given URL.""" try: if url.startswith("@"): actor_url = get_actor_url(url) if actor_url: return ap.fetch_remote_activity(actor_url) except NotAnActivityError: pass except requests.HTTPError: # Some websites may returns 404, 503 or others when they don't support webfinger, and we're just taking a guess # when performing the lookup. pass except requests.RequestException as err: raise RemoteServerUnavailableError(f"failed to fetch {url}: {err!r}") backend = ap.get_backend() try: resp = requests.head( url, timeout=10, allow_redirects=True, headers={"User-Agent": backend.user_agent()}, ) except requests.RequestException as err: raise RemoteServerUnavailableError(f"failed to GET {url}: {err!r}") try: resp.raise_for_status() except Exception: return ap.fetch_remote_activity(url) # If the page is HTML, maybe it contains an alternate link pointing to an AP object for alternate in mf2py.parse(resp.text).get("alternates", []): if alternate.get("type") == "application/activity+json": return ap.fetch_remote_activity(alternate["url"]) try: # Maybe the page was JSON-LD? data = resp.json() return ap.parse_activity(data) except Exception: pass # Try content negotiation (retry with the AP Accept header) return ap.fetch_remote_activity(url)
def admin_new() -> _Response: reply_id = None content = "" thread: List[Any] = [] print(request.args) default_visibility = None # ap.Visibility.PUBLIC if request.args.get("reply"): data = DB.activities.find_one( {"activity.object.id": request.args.get("reply")}) if data: reply = ap.parse_activity(data["activity"]) else: obj = ap.get_backend().fetch_iri(request.args.get("reply")) data = dict(meta=_meta(ap.parse_activity(obj)), activity=dict(object=obj)) data["_id"] = obj["id"] data["remote_id"] = obj["id"] reply = ap.parse_activity(data["activity"]["object"]) # Fetch the post visibility, in case it's follower only default_visibility = ap.get_visibility(reply) # If it's public, we default the reply to unlisted if default_visibility == ap.Visibility.PUBLIC: default_visibility = ap.Visibility.UNLISTED reply_id = reply.id if reply.ACTIVITY_TYPE == ap.ActivityType.CREATE: reply_id = reply.get_object().id actor = reply.get_actor() domain = urlparse(actor.id).netloc # FIXME(tsileo): if reply of reply, fetch all participants content = f"@{actor.preferredUsername}@{domain} " if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() for mention in reply.get_mentions(): if mention.href in [actor.id, ID]: continue m = ap.fetch_remote_activity(mention.href) if m.has_type(ap.ACTOR_TYPES): d = urlparse(m.id).netloc content += f"@{m.preferredUsername}@{d} " thread = _build_thread(data) return htmlify( render_template( "new.html", reply=reply_id, content=content, thread=thread, default_visibility=default_visibility, visibility=ap.Visibility, emojis=config.EMOJIS.split(" "), custom_emojis=sorted( [ap.Emoji(**dat) for name, dat in EMOJIS_BY_NAME.items()], key=lambda e: e.name, ), ))
def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") if activity.has_type(ap.ActivityType.CREATE): fetch_og_metadata.delay(iri) if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]): cache_object.delay(iri) actor = activity.get_actor() cache_actor_with_inbox = False if activity.has_type(ap.ActivityType.FOLLOW): if actor.id != ID: # It's a Follow from the Inbox cache_actor_with_inbox = True else: # It's a new following, cache the "object" (which is the actor we follow) DB.activities.update_one( {"remote_id": iri}, { "$set": { "meta.object": activitypub._actor_to_meta(activity.get_object()) } }, ) # Cache the actor info DB.activities.update_one( {"remote_id": iri}, { "$set": { "meta.actor": activitypub._actor_to_meta(actor, cache_actor_with_inbox) } }, ) log.info(f"actor cached for {iri}") if also_cache_attachments and activity.has_type( ap.ActivityType.CREATE): cache_attachments.delay(iri) except (ActivityGoneError, ActivityNotFoundError): DB.activities.update_one({"remote_id": iri}, {"$set": { "meta.deleted": True }}) log.exception(f"flagging activity {iri} as deleted, no actor caching") except Exception as err: log.exception(f"failed to cache actor for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def forward_activity(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) recipients = back.followers_as_recipients() log.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) for recp in recipients: log.debug(f"forwarding {activity!r} to {recp}") post_to_remote_inbox.delay(payload, recp) except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def admin_direct_messages() -> _Response: all_dms = DB.activities.find({ **not_poll_answer(), **by_type(ap.ActivityType.CREATE), **by_object_visibility(ap.Visibility.DIRECT), }).sort("meta.published", -1) # Group by threads _threads = defaultdict(list) # type: ignore for dm in all_dms: # Skip poll answers if dm["activity"].get("object", {}).get("name"): continue _threads[dm["meta"].get("thread_root_parent", dm["meta"]["object_id"])].append(dm) # Now build the data needed for the UI threads = [] for thread_root, thread in _threads.items(): # We need the list of participants participants = set() for raw_activity in thread: activity = ap.parse_activity(raw_activity["activity"]) actor = activity.get_actor() domain = urlparse(actor.id).netloc if actor.id != ID: participants.add(f"@{actor.preferredUsername}@{domain}") if activity.has_type(ap.ActivityType.CREATE): activity = activity.get_object() for mention in activity.get_mentions(): if mention.href in [actor.id, ID]: continue m = ap.fetch_remote_activity(mention.href) if m.has_type(ap.ACTOR_TYPES) and m.id != ID: d = urlparse(m.id).netloc participants.add(f"@{m.preferredUsername}@{d}") if not participants: continue # Build the UI data for this conversation oid = thread[-1]["meta"]["object_id"] threads.append({ "participants": list(participants), "oid": oid, "last_reply": thread[0], "len": len(thread), }) return htmlify(render_template("direct_messages.html", threads=threads))
def task_cache_actor() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload["iri"] try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") # Fetch the Open Grah metadata if it's a `Create` if activity.has_type(ap.ActivityType.CREATE): Tasks.fetch_og_meta(iri) actor = activity.get_actor() if actor.icon: if isinstance(actor.icon, dict) and "url" in actor.icon: config.MEDIA_CACHE.cache_actor_icon(actor.icon["url"]) else: app.logger.warning(f"failed to parse icon {actor.icon} for {iri}") if activity.has_type(ap.ActivityType.FOLLOW): if actor.id == config.ID: # It's a new following, cache the "object" (which is the actor we follow) DB.activities.update_one( {"remote_id": iri}, { "$set": { "meta.object": activity.get_object().to_dict(embed=True) } }, ) # Cache the actor info DB.activities.update_one( {"remote_id": iri}, {"$set": {"meta.actor": actor.to_dict(embed=True)}} ) app.logger.info(f"actor cached for {iri}") if activity.has_type([ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]): Tasks.cache_attachments(iri) except (ActivityGoneError, ActivityNotFoundError): DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}}) app.logger.exception(f"flagging activity {iri} as deleted, no actor caching") except Exception as err: app.logger.exception(f"failed to cache actor for {iri}") raise TaskError() from err return ""
def task_cache_attachments() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"caching attachment for activity={activity!r}") # Generates thumbnails for the actor's icon and the attachments if any if activity.has_type( [ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]): obj = activity.get_object() else: obj = activity if obj.content: content_html = BeautifulSoup(obj.content, "html5lib") for img in content_html.find_all("img"): src = img.attrs.get("src") if src: Tasks.cache_attachment({"url": src}, iri) if obj.has_type(ap.ActivityType.VIDEO): if isinstance(obj.url, list): # TODO: filter only videogt link = select_video_to_cache(obj.url) if link: Tasks.cache_attachment({"url": link["href"]}, iri) elif isinstance(obj.url, str): Tasks.cache_attachment({"url": obj.url}, iri) else: app.logger.warning( f"failed to parse video link {obj!r} for {iri}") # Iter the attachments for attachment in obj._data.get("attachment", []): Tasks.cache_attachment(attachment, iri) app.logger.info(f"attachments cached for {iri}") except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"dropping activity {iri}, no attachment caching") except Exception as err: app.logger.exception(f"failed to cache attachments for {iri}") raise TaskError() from err return ""
def task_finish_post_to_inbox() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) app.logger.info(f"activity={activity!r}") process_inbox(activity, {}) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): app.logger.exception(f"no retry") except Exception as err: app.logger.exception(f"failed to cfinish post to inbox for {iri}") raise TaskError() from err return ""
def forward_activity(self, iri: str) -> None: if not current_app.config["AP_ENABLED"]: return # not federating if not enabled try: activity = ap.fetch_remote_activity(iri) backend = ap.get_backend() recipients = backend.followers_as_recipients() current_app.logger.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) for recp in recipients: current_app.logger.debug(f"forwarding {activity!r} to {recp}") payload = json.dumps(activity) post_to_remote_inbox.delay(payload, recp) except Exception as err: # noqa: F841 current_app.logger.exception(f"failed to cache attachments for {iri}")
def finish_inbox_processing(self, iri: str) -> None: try: backend = ap.get_backend() activity = ap.fetch_remote_activity(iri) current_app.logger.info(f"activity={activity!r}") actor = activity.get_actor() id = activity.get_object_id() current_app.logger.debug(f"finish_inbox_processing actor {actor}") if activity.has_type(ap.ActivityType.DELETE): backend.inbox_delete(actor, activity) elif activity.has_type(ap.ActivityType.UPDATE): backend.inbox_update(actor, activity) elif activity.has_type(ap.ActivityType.CREATE): backend.inbox_create(actor, activity) elif activity.has_type(ap.ActivityType.ANNOUNCE): backend.inbox_announce(actor, activity) elif activity.has_type(ap.ActivityType.LIKE): backend.inbox_like(actor, activity) elif activity.has_type(ap.ActivityType.FOLLOW): # Reply to a Follow with an Accept accept = ap.Accept(actor=id, object=activity.to_dict(embed=True)) post_to_outbox(accept) backend.new_follower(activity, activity.get_actor(), activity.get_object()) elif activity.has_type(ap.ActivityType.ACCEPT): obj = activity.get_object() # FIXME: probably other types to ACCEPT the Activity if obj.has_type(ap.ActivityType.FOLLOW): # Accept new follower backend.new_following(activity, obj) elif activity.has_type(ap.ActivityType.UNDO): obj = activity.get_object() if obj.has_type(ap.ActivityType.LIKE): backend.inbox_undo_like(actor, obj) elif obj.has_type(ap.ActivityType.ANNOUNCE): backend.inbox_undo_announce(actor, obj) elif obj.has_type(ap.ActivityType.FOLLOW): backend.undo_new_follower(actor, obj) except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): current_app.logger.exception(f"no retry") except Exception as err: # noqa: F841 current_app.logger.exception(f"failed to cache attachments for" f" {iri}")
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None: _logger.info(f"process_inbox activity={announce!r}") # TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else # or remove it? try: obj = announce.get_object() except NotAnActivityError: _logger.exception( f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message' ) return if obj.has_type(ap.ActivityType.QUESTION): Tasks.fetch_remote_question(obj) # Cache the announced object Tasks.cache_object(announce.id) # Process the reply of the announced object if any in_reply_to = obj.get_in_reply_to() if in_reply_to: reply = ap.fetch_remote_activity(in_reply_to) if reply.has_type(ap.ActivityType.CREATE): reply = reply.get_object() in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to} # Update the activity to save some data about the reply if reply.get_actor().id == obj.get_actor().id: in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True}) else: in_reply_to_data.update({ MetaKey.IN_REPLY_TO_ACTOR: reply.get_actor().to_dict(embed=True) }) update_one_activity(by_remote_id(announce.id), upsert(in_reply_to_data)) # Spawn a task to process it (and determine if it needs to be saved) Tasks.process_reply(reply.id) update_one_activity( { **by_type(ap.ActivityType.CREATE), **by_object_id(obj.id) }, inc(MetaKey.COUNT_BOOST, 1), )
def task_forward_activity() -> _Response: task = p.parse(flask.request) app.logger.info(f"task={task!r}") iri = task.payload try: activity = ap.fetch_remote_activity(iri) recipients = back.followers_as_recipients() app.logger.debug(f"Forwarding {activity!r} to {recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) for recp in recipients: app.logger.debug(f"forwarding {activity!r} to {recp}") Tasks.post_to_remote_inbox(payload, recp) except Exception as err: app.logger.exception("task failed") raise TaskError() from err return ""
def admin_profile() -> _Response: if not request.args.get("actor_id"): abort(404) actor_id = request.args.get("actor_id") actor = ap.fetch_remote_activity(actor_id) q = { "meta.actor_id": actor_id, "box": "inbox", **not_deleted(), "type": { "$in": [ap.ActivityType.CREATE.value, ap.ActivityType.ANNOUNCE.value] }, } inbox_data, older_than, newer_than = paginated_query( DB.activities, q, limit=int(request.args.get("limit", 25))) follower = find_one_activity({ "box": "inbox", "type": ap.ActivityType.FOLLOW.value, "meta.actor_id": actor.id, "meta.undo": False, }) following = find_one_activity({ **by_type(ap.ActivityType.FOLLOW), **by_object_id(actor.id), **not_undo(), **in_outbox(), **follow_request_accepted(), }) return htmlify( render_template( "stream.html", actor_id=actor_id, actor=actor.to_dict(), inbox_data=inbox_data, older_than=older_than, newer_than=newer_than, follower=follower, following=following, lists=list(DB.lists.find()), ))
def finish_post_to_outbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) backend = ap.get_backend() current_app.logger.info(f"finish_post_to_outbox {activity}") recipients = activity.recipients() actor = activity.get_actor() current_app.logger.debug(f"finish_post_to_outbox actor {actor!r}") if activity.has_type(ap.ActivityType.DELETE): backend.outbox_delete(actor, activity) elif activity.has_type(ap.ActivityType.UPDATE): backend.outbox_update(actor, activity) elif activity.has_type(ap.ActivityType.CREATE): backend.outbox_create(actor, activity) elif activity.has_type(ap.ActivityType.ANNOUNCE): backend.outbox_announce(actor, activity) elif activity.has_type(ap.ActivityType.LIKE): backend.outbox_like(actor, activity) elif activity.has_type(ap.ActivityType.UNDO): obj = activity.get_object() if obj.has_type(ap.ActivityType.LIKE): backend.outbox_undo_like(actor, obj) elif obj.has_type(ap.ActivityType.ANNOUNCE): backend.outbox_undo_announce(actor, obj) elif obj.has_type(ap.ActivityType.FOLLOW): backend.undo_new_following(actor, obj) current_app.logger.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) payload = json.dumps(activity) for recp in recipients: current_app.logger.debug(f"posting to {recp}") post_to_remote_inbox.delay(payload, recp) except (ActivityGoneError, ActivityNotFoundError): current_app.logger.exception(f"no retry") except Exception as err: # noqa: F841 current_app.logger.exception(f"failed to post " f"to remote inbox for {iri}")
def finish_post_to_outbox(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") recipients = activity.recipients() if activity.has_type(ap.ActivityType.DELETE): back.outbox_delete(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UPDATE): back.outbox_update(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.CREATE): back.outbox_create(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.ANNOUNCE): back.outbox_announce(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.LIKE): back.outbox_like(MY_PERSON, activity) elif activity.has_type(ap.ActivityType.UNDO): obj = activity.get_object() if obj.has_type(ap.ActivityType.LIKE): back.outbox_undo_like(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.ANNOUNCE): back.outbox_undo_announce(MY_PERSON, obj) elif obj.has_type(ap.ActivityType.FOLLOW): back.undo_new_following(MY_PERSON, obj) log.info(f"recipients={recipients}") activity = ap.clean_activity(activity.to_dict()) DB.cache2.remove() payload = json.dumps(activity) for recp in recipients: log.debug(f"posting to {recp}") post_to_remote_inbox.delay(payload, recp) except (ActivityGoneError, ActivityNotFoundError): log.exception(f"no retry") except Exception as err: log.exception(f"failed to post to remote inbox for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))
def cache_attachments(self, iri: str) -> None: try: activity = ap.fetch_remote_activity(iri) log.info(f"activity={activity!r}") # Generates thumbnails for the actor's icon and the attachments if any actor = activity.get_actor() # Update the cached actor DB.actors.update_one( {"remote_id": iri}, {"$set": { "remote_id": iri, "data": actor.to_dict(embed=True) }}, upsert=True, ) if actor.icon: MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON) if activity.has_type(ap.ActivityType.CREATE): for attachment in activity.get_object()._data.get( "attachment", []): if (attachment.get("mediaType", "").startswith("image/") or attachment.get("type") == ap.ActivityType.IMAGE.value): try: MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT) except ValueError: log.exception(f"failed to cache {attachment}") log.info(f"attachments cached for {iri}") except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError): log.exception(f"dropping activity {iri}, no attachment caching") except Exception as err: log.exception(f"failed to cache attachments for {iri}") self.retry(exc=err, countdown=int(random.uniform(2, 4)**self.request.retries))