def task_cache_actor() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload["iri"]
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
# Reload the actor without caching (in case it got upated)
actor = ap.fetch_remote_activity(activity.get_actor().id,
no_cache=True)
# Fetch the Open Grah metadata if it's a `Create`
if activity.has_type(ap.ActivityType.CREATE):
obj = activity.get_object()
links = opengraph.links_from_note(obj.to_dict())
if links:
Tasks.fetch_og_meta(iri)
# Send Webmentions only if it's from the outbox, and public
if (is_from_outbox(obj)
and ap.get_visibility(obj) == ap.Visibility.PUBLIC):
Tasks.send_webmentions(activity, links)
if activity.has_type(ap.ActivityType.FOLLOW):
if actor.id == config.ID:
# It's a new following, cache the "object" (which is the actor we follow)
DB.activities.update_one(
by_remote_id(iri),
upsert({
MetaKey.OBJECT:
activity.get_object().to_dict(embed=True)
}),
)
# Cache the actor info
update_cached_actor(actor)
app.logger.info(f"actor cached for {iri}")
if not activity.has_type(
[ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]):
return ""
if activity.get_object()._data.get(
"attachment", []) or activity.get_object().has_type(
ap.ActivityType.VIDEO):
Tasks.cache_attachments(iri)
except (ActivityGoneError, ActivityNotFoundError):
DB.activities.update_one({"remote_id": iri},
{"$set": {
"meta.deleted": True
}})
app.logger.exception(
f"flagging activity {iri} as deleted, no actor caching")
except Exception as err:
app.logger.exception(f"failed to cache actor for {iri}")
raise TaskError() from err
return ""
def task_cache_object() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
obj = activity.get_object()
# Refetch the object actor (without cache)
with no_cache():
obj_actor = ap.fetch_remote_activity(obj.get_actor().id)
cache = {MetaKey.OBJECT: obj.to_dict(embed=True)}
if activity.get_actor().id != obj_actor.id:
# Cache the object actor
obj_actor_hash = _actor_hash(obj_actor)
cache[MetaKey.OBJECT_ACTOR] = obj_actor.to_dict(embed=True)
cache[MetaKey.OBJECT_ACTOR_ID] = obj_actor.id
cache[MetaKey.OBJECT_ACTOR_HASH] = obj_actor_hash
# Update the actor cache for the other activities
update_cached_actor(obj_actor)
update_one_activity(by_remote_id(activity.id), upsert(cache))
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}})
app.logger.exception(f"flagging activity {iri} as deleted, no object caching")
except Exception as err:
app.logger.exception(f"failed to cache object for {iri}")
raise TaskError() from err
return ""
def _handle_replies(self, as_actor: ap.Person, create: ap.Create) -> None:
"""Go up to the root reply, store unknown replies in the `threads` DB and set the "meta.thread_root_parent"
key to make it easy to query a whole thread."""
in_reply_to = create.get_object().inReplyTo
if not in_reply_to:
return
new_threads = []
root_reply = in_reply_to
reply = ap.fetch_remote_activity(root_reply)
creply = DB.activities.find_one_and_update(
{"activity.object.id": in_reply_to},
{"$inc": {
"meta.count_reply": 1,
"meta.count_direct_reply": 1
}},
)
if not creply:
# It means the activity is not in the inbox, and not in the outbox, we want to save it
self.save(Box.REPLIES, reply)
new_threads.append(reply.id)
while reply is not None:
in_reply_to = reply.inReplyTo
if not in_reply_to:
break
root_reply = in_reply_to
reply = ap.fetch_remote_activity(root_reply)
q = {"activity.object.id": root_reply}
if not DB.activities.count(q):
self.save(Box.REPLIES, reply)
new_threads.append(reply.id)
DB.activities.update_one(
{"remote_id": create.id},
{"$set": {
"meta.thread_root_parent": root_reply
}})
DB.activities.update(
{
"box": Box.REPLIES.value,
"remote_id": {
"$in": new_threads
}
},
{"$set": {
"meta.thread_root_parent": root_reply
}},
)
def task_process_new_activity() -> _Response:
"""Process an activity received in the inbox"""
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
flags = {}
if not activity.published:
flags[_meta(MetaKey.PUBLISHED)] = now()
else:
flags[_meta(MetaKey.PUBLISHED)] = activity.published
set_inbox_flags(activity, flags)
app.logger.info(f"a={activity}, flags={flags!r}")
if flags:
DB.activities.update_one({"remote_id": activity.id}, {"$set": flags})
app.logger.info(f"new activity {iri} processed")
if not activity.has_type(ap.ActivityType.DELETE):
Tasks.cache_actor(iri)
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"dropping activity {iri}, skip processing")
return ""
except Exception as err:
app.logger.exception(f"failed to process new activity {iri}")
raise TaskError() from err
return ""
def task_cache_attachments() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
# Generates thumbnails for the actor's icon and the attachments if any
obj = activity.get_object()
# Iter the attachments
for attachment in obj._data.get("attachment", []):
try:
config.MEDIA_CACHE.cache_attachment(attachment, iri)
except ValueError:
app.logger.exception(f"failed to cache {attachment}")
app.logger.info(f"attachments cached for {iri}")
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
app.logger.exception(f"dropping activity {iri}, no attachment caching")
except Exception as err:
app.logger.exception(f"failed to cache attachments for {iri}")
raise TaskError() from err
return ""
def api_ack_reply() -> _Response:
reply_iri = _user_api_arg("reply_iri")
obj = ap.fetch_remote_activity(reply_iri)
if obj.has_type(ap.ActivityType.CREATE):
obj = obj.get_object()
# TODO(tsileo): tweak the adressing?
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
{"$set": {
"meta.reply_acked": True
}},
)
read = ap.Read(
actor=MY_PERSON.id,
object=obj.id,
to=[MY_PERSON.followers],
cc=[ap.AS_PUBLIC, obj.get_actor().id],
published=now(),
context=new_context(obj),
)
read_id = post_to_outbox(read)
return _user_api_response(activity=read_id)
def _delete_process_inbox(delete: ap.Delete, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={delete!r}")
obj_id = delete.get_object_id()
_logger.debug(f"delete object={obj_id}")
try:
# FIXME(tsileo): call the DB here instead? like for the oubox
obj = ap.fetch_remote_activity(obj_id)
_logger.info(f"inbox_delete handle_replies obj={obj!r}")
in_reply_to = obj.get_in_reply_to() if obj.inReplyTo else None
if obj.has_type(ap.CREATE_TYPES):
in_reply_to = ap._get_id(
DB.activities.find_one({
"meta.object_id": obj_id,
"type": ap.ActivityType.CREATE.value
})["activity"]["object"].get("inReplyTo"))
if in_reply_to:
back._handle_replies_delete(MY_PERSON, in_reply_to)
except Exception:
_logger.exception(f"failed to handle delete replies for {obj_id}")
update_one_activity(
{
**by_object_id(obj_id),
**by_type(ap.ActivityType.CREATE)
},
upsert({MetaKey.DELETED: True}),
)
# Foce undo other related activities
DB.activities.update(by_object_id(obj_id), upsert({MetaKey.UNDO: True}))
def task_process_new_activity() -> _Response:
"""Process an activity received in the inbox"""
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
flags: _NewMeta = {}
set_inbox_flags(activity, flags)
app.logger.info(f"a={activity}, flags={flags!r}")
if flags:
DB.activities.update_one({"remote_id": activity.id},
{"$set": flags})
app.logger.info(f"new activity {iri} processed")
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"dropping activity {iri}, skip processing")
return ""
except Exception as err:
app.logger.exception(f"failed to process new activity {iri}")
raise TaskError() from err
return ""
def fetch_og_metadata(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.CREATE):
note = activity.get_object()
links = opengraph.links_from_note(note.to_dict())
og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links)
for og in og_metadata:
if not og.get("image"):
continue
MEDIA_CACHE.cache_og_image(og["image"])
log.debug(f"OG metadata {og_metadata!r}")
DB.activities.update_one(
{"remote_id": iri},
{"$set": {
"meta.og_metadata": og_metadata
}})
log.info(f"OG metadata fetched for {iri}")
except (ActivityGoneError, ActivityNotFoundError):
log.exception(f"dropping activity {iri}, skip OG metedata")
except requests.exceptions.HTTPError as http_err:
if 400 <= http_err.response.status_code < 500:
log.exception("bad request, no retry")
return
log.exception("failed to fetch OG metadata")
self.retry(exc=http_err,
countdown=int(random.uniform(2, 4)**self.request.retries))
except Exception as err:
log.exception(f"failed to fetch OG metadata for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def task_finish_post_to_outbox() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
recipients = activity.recipients()
process_outbox(activity, {})
app.logger.info(f"recipients={recipients}")
activity = ap.clean_activity(activity.to_dict())
payload = json.dumps(activity)
for recp in recipients:
app.logger.debug(f"posting to {recp}")
Tasks.post_to_remote_inbox(payload, recp)
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"no retry")
except Exception as err:
app.logger.exception(f"failed to post to remote inbox for {iri}")
raise TaskError() from err
return ""
def cache_object(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
obj = activity.get_object()
DB.activities.update_one(
{"remote_id": activity.id},
{
"$set": {
"meta.object":
obj.to_dict(embed=True),
"meta.object_actor":
activitypub._actor_to_meta(obj.get_actor()),
}
},
)
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
DB.activities.update_one({"remote_id": iri},
{"$set": {
"meta.deleted": True
}})
log.exception(f"flagging activity {iri} as deleted, no object caching")
except Exception as err:
log.exception(f"failed to cache object for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def finish_post_to_inbox(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.DELETE):
back.inbox_delete(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.UPDATE):
back.inbox_update(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.CREATE):
back.inbox_create(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.ANNOUNCE):
back.inbox_announce(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.LIKE):
back.inbox_like(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.FOLLOW):
# Reply to a Follow with an Accept
accept = ap.Accept(actor=ID, object=activity.to_dict(embed=True))
post_to_outbox(accept)
elif activity.has_type(ap.ActivityType.UNDO):
obj = activity.get_object()
if obj.has_type(ap.ActivityType.LIKE):
back.inbox_undo_like(MY_PERSON, obj)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
back.inbox_undo_announce(MY_PERSON, obj)
elif obj.has_type(ap.ActivityType.FOLLOW):
back.undo_new_follower(MY_PERSON, obj)
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
log.exception(f"no retry")
except Exception as err:
log.exception(f"failed to cache attachments for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def task_cache_object() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
obj = activity.get_object()
DB.activities.update_one(
{"remote_id": activity.id},
{
"$set": {
"meta.object": obj.to_dict(embed=True),
# FIXME(tsileo): set object actor only if different from actor?
"meta.object_actor": obj.get_actor().to_dict(embed=True),
}
},
)
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}})
app.logger.exception(f"flagging activity {iri} as deleted, no object caching")
except Exception as err:
app.logger.exception(f"failed to cache object for {iri}")
raise TaskError() from err
return ""
def _update_process_inbox(update: ap.Update, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={update!r}")
obj = update.get_object()
if obj.ACTIVITY_TYPE == ap.ActivityType.NOTE:
update_one_activity({"activity.object.id": obj.id},
{"$set": {
"activity.object": obj.to_dict()
}})
elif obj.has_type(ap.ActivityType.QUESTION):
choices = obj._data.get("oneOf", obj.anyOf)
total_replies = 0
_set = {}
for choice in choices:
answer_key = _answer_key(choice["name"])
cnt = choice["replies"]["totalItems"]
total_replies += cnt
_set[f"meta.question_answers.{answer_key}"] = cnt
_set["meta.question_replies"] = total_replies
update_one_activity({
**in_inbox(),
**by_object_id(obj.id)
}, {"$set": _set})
# Also update the cached copies of the question (like Announce and Like)
DB.activities.update_many(by_object_id(obj.id),
upsert({MetaKey.OBJECT: obj.to_dict()}))
elif obj.has_type(ap.ACTOR_TYPES):
actor = ap.fetch_remote_activity(obj.id, no_cache=True)
update_cached_actor(actor)
else:
raise ValueError(f"don't know how to update {obj!r}")
def lookup(url: str) -> ap.BaseActivity:
"""Try to find an AP object related to the given URL."""
try:
if url.startswith("@"):
actor_url = get_actor_url(url)
if actor_url:
return ap.fetch_remote_activity(actor_url)
except NotAnActivityError:
pass
except requests.HTTPError:
# Some websites may returns 404, 503 or others when they don't support webfinger, and we're just taking a guess
# when performing the lookup.
pass
except requests.RequestException as err:
raise RemoteServerUnavailableError(f"failed to fetch {url}: {err!r}")
backend = ap.get_backend()
try:
resp = requests.head(
url,
timeout=10,
allow_redirects=True,
headers={"User-Agent": backend.user_agent()},
)
except requests.RequestException as err:
raise RemoteServerUnavailableError(f"failed to GET {url}: {err!r}")
try:
resp.raise_for_status()
except Exception:
return ap.fetch_remote_activity(url)
# If the page is HTML, maybe it contains an alternate link pointing to an AP object
for alternate in mf2py.parse(resp.text).get("alternates", []):
if alternate.get("type") == "application/activity+json":
return ap.fetch_remote_activity(alternate["url"])
try:
# Maybe the page was JSON-LD?
data = resp.json()
return ap.parse_activity(data)
except Exception:
pass
# Try content negotiation (retry with the AP Accept header)
return ap.fetch_remote_activity(url)
def admin_new() -> _Response:
reply_id = None
content = ""
thread: List[Any] = []
print(request.args)
default_visibility = None # ap.Visibility.PUBLIC
if request.args.get("reply"):
data = DB.activities.find_one(
{"activity.object.id": request.args.get("reply")})
if data:
reply = ap.parse_activity(data["activity"])
else:
obj = ap.get_backend().fetch_iri(request.args.get("reply"))
data = dict(meta=_meta(ap.parse_activity(obj)),
activity=dict(object=obj))
data["_id"] = obj["id"]
data["remote_id"] = obj["id"]
reply = ap.parse_activity(data["activity"]["object"])
# Fetch the post visibility, in case it's follower only
default_visibility = ap.get_visibility(reply)
# If it's public, we default the reply to unlisted
if default_visibility == ap.Visibility.PUBLIC:
default_visibility = ap.Visibility.UNLISTED
reply_id = reply.id
if reply.ACTIVITY_TYPE == ap.ActivityType.CREATE:
reply_id = reply.get_object().id
actor = reply.get_actor()
domain = urlparse(actor.id).netloc
# FIXME(tsileo): if reply of reply, fetch all participants
content = f"@{actor.preferredUsername}@{domain} "
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
for mention in reply.get_mentions():
if mention.href in [actor.id, ID]:
continue
m = ap.fetch_remote_activity(mention.href)
if m.has_type(ap.ACTOR_TYPES):
d = urlparse(m.id).netloc
content += f"@{m.preferredUsername}@{d} "
thread = _build_thread(data)
return htmlify(
render_template(
"new.html",
reply=reply_id,
content=content,
thread=thread,
default_visibility=default_visibility,
visibility=ap.Visibility,
emojis=config.EMOJIS.split(" "),
custom_emojis=sorted(
[ap.Emoji(**dat) for name, dat in EMOJIS_BY_NAME.items()],
key=lambda e: e.name,
),
))
def cache_actor(self, iri: str, also_cache_attachments: bool = True) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.CREATE):
fetch_og_metadata.delay(iri)
if activity.has_type([ap.ActivityType.LIKE, ap.ActivityType.ANNOUNCE]):
cache_object.delay(iri)
actor = activity.get_actor()
cache_actor_with_inbox = False
if activity.has_type(ap.ActivityType.FOLLOW):
if actor.id != ID:
# It's a Follow from the Inbox
cache_actor_with_inbox = True
else:
# It's a new following, cache the "object" (which is the actor we follow)
DB.activities.update_one(
{"remote_id": iri},
{
"$set": {
"meta.object":
activitypub._actor_to_meta(activity.get_object())
}
},
)
# Cache the actor info
DB.activities.update_one(
{"remote_id": iri},
{
"$set": {
"meta.actor":
activitypub._actor_to_meta(actor, cache_actor_with_inbox)
}
},
)
log.info(f"actor cached for {iri}")
if also_cache_attachments and activity.has_type(
ap.ActivityType.CREATE):
cache_attachments.delay(iri)
except (ActivityGoneError, ActivityNotFoundError):
DB.activities.update_one({"remote_id": iri},
{"$set": {
"meta.deleted": True
}})
log.exception(f"flagging activity {iri} as deleted, no actor caching")
except Exception as err:
log.exception(f"failed to cache actor for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def forward_activity(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
recipients = back.followers_as_recipients()
log.debug(f"Forwarding {activity!r} to {recipients}")
activity = ap.clean_activity(activity.to_dict())
payload = json.dumps(activity)
for recp in recipients:
log.debug(f"forwarding {activity!r} to {recp}")
post_to_remote_inbox.delay(payload, recp)
except Exception as err:
log.exception(f"failed to cache attachments for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def admin_direct_messages() -> _Response:
all_dms = DB.activities.find({
**not_poll_answer(),
**by_type(ap.ActivityType.CREATE),
**by_object_visibility(ap.Visibility.DIRECT),
}).sort("meta.published", -1)
# Group by threads
_threads = defaultdict(list) # type: ignore
for dm in all_dms:
# Skip poll answers
if dm["activity"].get("object", {}).get("name"):
continue
_threads[dm["meta"].get("thread_root_parent",
dm["meta"]["object_id"])].append(dm)
# Now build the data needed for the UI
threads = []
for thread_root, thread in _threads.items():
# We need the list of participants
participants = set()
for raw_activity in thread:
activity = ap.parse_activity(raw_activity["activity"])
actor = activity.get_actor()
domain = urlparse(actor.id).netloc
if actor.id != ID:
participants.add(f"@{actor.preferredUsername}@{domain}")
if activity.has_type(ap.ActivityType.CREATE):
activity = activity.get_object()
for mention in activity.get_mentions():
if mention.href in [actor.id, ID]:
continue
m = ap.fetch_remote_activity(mention.href)
if m.has_type(ap.ACTOR_TYPES) and m.id != ID:
d = urlparse(m.id).netloc
participants.add(f"@{m.preferredUsername}@{d}")
if not participants:
continue
# Build the UI data for this conversation
oid = thread[-1]["meta"]["object_id"]
threads.append({
"participants": list(participants),
"oid": oid,
"last_reply": thread[0],
"len": len(thread),
})
return htmlify(render_template("direct_messages.html", threads=threads))
def task_cache_actor() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload["iri"]
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
# Fetch the Open Grah metadata if it's a `Create`
if activity.has_type(ap.ActivityType.CREATE):
Tasks.fetch_og_meta(iri)
actor = activity.get_actor()
if actor.icon:
if isinstance(actor.icon, dict) and "url" in actor.icon:
config.MEDIA_CACHE.cache_actor_icon(actor.icon["url"])
else:
app.logger.warning(f"failed to parse icon {actor.icon} for {iri}")
if activity.has_type(ap.ActivityType.FOLLOW):
if actor.id == config.ID:
# It's a new following, cache the "object" (which is the actor we follow)
DB.activities.update_one(
{"remote_id": iri},
{
"$set": {
"meta.object": activity.get_object().to_dict(embed=True)
}
},
)
# Cache the actor info
DB.activities.update_one(
{"remote_id": iri}, {"$set": {"meta.actor": actor.to_dict(embed=True)}}
)
app.logger.info(f"actor cached for {iri}")
if activity.has_type([ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]):
Tasks.cache_attachments(iri)
except (ActivityGoneError, ActivityNotFoundError):
DB.activities.update_one({"remote_id": iri}, {"$set": {"meta.deleted": True}})
app.logger.exception(f"flagging activity {iri} as deleted, no actor caching")
except Exception as err:
app.logger.exception(f"failed to cache actor for {iri}")
raise TaskError() from err
return ""
def task_cache_attachments() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"caching attachment for activity={activity!r}")
# Generates thumbnails for the actor's icon and the attachments if any
if activity.has_type(
[ap.ActivityType.CREATE, ap.ActivityType.ANNOUNCE]):
obj = activity.get_object()
else:
obj = activity
if obj.content:
content_html = BeautifulSoup(obj.content, "html5lib")
for img in content_html.find_all("img"):
src = img.attrs.get("src")
if src:
Tasks.cache_attachment({"url": src}, iri)
if obj.has_type(ap.ActivityType.VIDEO):
if isinstance(obj.url, list):
# TODO: filter only videogt
link = select_video_to_cache(obj.url)
if link:
Tasks.cache_attachment({"url": link["href"]}, iri)
elif isinstance(obj.url, str):
Tasks.cache_attachment({"url": obj.url}, iri)
else:
app.logger.warning(
f"failed to parse video link {obj!r} for {iri}")
# Iter the attachments
for attachment in obj._data.get("attachment", []):
Tasks.cache_attachment(attachment, iri)
app.logger.info(f"attachments cached for {iri}")
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
app.logger.exception(f"dropping activity {iri}, no attachment caching")
except Exception as err:
app.logger.exception(f"failed to cache attachments for {iri}")
raise TaskError() from err
return ""
def task_finish_post_to_inbox() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
process_inbox(activity, {})
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
app.logger.exception(f"no retry")
except Exception as err:
app.logger.exception(f"failed to cfinish post to inbox for {iri}")
raise TaskError() from err
return ""
def forward_activity(self, iri: str) -> None:
if not current_app.config["AP_ENABLED"]:
return # not federating if not enabled
try:
activity = ap.fetch_remote_activity(iri)
backend = ap.get_backend()
recipients = backend.followers_as_recipients()
current_app.logger.debug(f"Forwarding {activity!r} to {recipients}")
activity = ap.clean_activity(activity.to_dict())
for recp in recipients:
current_app.logger.debug(f"forwarding {activity!r} to {recp}")
payload = json.dumps(activity)
post_to_remote_inbox.delay(payload, recp)
except Exception as err: # noqa: F841
current_app.logger.exception(f"failed to cache attachments for {iri}")
def finish_inbox_processing(self, iri: str) -> None:
try:
backend = ap.get_backend()
activity = ap.fetch_remote_activity(iri)
current_app.logger.info(f"activity={activity!r}")
actor = activity.get_actor()
id = activity.get_object_id()
current_app.logger.debug(f"finish_inbox_processing actor {actor}")
if activity.has_type(ap.ActivityType.DELETE):
backend.inbox_delete(actor, activity)
elif activity.has_type(ap.ActivityType.UPDATE):
backend.inbox_update(actor, activity)
elif activity.has_type(ap.ActivityType.CREATE):
backend.inbox_create(actor, activity)
elif activity.has_type(ap.ActivityType.ANNOUNCE):
backend.inbox_announce(actor, activity)
elif activity.has_type(ap.ActivityType.LIKE):
backend.inbox_like(actor, activity)
elif activity.has_type(ap.ActivityType.FOLLOW):
# Reply to a Follow with an Accept
accept = ap.Accept(actor=id, object=activity.to_dict(embed=True))
post_to_outbox(accept)
backend.new_follower(activity, activity.get_actor(),
activity.get_object())
elif activity.has_type(ap.ActivityType.ACCEPT):
obj = activity.get_object()
# FIXME: probably other types to ACCEPT the Activity
if obj.has_type(ap.ActivityType.FOLLOW):
# Accept new follower
backend.new_following(activity, obj)
elif activity.has_type(ap.ActivityType.UNDO):
obj = activity.get_object()
if obj.has_type(ap.ActivityType.LIKE):
backend.inbox_undo_like(actor, obj)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
backend.inbox_undo_announce(actor, obj)
elif obj.has_type(ap.ActivityType.FOLLOW):
backend.undo_new_follower(actor, obj)
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
current_app.logger.exception(f"no retry")
except Exception as err: # noqa: F841
current_app.logger.exception(f"failed to cache attachments for"
f" {iri}")
def _announce_process_inbox(announce: ap.Announce, new_meta: _NewMeta) -> None:
_logger.info(f"process_inbox activity={announce!r}")
# TODO(tsileo): actually drop it without storing it and better logging, also move the check somewhere else
# or remove it?
try:
obj = announce.get_object()
except NotAnActivityError:
_logger.exception(
f'received an Annouce referencing an OStatus notice ({announce._data["object"]}), dropping the message'
)
return
if obj.has_type(ap.ActivityType.QUESTION):
Tasks.fetch_remote_question(obj)
# Cache the announced object
Tasks.cache_object(announce.id)
# Process the reply of the announced object if any
in_reply_to = obj.get_in_reply_to()
if in_reply_to:
reply = ap.fetch_remote_activity(in_reply_to)
if reply.has_type(ap.ActivityType.CREATE):
reply = reply.get_object()
in_reply_to_data = {MetaKey.IN_REPLY_TO: in_reply_to}
# Update the activity to save some data about the reply
if reply.get_actor().id == obj.get_actor().id:
in_reply_to_data.update({MetaKey.IN_REPLY_TO_SELF: True})
else:
in_reply_to_data.update({
MetaKey.IN_REPLY_TO_ACTOR:
reply.get_actor().to_dict(embed=True)
})
update_one_activity(by_remote_id(announce.id),
upsert(in_reply_to_data))
# Spawn a task to process it (and determine if it needs to be saved)
Tasks.process_reply(reply.id)
update_one_activity(
{
**by_type(ap.ActivityType.CREATE),
**by_object_id(obj.id)
},
inc(MetaKey.COUNT_BOOST, 1),
)
def task_forward_activity() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
recipients = back.followers_as_recipients()
app.logger.debug(f"Forwarding {activity!r} to {recipients}")
activity = ap.clean_activity(activity.to_dict())
payload = json.dumps(activity)
for recp in recipients:
app.logger.debug(f"forwarding {activity!r} to {recp}")
Tasks.post_to_remote_inbox(payload, recp)
except Exception as err:
app.logger.exception("task failed")
raise TaskError() from err
return ""
def admin_profile() -> _Response:
if not request.args.get("actor_id"):
abort(404)
actor_id = request.args.get("actor_id")
actor = ap.fetch_remote_activity(actor_id)
q = {
"meta.actor_id": actor_id,
"box": "inbox",
**not_deleted(),
"type": {
"$in":
[ap.ActivityType.CREATE.value, ap.ActivityType.ANNOUNCE.value]
},
}
inbox_data, older_than, newer_than = paginated_query(
DB.activities, q, limit=int(request.args.get("limit", 25)))
follower = find_one_activity({
"box": "inbox",
"type": ap.ActivityType.FOLLOW.value,
"meta.actor_id": actor.id,
"meta.undo": False,
})
following = find_one_activity({
**by_type(ap.ActivityType.FOLLOW),
**by_object_id(actor.id),
**not_undo(),
**in_outbox(),
**follow_request_accepted(),
})
return htmlify(
render_template(
"stream.html",
actor_id=actor_id,
actor=actor.to_dict(),
inbox_data=inbox_data,
older_than=older_than,
newer_than=newer_than,
follower=follower,
following=following,
lists=list(DB.lists.find()),
))
def finish_post_to_outbox(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
backend = ap.get_backend()
current_app.logger.info(f"finish_post_to_outbox {activity}")
recipients = activity.recipients()
actor = activity.get_actor()
current_app.logger.debug(f"finish_post_to_outbox actor {actor!r}")
if activity.has_type(ap.ActivityType.DELETE):
backend.outbox_delete(actor, activity)
elif activity.has_type(ap.ActivityType.UPDATE):
backend.outbox_update(actor, activity)
elif activity.has_type(ap.ActivityType.CREATE):
backend.outbox_create(actor, activity)
elif activity.has_type(ap.ActivityType.ANNOUNCE):
backend.outbox_announce(actor, activity)
elif activity.has_type(ap.ActivityType.LIKE):
backend.outbox_like(actor, activity)
elif activity.has_type(ap.ActivityType.UNDO):
obj = activity.get_object()
if obj.has_type(ap.ActivityType.LIKE):
backend.outbox_undo_like(actor, obj)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
backend.outbox_undo_announce(actor, obj)
elif obj.has_type(ap.ActivityType.FOLLOW):
backend.undo_new_following(actor, obj)
current_app.logger.info(f"recipients={recipients}")
activity = ap.clean_activity(activity.to_dict())
payload = json.dumps(activity)
for recp in recipients:
current_app.logger.debug(f"posting to {recp}")
post_to_remote_inbox.delay(payload, recp)
except (ActivityGoneError, ActivityNotFoundError):
current_app.logger.exception(f"no retry")
except Exception as err: # noqa: F841
current_app.logger.exception(f"failed to post "
f"to remote inbox for {iri}")
def finish_post_to_outbox(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
recipients = activity.recipients()
if activity.has_type(ap.ActivityType.DELETE):
back.outbox_delete(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.UPDATE):
back.outbox_update(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.CREATE):
back.outbox_create(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.ANNOUNCE):
back.outbox_announce(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.LIKE):
back.outbox_like(MY_PERSON, activity)
elif activity.has_type(ap.ActivityType.UNDO):
obj = activity.get_object()
if obj.has_type(ap.ActivityType.LIKE):
back.outbox_undo_like(MY_PERSON, obj)
elif obj.has_type(ap.ActivityType.ANNOUNCE):
back.outbox_undo_announce(MY_PERSON, obj)
elif obj.has_type(ap.ActivityType.FOLLOW):
back.undo_new_following(MY_PERSON, obj)
log.info(f"recipients={recipients}")
activity = ap.clean_activity(activity.to_dict())
DB.cache2.remove()
payload = json.dumps(activity)
for recp in recipients:
log.debug(f"posting to {recp}")
post_to_remote_inbox.delay(payload, recp)
except (ActivityGoneError, ActivityNotFoundError):
log.exception(f"no retry")
except Exception as err:
log.exception(f"failed to post to remote inbox for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def cache_attachments(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
# Generates thumbnails for the actor's icon and the attachments if any
actor = activity.get_actor()
# Update the cached actor
DB.actors.update_one(
{"remote_id": iri},
{"$set": {
"remote_id": iri,
"data": actor.to_dict(embed=True)
}},
upsert=True,
)
if actor.icon:
MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON)
if activity.has_type(ap.ActivityType.CREATE):
for attachment in activity.get_object()._data.get(
"attachment", []):
if (attachment.get("mediaType", "").startswith("image/") or
attachment.get("type") == ap.ActivityType.IMAGE.value):
try:
MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT)
except ValueError:
log.exception(f"failed to cache {attachment}")
log.info(f"attachments cached for {iri}")
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
log.exception(f"dropping activity {iri}, no attachment caching")
except Exception as err:
log.exception(f"failed to cache attachments for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))