def fetch_og_metadata(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.CREATE):
note = activity.get_object()
links = opengraph.links_from_note(note.to_dict())
og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links)
for og in og_metadata:
if not og.get("image"):
continue
MEDIA_CACHE.cache_og_image(og["image"])
log.debug(f"OG metadata {og_metadata!r}")
DB.activities.update_one(
{"remote_id": iri},
{"$set": {
"meta.og_metadata": og_metadata
}})
log.info(f"OG metadata fetched for {iri}")
except (ActivityGoneError, ActivityNotFoundError):
log.exception(f"dropping activity {iri}, skip OG metedata")
except requests.exceptions.HTTPError as http_err:
if 400 <= http_err.response.status_code < 500:
log.exception("bad request, no retry")
return
log.exception("failed to fetch OG metadata")
self.retry(exc=http_err,
countdown=int(random.uniform(2, 4)**self.request.retries))
except Exception as err:
log.exception(f"failed to fetch OG metadata for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def task_cache_emoji() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload["iri"]
url = task.payload["url"]
try:
MEDIA_CACHE.cache_emoji(url, iri)
except Exception as exc:
err = f"failed to cache emoji {url} at {iri}"
app.logger.exception(err)
raise TaskError() from exc
return ""
def task_cache_actor_icon() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
actor_iri = task.payload["actor_iri"]
icon_url = task.payload["icon_url"]
try:
MEDIA_CACHE.cache_actor_icon(icon_url)
except Exception as exc:
err = f"failed to cache actor icon {icon_url} for {actor_iri}"
app.logger.exception(err)
raise TaskError() from exc
return ""
def cache_actor_icon(icon_url: str, actor_iri: str) -> None:
if MEDIA_CACHE.is_actor_icon_cached(icon_url):
return None
p.push({
"icon_url": icon_url,
"actor_iri": actor_iri
}, "/task/cache_actor_icon")
def cache_attachments(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
# Generates thumbnails for the actor's icon and the attachments if any
actor = activity.get_actor()
# Update the cached actor
DB.actors.update_one(
{"remote_id": iri},
{"$set": {
"remote_id": iri,
"data": actor.to_dict(embed=True)
}},
upsert=True,
)
if actor.icon:
MEDIA_CACHE.cache(actor.icon["url"], Kind.ACTOR_ICON)
if activity.has_type(ap.ActivityType.CREATE):
for attachment in activity.get_object()._data.get(
"attachment", []):
if (attachment.get("mediaType", "").startswith("image/") or
attachment.get("type") == ap.ActivityType.IMAGE.value):
try:
MEDIA_CACHE.cache(attachment["url"], Kind.ATTACHMENT)
except ValueError:
log.exception(f"failed to cache {attachment}")
log.info(f"attachments cached for {iri}")
except (ActivityGoneError, ActivityNotFoundError, NotAnActivityError):
log.exception(f"dropping activity {iri}, no attachment caching")
except Exception as err:
log.exception(f"failed to cache attachments for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def _get_file_url(url, size, kind):
k = (kind, url, size)
cached = _GRIDFS_CACHE.get(k)
if cached:
return cached
doc = MEDIA_CACHE.get_file(url, size, kind)
if doc:
u = f"/media/{str(doc._id)}"
_GRIDFS_CACHE[k] = u
return u
# MEDIA_CACHE.cache(url, kind)
_logger.error(f"cache not available for {url}/{size}/{kind}")
return url
def _get_file_url(url, size, kind) -> str:
k = (url, size, kind)
cached = _FILE_URL_CACHE.get(k)
if cached:
return cached
doc = MEDIA_CACHE.get_file(*k)
if doc:
out = f"/media/{str(doc._id)}"
_FILE_URL_CACHE[k] = out
return out
_logger.error(f"cache not available for {url}/{size}/{kind}")
if url.startswith(BASE_URL):
return url
p = urlparse(url)
return f"/p/{p.scheme}" + p._replace(scheme="").geturl()[1:]
def api_new_note() -> _Response:
# Basic Micropub (https://www.w3.org/TR/micropub/) query configuration support
if request.method == "GET" and request.args.get("q") == "config":
return jsonify({})
elif request.method == "GET":
abort(405)
source = None
summary = None
place_tags = []
# Basic Micropub (https://www.w3.org/TR/micropub/) "create" support
is_micropub = False
# First, check if the Micropub specific fields are present
if (
_user_api_arg("h", default=None) == "entry"
or _user_api_arg("type", default=[None])[0] == "h-entry"
):
is_micropub = True
# Ensure the "create" scope is set
if "jwt_payload" not in flask.g or "create" not in flask.g.jwt_payload["scope"]:
abort(403)
# Handle location sent via form-data
# `geo:28.5,9.0,0.0`
location = _user_api_arg("location", default="")
if location.startswith("geo:"):
slat, slng, *_ = location[4:].split(",")
place_tags.append(
{
"type": ap.ActivityType.PLACE.value,
"url": "",
"name": "",
"latitude": float(slat),
"longitude": float(slng),
}
)
# Handle JSON microformats2 data
if _user_api_arg("type", default=None):
_logger.info(f"Micropub request: {request.json}")
try:
source = request.json["properties"]["content"][0]
except (ValueError, KeyError):
pass
# Handle HTML
if isinstance(source, dict):
source = source.get("html")
try:
summary = request.json["properties"]["name"][0]
except (ValueError, KeyError):
pass
# Try to parse the name as summary if the payload is POSTed using form-data
if summary is None:
summary = _user_api_arg("name", default=None)
# This step will also parse content from Micropub request
if source is None:
source = _user_api_arg("content", default=None)
if not source:
raise ValueError("missing content")
if summary is None:
summary = _user_api_arg("summary", default="")
if not place_tags:
if _user_api_arg("location_lat", default=None):
lat = float(_user_api_arg("location_lat"))
lng = float(_user_api_arg("location_lng"))
loc_name = _user_api_arg("location_name", default="")
place_tags.append(
{
"type": ap.ActivityType.PLACE.value,
"url": "",
"name": loc_name,
"latitude": lat,
"longitude": lng,
}
)
# All the following fields are specific to the API (i.e. not Micropub related)
_reply, reply = None, None
try:
_reply = _user_api_arg("reply")
except ValueError:
pass
visibility = ap.Visibility[
_user_api_arg("visibility", default=ap.Visibility.PUBLIC.name)
]
content, tags = parse_markdown(source)
# Check for custom emojis
tags = tags + emojis.tags(content) + place_tags
to: List[str] = []
cc: List[str] = []
if visibility == ap.Visibility.PUBLIC:
to = [ap.AS_PUBLIC]
cc = [ID + "/followers"]
elif visibility == ap.Visibility.UNLISTED:
to = [ID + "/followers"]
cc = [ap.AS_PUBLIC]
elif visibility == ap.Visibility.FOLLOWERS_ONLY:
to = [ID + "/followers"]
cc = []
if _reply:
reply = ap.fetch_remote_activity(_reply)
if visibility == ap.Visibility.DIRECT:
to.append(reply.attributedTo)
else:
cc.append(reply.attributedTo)
context = new_context(reply)
for tag in tags:
if tag["type"] == "Mention":
to.append(tag["href"])
raw_note = dict(
attributedTo=MY_PERSON.id,
cc=list(set(cc) - set([MY_PERSON.id])),
to=list(set(to) - set([MY_PERSON.id])),
summary=summary,
content=content,
tag=tags,
source={"mediaType": "text/markdown", "content": source},
inReplyTo=reply.id if reply else None,
context=context,
)
if request.files:
for f in request.files.keys():
if not request.files[f].filename:
continue
file = request.files[f]
rfilename = secure_filename(file.filename)
with BytesIO() as buf:
file.save(buf)
oid = MEDIA_CACHE.save_upload(buf, rfilename)
mtype = mimetypes.guess_type(rfilename)[0]
raw_note["attachment"] = [
{
"mediaType": mtype,
"name": _user_api_arg("file_description", default=rfilename),
"type": "Document",
"url": f"{BASE_URL}/uploads/{oid}/{rfilename}",
}
]
note = ap.Note(**raw_note)
create = note.build_create()
create_id = post_to_outbox(create)
# Return a 201 with the note URL in the Location header if this was a Micropub request
if is_micropub:
resp = flask.Response("", headers={"Location": create_id})
resp.status_code = 201
return resp
return _user_api_response(activity=create_id)
def api_new_note() -> _Response:
source = _user_api_arg("content")
if not source:
raise ValueError("missing content")
_reply, reply = None, None
try:
_reply = _user_api_arg("reply")
except ValueError:
pass
visibility = ap.Visibility[_user_api_arg(
"visibility", default=ap.Visibility.PUBLIC.name)]
content, tags = parse_markdown(source)
to: List[str] = []
cc: List[str] = []
if visibility == ap.Visibility.PUBLIC:
to = [ap.AS_PUBLIC]
cc = [ID + "/followers"]
elif visibility == ap.Visibility.UNLISTED:
to = [ID + "/followers"]
cc = [ap.AS_PUBLIC]
elif visibility == ap.Visibility.FOLLOWERS_ONLY:
to = [ID + "/followers"]
cc = []
if _reply:
reply = ap.fetch_remote_activity(_reply)
if visibility == ap.Visibility.DIRECT:
to.append(reply.attributedTo)
else:
cc.append(reply.attributedTo)
for tag in tags:
if tag["type"] == "Mention":
if visibility == ap.Visibility.DIRECT:
to.append(tag["href"])
else:
cc.append(tag["href"])
raw_note = dict(
attributedTo=MY_PERSON.id,
cc=list(set(cc)),
to=list(set(to)),
content=content,
tag=tags,
source={
"mediaType": "text/markdown",
"content": source
},
inReplyTo=reply.id if reply else None,
)
if "file" in request.files and request.files["file"].filename:
file = request.files["file"]
rfilename = secure_filename(file.filename)
with BytesIO() as buf:
file.save(buf)
oid = MEDIA_CACHE.save_upload(buf, rfilename)
mtype = mimetypes.guess_type(rfilename)[0]
raw_note["attachment"] = [{
"mediaType":
mtype,
"name":
rfilename,
"type":
"Document",
"url":
f"{BASE_URL}/uploads/{oid}/{rfilename}",
}]
note = ap.Note(**raw_note)
create = note.build_create()
create_id = post_to_outbox(create)
return _user_api_response(activity=create_id)
def cache_emoji(url: str, iri: str) -> None:
if MEDIA_CACHE.is_emoji_cached(iri):
return None
p.push({"url": url, "iri": iri}, "/task/cache_emoji")
def cache_actor_icon(icon_url: str, actor_iri: str) -> None:
if MEDIA_CACHE.is_actor_icon_cached(icon_url):
return None