def fetch_og(self, col, remote_id):
try:
log.info('fetch_og_meta remote_id=%s col=%s', remote_id, col)
if col == 'INBOX':
log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.inbox, remote_id))
elif col == 'OUTBOX':
log.info('%d links saved', fetch_og_metadata(USER_AGENT, DB.outbox, remote_id))
except Exception as err:
self.log.exception('failed')
self.retry(exc=err, countdown=int(random.uniform(2, 4) ** self.request.retries))
def fetch_og_metadata(self, iri: str) -> None:
try:
activity = ap.fetch_remote_activity(iri)
log.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.CREATE):
note = activity.get_object()
links = opengraph.links_from_note(note.to_dict())
og_metadata = opengraph.fetch_og_metadata(USER_AGENT, links)
for og in og_metadata:
if not og.get("image"):
continue
MEDIA_CACHE.cache_og_image(og["image"])
log.debug(f"OG metadata {og_metadata!r}")
DB.activities.update_one(
{"remote_id": iri},
{"$set": {
"meta.og_metadata": og_metadata
}})
log.info(f"OG metadata fetched for {iri}")
except (ActivityGoneError, ActivityNotFoundError):
log.exception(f"dropping activity {iri}, skip OG metedata")
except requests.exceptions.HTTPError as http_err:
if 400 <= http_err.response.status_code < 500:
log.exception("bad request, no retry")
return
log.exception("failed to fetch OG metadata")
self.retry(exc=http_err,
countdown=int(random.uniform(2, 4)**self.request.retries))
except Exception as err:
log.exception(f"failed to fetch OG metadata for {iri}")
self.retry(exc=err,
countdown=int(random.uniform(2, 4)**self.request.retries))
def task_fetch_og_meta() -> _Response:
task = p.parse(flask.request)
app.logger.info(f"task={task!r}")
iri = task.payload
try:
activity = ap.fetch_remote_activity(iri)
app.logger.info(f"activity={activity!r}")
if activity.has_type(ap.ActivityType.CREATE):
note = activity.get_object()
links = opengraph.links_from_note(note.to_dict())
og_metadata = opengraph.fetch_og_metadata(config.USER_AGENT, links)
for og in og_metadata:
if not og.get("image"):
continue
config.MEDIA_CACHE.cache_og_image(og["image"], iri)
app.logger.debug(f"OG metadata {og_metadata!r}")
DB.activities.update_one(
{"remote_id": iri},
{"$set": {
"meta.og_metadata": og_metadata
}})
app.logger.info(f"OG metadata fetched for {iri}: {og_metadata}")
except (ActivityGoneError, ActivityNotFoundError):
app.logger.exception(f"dropping activity {iri}, skip OG metedata")
return ""
except requests.exceptions.HTTPError as http_err:
if 400 <= http_err.response.status_code < 500:
app.logger.exception("bad request, no retry")
return ""
app.logger.exception("failed to fetch OG metadata")
raise TaskError() from http_err
except Exception as err:
app.logger.exception(f"failed to fetch OG metadata for {iri}")
raise TaskError() from err
return ""