def retrieve(fid, actor=None, serializer_class=None, queryset=None):
if queryset:
try:
# queryset can also be a Model class
existing = queryset.filter(fid=fid).first()
except AttributeError:
existing = queryset.objects.filter(fid=fid).first()
if existing:
return existing
auth = (None if not actor else signing.get_auth(actor.private_key,
actor.private_key_id))
response = session.get_session().get(
fid,
auth=auth,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={
"Accept": "application/activity+json",
"Content-Type": "application/activity+json",
},
)
response.raise_for_status()
data = response.json()
if not serializer_class:
return data
serializer = serializer_class(data=data)
serializer.is_valid(raise_exception=True)
return serializer.save()
def deliver_to_remote(delivery):
if not preferences.get("federation__enabled"):
# federation is disabled, we only deliver to local recipients
return
actor = delivery.activity.actor
logger.info("Preparing activity delivery to %s", delivery.inbox_url)
auth = signing.get_auth(actor.private_key, actor.private_key_id)
try:
response = session.get_session().post(
auth=auth,
json=delivery.activity.payload,
url=delivery.inbox_url,
headers={"Content-Type": "application/activity+json"},
)
logger.debug("Remote answered with %s", response.status_code)
response.raise_for_status()
except Exception:
delivery.last_attempt_date = timezone.now()
delivery.attempts = F("attempts") + 1
delivery.save(update_fields=["last_attempt_date", "attempts"])
raise
else:
delivery.last_attempt_date = timezone.now()
delivery.attempts = F("attempts") + 1
delivery.is_delivered = True
delivery.save(
update_fields=["last_attempt_date", "attempts", "is_delivered"])
def download_audio_from_remote(self, actor):
from funkwhale_api.federation import signing
if actor:
auth = signing.get_auth(actor.private_key, actor.private_key_id)
else:
auth = None
remote_response = session.get_session().get(
self.source,
auth=auth,
stream=True,
timeout=20,
headers={"Content-Type": "application/octet-stream"},
)
with remote_response as r:
remote_response.raise_for_status()
extension = utils.get_ext_from_type(self.mimetype)
title_parts = []
title_parts.append(self.track.title)
if self.track.album:
title_parts.append(self.track.album.title)
title_parts.append(self.track.artist.name)
title = " - ".join(title_parts)
filename = "{}.{}".format(title, extension)
tmp_file = tempfile.TemporaryFile()
for chunk in r.iter_content(chunk_size=512):
tmp_file.write(chunk)
self.audio_file.save(filename, tmp_file, save=False)
self.save(update_fields=["audio_file"])
def download_audio_from_remote(self, user):
from funkwhale_api.common import session
from funkwhale_api.federation import signing
if user.is_authenticated and user.actor:
auth = signing.get_auth(user.actor.private_key,
user.actor.private_key_id)
else:
auth = None
remote_response = session.get_session().get(
self.source,
auth=auth,
stream=True,
timeout=20,
headers={"Content-Type": "application/octet-stream"},
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
)
with remote_response as r:
remote_response.raise_for_status()
extension = utils.get_ext_from_type(self.mimetype)
title = " - ".join([
self.track.title, self.track.album.title,
self.track.artist.name
])
filename = "{}.{}".format(title, extension)
tmp_file = tempfile.TemporaryFile()
for chunk in r.iter_content(chunk_size=512):
tmp_file.write(chunk)
self.audio_file.save(filename, tmp_file, save=False)
self.save(update_fields=["audio_file"])
def get_actor_data(actor_url):
response = session.get_session().get(
actor_url,
headers={"Accept": "application/activity+json"},
)
response.raise_for_status()
try:
return response.json()
except Exception:
raise ValueError("Invalid actor payload: {}".format(response.text))
def get_resource(resource_string):
resource_type, resource = clean_resource(resource_string)
username, hostname = clean_acct(resource, ensure_local=False)
url = "https://{}/.well-known/webfinger?resource={}".format(
hostname, resource_string)
response = session.get_session().get(
url, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL, timeout=5)
response.raise_for_status()
serializer = serializers.ActorWebfingerSerializer(data=response.json())
serializer.is_valid(raise_exception=True)
return serializer.validated_data
def get_actor_data(actor_url):
response = session.get_session().get(
actor_url,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={"Accept": "application/activity+json"},
)
response.raise_for_status()
try:
return response.json()
except Exception:
raise ValueError("Invalid actor payload: {}".format(response.text))
def handle(self, *args, **options):
logger = logging.getLogger("funkwhale.mrf")
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(stream=sys.stderr))
input = options["input"]
if not input:
registry = getattr(mrf, options["type"])
self.stdout.write(
"No input given, listing registered policies for '{}' MRF:".
format(options["type"]))
for name in registry.keys():
self.stdout.write("- {}".format(name))
return
raw_content = None
content = None
if input == "-":
raw_content = sys.stdin.read()
elif is_uuid(input):
self.stderr.write("UUID provided, retrieving payload from db")
content = models.Activity.objects.get(uuid=input).payload
elif is_url(input):
response = session.get_session().get(
input,
headers={"Accept": "application/activity+json"},
)
response.raise_for_status()
content = response.json()
else:
with open(input, "rb") as f:
raw_content = f.read()
content = json.loads(raw_content) if content is None else content
policies = options["policies"] or []
registry = getattr(mrf, options["type"])
for policy in policies:
if policy not in registry:
raise CommandError("Unknown policy '{}' for MRF '{}'".format(
policy, options["type"]))
payload, updated = registry.apply(content, policies=policies)
if not payload:
self.stderr.write("Payload was discarded by MRF")
elif updated:
self.stderr.write("Payload was modified by MRF")
self.stderr.write("Initial payload:\n")
self.stdout.write(json.dumps(content, indent=2, sort_keys=True))
self.stderr.write("Modified payload:\n")
self.stdout.write(json.dumps(payload, indent=2, sort_keys=True))
else:
self.stderr.write("Payload left untouched by MRF")
def get_library_page(library, page_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
response = session.get_session().get(
page_url,
auth=auth,
timeout=5,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
headers={"Content-Type": "application/activity+json"},
)
serializer = serializers.CollectionPageSerializer(
data=response.json(),
context={"library": library, "item_serializer": serializers.UploadSerializer},
)
serializer.is_valid(raise_exception=True)
return serializer.validated_data
def get_library_page(library, page_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
response = session.get_session().get(
page_url,
auth=auth,
headers={"Accept": "application/activity+json"},
)
serializer = serializers.CollectionPageSerializer(
data=response.json(),
context={
"library": library,
"item_serializer": serializers.UploadSerializer
},
)
serializer.is_valid(raise_exception=True)
return serializer.validated_data
def fetch_nodeinfo(domain_name):
s = session.get_session()
wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name)
response = s.get(url=wellknown_url)
response.raise_for_status()
serializer = serializers.NodeInfoSerializer(data=response.json())
serializer.is_valid(raise_exception=True)
nodeinfo_url = None
for link in serializer.validated_data["links"]:
if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0":
nodeinfo_url = link["href"]
break
response = s.get(url=nodeinfo_url)
response.raise_for_status()
return response.json()
def retrieve_ap_object(fid,
actor,
serializer_class=None,
queryset=None,
apply_instance_policies=True):
# we have a duplicate check here because it's less expensive to do those checks
# twice than to trigger a HTTP request
payload, updated = mrf.inbox.apply({"id": fid})
if not payload:
raise exceptions.BlockedActorOrDomain()
if queryset:
try:
# queryset can also be a Model class
existing = queryset.filter(fid=fid).first()
except AttributeError:
existing = queryset.objects.filter(fid=fid).first()
if existing:
return existing
auth = (None if not actor else signing.get_auth(actor.private_key,
actor.private_key_id))
response = session.get_session().get(
fid,
auth=auth,
headers={
"Accept": "application/activity+json",
"Content-Type": "application/activity+json",
},
)
response.raise_for_status()
data = response.json()
# we match against mrf here again, because new data may yield different
# results
data, updated = mrf.inbox.apply(data)
if not data:
raise exceptions.BlockedActorOrDomain()
if not serializer_class:
return data
serializer = serializer_class(data=data, context={"fetch_actor": actor})
serializer.is_valid(raise_exception=True)
try:
return serializer.save()
except NotImplementedError:
return serializer.validated_data
def download_audio(self):
from . import actors
auth = actors.SYSTEM_ACTORS["library"].get_request_auth()
remote_response = session.get_session().get(
self.audio_url,
auth=auth,
stream=True,
timeout=20,
headers={"Accept": "application/activity+json"},
)
with remote_response as r:
remote_response.raise_for_status()
extension = music_utils.get_ext_from_type(self.audio_mimetype)
title = " - ".join(
[self.title, self.album_title, self.artist_name])
filename = "{}.{}".format(title, extension)
tmp_file = tempfile.TemporaryFile()
for chunk in r.iter_content(chunk_size=512):
tmp_file.write(chunk)
self.audio_file.save(filename, tmp_file)
def retrieve_feed(url):
try:
logger.info("Fetching RSS feed at %s", url)
response = session.get_session().get(url)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response:
raise FeedFetchException(
"Error while fetching feed: HTTP {}".format(e.response.status_code)
)
raise FeedFetchException("Error while fetching feed: unknown error")
except requests.exceptions.Timeout:
raise FeedFetchException("Error while fetching feed: timeout")
except requests.exceptions.ConnectionError:
raise FeedFetchException("Error while fetching feed: connection error")
except requests.RequestException as e:
raise FeedFetchException("Error while fetching feed: {}".format(e))
except Exception as e:
raise FeedFetchException("Error while fetching feed: {}".format(e))
return response
def get_image(self, data=None):
if data:
extensions = {
"image/jpeg": "jpg",
"image/png": "png",
"image/gif": "gif"
}
extension = extensions.get(data["mimetype"], "jpg")
if data.get("content"):
# we have to cover itself
f = ContentFile(data["content"])
elif data.get("url"):
# we can fetch from a url
try:
response = session.get_session().get(
data.get("url"),
timeout=3,
verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL,
)
response.raise_for_status()
except Exception as e:
logger.warn("Cannot download cover at url %s: %s",
data.get("url"), e)
return
else:
f = ContentFile(response.content)
self.cover.save("{}.{}".format(self.uuid, extension),
f,
save=False)
self.save(update_fields=["cover"])
return self.cover.file
if self.mbid:
image_data = musicbrainz.api.images.get_front(str(self.mbid))
f = ContentFile(image_data)
self.cover.save("{0}.jpg".format(self.mbid), f, save=False)
self.save(update_fields=["cover"])
return self.cover.file
def get_library_data(library_url, actor):
auth = signing.get_auth(actor.private_key, actor.private_key_id)
try:
response = session.get_session().get(
library_url,
auth=auth,
headers={"Accept": "application/activity+json"},
)
except requests.ConnectionError:
return {"errors": ["This library is not reachable"]}
scode = response.status_code
if scode == 401:
return {"errors": ["This library requires authentication"]}
elif scode == 403:
return {"errors": ["Permission denied while scanning library"]}
elif scode >= 400:
return {
"errors": ["Error {} while fetching the library".format(scode)]
}
serializer = serializers.LibrarySerializer(data=response.json())
if not serializer.is_valid():
return {"errors": ["Invalid ActivityPub response from remote library"]}
return serializer.validated_data
def get_session():
from funkwhale_api.common import session
return session.get_session()
def fetch(fetch_obj):
def error(code, **kwargs):
fetch_obj.status = "errored"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"error_code": code}
fetch_obj.detail.update(kwargs)
fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
url = fetch_obj.url
mrf_check_url = url
if not mrf_check_url.startswith("webfinger://"):
payload, updated = mrf.inbox.apply({"id": mrf_check_url})
if not payload:
return error("blocked", message="Blocked by MRF")
actor = fetch_obj.actor
if settings.FEDERATION_AUTHENTIFY_FETCHES:
auth = signing.get_auth(actor.private_key, actor.private_key_id)
else:
auth = None
auth = None
try:
if url.startswith("webfinger://"):
# we first grab the correpsonding webfinger representation
# to get the ActivityPub actor ID
webfinger_data = webfinger.get_resource(
"acct:" + url.replace("webfinger://", ""))
url = webfinger.get_ap_url(webfinger_data["links"])
if not url:
return error("webfinger",
message="Invalid or missing webfinger data")
payload, updated = mrf.inbox.apply({"id": url})
if not payload:
return error("blocked", message="Blocked by MRF")
response = session.get_session().get(
auth=auth,
url=url,
headers={"Accept": "application/activity+json"},
)
logger.debug("Remote answered with %s", response.status_code)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
return error(
"http", status_code=e.response.status_code if e.response else None)
except requests.exceptions.Timeout:
return error("timeout")
except requests.exceptions.ConnectionError as e:
return error("connection", message=str(e))
except requests.RequestException as e:
return error("request", message=str(e))
except Exception as e:
return error("unhandled", message=str(e))
try:
payload = response.json()
except json.decoder.JSONDecodeError:
# we attempt to extract a <link rel=alternate> that points
# to an activity pub resource, if possible, and retry with this URL
alternate_url = utils.find_alternate(response.text)
if alternate_url:
fetch_obj.url = alternate_url
fetch_obj.save(update_fields=["url"])
return fetch(fetch_id=fetch_obj.pk)
return error("invalid_json")
payload, updated = mrf.inbox.apply(payload)
if not payload:
return error("blocked", message="Blocked by MRF")
try:
doc = jsonld.expand(payload)
except ValueError:
return error("invalid_jsonld")
try:
type = doc.get("@type", [])[0]
except IndexError:
return error("missing_jsonld_type")
try:
serializer_classes = fetch_obj.serializers[type]
model = serializer_classes[0].Meta.model
except (KeyError, AttributeError):
fetch_obj.status = "skipped"
fetch_obj.fetch_date = timezone.now()
fetch_obj.detail = {"reason": "unhandled_type", "type": type}
return fetch_obj.save(update_fields=["fetch_date", "status", "detail"])
try:
id = doc.get("@id")
except IndexError:
existing = None
else:
existing = model.objects.filter(fid=id).first()
serializer = None
for serializer_class in serializer_classes:
serializer = serializer_class(existing, data=payload)
if not serializer.is_valid():
continue
else:
break
if serializer.errors:
return error("validation", validation_errors=serializer.errors)
try:
obj = serializer.save()
except Exception as e:
error("save", message=str(e))
raise
# special case for channels
# when obj is an actor, we check if the actor has a channel associated with it
# if it is the case, we consider the fetch obj to be a channel instead
# and also trigger a fetch on the channel outbox
if isinstance(obj, models.Actor) and obj.get_channel():
obj = obj.get_channel()
if obj.actor.outbox_url:
try:
# first page fetch is synchronous, so that at least some data is available
# in the UI after subscription
result = fetch_collection(
obj.actor.outbox_url,
channel_id=obj.pk,
max_pages=1,
)
except Exception:
logger.exception("Error while fetching actor outbox: %s",
obj.actor.outbox.url)
else:
if result.get("next_page"):
# additional pages are fetched in the background
result = fetch_collection.delay(
result["next_page"],
channel_id=obj.pk,
max_pages=settings.FEDERATION_COLLECTION_MAX_PAGES - 1,
is_page=True,
)
fetch_obj.object = obj
fetch_obj.status = "finished"
fetch_obj.fetch_date = timezone.now()
return fetch_obj.save(update_fields=[
"fetch_date", "status", "object_id", "object_content_type"
])
def test_get_session():
expected = session.get_user_agent()
assert session.get_session().headers["User-Agent"] == expected
