def refetch_obj(obj, queryset):
"""
Given an Artist/Album/Track instance, if the instance is from a remote pod,
will attempt to update local data with the latest ActivityPub representation.
"""
if obj.is_local:
return obj
now = timezone.now()
limit = now - datetime.timedelta(minutes=settings.FEDERATION_OBJECT_FETCH_DELAY)
last_fetch = obj.fetches.order_by("-creation_date").first()
if last_fetch is not None and last_fetch.creation_date > limit:
# we fetched recently, no need to do it again
return obj
logger.info("Refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid)
actor = actors.get_service_actor()
fetch = federation_models.Fetch.objects.create(actor=actor, url=obj.fid, object=obj)
try:
federation_tasks.fetch(fetch_id=fetch.pk)
except Exception:
logger.exception(
"Error while refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid
)
else:
fetch.refresh_from_db()
if fetch.status == "finished":
obj = queryset.get(pk=obj.pk)
return obj
def external_rss(self, include=True):
from funkwhale_api.federation import actors
query = models.Q(
attributed_to=actors.get_service_actor(),
actor__preferred_username__startswith="rssfeed-",
)
if include:
return self.filter(query)
return self.exclude(query)
def test_get_service_actor(db, settings):
settings.FEDERATION_HOSTNAME = "test.hello"
settings.FEDERATION_SERVICE_ACTOR_USERNAME = "******"
actor = actors.get_service_actor()
assert actor.preferred_username == "bob"
assert actor.domain.name == "test.hello"
assert actor.private_key is not None
assert actor.type == "Service"
assert actor.public_key is not None
def test_service_actor_detail(factories, api_client):
actor = actors.get_service_actor()
url = reverse(
"federation:actors-detail",
kwargs={"preferred_username": actor.preferred_username},
)
serializer = serializers.ActorSerializer(actor)
response = api_client.get(url)
assert response.status_code == 200
assert response.data == serializer.data
def test_authenticate_skips_anonymous_fetch_when_allow_list_enabled(
preferences, api_client):
preferences["moderation__allow_list_enabled"] = True
actor = actors.get_service_actor()
url = reverse(
"federation:actors-detail",
kwargs={"preferred_username": actor.preferred_username},
)
response = api_client.get(url)
assert response.status_code == 403
def filter_external(self, queryset, name, value):
query = Q(
attributed_to=actors.get_service_actor(),
actor__preferred_username__startswith="rssfeed-",
)
if value is True:
queryset = queryset.filter(query)
if value is False:
queryset = queryset.exclude(query)
return queryset
def test_outbox_update_track(factories):
track = factories["music.Track"]()
activity = list(routes.outbox_update_track({"track": track}))[0]
expected = serializers.ActivitySerializer({
"type":
"Update",
"object":
serializers.TrackSerializer(track).data
}).data
expected["to"] = [contexts.AS.Public, {"type": "instances_with_followers"}]
assert dict(activity["payload"]) == dict(expected)
assert activity["actor"] == actors.get_service_actor()
def test_outbox_flag(factory_name, factory_kwargs, factories, mocker):
target = factories[factory_name](**factory_kwargs)
report = factories["moderation.Report"](
target=target,
local=True,
target_owner=factories["federation.Actor"]())
activity = list(routes.outbox_flag({"report": report}))[0]
serializer = serializers.FlagSerializer(report)
expected = serializer.data
expected["to"] = [{"type": "actor_inbox", "actor": report.target_owner}]
assert activity["payload"] == expected
assert activity["actor"] == actors.get_service_actor()
def rss(self, request, *args, **kwargs):
object = self.get_object()
if not object.attributed_to.is_local:
return response.Response({"detail": "Not found"}, status=404)
if object.attributed_to == actors.get_service_actor():
# external feed, we redirect to the canonical one
return http.HttpResponseRedirect(object.rss_url)
uploads = (object.library.uploads.playable_by(None).prefetch_related(
Prefetch(
"track",
queryset=music_models.Track.objects.select_related(
"attachment_cover", "description").prefetch_related(
music_views.TAG_PREFETCH, ),
), ).select_related(
"track__attachment_cover",
"track__description").order_by("-creation_date"))[:50]
data = serializers.rss_serialize_channel_full(channel=object,
uploads=uploads)
return response.Response(data, status=200)
def test_nodeinfo_dump(preferences, mocker, avatar):
preferences["instance__banner"] = avatar
preferences["instance__nodeinfo_stats_enabled"] = True
preferences["common__api_authentication_required"] = False
preferences["moderation__unauthenticated_report_types"] = [
"takedown_request",
"other",
"other_category_that_doesnt_exist",
]
stats = {
"users": {
"total": 1,
"active_halfyear": 12,
"active_month": 13
},
"tracks": 2,
"albums": 3,
"artists": 4,
"track_favorites": 5,
"music_duration": 6,
"listenings": 7,
"downloads": 42,
}
mocker.patch("funkwhale_api.instance.stats.get", return_value=stats)
expected = {
"version": "2.0",
"software": {
"name": "funkwhale",
"version": funkwhale_api.__version__
},
"protocols": ["activitypub"],
"services": {
"inbound": [],
"outbound": []
},
"openRegistrations": preferences["users__registration_enabled"],
"usage": {
"users": {
"total": 1,
"activeHalfyear": 12,
"activeMonth": 13
}
},
"metadata": {
"actorId":
actors.get_service_actor().fid,
"private":
preferences["instance__nodeinfo_private"],
"shortDescription":
preferences["instance__short_description"],
"longDescription":
preferences["instance__long_description"],
"nodeName":
preferences["instance__name"],
"rules":
preferences["instance__rules"],
"contactEmail":
preferences["instance__contact_email"],
"defaultUploadQuota":
preferences["users__upload_quota"],
"terms":
preferences["instance__terms"],
"banner":
federation_utils.full_url(preferences["instance__banner"].url),
"library": {
"federationEnabled":
preferences["federation__enabled"],
"anonymousCanListen":
not preferences["common__api_authentication_required"],
"tracks": {
"total": stats["tracks"]
},
"artists": {
"total": stats["artists"]
},
"albums": {
"total": stats["albums"]
},
"music": {
"hours": stats["music_duration"]
},
},
"usage": {
"favorites": {
"tracks": {
"total": stats["track_favorites"]
}
},
"listenings": {
"total": stats["listenings"]
},
"downloads": {
"total": stats["downloads"]
},
},
"supportedUploadExtensions":
music_utils.SUPPORTED_EXTENSIONS,
"allowList": {
"enabled": False,
"domains": None
},
"reportTypes": [
{
"type": "takedown_request",
"label": "Takedown request",
"anonymous": True,
},
{
"type": "invalid_metadata",
"label": "Invalid metadata",
"anonymous": False,
},
{
"type": "illegal_content",
"label": "Illegal content",
"anonymous": False,
},
{
"type": "offensive_content",
"label": "Offensive content",
"anonymous": False,
},
{
"type": "other",
"label": "Other",
"anonymous": True
},
],
"funkwhaleSupportMessageEnabled":
preferences["instance__funkwhale_support_message_enabled"],
"instanceSupportMessage":
preferences["instance__support_message"],
"knownNodesListUrl":
federation_utils.full_url(
reverse("api:v1:federation:domains-list")),
},
}
assert nodeinfo.get() == expected
def save(self, rss_url):
validated_data = self.validated_data
# because there may be redirections from the original feed URL
real_rss_url = validated_data.get("atom_link", rss_url) or rss_url
service_actor = actors.get_service_actor()
author = validated_data.get("author_detail", {})
categories = validated_data.get("tags", {})
metadata = {
"explicit": validated_data.get("itunes_explicit", False),
"copyright": validated_data.get("rights"),
"owner_name": author.get("name"),
"owner_email": author.get("email"),
"itunes_category": categories.get("parent"),
"itunes_subcategory": categories.get("child"),
"language": validated_data.get("language"),
}
public_url = validated_data["link"]
existing = (
models.Channel.objects.external_rss()
.filter(
Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url)
)
.first()
)
channel_defaults = {
"rss_url": real_rss_url,
"metadata": metadata,
}
if existing:
artist_kwargs = {"channel": existing}
actor_kwargs = {"channel": existing}
actor_defaults = {"url": public_url}
else:
artist_kwargs = {"pk": None}
actor_kwargs = {"pk": None}
preferred_username = "******".format(uuid.uuid4())
actor_defaults = {
"preferred_username": preferred_username,
"type": "Application",
"domain": service_actor.domain,
"url": public_url,
"fid": federation_utils.full_url(
reverse(
"federation:actors-detail",
kwargs={"preferred_username": preferred_username},
)
),
}
channel_defaults["attributed_to"] = service_actor
actor_defaults["last_fetch_date"] = timezone.now()
# create/update the artist profile
artist, created = music_models.Artist.objects.update_or_create(
**artist_kwargs,
defaults={
"attributed_to": service_actor,
"name": validated_data["title"][
: music_models.MAX_LENGTHS["ARTIST_NAME"]
],
"content_category": "podcast",
},
)
cover = validated_data.get("image")
if cover:
common_utils.attach_file(artist, "attachment_cover", cover)
tags = categories.get("tags", [])
if tags:
tags_models.set_tags(artist, *tags)
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(artist, "description", summary)
if created:
channel_defaults["artist"] = artist
# create/update the actor
actor, created = federation_models.Actor.objects.update_or_create(
**actor_kwargs, defaults=actor_defaults
)
if created:
channel_defaults["actor"] = actor
# create the library
if not existing:
channel_defaults["library"] = music_models.Library.objects.create(
actor=service_actor,
privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY,
name=actor_defaults["preferred_username"],
)
# create/update the channel
channel, created = models.Channel.objects.update_or_create(
pk=existing.pk if existing else None, defaults=channel_defaults,
)
return channel
def get_actor(self, obj):
if obj.attributed_to == actors.get_service_actor():
return None
return federation_serializers.APIActorSerializer(obj.actor).data
def service_actor(db):
return actors.get_service_actor()
def test_rss_feed_serializer_update(factories, now):
rss_url = "http://example.rss/"
channel = factories["audio.Channel"](rss_url=rss_url, external=True)
xml_payload = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Hello</title>
<description>Description</description>
<link>http://public.url</link>
<atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/>
<lastBuildDate>Wed, 11 Mar 2020 16:01:08 GMT</lastBuildDate>
<pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate>
<ttl>30</ttl>
<language>en</language>
<copyright>2019 Tests</copyright>
<itunes:keywords>pop rock</itunes:keywords>
<image>
<url>
https://image.url
</url>
<title>Image caption</title>
</image>
<itunes:image href="https://image.url"/>
<itunes:subtitle>Subtitle</itunes:subtitle>
<itunes:type>episodic</itunes:type>
<itunes:author>Author</itunes:author>
<itunes:summary><![CDATA[Some content]]></itunes:summary>
<itunes:owner>
<itunes:name>Name</itunes:name>
<itunes:email>email@domain</itunes:email>
</itunes:owner>
<itunes:explicit>yes</itunes:explicit>
<itunes:keywords/>
<itunes:category text="Business">
<itunes:category text="Entrepreneurship">
</itunes:category>
</channel>
</rss>
"""
parsed_feed = feedparser.parse(xml_payload)
serializer = serializers.RssFeedSerializer(data=parsed_feed.feed)
assert serializer.is_valid(raise_exception=True) is True
serializer.save(rss_url)
channel.refresh_from_db()
assert channel.rss_url == "http://real.rss.url"
assert channel.attributed_to == actors.get_service_actor()
assert channel.library.actor == actors.get_service_actor()
assert channel.library.fid is not None
assert channel.artist.name == "Hello"
assert channel.artist.attributed_to == actors.get_service_actor()
assert channel.artist.description.content_type == "text/plain"
assert channel.artist.description.text == "Some content"
assert channel.artist.attachment_cover.url == "https://image.url"
assert channel.artist.get_tags() == ["pop", "rock"]
assert channel.actor.url == "http://public.url"
assert channel.actor.last_fetch_date == now
assert channel.metadata == {
"explicit": True,
"copyright": "2019 Tests",
"owner_name": "Name",
"owner_email": "email@domain",
"itunes_category": "Business",
"itunes_subcategory": "Entrepreneurship",
"language": "en",
}
def test_nodeinfo_dump_stats_disabled(preferences, mocker):
preferences["instance__nodeinfo_stats_enabled"] = False
preferences["moderation__unauthenticated_report_types"] = [
"takedown_request",
"other",
]
expected = {
"version": "2.0",
"software": {
"name": "funkwhale",
"version": funkwhale_api.__version__
},
"protocols": ["activitypub"],
"services": {
"inbound": [],
"outbound": []
},
"openRegistrations": preferences["users__registration_enabled"],
"usage": {
"users": {
"total": 0,
"activeHalfyear": 0,
"activeMonth": 0
}
},
"metadata": {
"actorId":
actors.get_service_actor().fid,
"private":
preferences["instance__nodeinfo_private"],
"shortDescription":
preferences["instance__short_description"],
"longDescription":
preferences["instance__long_description"],
"nodeName":
preferences["instance__name"],
"rules":
preferences["instance__rules"],
"contactEmail":
preferences["instance__contact_email"],
"defaultUploadQuota":
preferences["users__upload_quota"],
"terms":
preferences["instance__terms"],
"banner":
None,
"library": {
"federationEnabled":
preferences["federation__enabled"],
"anonymousCanListen":
not preferences["common__api_authentication_required"],
},
"supportedUploadExtensions":
music_utils.SUPPORTED_EXTENSIONS,
"allowList": {
"enabled": False,
"domains": None
},
"reportTypes": [
{
"type": "takedown_request",
"label": "Takedown request",
"anonymous": True,
},
{
"type": "invalid_metadata",
"label": "Invalid metadata",
"anonymous": False,
},
{
"type": "illegal_content",
"label": "Illegal content",
"anonymous": False,
},
{
"type": "offensive_content",
"label": "Offensive content",
"anonymous": False,
},
{
"type": "other",
"label": "Other",
"anonymous": True
},
],
"funkwhaleSupportMessageEnabled":
preferences["instance__funkwhale_support_message_enabled"],
"instanceSupportMessage":
preferences["instance__support_message"],
"knownNodesListUrl":
None,
},
}
assert nodeinfo.get() == expected
def get():
all_preferences = preferences.all()
share_stats = all_preferences.get("instance__nodeinfo_stats_enabled")
allow_list_enabled = all_preferences.get("moderation__allow_list_enabled")
allow_list_public = all_preferences.get("moderation__allow_list_public")
auth_required = all_preferences.get("common__api_authentication_required")
banner = all_preferences.get("instance__banner")
unauthenticated_report_types = all_preferences.get(
"moderation__unauthenticated_report_types"
)
if allow_list_enabled and allow_list_public:
allowed_domains = list(
federation_models.Domain.objects.filter(allowed=True)
.order_by("name")
.values_list("name", flat=True)
)
else:
allowed_domains = None
data = {
"version": "2.0",
"software": {"name": "funkwhale", "version": funkwhale_api.__version__},
"protocols": ["activitypub"],
"services": {"inbound": [], "outbound": []},
"openRegistrations": all_preferences.get("users__registration_enabled"),
"usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}},
"metadata": {
"actorId": actors.get_service_actor().fid,
"private": all_preferences.get("instance__nodeinfo_private"),
"shortDescription": all_preferences.get("instance__short_description"),
"longDescription": all_preferences.get("instance__long_description"),
"rules": all_preferences.get("instance__rules"),
"contactEmail": all_preferences.get("instance__contact_email"),
"terms": all_preferences.get("instance__terms"),
"nodeName": all_preferences.get("instance__name"),
"banner": federation_utils.full_url(banner.url) if banner else None,
"defaultUploadQuota": all_preferences.get("users__upload_quota"),
"library": {
"federationEnabled": all_preferences.get("federation__enabled"),
"anonymousCanListen": not all_preferences.get(
"common__api_authentication_required"
),
},
"supportedUploadExtensions": music_utils.SUPPORTED_EXTENSIONS,
"allowList": {"enabled": allow_list_enabled, "domains": allowed_domains},
"reportTypes": [
{"type": t, "label": l, "anonymous": t in unauthenticated_report_types}
for t, l in moderation_models.REPORT_TYPES
],
"funkwhaleSupportMessageEnabled": all_preferences.get(
"instance__funkwhale_support_message_enabled"
),
"instanceSupportMessage": all_preferences.get("instance__support_message"),
"endpoints": {"knownNodes": None, "channels": None, "libraries": None},
},
}
if share_stats:
getter = memo(lambda: stats.get(), max_age=600)
statistics = getter()
data["usage"]["users"]["total"] = statistics["users"]["total"]
data["usage"]["users"]["activeHalfyear"] = statistics["users"][
"active_halfyear"
]
data["usage"]["users"]["activeMonth"] = statistics["users"]["active_month"]
data["metadata"]["library"]["tracks"] = {"total": statistics["tracks"]}
data["metadata"]["library"]["artists"] = {"total": statistics["artists"]}
data["metadata"]["library"]["albums"] = {"total": statistics["albums"]}
data["metadata"]["library"]["music"] = {"hours": statistics["music_duration"]}
data["metadata"]["usage"] = {
"favorites": {"tracks": {"total": statistics["track_favorites"]}},
"listenings": {"total": statistics["listenings"]},
"downloads": {"total": statistics["downloads"]},
}
if not auth_required:
data["metadata"]["endpoints"]["knownNodes"] = federation_utils.full_url(
reverse("api:v1:federation:domains-list")
)
if not auth_required and preferences.get("federation__public_index"):
data["metadata"]["endpoints"]["libraries"] = federation_utils.full_url(
reverse("federation:index:index-libraries")
)
data["metadata"]["endpoints"]["channels"] = federation_utils.full_url(
reverse("federation:index:index-channels")
)
return data
def handle_serve(
upload, user, format=None, max_bitrate=None, proxy_media=True, download=True
):
f = upload
# we update the accessed_date
now = timezone.now()
upload.accessed_date = now
upload.save(update_fields=["accessed_date"])
f = upload
if f.audio_file:
file_path = get_file_path(f.audio_file)
elif f.source and (
f.source.startswith("http://") or f.source.startswith("https://")
):
# we need to populate from cache
with transaction.atomic():
# why the transaction/select_for_update?
# this is because browsers may send multiple requests
# in a short time range, for partial content,
# thus resulting in multiple downloads from the remote
qs = f.__class__.objects.select_for_update()
f = qs.get(pk=f.pk)
if user.is_authenticated:
actor = user.actor
else:
actor = actors.get_service_actor()
f.download_audio_from_remote(actor=actor)
data = f.get_audio_data()
if data:
f.duration = data["duration"]
f.size = data["size"]
f.bitrate = data["bitrate"]
f.save(update_fields=["bitrate", "duration", "size"])
file_path = get_file_path(f.audio_file)
elif f.source and f.source.startswith("file://"):
file_path = get_file_path(f.source.replace("file://", "", 1))
mt = f.mimetype
if should_transcode(f, format, max_bitrate=max_bitrate):
transcoded_version = f.get_transcoded_version(format, max_bitrate=max_bitrate)
transcoded_version.accessed_date = now
transcoded_version.save(update_fields=["accessed_date"])
f = transcoded_version
file_path = get_file_path(f.audio_file)
mt = f.mimetype
if not proxy_media and f.audio_file:
# we simply issue a 302 redirect to the real URL
response = Response(status=302)
response["Location"] = f.audio_file.url
return response
if mt:
response = Response(content_type=mt)
else:
response = Response()
filename = f.filename
mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"}
file_header = mapping[settings.REVERSE_PROXY_TYPE]
response[file_header] = file_path
if download:
response["Content-Disposition"] = get_content_disposition(filename)
if mt:
response["Content-Type"] = mt
return response
