def refetch_obj(obj, queryset): """ Given an Artist/Album/Track instance, if the instance is from a remote pod, will attempt to update local data with the latest ActivityPub representation. """ if obj.is_local: return obj now = timezone.now() limit = now - datetime.timedelta(minutes=settings.FEDERATION_OBJECT_FETCH_DELAY) last_fetch = obj.fetches.order_by("-creation_date").first() if last_fetch is not None and last_fetch.creation_date > limit: # we fetched recently, no need to do it again return obj logger.info("Refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid) actor = actors.get_service_actor() fetch = federation_models.Fetch.objects.create(actor=actor, url=obj.fid, object=obj) try: federation_tasks.fetch(fetch_id=fetch.pk) except Exception: logger.exception( "Error while refetching %s:%s at %s…", obj._meta.label, obj.pk, obj.fid ) else: fetch.refresh_from_db() if fetch.status == "finished": obj = queryset.get(pk=obj.pk) return obj
def external_rss(self, include=True): from funkwhale_api.federation import actors query = models.Q( attributed_to=actors.get_service_actor(), actor__preferred_username__startswith="rssfeed-", ) if include: return self.filter(query) return self.exclude(query)
def test_get_service_actor(db, settings): settings.FEDERATION_HOSTNAME = "test.hello" settings.FEDERATION_SERVICE_ACTOR_USERNAME = "******" actor = actors.get_service_actor() assert actor.preferred_username == "bob" assert actor.domain.name == "test.hello" assert actor.private_key is not None assert actor.type == "Service" assert actor.public_key is not None
def test_service_actor_detail(factories, api_client): actor = actors.get_service_actor() url = reverse( "federation:actors-detail", kwargs={"preferred_username": actor.preferred_username}, ) serializer = serializers.ActorSerializer(actor) response = api_client.get(url) assert response.status_code == 200 assert response.data == serializer.data
def test_authenticate_skips_anonymous_fetch_when_allow_list_enabled( preferences, api_client): preferences["moderation__allow_list_enabled"] = True actor = actors.get_service_actor() url = reverse( "federation:actors-detail", kwargs={"preferred_username": actor.preferred_username}, ) response = api_client.get(url) assert response.status_code == 403
def filter_external(self, queryset, name, value): query = Q( attributed_to=actors.get_service_actor(), actor__preferred_username__startswith="rssfeed-", ) if value is True: queryset = queryset.filter(query) if value is False: queryset = queryset.exclude(query) return queryset
def test_outbox_update_track(factories): track = factories["music.Track"]() activity = list(routes.outbox_update_track({"track": track}))[0] expected = serializers.ActivitySerializer({ "type": "Update", "object": serializers.TrackSerializer(track).data }).data expected["to"] = [contexts.AS.Public, {"type": "instances_with_followers"}] assert dict(activity["payload"]) == dict(expected) assert activity["actor"] == actors.get_service_actor()
def test_outbox_flag(factory_name, factory_kwargs, factories, mocker): target = factories[factory_name](**factory_kwargs) report = factories["moderation.Report"]( target=target, local=True, target_owner=factories["federation.Actor"]()) activity = list(routes.outbox_flag({"report": report}))[0] serializer = serializers.FlagSerializer(report) expected = serializer.data expected["to"] = [{"type": "actor_inbox", "actor": report.target_owner}] assert activity["payload"] == expected assert activity["actor"] == actors.get_service_actor()
def rss(self, request, *args, **kwargs): object = self.get_object() if not object.attributed_to.is_local: return response.Response({"detail": "Not found"}, status=404) if object.attributed_to == actors.get_service_actor(): # external feed, we redirect to the canonical one return http.HttpResponseRedirect(object.rss_url) uploads = (object.library.uploads.playable_by(None).prefetch_related( Prefetch( "track", queryset=music_models.Track.objects.select_related( "attachment_cover", "description").prefetch_related( music_views.TAG_PREFETCH, ), ), ).select_related( "track__attachment_cover", "track__description").order_by("-creation_date"))[:50] data = serializers.rss_serialize_channel_full(channel=object, uploads=uploads) return response.Response(data, status=200)
def test_nodeinfo_dump(preferences, mocker, avatar): preferences["instance__banner"] = avatar preferences["instance__nodeinfo_stats_enabled"] = True preferences["common__api_authentication_required"] = False preferences["moderation__unauthenticated_report_types"] = [ "takedown_request", "other", "other_category_that_doesnt_exist", ] stats = { "users": { "total": 1, "active_halfyear": 12, "active_month": 13 }, "tracks": 2, "albums": 3, "artists": 4, "track_favorites": 5, "music_duration": 6, "listenings": 7, "downloads": 42, } mocker.patch("funkwhale_api.instance.stats.get", return_value=stats) expected = { "version": "2.0", "software": { "name": "funkwhale", "version": funkwhale_api.__version__ }, "protocols": ["activitypub"], "services": { "inbound": [], "outbound": [] }, "openRegistrations": preferences["users__registration_enabled"], "usage": { "users": { "total": 1, "activeHalfyear": 12, "activeMonth": 13 } }, "metadata": { "actorId": actors.get_service_actor().fid, "private": preferences["instance__nodeinfo_private"], "shortDescription": preferences["instance__short_description"], "longDescription": preferences["instance__long_description"], "nodeName": preferences["instance__name"], "rules": preferences["instance__rules"], "contactEmail": preferences["instance__contact_email"], "defaultUploadQuota": preferences["users__upload_quota"], "terms": preferences["instance__terms"], "banner": federation_utils.full_url(preferences["instance__banner"].url), "library": { "federationEnabled": preferences["federation__enabled"], "anonymousCanListen": not preferences["common__api_authentication_required"], "tracks": { "total": stats["tracks"] }, "artists": { "total": stats["artists"] }, "albums": { "total": stats["albums"] }, "music": { "hours": stats["music_duration"] }, }, "usage": { "favorites": { "tracks": { "total": stats["track_favorites"] } }, "listenings": { "total": stats["listenings"] }, "downloads": { "total": stats["downloads"] }, }, "supportedUploadExtensions": music_utils.SUPPORTED_EXTENSIONS, "allowList": { "enabled": False, "domains": None }, "reportTypes": [ { "type": "takedown_request", "label": "Takedown request", "anonymous": True, }, { "type": "invalid_metadata", "label": "Invalid metadata", "anonymous": False, }, { "type": "illegal_content", "label": "Illegal content", "anonymous": False, }, { "type": "offensive_content", "label": "Offensive content", "anonymous": False, }, { "type": "other", "label": "Other", "anonymous": True }, ], "funkwhaleSupportMessageEnabled": preferences["instance__funkwhale_support_message_enabled"], "instanceSupportMessage": preferences["instance__support_message"], "knownNodesListUrl": federation_utils.full_url( reverse("api:v1:federation:domains-list")), }, } assert nodeinfo.get() == expected
def save(self, rss_url): validated_data = self.validated_data # because there may be redirections from the original feed URL real_rss_url = validated_data.get("atom_link", rss_url) or rss_url service_actor = actors.get_service_actor() author = validated_data.get("author_detail", {}) categories = validated_data.get("tags", {}) metadata = { "explicit": validated_data.get("itunes_explicit", False), "copyright": validated_data.get("rights"), "owner_name": author.get("name"), "owner_email": author.get("email"), "itunes_category": categories.get("parent"), "itunes_subcategory": categories.get("child"), "language": validated_data.get("language"), } public_url = validated_data["link"] existing = ( models.Channel.objects.external_rss() .filter( Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url) ) .first() ) channel_defaults = { "rss_url": real_rss_url, "metadata": metadata, } if existing: artist_kwargs = {"channel": existing} actor_kwargs = {"channel": existing} actor_defaults = {"url": public_url} else: artist_kwargs = {"pk": None} actor_kwargs = {"pk": None} preferred_username = "******".format(uuid.uuid4()) actor_defaults = { "preferred_username": preferred_username, "type": "Application", "domain": service_actor.domain, "url": public_url, "fid": federation_utils.full_url( reverse( "federation:actors-detail", kwargs={"preferred_username": preferred_username}, ) ), } channel_defaults["attributed_to"] = service_actor actor_defaults["last_fetch_date"] = timezone.now() # create/update the artist profile artist, created = music_models.Artist.objects.update_or_create( **artist_kwargs, defaults={ "attributed_to": service_actor, "name": validated_data["title"][ : music_models.MAX_LENGTHS["ARTIST_NAME"] ], "content_category": "podcast", }, ) cover = validated_data.get("image") if cover: common_utils.attach_file(artist, "attachment_cover", cover) tags = categories.get("tags", []) if tags: tags_models.set_tags(artist, *tags) summary = validated_data.get("summary_detail") if summary: common_utils.attach_content(artist, "description", summary) if created: channel_defaults["artist"] = artist # create/update the actor actor, created = federation_models.Actor.objects.update_or_create( **actor_kwargs, defaults=actor_defaults ) if created: channel_defaults["actor"] = actor # create the library if not existing: channel_defaults["library"] = music_models.Library.objects.create( actor=service_actor, privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY, name=actor_defaults["preferred_username"], ) # create/update the channel channel, created = models.Channel.objects.update_or_create( pk=existing.pk if existing else None, defaults=channel_defaults, ) return channel
def get_actor(self, obj): if obj.attributed_to == actors.get_service_actor(): return None return federation_serializers.APIActorSerializer(obj.actor).data
def service_actor(db): return actors.get_service_actor()
def test_rss_feed_serializer_update(factories, now): rss_url = "http://example.rss/" channel = factories["audio.Channel"](rss_url=rss_url, external=True) xml_payload = """<?xml version="1.0" encoding="UTF-8"?> <rss version="2.0"> <channel> <title>Hello</title> <description>Description</description> <link>http://public.url</link> <atom:link rel="self" type="application/rss+xml" href="http://real.rss.url"/> <lastBuildDate>Wed, 11 Mar 2020 16:01:08 GMT</lastBuildDate> <pubDate>Wed, 11 Mar 2020 16:00:00 GMT</pubDate> <ttl>30</ttl> <language>en</language> <copyright>2019 Tests</copyright> <itunes:keywords>pop rock</itunes:keywords> <image> <url> https://image.url </url> <title>Image caption</title> </image> <itunes:image href="https://image.url"/> <itunes:subtitle>Subtitle</itunes:subtitle> <itunes:type>episodic</itunes:type> <itunes:author>Author</itunes:author> <itunes:summary><![CDATA[Some content]]></itunes:summary> <itunes:owner> <itunes:name>Name</itunes:name> <itunes:email>email@domain</itunes:email> </itunes:owner> <itunes:explicit>yes</itunes:explicit> <itunes:keywords/> <itunes:category text="Business"> <itunes:category text="Entrepreneurship"> </itunes:category> </channel> </rss> """ parsed_feed = feedparser.parse(xml_payload) serializer = serializers.RssFeedSerializer(data=parsed_feed.feed) assert serializer.is_valid(raise_exception=True) is True serializer.save(rss_url) channel.refresh_from_db() assert channel.rss_url == "http://real.rss.url" assert channel.attributed_to == actors.get_service_actor() assert channel.library.actor == actors.get_service_actor() assert channel.library.fid is not None assert channel.artist.name == "Hello" assert channel.artist.attributed_to == actors.get_service_actor() assert channel.artist.description.content_type == "text/plain" assert channel.artist.description.text == "Some content" assert channel.artist.attachment_cover.url == "https://image.url" assert channel.artist.get_tags() == ["pop", "rock"] assert channel.actor.url == "http://public.url" assert channel.actor.last_fetch_date == now assert channel.metadata == { "explicit": True, "copyright": "2019 Tests", "owner_name": "Name", "owner_email": "email@domain", "itunes_category": "Business", "itunes_subcategory": "Entrepreneurship", "language": "en", }
def test_nodeinfo_dump_stats_disabled(preferences, mocker): preferences["instance__nodeinfo_stats_enabled"] = False preferences["moderation__unauthenticated_report_types"] = [ "takedown_request", "other", ] expected = { "version": "2.0", "software": { "name": "funkwhale", "version": funkwhale_api.__version__ }, "protocols": ["activitypub"], "services": { "inbound": [], "outbound": [] }, "openRegistrations": preferences["users__registration_enabled"], "usage": { "users": { "total": 0, "activeHalfyear": 0, "activeMonth": 0 } }, "metadata": { "actorId": actors.get_service_actor().fid, "private": preferences["instance__nodeinfo_private"], "shortDescription": preferences["instance__short_description"], "longDescription": preferences["instance__long_description"], "nodeName": preferences["instance__name"], "rules": preferences["instance__rules"], "contactEmail": preferences["instance__contact_email"], "defaultUploadQuota": preferences["users__upload_quota"], "terms": preferences["instance__terms"], "banner": None, "library": { "federationEnabled": preferences["federation__enabled"], "anonymousCanListen": not preferences["common__api_authentication_required"], }, "supportedUploadExtensions": music_utils.SUPPORTED_EXTENSIONS, "allowList": { "enabled": False, "domains": None }, "reportTypes": [ { "type": "takedown_request", "label": "Takedown request", "anonymous": True, }, { "type": "invalid_metadata", "label": "Invalid metadata", "anonymous": False, }, { "type": "illegal_content", "label": "Illegal content", "anonymous": False, }, { "type": "offensive_content", "label": "Offensive content", "anonymous": False, }, { "type": "other", "label": "Other", "anonymous": True }, ], "funkwhaleSupportMessageEnabled": preferences["instance__funkwhale_support_message_enabled"], "instanceSupportMessage": preferences["instance__support_message"], "knownNodesListUrl": None, }, } assert nodeinfo.get() == expected
def get(): all_preferences = preferences.all() share_stats = all_preferences.get("instance__nodeinfo_stats_enabled") allow_list_enabled = all_preferences.get("moderation__allow_list_enabled") allow_list_public = all_preferences.get("moderation__allow_list_public") auth_required = all_preferences.get("common__api_authentication_required") banner = all_preferences.get("instance__banner") unauthenticated_report_types = all_preferences.get( "moderation__unauthenticated_report_types" ) if allow_list_enabled and allow_list_public: allowed_domains = list( federation_models.Domain.objects.filter(allowed=True) .order_by("name") .values_list("name", flat=True) ) else: allowed_domains = None data = { "version": "2.0", "software": {"name": "funkwhale", "version": funkwhale_api.__version__}, "protocols": ["activitypub"], "services": {"inbound": [], "outbound": []}, "openRegistrations": all_preferences.get("users__registration_enabled"), "usage": {"users": {"total": 0, "activeHalfyear": 0, "activeMonth": 0}}, "metadata": { "actorId": actors.get_service_actor().fid, "private": all_preferences.get("instance__nodeinfo_private"), "shortDescription": all_preferences.get("instance__short_description"), "longDescription": all_preferences.get("instance__long_description"), "rules": all_preferences.get("instance__rules"), "contactEmail": all_preferences.get("instance__contact_email"), "terms": all_preferences.get("instance__terms"), "nodeName": all_preferences.get("instance__name"), "banner": federation_utils.full_url(banner.url) if banner else None, "defaultUploadQuota": all_preferences.get("users__upload_quota"), "library": { "federationEnabled": all_preferences.get("federation__enabled"), "anonymousCanListen": not all_preferences.get( "common__api_authentication_required" ), }, "supportedUploadExtensions": music_utils.SUPPORTED_EXTENSIONS, "allowList": {"enabled": allow_list_enabled, "domains": allowed_domains}, "reportTypes": [ {"type": t, "label": l, "anonymous": t in unauthenticated_report_types} for t, l in moderation_models.REPORT_TYPES ], "funkwhaleSupportMessageEnabled": all_preferences.get( "instance__funkwhale_support_message_enabled" ), "instanceSupportMessage": all_preferences.get("instance__support_message"), "endpoints": {"knownNodes": None, "channels": None, "libraries": None}, }, } if share_stats: getter = memo(lambda: stats.get(), max_age=600) statistics = getter() data["usage"]["users"]["total"] = statistics["users"]["total"] data["usage"]["users"]["activeHalfyear"] = statistics["users"][ "active_halfyear" ] data["usage"]["users"]["activeMonth"] = statistics["users"]["active_month"] data["metadata"]["library"]["tracks"] = {"total": statistics["tracks"]} data["metadata"]["library"]["artists"] = {"total": statistics["artists"]} data["metadata"]["library"]["albums"] = {"total": statistics["albums"]} data["metadata"]["library"]["music"] = {"hours": statistics["music_duration"]} data["metadata"]["usage"] = { "favorites": {"tracks": {"total": statistics["track_favorites"]}}, "listenings": {"total": statistics["listenings"]}, "downloads": {"total": statistics["downloads"]}, } if not auth_required: data["metadata"]["endpoints"]["knownNodes"] = federation_utils.full_url( reverse("api:v1:federation:domains-list") ) if not auth_required and preferences.get("federation__public_index"): data["metadata"]["endpoints"]["libraries"] = federation_utils.full_url( reverse("federation:index:index-libraries") ) data["metadata"]["endpoints"]["channels"] = federation_utils.full_url( reverse("federation:index:index-channels") ) return data
def handle_serve( upload, user, format=None, max_bitrate=None, proxy_media=True, download=True ): f = upload # we update the accessed_date now = timezone.now() upload.accessed_date = now upload.save(update_fields=["accessed_date"]) f = upload if f.audio_file: file_path = get_file_path(f.audio_file) elif f.source and ( f.source.startswith("http://") or f.source.startswith("https://") ): # we need to populate from cache with transaction.atomic(): # why the transaction/select_for_update? # this is because browsers may send multiple requests # in a short time range, for partial content, # thus resulting in multiple downloads from the remote qs = f.__class__.objects.select_for_update() f = qs.get(pk=f.pk) if user.is_authenticated: actor = user.actor else: actor = actors.get_service_actor() f.download_audio_from_remote(actor=actor) data = f.get_audio_data() if data: f.duration = data["duration"] f.size = data["size"] f.bitrate = data["bitrate"] f.save(update_fields=["bitrate", "duration", "size"]) file_path = get_file_path(f.audio_file) elif f.source and f.source.startswith("file://"): file_path = get_file_path(f.source.replace("file://", "", 1)) mt = f.mimetype if should_transcode(f, format, max_bitrate=max_bitrate): transcoded_version = f.get_transcoded_version(format, max_bitrate=max_bitrate) transcoded_version.accessed_date = now transcoded_version.save(update_fields=["accessed_date"]) f = transcoded_version file_path = get_file_path(f.audio_file) mt = f.mimetype if not proxy_media and f.audio_file: # we simply issue a 302 redirect to the real URL response = Response(status=302) response["Location"] = f.audio_file.url return response if mt: response = Response(content_type=mt) else: response = Response() filename = f.filename mapping = {"nginx": "X-Accel-Redirect", "apache2": "X-Sendfile"} file_header = mapping[settings.REVERSE_PROXY_TYPE] response[file_header] = file_path if download: response["Content-Disposition"] = get_content_disposition(filename) if mt: response["Content-Type"] = mt return response