def populate_album_cover(album, source=None, replace=False): if album.attachment_cover and not replace: return if source and source.startswith("file://"): # let's look for a cover in the same directory path = os.path.dirname(source.replace("file://", "", 1)) logger.info("[Album %s] scanning covers from %s", album.pk, path) cover = get_cover_from_fs(path) return common_utils.attach_file(album, "attachment_cover", cover) if album.mbid: logger.info( "[Album %s] Fetching cover from musicbrainz release %s", album.pk, str(album.mbid), ) try: image_data = musicbrainz.api.images.get_front(str(album.mbid)) except ResponseError as exc: logger.warning( "[Album %s] cannot fetch cover from musicbrainz: %s", album.pk, str(exc)) else: return common_utils.attach_file( album, "attachment_cover", { "content": image_data, "mimetype": "image/jpeg" }, fetch=True, )
def get_artist(artist_data, attributed_to, from_activity_id): artist_mbid = artist_data.get("mbid", None) artist_fid = artist_data.get("fid", None) artist_name = truncate(artist_data["name"], models.MAX_LENGTHS["ARTIST_NAME"]) if artist_mbid: query = Q(mbid=artist_mbid) else: query = Q(name__iexact=artist_name) if artist_fid: query |= Q(fid=artist_fid) defaults = { "name": artist_name, "mbid": artist_mbid, "fid": artist_fid, "from_activity_id": from_activity_id, "attributed_to": artist_data.get("attributed_to", attributed_to), } if artist_data.get("fdate"): defaults["creation_date"] = artist_data.get("fdate") artist, created = get_best_candidate_or_create(models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]) if created: tags_models.add_tags(artist, *artist_data.get("tags", [])) common_utils.attach_content(artist, "description", artist_data.get("description")) common_utils.attach_file(artist, "attachment_cover", artist_data.get("cover_data")) return artist
def test_attach_file_attachment(factories, r_mock): album = factories["music.Album"]() data = factories["common.Attachment"]() utils.attach_file(album, "attachment_cover", data) album.refresh_from_db() assert album.attachment_cover == data
def update_track_metadata(audio_metadata, track): serializer = metadata.TrackMetadataSerializer(data=audio_metadata) serializer.is_valid(raise_exception=True) new_data = serializer.validated_data to_update = [ ("track", track, lambda data: data), ("album", track.album, lambda data: data["album"]), ("artist", track.artist, lambda data: data["artists"][0]), ( "album_artist", track.album.artist if track.album else None, lambda data: data["album"]["artists"][0], ), ] for id, obj, data_getter in to_update: if not obj: continue obj_updated_fields = [] try: obj_data = data_getter(new_data) except IndexError: continue for field, config in UPDATE_CONFIG[id].items(): getter = config.get( "getter", lambda data, field: data[config.get("field", field)]) try: new_value = getter(obj_data, field) except KeyError: continue old_value = getattr(obj, field) if new_value == old_value: continue obj_updated_fields.append(field) setattr(obj, field, new_value) if obj_updated_fields: obj.save(update_fields=obj_updated_fields) tags_models.set_tags(track, *new_data.get("tags", [])) if track.album and "album" in new_data and new_data["album"].get( "cover_data"): common_utils.attach_file(track.album, "attachment_cover", new_data["album"].get("cover_data"))
def test_attach_file_content(factories, r_mock): album = factories["music.Album"]() data = {"mimetype": "image/jpeg", "content": b"content"} new_attachment = utils.attach_file(album, "attachment_cover", data) album.refresh_from_db() assert album.attachment_cover == new_attachment assert new_attachment.file.read() == b"content" assert new_attachment.url is None assert new_attachment.mimetype == data["mimetype"]
def test_attach_file_url_fetch(factories, r_mock): album = factories["music.Album"](with_cover=True) data = {"mimetype": "image/jpeg", "url": "https://example.com/test.jpg"} r_mock.get(data["url"], body=io.BytesIO(b"content")) new_attachment = utils.attach_file(album, "attachment_cover", data, fetch=True) album.refresh_from_db() assert album.attachment_cover == new_attachment assert new_attachment.file.read() == b"content" assert new_attachment.url == data["url"] assert new_attachment.mimetype == data["mimetype"]
def test_attach_file_url(factories): album = factories["music.Album"](with_cover=True) existing_attachment = album.attachment_cover assert existing_attachment is not None data = {"mimetype": "image/jpeg", "url": "https://example.com/test.jpg"} new_attachment = utils.attach_file(album, "attachment_cover", data) album.refresh_from_db() with pytest.raises(existing_attachment.DoesNotExist): existing_attachment.refresh_from_db() assert album.attachment_cover == new_attachment assert not new_attachment.file assert new_attachment.url == data["url"] assert new_attachment.mimetype == data["mimetype"]
def save(self, channel, existing_uploads=[], **track_defaults): validated_data = self.validated_data categories = validated_data.get("tags", {}) expected_uuid = uuid.uuid3( uuid.NAMESPACE_URL, "rss://{}-{}".format(channel.pk, validated_data["id"]) ) existing_upload = get_cached_upload(existing_uploads, expected_uuid) if existing_upload: existing_track = existing_upload.track else: existing_track = ( music_models.Track.objects.filter( uuid=expected_uuid, artist__channel=channel ) .select_related("description", "attachment_cover") .first() ) if existing_track: existing_upload = existing_track.uploads.filter( library=channel.library ).first() track_defaults = track_defaults track_defaults.update( { "disc_number": validated_data.get("itunes_season", 1) or 1, "position": validated_data.get("itunes_episode", 1) or 1, "title": validated_data["title"][ : music_models.MAX_LENGTHS["TRACK_TITLE"] ], "artist": channel.artist, } ) if "rights" in validated_data: track_defaults["copyright"] = validated_data["rights"][ : music_models.MAX_LENGTHS["COPYRIGHT"] ] if "published_parsed" in validated_data: track_defaults["creation_date"] = datetime.datetime.fromtimestamp( time.mktime(validated_data["published_parsed"]) ).replace(tzinfo=pytz.utc) upload_defaults = { "source": validated_data["links"]["audio"]["source"], "size": validated_data["links"]["audio"]["size"], "mimetype": validated_data["links"]["audio"]["mimetype"], "duration": validated_data.get("itunes_duration") or None, "import_status": "finished", "library": channel.library, } if existing_track: track_kwargs = {"pk": existing_track.pk} upload_kwargs = {"track": existing_track} else: track_kwargs = {"pk": None} track_defaults["uuid"] = expected_uuid upload_kwargs = {"pk": None} if existing_upload and existing_upload.source != upload_defaults["source"]: # delete existing upload, the url to the audio file has changed existing_upload.delete() # create/update the track track, created = music_models.Track.objects.update_or_create( **track_kwargs, defaults=track_defaults, ) # optimisation for reducing SQL queries, because we cannot use select_related with # update or create, so we restore the cache by hand if existing_track: for field in ["attachment_cover", "description"]: cached_id_value = getattr(existing_track, "{}_id".format(field)) new_id_value = getattr(track, "{}_id".format(field)) if new_id_value and cached_id_value == new_id_value: setattr(track, field, getattr(existing_track, field)) cover = validated_data.get("image") if cover: common_utils.attach_file(track, "attachment_cover", cover) tags = categories.get("tags", []) if tags: tags_models.set_tags(track, *tags) summary = validated_data.get("summary_detail") if summary: common_utils.attach_content(track, "description", summary) if created: upload_defaults["track"] = track # create/update the upload upload, created = music_models.Upload.objects.update_or_create( **upload_kwargs, defaults=upload_defaults ) return upload
def save(self, rss_url): validated_data = self.validated_data # because there may be redirections from the original feed URL real_rss_url = validated_data.get("atom_link", rss_url) or rss_url service_actor = actors.get_service_actor() author = validated_data.get("author_detail", {}) categories = validated_data.get("tags", {}) metadata = { "explicit": validated_data.get("itunes_explicit", False), "copyright": validated_data.get("rights"), "owner_name": author.get("name"), "owner_email": author.get("email"), "itunes_category": categories.get("parent"), "itunes_subcategory": categories.get("child"), "language": validated_data.get("language"), } public_url = validated_data["link"] existing = ( models.Channel.objects.external_rss() .filter( Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url) ) .first() ) channel_defaults = { "rss_url": real_rss_url, "metadata": metadata, } if existing: artist_kwargs = {"channel": existing} actor_kwargs = {"channel": existing} actor_defaults = {"url": public_url} else: artist_kwargs = {"pk": None} actor_kwargs = {"pk": None} preferred_username = "******".format(uuid.uuid4()) actor_defaults = { "preferred_username": preferred_username, "type": "Application", "domain": service_actor.domain, "url": public_url, "fid": federation_utils.full_url( reverse( "federation:actors-detail", kwargs={"preferred_username": preferred_username}, ) ), } channel_defaults["attributed_to"] = service_actor actor_defaults["last_fetch_date"] = timezone.now() # create/update the artist profile artist, created = music_models.Artist.objects.update_or_create( **artist_kwargs, defaults={ "attributed_to": service_actor, "name": validated_data["title"][ : music_models.MAX_LENGTHS["ARTIST_NAME"] ], "content_category": "podcast", }, ) cover = validated_data.get("image") if cover: common_utils.attach_file(artist, "attachment_cover", cover) tags = categories.get("tags", []) if tags: tags_models.set_tags(artist, *tags) summary = validated_data.get("summary_detail") if summary: common_utils.attach_content(artist, "description", summary) if created: channel_defaults["artist"] = artist # create/update the actor actor, created = federation_models.Actor.objects.update_or_create( **actor_kwargs, defaults=actor_defaults ) if created: channel_defaults["actor"] = actor # create the library if not existing: channel_defaults["library"] = music_models.Library.objects.create( actor=service_actor, privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY, name=actor_defaults["preferred_username"], ) # create/update the channel channel, created = models.Channel.objects.update_or_create( pk=existing.pk if existing else None, defaults=channel_defaults, ) return channel
def _get_track(data, attributed_to=None, **forced_values): track_uuid = getter(data, "funkwhale", "track", "uuid") if track_uuid: # easy case, we have a reference to a uuid of a track that # already exists in our database try: track = models.Track.objects.get(uuid=track_uuid) except models.Track.DoesNotExist: raise UploadImportError(code="track_uuid_not_found") return track from_activity_id = data.get("from_activity_id", None) track_mbid = (forced_values["mbid"] if "mbid" in forced_values else data.get("mbid", None)) try: album_mbid = getter(data, "album", "mbid") except TypeError: # album is forced album_mbid = None track_fid = getter(data, "fid") query = None if album_mbid and track_mbid: query = Q(mbid=track_mbid, album__mbid=album_mbid) if track_fid: query = query | Q(fid=track_fid) if query else Q(fid=track_fid) if query: # second easy case: we have a (track_mbid, album_mbid) pair or # a federation uuid we can check on try: return sort_candidates(models.Track.objects.filter(query), ["mbid", "fid"])[0] except IndexError: pass # get / create artist and album artist artists = getter(data, "artists", default=[]) if "artist" in forced_values: artist = forced_values["artist"] else: artist_data = artists[0] artist = get_artist(artist_data, attributed_to=attributed_to, from_activity_id=from_activity_id) artist_name = artist.name if "album" in forced_values: album = forced_values["album"] else: if "artist" in forced_values: album_artist = forced_values["artist"] else: album_artists = getter(data, "album", "artists", default=artists) or artists album_artist_data = album_artists[0] album_artist_name = truncate(album_artist_data.get("name"), models.MAX_LENGTHS["ARTIST_NAME"]) if album_artist_name == artist_name: album_artist = artist else: query = Q(name__iexact=album_artist_name) album_artist_mbid = album_artist_data.get("mbid", None) album_artist_fid = album_artist_data.get("fid", None) if album_artist_mbid: query |= Q(mbid=album_artist_mbid) if album_artist_fid: query |= Q(fid=album_artist_fid) defaults = { "name": album_artist_name, "mbid": album_artist_mbid, "fid": album_artist_fid, "from_activity_id": from_activity_id, "attributed_to": album_artist_data.get("attributed_to", attributed_to), } if album_artist_data.get("fdate"): defaults["creation_date"] = album_artist_data.get("fdate") album_artist, created = get_best_candidate_or_create( models.Artist, query, defaults=defaults, sort_fields=["mbid", "fid"]) if created: tags_models.add_tags(album_artist, *album_artist_data.get("tags", [])) common_utils.attach_content( album_artist, "description", album_artist_data.get("description"), ) common_utils.attach_file( album_artist, "attachment_cover", album_artist_data.get("cover_data"), ) # get / create album if "album" in data: album_data = data["album"] album_title = truncate(album_data["title"], models.MAX_LENGTHS["ALBUM_TITLE"]) album_fid = album_data.get("fid", None) if album_mbid: query = Q(mbid=album_mbid) else: query = Q(title__iexact=album_title, artist=album_artist) if album_fid: query |= Q(fid=album_fid) defaults = { "title": album_title, "artist": album_artist, "mbid": album_mbid, "release_date": album_data.get("release_date"), "fid": album_fid, "from_activity_id": from_activity_id, "attributed_to": album_data.get("attributed_to", attributed_to), } if album_data.get("fdate"): defaults["creation_date"] = album_data.get("fdate") album, created = get_best_candidate_or_create( models.Album, query, defaults=defaults, sort_fields=["mbid", "fid"]) if created: tags_models.add_tags(album, *album_data.get("tags", [])) common_utils.attach_content(album, "description", album_data.get("description")) common_utils.attach_file(album, "attachment_cover", album_data.get("cover_data")) else: album = None # get / create track track_title = (forced_values["title"] if "title" in forced_values else truncate(data["title"], models.MAX_LENGTHS["TRACK_TITLE"])) position = (forced_values["position"] if "position" in forced_values else data.get("position", 1)) disc_number = (forced_values["disc_number"] if "disc_number" in forced_values else data.get("disc_number")) license = (forced_values["license"] if "license" in forced_values else licenses.match(data.get("license"), data.get("copyright"))) copyright = (forced_values["copyright"] if "copyright" in forced_values else truncate( data.get("copyright"), models.MAX_LENGTHS["COPYRIGHT"])) description = ({ "text": forced_values["description"], "content_type": "text/markdown" } if "description" in forced_values else data.get("description")) cover_data = (forced_values["cover"] if "cover" in forced_values else data.get("cover_data")) query = Q( title__iexact=track_title, artist=artist, album=album, position=position, disc_number=disc_number, ) if track_mbid: if album_mbid: query |= Q(mbid=track_mbid, album__mbid=album_mbid) else: query |= Q(mbid=track_mbid) if track_fid: query |= Q(fid=track_fid) if album and len(artists) > 1: # we use the second artist to preserve featuring information artist = artist = get_artist(artists[1], attributed_to=attributed_to, from_activity_id=from_activity_id) defaults = { "title": track_title, "album": album, "mbid": track_mbid, "artist": artist, "position": position, "disc_number": disc_number, "fid": track_fid, "from_activity_id": from_activity_id, "attributed_to": data.get("attributed_to", attributed_to), "license": license, "copyright": copyright, } if data.get("fdate"): defaults["creation_date"] = data.get("fdate") track, created = get_best_candidate_or_create(models.Track, query, defaults=defaults, sort_fields=["mbid", "fid"]) if created: tags = (forced_values["tags"] if "tags" in forced_values else data.get( "tags", [])) tags_models.add_tags(track, *tags) common_utils.attach_content(track, "description", description) common_utils.attach_file(track, "attachment_cover", cover_data) return track