def populate_album_cover(album, source=None, replace=False):
if album.attachment_cover and not replace:
return
if source and source.startswith("file://"):
# let's look for a cover in the same directory
path = os.path.dirname(source.replace("file://", "", 1))
logger.info("[Album %s] scanning covers from %s", album.pk, path)
cover = get_cover_from_fs(path)
return common_utils.attach_file(album, "attachment_cover", cover)
if album.mbid:
logger.info(
"[Album %s] Fetching cover from musicbrainz release %s",
album.pk,
str(album.mbid),
)
try:
image_data = musicbrainz.api.images.get_front(str(album.mbid))
except ResponseError as exc:
logger.warning(
"[Album %s] cannot fetch cover from musicbrainz: %s", album.pk,
str(exc))
else:
return common_utils.attach_file(
album,
"attachment_cover",
{
"content": image_data,
"mimetype": "image/jpeg"
},
fetch=True,
)
def get_artist(artist_data, attributed_to, from_activity_id):
artist_mbid = artist_data.get("mbid", None)
artist_fid = artist_data.get("fid", None)
artist_name = truncate(artist_data["name"],
models.MAX_LENGTHS["ARTIST_NAME"])
if artist_mbid:
query = Q(mbid=artist_mbid)
else:
query = Q(name__iexact=artist_name)
if artist_fid:
query |= Q(fid=artist_fid)
defaults = {
"name": artist_name,
"mbid": artist_mbid,
"fid": artist_fid,
"from_activity_id": from_activity_id,
"attributed_to": artist_data.get("attributed_to", attributed_to),
}
if artist_data.get("fdate"):
defaults["creation_date"] = artist_data.get("fdate")
artist, created = get_best_candidate_or_create(models.Artist,
query,
defaults=defaults,
sort_fields=["mbid", "fid"])
if created:
tags_models.add_tags(artist, *artist_data.get("tags", []))
common_utils.attach_content(artist, "description",
artist_data.get("description"))
common_utils.attach_file(artist, "attachment_cover",
artist_data.get("cover_data"))
return artist
def test_attach_file_attachment(factories, r_mock):
album = factories["music.Album"]()
data = factories["common.Attachment"]()
utils.attach_file(album, "attachment_cover", data)
album.refresh_from_db()
assert album.attachment_cover == data
def update_track_metadata(audio_metadata, track):
serializer = metadata.TrackMetadataSerializer(data=audio_metadata)
serializer.is_valid(raise_exception=True)
new_data = serializer.validated_data
to_update = [
("track", track, lambda data: data),
("album", track.album, lambda data: data["album"]),
("artist", track.artist, lambda data: data["artists"][0]),
(
"album_artist",
track.album.artist if track.album else None,
lambda data: data["album"]["artists"][0],
),
]
for id, obj, data_getter in to_update:
if not obj:
continue
obj_updated_fields = []
try:
obj_data = data_getter(new_data)
except IndexError:
continue
for field, config in UPDATE_CONFIG[id].items():
getter = config.get(
"getter", lambda data, field: data[config.get("field", field)])
try:
new_value = getter(obj_data, field)
except KeyError:
continue
old_value = getattr(obj, field)
if new_value == old_value:
continue
obj_updated_fields.append(field)
setattr(obj, field, new_value)
if obj_updated_fields:
obj.save(update_fields=obj_updated_fields)
tags_models.set_tags(track, *new_data.get("tags", []))
if track.album and "album" in new_data and new_data["album"].get(
"cover_data"):
common_utils.attach_file(track.album, "attachment_cover",
new_data["album"].get("cover_data"))
def test_attach_file_content(factories, r_mock):
album = factories["music.Album"]()
data = {"mimetype": "image/jpeg", "content": b"content"}
new_attachment = utils.attach_file(album, "attachment_cover", data)
album.refresh_from_db()
assert album.attachment_cover == new_attachment
assert new_attachment.file.read() == b"content"
assert new_attachment.url is None
assert new_attachment.mimetype == data["mimetype"]
def test_attach_file_url_fetch(factories, r_mock):
album = factories["music.Album"](with_cover=True)
data = {"mimetype": "image/jpeg", "url": "https://example.com/test.jpg"}
r_mock.get(data["url"], body=io.BytesIO(b"content"))
new_attachment = utils.attach_file(album, "attachment_cover", data, fetch=True)
album.refresh_from_db()
assert album.attachment_cover == new_attachment
assert new_attachment.file.read() == b"content"
assert new_attachment.url == data["url"]
assert new_attachment.mimetype == data["mimetype"]
def test_attach_file_url(factories):
album = factories["music.Album"](with_cover=True)
existing_attachment = album.attachment_cover
assert existing_attachment is not None
data = {"mimetype": "image/jpeg", "url": "https://example.com/test.jpg"}
new_attachment = utils.attach_file(album, "attachment_cover", data)
album.refresh_from_db()
with pytest.raises(existing_attachment.DoesNotExist):
existing_attachment.refresh_from_db()
assert album.attachment_cover == new_attachment
assert not new_attachment.file
assert new_attachment.url == data["url"]
assert new_attachment.mimetype == data["mimetype"]
def save(self, channel, existing_uploads=[], **track_defaults):
validated_data = self.validated_data
categories = validated_data.get("tags", {})
expected_uuid = uuid.uuid3(
uuid.NAMESPACE_URL, "rss://{}-{}".format(channel.pk, validated_data["id"])
)
existing_upload = get_cached_upload(existing_uploads, expected_uuid)
if existing_upload:
existing_track = existing_upload.track
else:
existing_track = (
music_models.Track.objects.filter(
uuid=expected_uuid, artist__channel=channel
)
.select_related("description", "attachment_cover")
.first()
)
if existing_track:
existing_upload = existing_track.uploads.filter(
library=channel.library
).first()
track_defaults = track_defaults
track_defaults.update(
{
"disc_number": validated_data.get("itunes_season", 1) or 1,
"position": validated_data.get("itunes_episode", 1) or 1,
"title": validated_data["title"][
: music_models.MAX_LENGTHS["TRACK_TITLE"]
],
"artist": channel.artist,
}
)
if "rights" in validated_data:
track_defaults["copyright"] = validated_data["rights"][
: music_models.MAX_LENGTHS["COPYRIGHT"]
]
if "published_parsed" in validated_data:
track_defaults["creation_date"] = datetime.datetime.fromtimestamp(
time.mktime(validated_data["published_parsed"])
).replace(tzinfo=pytz.utc)
upload_defaults = {
"source": validated_data["links"]["audio"]["source"],
"size": validated_data["links"]["audio"]["size"],
"mimetype": validated_data["links"]["audio"]["mimetype"],
"duration": validated_data.get("itunes_duration") or None,
"import_status": "finished",
"library": channel.library,
}
if existing_track:
track_kwargs = {"pk": existing_track.pk}
upload_kwargs = {"track": existing_track}
else:
track_kwargs = {"pk": None}
track_defaults["uuid"] = expected_uuid
upload_kwargs = {"pk": None}
if existing_upload and existing_upload.source != upload_defaults["source"]:
# delete existing upload, the url to the audio file has changed
existing_upload.delete()
# create/update the track
track, created = music_models.Track.objects.update_or_create(
**track_kwargs, defaults=track_defaults,
)
# optimisation for reducing SQL queries, because we cannot use select_related with
# update or create, so we restore the cache by hand
if existing_track:
for field in ["attachment_cover", "description"]:
cached_id_value = getattr(existing_track, "{}_id".format(field))
new_id_value = getattr(track, "{}_id".format(field))
if new_id_value and cached_id_value == new_id_value:
setattr(track, field, getattr(existing_track, field))
cover = validated_data.get("image")
if cover:
common_utils.attach_file(track, "attachment_cover", cover)
tags = categories.get("tags", [])
if tags:
tags_models.set_tags(track, *tags)
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(track, "description", summary)
if created:
upload_defaults["track"] = track
# create/update the upload
upload, created = music_models.Upload.objects.update_or_create(
**upload_kwargs, defaults=upload_defaults
)
return upload
def save(self, rss_url):
validated_data = self.validated_data
# because there may be redirections from the original feed URL
real_rss_url = validated_data.get("atom_link", rss_url) or rss_url
service_actor = actors.get_service_actor()
author = validated_data.get("author_detail", {})
categories = validated_data.get("tags", {})
metadata = {
"explicit": validated_data.get("itunes_explicit", False),
"copyright": validated_data.get("rights"),
"owner_name": author.get("name"),
"owner_email": author.get("email"),
"itunes_category": categories.get("parent"),
"itunes_subcategory": categories.get("child"),
"language": validated_data.get("language"),
}
public_url = validated_data["link"]
existing = (
models.Channel.objects.external_rss()
.filter(
Q(rss_url=real_rss_url) | Q(rss_url=rss_url) | Q(actor__url=public_url)
)
.first()
)
channel_defaults = {
"rss_url": real_rss_url,
"metadata": metadata,
}
if existing:
artist_kwargs = {"channel": existing}
actor_kwargs = {"channel": existing}
actor_defaults = {"url": public_url}
else:
artist_kwargs = {"pk": None}
actor_kwargs = {"pk": None}
preferred_username = "******".format(uuid.uuid4())
actor_defaults = {
"preferred_username": preferred_username,
"type": "Application",
"domain": service_actor.domain,
"url": public_url,
"fid": federation_utils.full_url(
reverse(
"federation:actors-detail",
kwargs={"preferred_username": preferred_username},
)
),
}
channel_defaults["attributed_to"] = service_actor
actor_defaults["last_fetch_date"] = timezone.now()
# create/update the artist profile
artist, created = music_models.Artist.objects.update_or_create(
**artist_kwargs,
defaults={
"attributed_to": service_actor,
"name": validated_data["title"][
: music_models.MAX_LENGTHS["ARTIST_NAME"]
],
"content_category": "podcast",
},
)
cover = validated_data.get("image")
if cover:
common_utils.attach_file(artist, "attachment_cover", cover)
tags = categories.get("tags", [])
if tags:
tags_models.set_tags(artist, *tags)
summary = validated_data.get("summary_detail")
if summary:
common_utils.attach_content(artist, "description", summary)
if created:
channel_defaults["artist"] = artist
# create/update the actor
actor, created = federation_models.Actor.objects.update_or_create(
**actor_kwargs, defaults=actor_defaults
)
if created:
channel_defaults["actor"] = actor
# create the library
if not existing:
channel_defaults["library"] = music_models.Library.objects.create(
actor=service_actor,
privacy_level=settings.PODCASTS_THIRD_PARTY_VISIBILITY,
name=actor_defaults["preferred_username"],
)
# create/update the channel
channel, created = models.Channel.objects.update_or_create(
pk=existing.pk if existing else None, defaults=channel_defaults,
)
return channel
def _get_track(data, attributed_to=None, **forced_values):
track_uuid = getter(data, "funkwhale", "track", "uuid")
if track_uuid:
# easy case, we have a reference to a uuid of a track that
# already exists in our database
try:
track = models.Track.objects.get(uuid=track_uuid)
except models.Track.DoesNotExist:
raise UploadImportError(code="track_uuid_not_found")
return track
from_activity_id = data.get("from_activity_id", None)
track_mbid = (forced_values["mbid"]
if "mbid" in forced_values else data.get("mbid", None))
try:
album_mbid = getter(data, "album", "mbid")
except TypeError:
# album is forced
album_mbid = None
track_fid = getter(data, "fid")
query = None
if album_mbid and track_mbid:
query = Q(mbid=track_mbid, album__mbid=album_mbid)
if track_fid:
query = query | Q(fid=track_fid) if query else Q(fid=track_fid)
if query:
# second easy case: we have a (track_mbid, album_mbid) pair or
# a federation uuid we can check on
try:
return sort_candidates(models.Track.objects.filter(query),
["mbid", "fid"])[0]
except IndexError:
pass
# get / create artist and album artist
artists = getter(data, "artists", default=[])
if "artist" in forced_values:
artist = forced_values["artist"]
else:
artist_data = artists[0]
artist = get_artist(artist_data,
attributed_to=attributed_to,
from_activity_id=from_activity_id)
artist_name = artist.name
if "album" in forced_values:
album = forced_values["album"]
else:
if "artist" in forced_values:
album_artist = forced_values["artist"]
else:
album_artists = getter(data, "album", "artists",
default=artists) or artists
album_artist_data = album_artists[0]
album_artist_name = truncate(album_artist_data.get("name"),
models.MAX_LENGTHS["ARTIST_NAME"])
if album_artist_name == artist_name:
album_artist = artist
else:
query = Q(name__iexact=album_artist_name)
album_artist_mbid = album_artist_data.get("mbid", None)
album_artist_fid = album_artist_data.get("fid", None)
if album_artist_mbid:
query |= Q(mbid=album_artist_mbid)
if album_artist_fid:
query |= Q(fid=album_artist_fid)
defaults = {
"name":
album_artist_name,
"mbid":
album_artist_mbid,
"fid":
album_artist_fid,
"from_activity_id":
from_activity_id,
"attributed_to":
album_artist_data.get("attributed_to", attributed_to),
}
if album_artist_data.get("fdate"):
defaults["creation_date"] = album_artist_data.get("fdate")
album_artist, created = get_best_candidate_or_create(
models.Artist,
query,
defaults=defaults,
sort_fields=["mbid", "fid"])
if created:
tags_models.add_tags(album_artist,
*album_artist_data.get("tags", []))
common_utils.attach_content(
album_artist,
"description",
album_artist_data.get("description"),
)
common_utils.attach_file(
album_artist,
"attachment_cover",
album_artist_data.get("cover_data"),
)
# get / create album
if "album" in data:
album_data = data["album"]
album_title = truncate(album_data["title"],
models.MAX_LENGTHS["ALBUM_TITLE"])
album_fid = album_data.get("fid", None)
if album_mbid:
query = Q(mbid=album_mbid)
else:
query = Q(title__iexact=album_title, artist=album_artist)
if album_fid:
query |= Q(fid=album_fid)
defaults = {
"title": album_title,
"artist": album_artist,
"mbid": album_mbid,
"release_date": album_data.get("release_date"),
"fid": album_fid,
"from_activity_id": from_activity_id,
"attributed_to": album_data.get("attributed_to",
attributed_to),
}
if album_data.get("fdate"):
defaults["creation_date"] = album_data.get("fdate")
album, created = get_best_candidate_or_create(
models.Album,
query,
defaults=defaults,
sort_fields=["mbid", "fid"])
if created:
tags_models.add_tags(album, *album_data.get("tags", []))
common_utils.attach_content(album, "description",
album_data.get("description"))
common_utils.attach_file(album, "attachment_cover",
album_data.get("cover_data"))
else:
album = None
# get / create track
track_title = (forced_values["title"] if "title" in forced_values else
truncate(data["title"], models.MAX_LENGTHS["TRACK_TITLE"]))
position = (forced_values["position"]
if "position" in forced_values else data.get("position", 1))
disc_number = (forced_values["disc_number"] if "disc_number"
in forced_values else data.get("disc_number"))
license = (forced_values["license"] if "license" in forced_values else
licenses.match(data.get("license"), data.get("copyright")))
copyright = (forced_values["copyright"]
if "copyright" in forced_values else truncate(
data.get("copyright"), models.MAX_LENGTHS["COPYRIGHT"]))
description = ({
"text": forced_values["description"],
"content_type": "text/markdown"
} if "description" in forced_values else data.get("description"))
cover_data = (forced_values["cover"]
if "cover" in forced_values else data.get("cover_data"))
query = Q(
title__iexact=track_title,
artist=artist,
album=album,
position=position,
disc_number=disc_number,
)
if track_mbid:
if album_mbid:
query |= Q(mbid=track_mbid, album__mbid=album_mbid)
else:
query |= Q(mbid=track_mbid)
if track_fid:
query |= Q(fid=track_fid)
if album and len(artists) > 1:
# we use the second artist to preserve featuring information
artist = artist = get_artist(artists[1],
attributed_to=attributed_to,
from_activity_id=from_activity_id)
defaults = {
"title": track_title,
"album": album,
"mbid": track_mbid,
"artist": artist,
"position": position,
"disc_number": disc_number,
"fid": track_fid,
"from_activity_id": from_activity_id,
"attributed_to": data.get("attributed_to", attributed_to),
"license": license,
"copyright": copyright,
}
if data.get("fdate"):
defaults["creation_date"] = data.get("fdate")
track, created = get_best_candidate_or_create(models.Track,
query,
defaults=defaults,
sort_fields=["mbid", "fid"])
if created:
tags = (forced_values["tags"] if "tags" in forced_values else data.get(
"tags", []))
tags_models.add_tags(track, *tags)
common_utils.attach_content(track, "description", description)
common_utils.attach_file(track, "attachment_cover", cover_data)
return track