def _save_process_results(version, instance, document, worker, results, starttime, endtime):
total_time = int(endtime - starttime)
module = version.module
with transaction.atomic():
for dataslug, contents in results.items():
outputdata = instance._output[dataslug]
extension = outputdata["extension"]
mimetype = outputdata["mimetype"]
multipart = outputdata.get("parts", False)
logger.info("data %s (%s)" % (dataslug, type(contents)))
logger.info("multiparts %s" % multipart)
if not multipart:
contents = [contents]
df, created = models.DerivedFile.objects.get_or_create(
document=document,
module_version=version, outputname=dataslug, extension=extension,
mimetype=mimetype, defaults={'computation_time': total_time, 'num_parts': len(contents)})
if not created:
df.date = django.utils.timezone.now()
df.num_parts = len(contents)
df.save()
if worker:
df.essentia = worker.essentia
df.pycompmusic = worker.pycompmusic
df.save()
for i, partdata in enumerate(contents, 1):
_save_file(df, i, extension, partdata)
# When we've finished, log that we processed the file. If this throws an
# exception, we won't do the log.
log.log_processed_file(instance.hostname, document.external_identifier, version.pk)
def _add_release_performance(self, releasembid, artistid, perf_type, attrs):
# release.mbid, artistid, perf_type, attrs
logger.info(" Adding release performance...")
release = andalusian.models.Album.objects.get(mbid=releasembid)
for rec in release.recordings.all():
self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def get_latest_module_version(themod_id=None):
""" Create a new ModuleVersion if this module has been
updated.
If no argument is given, update all modules. """
if themod_id:
modules = models.Module.objects.filter(pk=themod_id)
else:
modules = models.Module.objects.filter(disabled=False)
for m in modules:
logger.info("module %s" % m)
instance = _get_module_instance_by_path(m.module)
logger.info("instance %s" % instance)
if instance:
if m.disabled: # if we were disabled and exist again, reenable.
m.disabled = False
m.save()
version = instance._version
v = "%s" % version
versions = m.versions.filter(version=version)
if not len(versions):
models.ModuleVersion.objects.create(module=m, version=v)
else:
m.disabled = True
m.save()
def _create_release_object(self, mbrelease):
concert, created = carnatic.models.Concert.objects.get_or_create(
mbid=mbid, defaults={"title": rel["title"]})
if not created:
if releaseid in self.imported_releases:
print "Release already updated in this import. Not doing it again"
return concert
if created or self.overwrite:
year = rel.get("date", "")[:4]
if year:
year = int(year)
else:
year = None
concert.title = rel["title"]
concert.year = year
credit_phrase = rel.get("artist-credit-phrase")
concert.artistcredit = credit_phrase
source = self.make_mb_source("http://musicbrainz.org/release/%s" % mbid)
concert.source = source
concert.save()
if not created and self.overwrite:
# If it already exists and we're doing an overwrite
concert.artists.clear()
for a in rel["artist-credit"]:
if isinstance(a, dict):
artistid = a["artist"]["id"]
artist = self.add_and_get_artist(artistid)
logger.info(" artist: %s" % artist)
if not concert.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list")
concert.artists.add(artist)
return concert
def _add_release_performance(self, releaseid, artistid, perf_type, attrs):
logger.info(" Adding concert performance...")
concert = carnatic.models.Concert.objects.get(mbid=releaseid)
for rec in concert.recordings.all():
self._add_recording_performance(rec.mbid, artistid, perf_type,
attrs)
def import_release(self, releaseid, directories):
rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels"])
rel = rel["release"]
mbid = rel["id"]
logger.info("Adding release %s" % mbid)
if mbid in self.imported_releases:
print "Release already updated in this import. Not doing it again"
return
release = self._create_release_object(rel)
recordings = []
for medium in rel["medium-list"]:
for track in medium["track-list"]:
recordings.append(track["recording"]["id"])
if not created and self.overwrite:
concert.tracks.clear()
trackorder = 1
for recid in recordings:
recording = self.add_and_get_recording(recid)
if not concert.tracks.filter(pk=recording.pk).exists():
carnatic.models.ConcertRecording.objects.create(
concert=concert, recording=recording, track=trackorder)
trackorder += 1
if not created and self.overwrite:
concert.performance.clear()
for perf in self._get_artist_performances(rel.get("artist-relation-list", [])):
artistid, instrument, is_lead = perf
self.add_release_performance(releaseid, artistid, instrument, is_lead)
external_data.import_concert_image(concert, directories, self.overwrite)
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
# We don't expect to see these two relations here, as we know that all releases in MusicBrainz
# for this collection have recording-level relationships.
if perf_type in [release_importer.RELATION_RELEASE_VOCAL, release_importer.RELATION_RELEASE_INSTRUMENT]:
raise release_importer.ImportFailedException("Found a release-level artist instrument relation on "
"recording {} which isn't supported".format(recordingid))
is_lead = False
instr_name = attrs[-1]
if perf_type == release_importer.RELATION_RECORDING_VOCAL:
instr_name = "voice"
if "lead vocals" in attrs:
is_lead = True
elif perf_type == release_importer.RELATION_RECORDING_INSTRUMENT:
instr_name = attrs[-1]
attrs = attrs[:-1]
if "lead" in attrs:
is_lead = True
instrument = self._get_instrument(instr_name)
if instrument:
recording = andalusian.models.Recording.objects.get(mbid=recordingid)
perf = andalusian.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def add_and_get_recording(self, recordingid):
mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags", "work-rels", "artist-rels"])
mbrec = mbrec["recording"]
rec, created = self._RecordingClass.objects.get_or_create(mbid=recordingid)
if created or self.overwrite:
logger.info(" adding recording %s" % (recordingid,))
source = self.make_mb_source("http://musicbrainz.org/recording/%s" % recordingid)
rec.source = source
rec.length = mbrec.get("length")
rec.title = mbrec["title"]
rec.save()
works = []
for work in mbrec.get("work-relation-list", []):
if work["type"] == "performance":
w = self.add_and_get_work(work["target"])
works.append(w)
tags = mbrec.get("tag-list", [])
# Join recording and works in a subclass because some models
# have 1 work per recording and others have many
self._join_recording_and_works(rec, works)
# Sometime we attach tags to works, sometimes to recordings
self._apply_tags(rec, works, tags)
if self.overwrite:
IPClass = rec.get_object_map("performance")
IPClass.objects.filter(recording=rec).delete()
for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])):
artistid, instrument, is_lead = perf
self._add_recording_performance(recordingid, artistid, instrument, is_lead)
return rec
def _create_artist_object(self,
ArtistKlass,
AliasKlass,
mbartist,
alias_ref="artist"):
artistid = mbartist["id"]
artist, created = ArtistKlass.objects.get_or_create(
mbid=artistid, defaults={"name": mbartist["name"]})
logger.info(" adding artist/composer %s" % (artistid, ))
source = self.make_mb_source("http://musicbrainz.org/artist/%s" %
artistid)
artist.source = source
artist.name = mbartist["name"]
if mbartist.get("type") == "Person":
artist.artist_type = "P"
elif mbartist.get("type") == "Group":
artist.artist_type = "G"
if mbartist.get("gender") == "Male":
artist.gender = "M"
elif mbartist.get("gender") == "Female":
artist.gender = "F"
dates = mbartist.get("life-span")
if dates:
artist.begin = dates.get("begin")
artist.end = dates.get("end")
artist.save()
# add wikipedia references if they exist
artist.references.clear()
for rel in mbartist.get("url-relation-list", []):
if rel["type-id"] == RELEASE_TYPE_WIKIPEDIA:
source = self.make_wikipedia_source(rel["target"])
if not artist.references.filter(pk=source.pk).exists():
artist.references.add(source)
# We can't 'clear' an alias list from artist because an alias
# object requires an artist.
# TODO Deleting these each time we overwrite means we churn the
# alias ids. This may or may not be a good idea
args = {}
args[alias_ref] = artist
AliasKlass.objects.filter(**args).delete()
for alias in mbartist.get("alias-list", []):
a = alias["alias"]
primary = alias.get("primary")
locale = alias.get("locale")
args = {"alias": a}
args[alias_ref] = artist
aob, created = AliasKlass.objects.get_or_create(**args)
if primary:
aob.primary = True
if locale:
aob.locale = locale
aob.save()
external_data.import_artist_wikipedia(artist)
return artist
def _add_release_performance(self, releaseid, artistid, perf_type, attrs):
"""If there is a relationship between a release and artist, apply the relationship to all
recordings in the release"""
logger.info(" Adding concert performance...")
concert = carnatic.models.Concert.objects.get(mbid=releaseid)
for rec in concert.recordings.all():
self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def _add_release_performance(self, releaseid, artistid, instrument, is_lead):
logger.info(" Adding concert performance...")
artist = self.add_and_get_artist(artistid)
instrument = self.get_instrument(instrument)
if instrument:
concert = makam.models.Concert.objects.get(mbid=releaseid)
perf = makam.models.InstrumentConcertPerformance(concert=concert, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def _add_recording_artists(self, rec, artistids):
rec.performers.clear()
for a in artistids:
# If the artist is [dialogue] the we don't show analysis.
artist = self.add_and_get_artist(a)
logger.info(" artist: %s" % artist)
if not rec.performers.filter(pk=artist.pk).exists():
rec.performers.add(artist)
def _add_recording_artists(self, rec, artistids):
rec.performers .clear()
for a in artistids:
# If the artist is [dialogue] the we don't show analysis.
artist = self.add_and_get_artist(a)
logger.info(" artist: %s" % artist)
if not rec.performers.filter(pk=artist.pk).exists():
rec.performers.add(artist)
def add_recording_performance(self, recordingid, artistid, instrument, is_lead):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
instrument = self.get_instrument(instrument)
if instrument:
recording = carnatic.models.Recording.objects.get(mbid=recordingid)
perf = carnatic.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def _create_artist_object(self, ArtistKlass, AliasKlass, mbartist, alias_ref="artist"):
artistid = mbartist["id"]
artist, created = ArtistKlass.objects.get_or_create(
mbid=artistid,
defaults={"name": mbartist["name"]})
if created or self.overwrite:
logger.info(" adding artist/composer %s" % (artistid, ))
source = self.make_mb_source("http://musicbrainz.org/artist/%s" % artistid)
artist.source = source
artist.name = mbartist["name"]
if mbartist.get("type") == "Person":
artist.artist_type = "P"
elif mbartist.get("type") == "Group":
artist.artist_type = "G"
if mbartist.get("gender") == "Male":
artist.gender = "M"
elif mbartist.get("gender") == "Female":
artist.gender = "F"
dates = mbartist.get("life-span")
if dates:
artist.begin = dates.get("begin")
artist.end = dates.get("end")
artist.save()
# add wikipedia references if they exist
if self.overwrite:
artist.references.clear()
for rel in mbartist.get("url-relation-list", []):
if rel["type-id"] == RELEASE_TYPE_WIKIPEDIA:
source = self.make_wikipedia_source(rel["target"])
if not artist.references.filter(pk=source.pk).exists():
artist.references.add(source)
if self.overwrite:
# We can't 'clear' an alias list from artist because an alias
# object requires an artist.
# TODO Deleting these each time we overwrite means we churn the
# alias ids. This may or may not be a good idea
args = {}
args[alias_ref] = artist
AliasKlass.objects.filter(**args).delete()
for alias in mbartist.get("alias-list", []):
a = alias["alias"]
primary = alias.get("primary")
locale = alias.get("locale")
args = {"alias": a}
args[alias_ref] = artist
aob, created = AliasKlass.objects.get_or_create(**args)
if primary:
aob.primary = True
if locale:
aob.locale = locale
aob.save()
external_data.import_artist_wikipedia(artist, self.overwrite)
return artist
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
recording = carnatic.models.Recording.objects.get(mbid=recordingid)
instrument, attributes, is_lead = self._performance_type_to_instrument(perf_type, attrs)
if instrument:
carnatic.models.InstrumentPerformance.objects.create(recording=recording, instrument=instrument, artist=artist, lead=is_lead, attributes=attributes)
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
recording = carnatic.models.Recording.objects.get(mbid=recordingid)
instrument, attributes, is_lead = self._performance_type_to_instrument(perf_type, attrs)
if instrument:
carnatic.models.InstrumentPerformance.objects.create(recording=recording, instrument=instrument, artist=artist, lead=is_lead, attributes=attributes)
def add_release_performance(self, releaseid, artistid, instrument, is_lead):
# TODO: Can we de-duplicate this with the recording stuff above
logger.info(" Adding concert performance...")
artist = self.add_and_get_artist(artistid)
instrument = self.get_instrument(instrument)
if instrument:
concert = carnatic.models.Concert.objects.get(mbid=releaseid)
perf = carnatic.models.InstrumentConcertPerformance(concert=concert, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def _add_recording_performance(self, recordingid, artistid, instrument, is_lead):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
# Musicbrainz calls it 'vocal', but we want it to be 'voice'
if instrument == "vocal":
instrument = "voice"
instrument = self.get_instrument(instrument)
if instrument:
recording = hindustani.models.Recording.objects.get(mbid=recordingid)
perf = hindustani.models.InstrumentPerformance(recording=recording, instrument=instrument, artist=artist, lead=is_lead)
perf.save()
def _add_recording_artists(self, rec, artistids):
rec.artists.clear()
for a in artistids:
# If the artist is [dialogue] the we don't show analysis.
if a == DIALOGUE_ARTIST:
rec.analyse = False
rec.save()
artist = self.add_and_get_artist(a)
logger.info(" artist: %s" % artist)
if not rec.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list 2")
rec.artists.add(artist)
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
is_lead = False
if "lead" in attrs:
is_lead = True
instrument, attributes = self._performance_type_to_instrument(perf_type, attrs)
recording = makam.models.Recording.objects.get(mbid=recordingid)
makam.models.InstrumentPerformance.objects.create(recording=recording, instrument=instrument, artist=artist, attributes=attributes, lead=is_lead)
def _add_recording_artists(self, rec, artistids):
rec.artists.clear()
for a in artistids:
# If the artist is [dialogue] the we don't show analysis.
if a == DIALOGUE_ARTIST:
rec.analyse = False
rec.save()
artist = self.add_and_get_artist(a)
logger.info(" artist: %s" % artist)
if not rec.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list 2")
rec.artists.add(artist)
def _add_release_performance(self, releaseid, artistid, instrument, is_lead):
logger.info(" Adding release performance...")
artist = self.add_and_get_artist(artistid)
instrument = self._get_instrument(instrument)
if instrument:
release = andalusian.models.Album.objects.get(mbid=releaseid)
# For each recording in the release, see if the relationship
# already exists. If not, create it.
for rec in release.recordings.all():
if not andalusian.models.InstrumentPerformance.objects.filter(
recording=rec, instrument=instrument, performer=artist).exists():
perf = andalusian.models.InstrumentPerformance(recording=rec, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead):
logger.info(" Adding release performance...")
artist = self.add_and_get_artist(artistid)
instrument = self._get_instrument(instrument)
if instrument:
release = makam.models.Release.objects.get(mbid=releaseid)
# For each recording in the release, see if the relationship
# already exists. If not, create it.
for rec in release.recordings.all():
if not makam.models.InstrumentPerformance.objects.filter(
recording=rec, instrument=instrument, artist=artist).exists():
perf = makam.models.InstrumentPerformance(recording=rec, instrument=instrument, artist=artist, lead=is_lead)
perf.save()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead):
logger.info(" Adding concert performance...")
artist = self.add_and_get_artist(artistid)
instrument = self._get_instrument(instrument)
concert = carnatic.models.Concert.objects.get(mbid=releaseid)
# Instrument could be None if we don't know it
for rec in concert.recordings.all():
if not carnatic.models.InstrumentPerformance.objects.filter(
recording=rec, instrument=instrument, artist=artist).exists():
perf = carnatic.models.InstrumentPerformance(recording=rec, instrument=instrument, artist=artist, lead=is_lead)
perf.save()
def _add_and_get_artist_composer(self, ArtistClass, artistid, addgroups=True):
"""
arguments: ArtistClass: models.Artist or models.Composer
"""
a = compmusic.mb.get_artist_by_id(artistid, includes=["url-rels", "artist-rels"])["artist"]
artist, created = ArtistClass.objects.get_or_create(mbid=artistid,
defaults={"name": a["name"]})
if not created:
if artistid in self.imported_artists:
print "Artist already updated in this import. Not doing it again"
return artist
if created or self.overwrite:
logger.info(" adding artist/composer %s" % (artistid, ))
source = self.make_mb_source("http://musicbrainz.org/artist/%s" % artistid)
artist.source = source
artist.name = a["name"]
if a.get("type") == "Person":
artist.artist_type = "P"
elif a.get("type") == "Group":
artist.artist_type = "G"
if a.get("gender") == "Male":
artist.gender = "M"
elif a.get("gender") == "Female":
artist.gender = "F"
dates = a.get("life-span")
if dates:
artist.begin = dates.get("begin")
artist.end = dates.get("end")
artist.save()
# TODO: Annoying hack to only work on artists
if ArtistClass == carnatic.models.Artist and addgroups:
if self.overwrite:
artist.group_members.clear()
for member in a.get("artist-relation-list", []):
if "member" in member["type"] and member.get("direction") == "backward":
memberartist = self._add_and_get_artist_composer(ArtistClass, member["target"])
artist.group_members.add(memberartist)
# add wikipedia references if they exist
for rel in a.get("url-relation-list", []):
if rel["type"] == ["wikipedia"]:
source = self.make_wikipedia_source(rel["url"])
if not artist.references.filter(pk=source.pk).exists():
artist.references.add(source)
external_data.import_artist_bio(artist, self.overwrite)
return artist
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
instr_name = attrs[-1]
attrs = attrs[:-1]
is_lead = False
if "lead" in attrs:
is_lead = True
instrument = self._get_instrument(instr_name)
if instrument:
recording = andalusian.models.Recording.objects.get(mbid=recordingid)
perf = andalusian.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead)
perf.save()
def add_and_get_recording(self, recordingid):
mbrec = compmusic.mb.get_recording_by_id(
recordingid,
includes=["tags", "work-rels", "artist-rels", "artists"])
mbrec = mbrec["recording"]
rec, created = self._RecordingClass.objects.get_or_create(
mbid=recordingid)
logger.info(" adding recording %s" % (recordingid, ))
import_logger.info("importing recording %s", mbrec["title"])
source = self.make_mb_source("http://musicbrainz.org/recording/%s" %
recordingid)
rec.source = source
rec.length = mbrec.get("length")
rec.title = mbrec["title"]
rec.save()
artistids = []
# Create recording primary artists
for a in mbrec.get("artist-credit", []):
if isinstance(a, dict):
artistid = a["artist"]["id"]
artistids.append(artistid)
self._add_recording_artists(rec, artistids)
works = []
for work in mbrec.get("work-relation-list", []):
if work["type"] == "performance":
w = self.add_and_get_work(work["target"])
works.append(w)
tags = mbrec.get("tag-list", [])
# Join recording and works in a subclass because some models
# have 1 work per recording and others have many
self._join_recording_and_works(rec, works)
# Sometime we attach tags to works, sometimes to recordings
self._apply_tags(rec, works, tags)
IPClass = rec.get_object_map("performance")
IPClass.objects.filter(recording=rec).delete()
for perf in self._get_artist_performances(
mbrec.get("artist-relation-list", [])):
artistid, perf_type, attrs = perf
self._add_recording_performance(recordingid, artistid, perf_type,
attrs)
return rec
def _add_recording_performance(self, recordingid, artistid, perf_type,
attrs):
logger.info(" Adding recording performance...")
artist = self.add_and_get_artist(artistid)
is_lead = False
if "lead" in attrs:
is_lead = True
instrument, attributes = self._performance_type_to_instrument(
perf_type, attrs)
recording = makam.models.Recording.objects.get(mbid=recordingid)
makam.models.InstrumentPerformance.objects.create(
recording=recording,
instrument=instrument,
artist=artist,
attributes=attributes,
lead=is_lead)
def _performance_type_to_instrument(self, perf_type, attrs):
is_lead = False
if perf_type in [release_importer.RELATION_RECORDING_VOCAL, release_importer.RELATION_RELEASE_VOCAL]:
instr_name = "voice"
if "lead vocals" in attrs:
is_lead = True
elif perf_type in [release_importer.RELATION_RECORDING_INSTRUMENT, release_importer.RELATION_RELEASE_INSTRUMENT]:
instr_name = attrs[-1]
attrs = attrs[:-1]
if "lead" in attrs:
is_lead = True
else:
logger.info(" Unknown performance type: {}".format(perf_type))
instr_name = None
attributes = " ".join(attrs)
instrument = self._get_instrument(instr_name)
return instrument, attributes, is_lead
def _create_recording_and_works(self, recording, works, tags):
rec, created = carnatic.models.Recording.objects.get_or_create(mbid=recordingid)
if created or self.overwrite:
logger.info(" adding recording %s" % (recordingid,))
raagas = self._get_raaga(mbrec.get("tag-list", []))
taalas = self._get_taala(mbrec.get("tag-list", []))
source = self.make_mb_source("http://musicbrainz.org/recording/%s" % recordingid)
rec.source = source
rec.length = mbrec.get("length")
rec.title = mbrec["title"]
rec.save()
if not created and self.overwrite:
rec.performance.clear()
for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])):
artistid, instrument, is_lead = perf
self.add_recording_performance(recordingid, artistid, instrument, is_lead)
return rec
def delete_moduleversion(vid):
version = models.ModuleVersion.objects.get(pk=vid)
logger.info("deleting moduleversion %s" % version)
files = version.derivedfile_set.all()
for f in files:
for pn in range(1, f.num_parts + 1):
path = f.full_path_for_part(pn)
try:
os.unlink(path)
dirname = os.path.dirname(path)
if len(os.listdir(dirname)) == 0:
# If this directory is empty, remove it and all empty
# parent dirs. May throw OSError (not really an error)
os.removedirs(dirname)
except OSError:
pass # if the file doesn't exist, not really an error
f.delete()
module = version.module
version.delete()
logger.info("done")
def add_and_get_recording(self, recordingid):
mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags", "work-rels", "artist-rels", "artists"])
mbrec = mbrec["recording"]
rec, created = self._RecordingClass.objects.get_or_create(mbid=recordingid)
logger.info(" adding recording %s" % (recordingid,))
import_logger.info("importing recording %s", mbrec["title"])
rec.length = mbrec.get("length")
rec.title = mbrec["title"]
rec.save()
artistids = []
# Create recording primary artists
for a in mbrec.get("artist-credit", []):
if isinstance(a, dict):
artistid = a["artist"]["id"]
artistids.append(artistid)
self._add_recording_artists(rec, artistids)
works = []
for work in mbrec.get("work-relation-list", []):
if work["type"] == "performance":
w = self.add_and_get_work(work["target"])
works.append(w)
tags = mbrec.get("tag-list", [])
# Join recording and works in a subclass because some models
# have 1 work per recording and others have many
self._join_recording_and_works(rec, works)
# Sometime we attach tags to works, sometimes to recordings
self._apply_tags(rec, works, tags)
IPClass = rec.get_object_map("performance")
IPClass.objects.filter(recording=rec).delete()
for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])):
artistid, perf_type, attrs = perf
self._add_recording_performance(recordingid, artistid, perf_type, attrs)
return rec
def run_module_on_recordings(moduleid, recids):
module = models.Module.objects.get(pk=moduleid)
version = module.get_latest_version()
logger.info("running module %s on %s files" % (module, len(recids)))
if version:
logger.info("version %s %s" % (version, version.pk))
# All documents that don't already have a derived file for this module version
docs = models.Document.objects.filter(
sourcefiles__file_type=module.source_type,
external_identifier__in=recids,
).exclude(derivedfiles__module_version=version)
for d in docs:
logger.info(" document %s" % d)
logger.info(" docid %s" % d.pk)
process_document.delay(d.pk, version.pk)
def import_release(self, releaseid, directories):
if releaseid in self.imported_releases:
print("Release already updated in this import. Not doing it again")
return self._ReleaseClass.objects.get(mbid=releaseid)
rel = compmusic.mb.get_release_by_id(releaseid,
includes=[
"artists", "recordings",
"artist-rels",
"release-groups"
])
rel = rel["release"]
mbid = rel["id"]
logger.info("Adding release %s" % mbid)
release = self._create_release_object(rel)
# Create release primary artists
release.artists.clear()
for a in rel["artist-credit"]:
if isinstance(a, dict):
artistid = a["artist"]["id"]
artist = self.add_and_get_release_artist(artistid)
logger.info(" artist: %s" % artist)
if not release.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list")
release.artists.add(artist)
recordings = []
for mnum, medium in enumerate(rel["medium-list"], 1):
for tnum, track in enumerate(medium["track-list"], 1):
recordings.append((track["recording"]["id"], mnum, tnum))
release.recordings.clear()
trackorder = 1
for recid, mnum, tnum in recordings:
recob = self.add_and_get_recording(recid)
self._link_release_recording(release, recob, trackorder, mnum,
tnum)
trackorder += 1
for perf in self._get_artist_performances(
rel.get("artist-relation-list", [])):
artistid, perf_type, attrs = perf
self._add_release_performance(release.mbid, artistid, perf_type,
attrs)
self._add_release_artists_as_relationship(release,
rel["artist-credit"])
self._add_image_to_release(release, directories)
self.imported_releases.append(releaseid)
return release
def import_release(self, releaseid, directories):
if releaseid in self.imported_releases:
print "Release already updated in this import. Not doing it again"
return self._ReleaseClass.objects.get(mbid=releaseid)
rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels"])
rel = rel["release"]
mbid = rel["id"]
logger.info("Adding release %s" % mbid)
release = self._create_release_object(rel)
# Create release primary artists
if self.overwrite:
# If it already exists and we're doing an overwrite
release.artists.clear()
for a in rel["artist-credit"]:
if isinstance(a, dict):
artistid = a["artist"]["id"]
artist = self.add_and_get_artist(artistid)
logger.info(" artist: %s" % artist)
if not release.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list")
release.artists.add(artist)
recordings = []
for medium in rel["medium-list"]:
for track in medium["track-list"]:
recordings.append(track["recording"]["id"])
print recordings
if self.overwrite:
release.tracks.clear()
trackorder = 1
for recid in recordings:
recob = self.add_and_get_recording(recid)
self._link_release_recording(release, recob, trackorder)
trackorder += 1
if self.overwrite:
self._clear_release_performances(release)
for perf in self._get_artist_performances(rel.get("artist-relation-list", [])):
artistid, instrument, is_lead = perf
self._add_release_performance(release.mbid, artistid, instrument, is_lead)
external_data.import_release_image(release, directories, self.overwrite)
self.imported_releases.append(releaseid)
return release
def delete_moduleversion(vid):
version = models.ModuleVersion.objects.get(pk=vid)
logger.info("deleting moduleversion %s" % version)
files = version.derivedfile_set.all()
for f in files:
for p in f.parts.all():
path = p.fullpath
try:
os.unlink(path)
dirname = os.path.dirname(path)
if len(os.listdir(dirname)) == 0:
# If this directory is empty, remove it and all empty
# parent dirs. May throw OSError (not really an error)
os.removedirs(dirname)
except OSError:
pass # if the file doesn't exist, not really an error
f.delete()
module = version.module
version.delete()
if module.versions.count() == 0:
logger.info("No more moduleversions for this module, deleting the module")
module.delete()
logger.info(" .. module deleted")
logger.info("done")
def import_release(self, releaseid, directories):
if releaseid in self.imported_releases:
print "Release already updated in this import. Not doing it again"
return self._ReleaseClass.objects.get(mbid=releaseid)
rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels", "release-groups"])
rel = rel["release"]
mbid = rel["id"]
logger.info("Adding release %s" % mbid)
release = self._create_release_object(rel)
# Create release primary artists
release.artists.clear()
for a in rel["artist-credit"]:
if isinstance(a, dict):
artistid = a["artist"]["id"]
artist = self.add_and_get_release_artist(artistid)
logger.info(" artist: %s" % artist)
if not release.artists.filter(pk=artist.pk).exists():
logger.info(" - adding to artist list")
release.artists.add(artist)
recordings = []
for mnum, medium in enumerate(rel["medium-list"], 1):
for tnum, track in enumerate(medium["track-list"], 1):
recordings.append( (track["recording"]["id"], mnum, tnum))
release.recordings.clear()
trackorder = 1
for recid, mnum, tnum in recordings:
recob = self.add_and_get_recording(recid)
self._link_release_recording(release, recob, trackorder, mnum, tnum)
trackorder += 1
for perf in self._get_artist_performances(rel.get("artist-relation-list", [])):
artistid, perf_type, attrs = perf
self._add_release_performance(release.mbid, artistid, perf_type, attrs)
self._add_release_artists_as_relationship(release, rel["artist-credit"])
external_data.import_release_image(release, directories)
self.imported_releases.append(releaseid)
return release
def run_module_on_collection(collectionid, moduleid, versionid=None):
collection = models.Collection.objects.get(pk=collectionid)
module = models.Module.objects.get(pk=moduleid)
if versionid:
version = module.versions.get(pk=versionid)
else:
version = module.get_latest_version()
logger.info("running module %s on collection %s" % (module, collection))
if version:
logger.info("version %s" % version)
if module.many_files:
process_collection.delay(collection.pk, version.pk)
else:
# All documents that don't already have a derived file for this module version
docs = models.Document.objects.filter(
sourcefiles__file_type=module.source_type,
collections=collection).exclude(derivedfiles__module_version=version)
for i, d in enumerate(docs, 1):
logger.info(" document %s/%s - %s" % (i, len(docs), d))
process_document.delay(d.pk, version.pk)
def remove_deleted_items():
""" Search musicbrainz for all items in the database and if they
have been deleted then remove them """
logger.info("Scanning works...")
for w in carnatic.models.Work.objects.all():
try:
mbwork = mb.get_work_by_id(w.mbid)
except mb.ResponseError:
logger.info("work %s (%s) missing; deleting" % (w, w.mbid))
w.delete()
logger.info("Scanning recordings...")
for r in carnatic.models.Recording.objects.all():
try:
mbrec = mb.get_recording_by_id(r.mbid)
except mb.ResponseError:
logger.info("recording %s (%s) missing; deleting" % (r, r.mbid))
r.delete()
logger.info("Scanning concerts...")
for c in carnatic.models.Concert.objects.all():
try:
mbcon = mb.get_release_by_id(c.mbid)
except mb.ResponseError:
logger.info("release %s (%s) missing; deleting" % (c, c.mbid))
c.delete()
logger.info("Scanning artists...")
for a in carnatic.models.Artist.objects.all():
try:
mbart = mb.get_artist_by_id(a.mbid)
except mb.ResponseError:
# We import dummy artists to be gurus, leave them here
if not a.dummy:
logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
a.delete()
logger.info("Scanning composers...")
for a in carnatic.models.Composer.objects.all():
try:
mbart = mb.get_artist_by_id(a.mbid)
except mb.ResponseError:
logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
a.delete()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead):
logger.info(" Adding concert performance to all tracks...")
release = hindustani.models.Release.objects.get(mbid=releaseid)
for t in release.tracks:
self._add_recording_performance(t.mbid, artistid, instrument, is_lead)
def delete_module(mid):
module = models.Module.objects.get(pk=mid)
logger.info("deleting entire module %s" % module)
for v in module.versions.all():
delete_moduleversion(v.pk)
logger.info("done")
def _add_release_performance(self, releaseid, artistid, perf_type, attrs):
logger.info(" Adding concert performance...")
concert = carnatic.models.Concert.objects.get(mbid=releaseid)
for rec in concert.recordings.all():
self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def _add_release_performance(self, releaseid, artistid, perf_type, attrs):
logger.info(" Adding concert performance to all recordings...")
release = hindustani.models.Release.objects.get(mbid=releaseid)
for t in release.recordings.all():
self._add_recording_performance(t.mbid, artistid, perf_type, attrs)
def _add_release_performance(self, releaseid, artistid, perf_type, attrs):
logger.info(" Adding concert performance to all recordings...")
release = hindustani.models.Release.objects.get(mbid=releaseid)
for t in release.recordings.all():
self._add_recording_performance(t.mbid, artistid, perf_type, attrs)
