def _save_process_results(version, instance, document, worker, results, starttime, endtime): total_time = int(endtime - starttime) module = version.module with transaction.atomic(): for dataslug, contents in results.items(): outputdata = instance._output[dataslug] extension = outputdata["extension"] mimetype = outputdata["mimetype"] multipart = outputdata.get("parts", False) logger.info("data %s (%s)" % (dataslug, type(contents))) logger.info("multiparts %s" % multipart) if not multipart: contents = [contents] df, created = models.DerivedFile.objects.get_or_create( document=document, module_version=version, outputname=dataslug, extension=extension, mimetype=mimetype, defaults={'computation_time': total_time, 'num_parts': len(contents)}) if not created: df.date = django.utils.timezone.now() df.num_parts = len(contents) df.save() if worker: df.essentia = worker.essentia df.pycompmusic = worker.pycompmusic df.save() for i, partdata in enumerate(contents, 1): _save_file(df, i, extension, partdata) # When we've finished, log that we processed the file. If this throws an # exception, we won't do the log. log.log_processed_file(instance.hostname, document.external_identifier, version.pk)
def _add_release_performance(self, releasembid, artistid, perf_type, attrs): # release.mbid, artistid, perf_type, attrs logger.info(" Adding release performance...") release = andalusian.models.Album.objects.get(mbid=releasembid) for rec in release.recordings.all(): self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def get_latest_module_version(themod_id=None): """ Create a new ModuleVersion if this module has been updated. If no argument is given, update all modules. """ if themod_id: modules = models.Module.objects.filter(pk=themod_id) else: modules = models.Module.objects.filter(disabled=False) for m in modules: logger.info("module %s" % m) instance = _get_module_instance_by_path(m.module) logger.info("instance %s" % instance) if instance: if m.disabled: # if we were disabled and exist again, reenable. m.disabled = False m.save() version = instance._version v = "%s" % version versions = m.versions.filter(version=version) if not len(versions): models.ModuleVersion.objects.create(module=m, version=v) else: m.disabled = True m.save()
def _create_release_object(self, mbrelease): concert, created = carnatic.models.Concert.objects.get_or_create( mbid=mbid, defaults={"title": rel["title"]}) if not created: if releaseid in self.imported_releases: print "Release already updated in this import. Not doing it again" return concert if created or self.overwrite: year = rel.get("date", "")[:4] if year: year = int(year) else: year = None concert.title = rel["title"] concert.year = year credit_phrase = rel.get("artist-credit-phrase") concert.artistcredit = credit_phrase source = self.make_mb_source("http://musicbrainz.org/release/%s" % mbid) concert.source = source concert.save() if not created and self.overwrite: # If it already exists and we're doing an overwrite concert.artists.clear() for a in rel["artist-credit"]: if isinstance(a, dict): artistid = a["artist"]["id"] artist = self.add_and_get_artist(artistid) logger.info(" artist: %s" % artist) if not concert.artists.filter(pk=artist.pk).exists(): logger.info(" - adding to artist list") concert.artists.add(artist) return concert
def _add_release_performance(self, releaseid, artistid, perf_type, attrs): logger.info(" Adding concert performance...") concert = carnatic.models.Concert.objects.get(mbid=releaseid) for rec in concert.recordings.all(): self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def import_release(self, releaseid, directories): rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels"]) rel = rel["release"] mbid = rel["id"] logger.info("Adding release %s" % mbid) if mbid in self.imported_releases: print "Release already updated in this import. Not doing it again" return release = self._create_release_object(rel) recordings = [] for medium in rel["medium-list"]: for track in medium["track-list"]: recordings.append(track["recording"]["id"]) if not created and self.overwrite: concert.tracks.clear() trackorder = 1 for recid in recordings: recording = self.add_and_get_recording(recid) if not concert.tracks.filter(pk=recording.pk).exists(): carnatic.models.ConcertRecording.objects.create( concert=concert, recording=recording, track=trackorder) trackorder += 1 if not created and self.overwrite: concert.performance.clear() for perf in self._get_artist_performances(rel.get("artist-relation-list", [])): artistid, instrument, is_lead = perf self.add_release_performance(releaseid, artistid, instrument, is_lead) external_data.import_concert_image(concert, directories, self.overwrite)
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) # We don't expect to see these two relations here, as we know that all releases in MusicBrainz # for this collection have recording-level relationships. if perf_type in [release_importer.RELATION_RELEASE_VOCAL, release_importer.RELATION_RELEASE_INSTRUMENT]: raise release_importer.ImportFailedException("Found a release-level artist instrument relation on " "recording {} which isn't supported".format(recordingid)) is_lead = False instr_name = attrs[-1] if perf_type == release_importer.RELATION_RECORDING_VOCAL: instr_name = "voice" if "lead vocals" in attrs: is_lead = True elif perf_type == release_importer.RELATION_RECORDING_INSTRUMENT: instr_name = attrs[-1] attrs = attrs[:-1] if "lead" in attrs: is_lead = True instrument = self._get_instrument(instr_name) if instrument: recording = andalusian.models.Recording.objects.get(mbid=recordingid) perf = andalusian.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def add_and_get_recording(self, recordingid): mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags", "work-rels", "artist-rels"]) mbrec = mbrec["recording"] rec, created = self._RecordingClass.objects.get_or_create(mbid=recordingid) if created or self.overwrite: logger.info(" adding recording %s" % (recordingid,)) source = self.make_mb_source("http://musicbrainz.org/recording/%s" % recordingid) rec.source = source rec.length = mbrec.get("length") rec.title = mbrec["title"] rec.save() works = [] for work in mbrec.get("work-relation-list", []): if work["type"] == "performance": w = self.add_and_get_work(work["target"]) works.append(w) tags = mbrec.get("tag-list", []) # Join recording and works in a subclass because some models # have 1 work per recording and others have many self._join_recording_and_works(rec, works) # Sometime we attach tags to works, sometimes to recordings self._apply_tags(rec, works, tags) if self.overwrite: IPClass = rec.get_object_map("performance") IPClass.objects.filter(recording=rec).delete() for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])): artistid, instrument, is_lead = perf self._add_recording_performance(recordingid, artistid, instrument, is_lead) return rec
def _create_artist_object(self, ArtistKlass, AliasKlass, mbartist, alias_ref="artist"): artistid = mbartist["id"] artist, created = ArtistKlass.objects.get_or_create( mbid=artistid, defaults={"name": mbartist["name"]}) logger.info(" adding artist/composer %s" % (artistid, )) source = self.make_mb_source("http://musicbrainz.org/artist/%s" % artistid) artist.source = source artist.name = mbartist["name"] if mbartist.get("type") == "Person": artist.artist_type = "P" elif mbartist.get("type") == "Group": artist.artist_type = "G" if mbartist.get("gender") == "Male": artist.gender = "M" elif mbartist.get("gender") == "Female": artist.gender = "F" dates = mbartist.get("life-span") if dates: artist.begin = dates.get("begin") artist.end = dates.get("end") artist.save() # add wikipedia references if they exist artist.references.clear() for rel in mbartist.get("url-relation-list", []): if rel["type-id"] == RELEASE_TYPE_WIKIPEDIA: source = self.make_wikipedia_source(rel["target"]) if not artist.references.filter(pk=source.pk).exists(): artist.references.add(source) # We can't 'clear' an alias list from artist because an alias # object requires an artist. # TODO Deleting these each time we overwrite means we churn the # alias ids. This may or may not be a good idea args = {} args[alias_ref] = artist AliasKlass.objects.filter(**args).delete() for alias in mbartist.get("alias-list", []): a = alias["alias"] primary = alias.get("primary") locale = alias.get("locale") args = {"alias": a} args[alias_ref] = artist aob, created = AliasKlass.objects.get_or_create(**args) if primary: aob.primary = True if locale: aob.locale = locale aob.save() external_data.import_artist_wikipedia(artist) return artist
def _add_release_performance(self, releaseid, artistid, perf_type, attrs): """If there is a relationship between a release and artist, apply the relationship to all recordings in the release""" logger.info(" Adding concert performance...") concert = carnatic.models.Concert.objects.get(mbid=releaseid) for rec in concert.recordings.all(): self._add_recording_performance(rec.mbid, artistid, perf_type, attrs)
def _add_release_performance(self, releaseid, artistid, instrument, is_lead): logger.info(" Adding concert performance...") artist = self.add_and_get_artist(artistid) instrument = self.get_instrument(instrument) if instrument: concert = makam.models.Concert.objects.get(mbid=releaseid) perf = makam.models.InstrumentConcertPerformance(concert=concert, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def _add_recording_artists(self, rec, artistids): rec.performers.clear() for a in artistids: # If the artist is [dialogue] the we don't show analysis. artist = self.add_and_get_artist(a) logger.info(" artist: %s" % artist) if not rec.performers.filter(pk=artist.pk).exists(): rec.performers.add(artist)
def _add_recording_artists(self, rec, artistids): rec.performers .clear() for a in artistids: # If the artist is [dialogue] the we don't show analysis. artist = self.add_and_get_artist(a) logger.info(" artist: %s" % artist) if not rec.performers.filter(pk=artist.pk).exists(): rec.performers.add(artist)
def add_recording_performance(self, recordingid, artistid, instrument, is_lead): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) instrument = self.get_instrument(instrument) if instrument: recording = carnatic.models.Recording.objects.get(mbid=recordingid) perf = carnatic.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def _create_artist_object(self, ArtistKlass, AliasKlass, mbartist, alias_ref="artist"): artistid = mbartist["id"] artist, created = ArtistKlass.objects.get_or_create( mbid=artistid, defaults={"name": mbartist["name"]}) if created or self.overwrite: logger.info(" adding artist/composer %s" % (artistid, )) source = self.make_mb_source("http://musicbrainz.org/artist/%s" % artistid) artist.source = source artist.name = mbartist["name"] if mbartist.get("type") == "Person": artist.artist_type = "P" elif mbartist.get("type") == "Group": artist.artist_type = "G" if mbartist.get("gender") == "Male": artist.gender = "M" elif mbartist.get("gender") == "Female": artist.gender = "F" dates = mbartist.get("life-span") if dates: artist.begin = dates.get("begin") artist.end = dates.get("end") artist.save() # add wikipedia references if they exist if self.overwrite: artist.references.clear() for rel in mbartist.get("url-relation-list", []): if rel["type-id"] == RELEASE_TYPE_WIKIPEDIA: source = self.make_wikipedia_source(rel["target"]) if not artist.references.filter(pk=source.pk).exists(): artist.references.add(source) if self.overwrite: # We can't 'clear' an alias list from artist because an alias # object requires an artist. # TODO Deleting these each time we overwrite means we churn the # alias ids. This may or may not be a good idea args = {} args[alias_ref] = artist AliasKlass.objects.filter(**args).delete() for alias in mbartist.get("alias-list", []): a = alias["alias"] primary = alias.get("primary") locale = alias.get("locale") args = {"alias": a} args[alias_ref] = artist aob, created = AliasKlass.objects.get_or_create(**args) if primary: aob.primary = True if locale: aob.locale = locale aob.save() external_data.import_artist_wikipedia(artist, self.overwrite) return artist
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) recording = carnatic.models.Recording.objects.get(mbid=recordingid) instrument, attributes, is_lead = self._performance_type_to_instrument(perf_type, attrs) if instrument: carnatic.models.InstrumentPerformance.objects.create(recording=recording, instrument=instrument, artist=artist, lead=is_lead, attributes=attributes)
def add_release_performance(self, releaseid, artistid, instrument, is_lead): # TODO: Can we de-duplicate this with the recording stuff above logger.info(" Adding concert performance...") artist = self.add_and_get_artist(artistid) instrument = self.get_instrument(instrument) if instrument: concert = carnatic.models.Concert.objects.get(mbid=releaseid) perf = carnatic.models.InstrumentConcertPerformance(concert=concert, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def _add_recording_performance(self, recordingid, artistid, instrument, is_lead): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) # Musicbrainz calls it 'vocal', but we want it to be 'voice' if instrument == "vocal": instrument = "voice" instrument = self.get_instrument(instrument) if instrument: recording = hindustani.models.Recording.objects.get(mbid=recordingid) perf = hindustani.models.InstrumentPerformance(recording=recording, instrument=instrument, artist=artist, lead=is_lead) perf.save()
def _add_recording_artists(self, rec, artistids): rec.artists.clear() for a in artistids: # If the artist is [dialogue] the we don't show analysis. if a == DIALOGUE_ARTIST: rec.analyse = False rec.save() artist = self.add_and_get_artist(a) logger.info(" artist: %s" % artist) if not rec.artists.filter(pk=artist.pk).exists(): logger.info(" - adding to artist list 2") rec.artists.add(artist)
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) is_lead = False if "lead" in attrs: is_lead = True instrument, attributes = self._performance_type_to_instrument(perf_type, attrs) recording = makam.models.Recording.objects.get(mbid=recordingid) makam.models.InstrumentPerformance.objects.create(recording=recording, instrument=instrument, artist=artist, attributes=attributes, lead=is_lead)
def _add_release_performance(self, releaseid, artistid, instrument, is_lead): logger.info(" Adding release performance...") artist = self.add_and_get_artist(artistid) instrument = self._get_instrument(instrument) if instrument: release = andalusian.models.Album.objects.get(mbid=releaseid) # For each recording in the release, see if the relationship # already exists. If not, create it. for rec in release.recordings.all(): if not andalusian.models.InstrumentPerformance.objects.filter( recording=rec, instrument=instrument, performer=artist).exists(): perf = andalusian.models.InstrumentPerformance(recording=rec, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead): logger.info(" Adding release performance...") artist = self.add_and_get_artist(artistid) instrument = self._get_instrument(instrument) if instrument: release = makam.models.Release.objects.get(mbid=releaseid) # For each recording in the release, see if the relationship # already exists. If not, create it. for rec in release.recordings.all(): if not makam.models.InstrumentPerformance.objects.filter( recording=rec, instrument=instrument, artist=artist).exists(): perf = makam.models.InstrumentPerformance(recording=rec, instrument=instrument, artist=artist, lead=is_lead) perf.save()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead): logger.info(" Adding concert performance...") artist = self.add_and_get_artist(artistid) instrument = self._get_instrument(instrument) concert = carnatic.models.Concert.objects.get(mbid=releaseid) # Instrument could be None if we don't know it for rec in concert.recordings.all(): if not carnatic.models.InstrumentPerformance.objects.filter( recording=rec, instrument=instrument, artist=artist).exists(): perf = carnatic.models.InstrumentPerformance(recording=rec, instrument=instrument, artist=artist, lead=is_lead) perf.save()
def _add_and_get_artist_composer(self, ArtistClass, artistid, addgroups=True): """ arguments: ArtistClass: models.Artist or models.Composer """ a = compmusic.mb.get_artist_by_id(artistid, includes=["url-rels", "artist-rels"])["artist"] artist, created = ArtistClass.objects.get_or_create(mbid=artistid, defaults={"name": a["name"]}) if not created: if artistid in self.imported_artists: print "Artist already updated in this import. Not doing it again" return artist if created or self.overwrite: logger.info(" adding artist/composer %s" % (artistid, )) source = self.make_mb_source("http://musicbrainz.org/artist/%s" % artistid) artist.source = source artist.name = a["name"] if a.get("type") == "Person": artist.artist_type = "P" elif a.get("type") == "Group": artist.artist_type = "G" if a.get("gender") == "Male": artist.gender = "M" elif a.get("gender") == "Female": artist.gender = "F" dates = a.get("life-span") if dates: artist.begin = dates.get("begin") artist.end = dates.get("end") artist.save() # TODO: Annoying hack to only work on artists if ArtistClass == carnatic.models.Artist and addgroups: if self.overwrite: artist.group_members.clear() for member in a.get("artist-relation-list", []): if "member" in member["type"] and member.get("direction") == "backward": memberartist = self._add_and_get_artist_composer(ArtistClass, member["target"]) artist.group_members.add(memberartist) # add wikipedia references if they exist for rel in a.get("url-relation-list", []): if rel["type"] == ["wikipedia"]: source = self.make_wikipedia_source(rel["url"]) if not artist.references.filter(pk=source.pk).exists(): artist.references.add(source) external_data.import_artist_bio(artist, self.overwrite) return artist
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) instr_name = attrs[-1] attrs = attrs[:-1] is_lead = False if "lead" in attrs: is_lead = True instrument = self._get_instrument(instr_name) if instrument: recording = andalusian.models.Recording.objects.get(mbid=recordingid) perf = andalusian.models.InstrumentPerformance(recording=recording, instrument=instrument, performer=artist, lead=is_lead) perf.save()
def add_and_get_recording(self, recordingid): mbrec = compmusic.mb.get_recording_by_id( recordingid, includes=["tags", "work-rels", "artist-rels", "artists"]) mbrec = mbrec["recording"] rec, created = self._RecordingClass.objects.get_or_create( mbid=recordingid) logger.info(" adding recording %s" % (recordingid, )) import_logger.info("importing recording %s", mbrec["title"]) source = self.make_mb_source("http://musicbrainz.org/recording/%s" % recordingid) rec.source = source rec.length = mbrec.get("length") rec.title = mbrec["title"] rec.save() artistids = [] # Create recording primary artists for a in mbrec.get("artist-credit", []): if isinstance(a, dict): artistid = a["artist"]["id"] artistids.append(artistid) self._add_recording_artists(rec, artistids) works = [] for work in mbrec.get("work-relation-list", []): if work["type"] == "performance": w = self.add_and_get_work(work["target"]) works.append(w) tags = mbrec.get("tag-list", []) # Join recording and works in a subclass because some models # have 1 work per recording and others have many self._join_recording_and_works(rec, works) # Sometime we attach tags to works, sometimes to recordings self._apply_tags(rec, works, tags) IPClass = rec.get_object_map("performance") IPClass.objects.filter(recording=rec).delete() for perf in self._get_artist_performances( mbrec.get("artist-relation-list", [])): artistid, perf_type, attrs = perf self._add_recording_performance(recordingid, artistid, perf_type, attrs) return rec
def _add_recording_performance(self, recordingid, artistid, perf_type, attrs): logger.info(" Adding recording performance...") artist = self.add_and_get_artist(artistid) is_lead = False if "lead" in attrs: is_lead = True instrument, attributes = self._performance_type_to_instrument( perf_type, attrs) recording = makam.models.Recording.objects.get(mbid=recordingid) makam.models.InstrumentPerformance.objects.create( recording=recording, instrument=instrument, artist=artist, attributes=attributes, lead=is_lead)
def _performance_type_to_instrument(self, perf_type, attrs): is_lead = False if perf_type in [release_importer.RELATION_RECORDING_VOCAL, release_importer.RELATION_RELEASE_VOCAL]: instr_name = "voice" if "lead vocals" in attrs: is_lead = True elif perf_type in [release_importer.RELATION_RECORDING_INSTRUMENT, release_importer.RELATION_RELEASE_INSTRUMENT]: instr_name = attrs[-1] attrs = attrs[:-1] if "lead" in attrs: is_lead = True else: logger.info(" Unknown performance type: {}".format(perf_type)) instr_name = None attributes = " ".join(attrs) instrument = self._get_instrument(instr_name) return instrument, attributes, is_lead
def _create_recording_and_works(self, recording, works, tags): rec, created = carnatic.models.Recording.objects.get_or_create(mbid=recordingid) if created or self.overwrite: logger.info(" adding recording %s" % (recordingid,)) raagas = self._get_raaga(mbrec.get("tag-list", [])) taalas = self._get_taala(mbrec.get("tag-list", [])) source = self.make_mb_source("http://musicbrainz.org/recording/%s" % recordingid) rec.source = source rec.length = mbrec.get("length") rec.title = mbrec["title"] rec.save() if not created and self.overwrite: rec.performance.clear() for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])): artistid, instrument, is_lead = perf self.add_recording_performance(recordingid, artistid, instrument, is_lead) return rec
def delete_moduleversion(vid): version = models.ModuleVersion.objects.get(pk=vid) logger.info("deleting moduleversion %s" % version) files = version.derivedfile_set.all() for f in files: for pn in range(1, f.num_parts + 1): path = f.full_path_for_part(pn) try: os.unlink(path) dirname = os.path.dirname(path) if len(os.listdir(dirname)) == 0: # If this directory is empty, remove it and all empty # parent dirs. May throw OSError (not really an error) os.removedirs(dirname) except OSError: pass # if the file doesn't exist, not really an error f.delete() module = version.module version.delete() logger.info("done")
def add_and_get_recording(self, recordingid): mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags", "work-rels", "artist-rels", "artists"]) mbrec = mbrec["recording"] rec, created = self._RecordingClass.objects.get_or_create(mbid=recordingid) logger.info(" adding recording %s" % (recordingid,)) import_logger.info("importing recording %s", mbrec["title"]) rec.length = mbrec.get("length") rec.title = mbrec["title"] rec.save() artistids = [] # Create recording primary artists for a in mbrec.get("artist-credit", []): if isinstance(a, dict): artistid = a["artist"]["id"] artistids.append(artistid) self._add_recording_artists(rec, artistids) works = [] for work in mbrec.get("work-relation-list", []): if work["type"] == "performance": w = self.add_and_get_work(work["target"]) works.append(w) tags = mbrec.get("tag-list", []) # Join recording and works in a subclass because some models # have 1 work per recording and others have many self._join_recording_and_works(rec, works) # Sometime we attach tags to works, sometimes to recordings self._apply_tags(rec, works, tags) IPClass = rec.get_object_map("performance") IPClass.objects.filter(recording=rec).delete() for perf in self._get_artist_performances(mbrec.get("artist-relation-list", [])): artistid, perf_type, attrs = perf self._add_recording_performance(recordingid, artistid, perf_type, attrs) return rec
def run_module_on_recordings(moduleid, recids): module = models.Module.objects.get(pk=moduleid) version = module.get_latest_version() logger.info("running module %s on %s files" % (module, len(recids))) if version: logger.info("version %s %s" % (version, version.pk)) # All documents that don't already have a derived file for this module version docs = models.Document.objects.filter( sourcefiles__file_type=module.source_type, external_identifier__in=recids, ).exclude(derivedfiles__module_version=version) for d in docs: logger.info(" document %s" % d) logger.info(" docid %s" % d.pk) process_document.delay(d.pk, version.pk)
def import_release(self, releaseid, directories): if releaseid in self.imported_releases: print("Release already updated in this import. Not doing it again") return self._ReleaseClass.objects.get(mbid=releaseid) rel = compmusic.mb.get_release_by_id(releaseid, includes=[ "artists", "recordings", "artist-rels", "release-groups" ]) rel = rel["release"] mbid = rel["id"] logger.info("Adding release %s" % mbid) release = self._create_release_object(rel) # Create release primary artists release.artists.clear() for a in rel["artist-credit"]: if isinstance(a, dict): artistid = a["artist"]["id"] artist = self.add_and_get_release_artist(artistid) logger.info(" artist: %s" % artist) if not release.artists.filter(pk=artist.pk).exists(): logger.info(" - adding to artist list") release.artists.add(artist) recordings = [] for mnum, medium in enumerate(rel["medium-list"], 1): for tnum, track in enumerate(medium["track-list"], 1): recordings.append((track["recording"]["id"], mnum, tnum)) release.recordings.clear() trackorder = 1 for recid, mnum, tnum in recordings: recob = self.add_and_get_recording(recid) self._link_release_recording(release, recob, trackorder, mnum, tnum) trackorder += 1 for perf in self._get_artist_performances( rel.get("artist-relation-list", [])): artistid, perf_type, attrs = perf self._add_release_performance(release.mbid, artistid, perf_type, attrs) self._add_release_artists_as_relationship(release, rel["artist-credit"]) self._add_image_to_release(release, directories) self.imported_releases.append(releaseid) return release
def import_release(self, releaseid, directories): if releaseid in self.imported_releases: print "Release already updated in this import. Not doing it again" return self._ReleaseClass.objects.get(mbid=releaseid) rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels"]) rel = rel["release"] mbid = rel["id"] logger.info("Adding release %s" % mbid) release = self._create_release_object(rel) # Create release primary artists if self.overwrite: # If it already exists and we're doing an overwrite release.artists.clear() for a in rel["artist-credit"]: if isinstance(a, dict): artistid = a["artist"]["id"] artist = self.add_and_get_artist(artistid) logger.info(" artist: %s" % artist) if not release.artists.filter(pk=artist.pk).exists(): logger.info(" - adding to artist list") release.artists.add(artist) recordings = [] for medium in rel["medium-list"]: for track in medium["track-list"]: recordings.append(track["recording"]["id"]) print recordings if self.overwrite: release.tracks.clear() trackorder = 1 for recid in recordings: recob = self.add_and_get_recording(recid) self._link_release_recording(release, recob, trackorder) trackorder += 1 if self.overwrite: self._clear_release_performances(release) for perf in self._get_artist_performances(rel.get("artist-relation-list", [])): artistid, instrument, is_lead = perf self._add_release_performance(release.mbid, artistid, instrument, is_lead) external_data.import_release_image(release, directories, self.overwrite) self.imported_releases.append(releaseid) return release
def delete_moduleversion(vid): version = models.ModuleVersion.objects.get(pk=vid) logger.info("deleting moduleversion %s" % version) files = version.derivedfile_set.all() for f in files: for p in f.parts.all(): path = p.fullpath try: os.unlink(path) dirname = os.path.dirname(path) if len(os.listdir(dirname)) == 0: # If this directory is empty, remove it and all empty # parent dirs. May throw OSError (not really an error) os.removedirs(dirname) except OSError: pass # if the file doesn't exist, not really an error f.delete() module = version.module version.delete() if module.versions.count() == 0: logger.info("No more moduleversions for this module, deleting the module") module.delete() logger.info(" .. module deleted") logger.info("done")
def import_release(self, releaseid, directories): if releaseid in self.imported_releases: print "Release already updated in this import. Not doing it again" return self._ReleaseClass.objects.get(mbid=releaseid) rel = compmusic.mb.get_release_by_id(releaseid, includes=["artists", "recordings", "artist-rels", "release-groups"]) rel = rel["release"] mbid = rel["id"] logger.info("Adding release %s" % mbid) release = self._create_release_object(rel) # Create release primary artists release.artists.clear() for a in rel["artist-credit"]: if isinstance(a, dict): artistid = a["artist"]["id"] artist = self.add_and_get_release_artist(artistid) logger.info(" artist: %s" % artist) if not release.artists.filter(pk=artist.pk).exists(): logger.info(" - adding to artist list") release.artists.add(artist) recordings = [] for mnum, medium in enumerate(rel["medium-list"], 1): for tnum, track in enumerate(medium["track-list"], 1): recordings.append( (track["recording"]["id"], mnum, tnum)) release.recordings.clear() trackorder = 1 for recid, mnum, tnum in recordings: recob = self.add_and_get_recording(recid) self._link_release_recording(release, recob, trackorder, mnum, tnum) trackorder += 1 for perf in self._get_artist_performances(rel.get("artist-relation-list", [])): artistid, perf_type, attrs = perf self._add_release_performance(release.mbid, artistid, perf_type, attrs) self._add_release_artists_as_relationship(release, rel["artist-credit"]) external_data.import_release_image(release, directories) self.imported_releases.append(releaseid) return release
def run_module_on_collection(collectionid, moduleid, versionid=None): collection = models.Collection.objects.get(pk=collectionid) module = models.Module.objects.get(pk=moduleid) if versionid: version = module.versions.get(pk=versionid) else: version = module.get_latest_version() logger.info("running module %s on collection %s" % (module, collection)) if version: logger.info("version %s" % version) if module.many_files: process_collection.delay(collection.pk, version.pk) else: # All documents that don't already have a derived file for this module version docs = models.Document.objects.filter( sourcefiles__file_type=module.source_type, collections=collection).exclude(derivedfiles__module_version=version) for i, d in enumerate(docs, 1): logger.info(" document %s/%s - %s" % (i, len(docs), d)) process_document.delay(d.pk, version.pk)
def remove_deleted_items(): """ Search musicbrainz for all items in the database and if they have been deleted then remove them """ logger.info("Scanning works...") for w in carnatic.models.Work.objects.all(): try: mbwork = mb.get_work_by_id(w.mbid) except mb.ResponseError: logger.info("work %s (%s) missing; deleting" % (w, w.mbid)) w.delete() logger.info("Scanning recordings...") for r in carnatic.models.Recording.objects.all(): try: mbrec = mb.get_recording_by_id(r.mbid) except mb.ResponseError: logger.info("recording %s (%s) missing; deleting" % (r, r.mbid)) r.delete() logger.info("Scanning concerts...") for c in carnatic.models.Concert.objects.all(): try: mbcon = mb.get_release_by_id(c.mbid) except mb.ResponseError: logger.info("release %s (%s) missing; deleting" % (c, c.mbid)) c.delete() logger.info("Scanning artists...") for a in carnatic.models.Artist.objects.all(): try: mbart = mb.get_artist_by_id(a.mbid) except mb.ResponseError: # We import dummy artists to be gurus, leave them here if not a.dummy: logger.info("artist %s (%s) missing; deleting" % (a, a.mbid)) a.delete() logger.info("Scanning composers...") for a in carnatic.models.Composer.objects.all(): try: mbart = mb.get_artist_by_id(a.mbid) except mb.ResponseError: logger.info("artist %s (%s) missing; deleting" % (a, a.mbid)) a.delete()
def _add_release_performance(self, releaseid, artistid, instrument, is_lead): logger.info(" Adding concert performance to all tracks...") release = hindustani.models.Release.objects.get(mbid=releaseid) for t in release.tracks: self._add_recording_performance(t.mbid, artistid, instrument, is_lead)
def delete_module(mid): module = models.Module.objects.get(pk=mid) logger.info("deleting entire module %s" % module) for v in module.versions.all(): delete_moduleversion(v.pk) logger.info("done")
def _add_release_performance(self, releaseid, artistid, perf_type, attrs): logger.info(" Adding concert performance to all recordings...") release = hindustani.models.Release.objects.get(mbid=releaseid) for t in release.recordings.all(): self._add_recording_performance(t.mbid, artistid, perf_type, attrs)