def task(self, collectionfile_id):
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
complete = True
data = {}
m = meta["meta"]
data.update(m)
if not m["artist"]:
complete = False
data["artist_missing"] = True
if not m["title"]:
complete = False
data["title_missing"] = True
if not m["release"]:
complete = False
data["release_missing"] = True
if not m["artistid"]:
complete = False
data["artistid_missing"] = True
if not m["releaseid"]:
complete = False
data["releaseid_missing"] = True
if not m["recordingid"]:
complete = False
data["recordingid_missing"] = True
return (complete, data)
def task(self, collectionfile_id):
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
complete = True
data = {}
m = meta["meta"]
data.update(m)
if not m["artist"]:
complete = False
data["artist_missing"] = True
if not m["title"]:
complete = False
data["title_missing"] = True
if not m["release"]:
complete = False
data["release_missing"] = True
if not m["artistid"]:
complete = False
data["artistid_missing"] = True
if not m["releaseid"]:
complete = False
data["releaseid_missing"] = True
if not m["recordingid"]:
complete = False
data["recordingid_missing"] = True
return (complete, data)
def _check_existing_directories(coll):
# For all of the matched directories, look at the contents of them and
# remove/add files as needed
# We don't remove from the docserver because it's not very important to
# remove them, and it's complex to cover all cases. We do this separately
for cd in coll.collectiondirectory_set.all():
files = os.listdir(cd.full_path)
mp3files = _get_mp3_files(files)
existing_f = cd.collectionfile_set.all()
existing_names = [f.name for f in existing_f]
to_remove = set(existing_names) - set(mp3files)
to_add = set(mp3files) - set(existing_names)
same_files = set(mp3files) & set(existing_names)
for rm in to_remove:
# If it was renamed, we will make a CollectionFile in
# the to_add block below
cd.collectionfile_set.get(name=rm).delete()
for f in same_files:
# Look through all files and see if the mbid has changed. If it's changed,
# update the reference
fileobject = cd.collectionfile_set.get(name=f)
meta = compmusic.file_metadata(os.path.join(cd.full_path, f))
recordingid = meta["meta"].get("recordingid")
if recordingid != fileobject.recordingid:
fileobject.recordingid = recordingid
fileobject.save()
if to_add:
# If there are new files in the directory, just run _match again and
# it will create the new file objects
_match_directory_to_release(coll.collectionid, cd.full_path)
def _check_existing_directories(coll):
# For all of the matched directories, look at the contents of them and
# remove/add files as needed
# We don't remove from the docserver because it's not very important to
# remove them, and it's complex to cover all cases. We do this separately
for cd in coll.collectiondirectory_set.all():
files = os.listdir(cd.full_path)
mp3files = _get_mp3_files(cd.full_path, files)
existing_f = cd.collectionfile_set.all()
existing_names = [f.name for f in existing_f]
to_remove = set(existing_names) - set(mp3files)
to_add = set(mp3files) - set(existing_names)
same_files = set(mp3files) & set(existing_names)
for rm in to_remove:
# If it was renamed, we will make a CollectionFile in
# the to_add block below
cd.collectionfile_set.get(name=rm).delete()
for f in same_files:
# Look through all files and see if the mbid has changed. If it's changed,
# update the reference
fileobject = cd.collectionfile_set.get(name=f)
meta = compmusic.file_metadata(os.path.join(cd.full_path, f))
recordingid = meta["meta"].get("recordingid")
if recordingid != fileobject.recordingid:
fileobject.recordingid = recordingid
fileobject.save()
if to_add:
# If there are new files in the directory, just run _match again and
# it will create the new file objects
_match_directory_to_release(coll.collectionid, cd.full_path)
def directory(request, dirid):
""" A directory that wasn't matched to a release in the collection.
This could be because it has no release tags, or the release isn't in
the collection.
We want to group together as much common information as possible, and
link to musicbrainz if we can.
"""
directory = get_object_or_404(models.CollectionDirectory, pk=dirid)
rematch = request.GET.get("rematch")
if rematch is not None:
# TODO: Change to celery
jobs.rematch_unknown_directory(dirid)
get_object_or_404(models.CollectionDirectory, pk=dirid)
return redirect('dashboard-directory', dirid)
full_path = directory.full_path
files = os.listdir(full_path)
releaseids = set()
releasename = set()
artistids = set()
artistname = set()
for f in files:
fname = os.path.join(full_path, f)
if compmusic.is_mp3_file(fname):
data = compmusic.file_metadata(fname)
relid = data["meta"]["releaseid"]
relname = data["meta"]["release"]
aname = data["meta"]["artist"]
aid = data["meta"]["artistid"]
if relid and relname:
releaseids.add(relid)
releasename.add(relname)
if aname and aid:
artistids.add(aid)
artistname.add(aname)
got_release_id = len(releaseids) == 1
# TODO: This won't work if there are more than 1 lead artist?
got_artist = len(artistids) == 1
if directory.musicbrainzrelease:
matched_release = directory.musicbrainzrelease
else:
matched_release = None
ret = {
"files": sorted(files),
"directory": directory,
"got_release_id": got_release_id,
"got_artist": got_artist,
"matched_release": matched_release
}
if got_release_id:
ret["releasename"] = list(releasename)[0]
ret["releaseid"] = list(releaseids)[0]
if got_artist:
ret["artistname"] = list(artistname)[0]
ret["artistid"] = list(artistids)[0]
return render(request, 'dashboard/directory.html', ret)
def directory(request, dirid):
""" A directory that wasn't matched to a release in the collection.
This could be because it has no release tags, or the release isn't in
the collection.
We want to group together as much common information as possible, and
link to musicbrainz if we can.
"""
directory = get_object_or_404(models.CollectionDirectory, pk=dirid)
rematch = request.GET.get("rematch")
if rematch is not None:
# TODO: Change to celery
jobs.rematch_unknown_directory(dirid)
directory = get_object_or_404(models.CollectionDirectory, pk=dirid)
return HttpResponseRedirect(reverse('dashboard.views.directory', args=[dirid]))
collection = directory.collection
full_path = os.path.join(collection.root_directory, directory.path)
files = os.listdir(full_path)
meta = {}
releaseids = set()
releasename = set()
artistids = set()
artistname = set()
for f in files:
fname = os.path.join(full_path, f)
if compmusic.is_mp3_file(fname):
data = compmusic.file_metadata(fname)
relid = data["meta"]["releaseid"]
relname = data["meta"]["release"]
aname = data["meta"]["artist"]
aid = data["meta"]["artistid"]
if relid and relname:
releaseids.add(relid)
releasename.add(relname)
if aname and aid:
artistids.add(aid)
artistname.add(aname)
got_release_id = len(releaseids) == 1
# TODO: This won't work if there are more than 1 lead artist?
got_artist = len(artistids) == 1
print "releaseids", releaseids
print "artistids", artistids
if directory.musicbrainzrelease:
matched_release = directory.musicbrainzrelease
else:
matched_release = None
ret = {"files": sorted(files), "directory": directory, "got_release_id": got_release_id, "got_artist": got_artist, "matched_release": matched_release}
if got_release_id:
ret["releasename"] = list(releasename)[0]
ret["releaseid"] = list(releaseids)[0]
if got_artist:
ret["artistname"] = list(artistname)[0]
ret["artistid"] = list(artistids)[0]
return render(request, 'dashboard/directory.html', ret)
def docserver_add_mp3(collectionid, releaseid, fpath, recordingid):
meta = compmusic.file_metadata(fpath)
mp3type = models.SourceFileType.objects.get_by_extension("mp3")
title = meta["meta"].get("title")
try:
doc = models.Document.objects.get_by_external_id(recordingid)
docserver_add_sourcefile(doc.pk, mp3type, fpath)
except models.Document.DoesNotExist:
docserver_add_document(collectionid, mp3type, title, fpath, recordingid)
def docserver_add_mp3(collectionid, releaseid, fpath, recordingid):
meta = compmusic.file_metadata(fpath)
mp3type = models.SourceFileType.objects.get_by_extension("mp3")
title = meta["meta"].get("title")
try:
doc = models.Document.objects.get_by_external_id(recordingid)
docserver_add_sourcefile(doc.pk, mp3type, fpath)
except models.Document.DoesNotExist:
docserver_add_document(collectionid, mp3type, title, fpath, recordingid)
def run_file(module, filename, mbid=None):
if not mbid:
if filename.lower().endswith(".mp3"):
md = compmusic.file_metadata(filename)
mbid = md["meta"]["recordingid"]
if mbid:
module.musicbrainz_id = mbid
ret = module.run(mbid, filename)
save_data(module, ret)
else:
logging.error("Cannot find a mbid in this file. Use the mbid argument")
def run_file(module, filename, mbid=None):
if not mbid:
if filename.lower().endswith(".mp3"):
md = compmusic.file_metadata(filename)
mbid = md["meta"]["recordingid"]
if mbid:
module.musicbrainz_id = mbid
ret = module.run(filename)
save_data(module, ret)
else:
logging.error("Cannot find a mbid in this file. Use the mbid argument")
def _get_musicbrainz_release_for_dir(dirname):
""" Get a unique list of all the musicbrainz release IDs that
are in tags in mp3 files in the given directory.
"""
release_ids = set()
for fname in _get_mp3_files(os.listdir(dirname)):
fpath = os.path.join(dirname, fname)
meta = compmusic.file_metadata(fpath)
rel = meta["meta"]["releaseid"]
if rel:
release_ids.add(rel)
return list(release_ids)
def _get_musicbrainz_release_for_dir(dirname):
""" Get a unique list of all the musicbrainz release IDs that
are in tags in mp3 files in the given directory.
"""
release_ids = set()
for fname in _get_mp3_files(os.listdir(dirname)):
fpath = os.path.join(dirname, fname)
meta = compmusic.file_metadata(fpath)
rel = meta["meta"]["releaseid"]
if rel:
release_ids.add(rel)
return list(release_ids)
def task(self, collectionfile_id):
import hindustani_importer
hi = hindustani_importer.HindustaniReleaseImporter()
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
m = meta["meta"]
recordingid = m["recordingid"]
if not recordingid:
thefile.add_log_message("This file has no recording id tag")
mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags"])
mbrec = mbrec["recording"]
tags = mbrec.get("tag-list", [])
res = {}
raags = hi._get_raag_tags(tags)
taals = hi._get_taal_tags(tags)
layas = hi._get_laya_tags(tags)
forms = hi._get_form_tags(tags)
res["recordingid"] = recordingid
missingr = [r for _, r in raags if hi._get_raag(r) is None]
missingt = [t for _, t in taals if hi._get_taal(t) is None]
missingf = [f for _, f in forms if hi._get_form(f) is None]
missingl = [l for _, l in layas if hi._get_laya(l) is None]
if missingr:
res["missingr"] = missingr
if missingt:
res["missingt"] = missingt
if missingf:
res["missingf"] = missingf
if missingl:
res["missingl"] = missingl
res["gotraag"] = len(raags) > 0
res["gottaal"] = len(taals) > 0
res["gotform"] = len(forms) > 0
res["gotlaya"] = len(layas) > 0
res["raag"] = [r for _, r in raags]
res["taal"] = [t for _, t in taals]
res["form"] = [f for _, f in forms]
res["laya"] = [l for _, l in layas]
r = raags and not missingr
t = taals and not missingt
f = forms and not missingf
l = layas and not missingl
if r and t and f and l:
return (True, res)
else:
return (False, res)
def _create_collectionfile(cd, name):
"""arguments:
cd: a collectiondirectory
name: the name of the file, with no path information
"""
path = os.path.join(cd.full_path, name)
meta = compmusic.file_metadata(path)
recordingid = meta["meta"].get("recordingid")
size = os.path.getsize(path)
cfile, created = models.CollectionFile.objects.get_or_create(name=name, directory=cd, recordingid=recordingid, defaults={'filesize': size})
if not created:
cfile.filesize = size
cfile.save()
def docserver_add_mp3(collectionid, releaseid, fpath, recordingid):
meta = compmusic.file_metadata(fpath)
mp3type = models.SourceFileType.objects.get_by_slug("mp3")
title = meta["meta"].get("title")
try:
document = models.Document.objects.get_by_external_id(recordingid)
collection = models.Collection.objects.get(collectionid=collectionid)
if collection not in document.collections.all():
document.collections.add(collection)
except models.Document.DoesNotExist:
document = docserver_create_document(collectionid, recordingid, title)
docserver_add_sourcefile(document.pk, mp3type.pk, fpath)
def task(self, collectionfile_id):
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
m = meta["meta"]
recordingid = m["recordingid"]
mbrec = compmusic.mb.get_recording_by_id(recordingid, includes=["tags"])
mbrec = mbrec["recording"]
tags = mbrec.get("tag-list", [])
res = {}
raagas = []
taalas = []
for t in tags:
tag = t["name"]
if compmusic.tags.has_raaga(tag):
raagas.append(compmusic.tags.parse_raaga(tag))
if compmusic.tags.has_taala(tag):
taalas.append(compmusic.tags.parse_taala(tag))
res["recordingid"] = recordingid
missingr = []
missingt = []
for r in raagas:
try:
carnatic.models.Raaga.objects.fuzzy(r)
except carnatic.models.Raaga.DoesNotExist:
try:
carnatic.models.RaagaAlias.objects.fuzzy(r)
except carnatic.models.RaagaAlias.DoesNotExist:
missingr.append(r)
if missingr:
res["missingr"] = missingr
for t in taalas:
try:
carnatic.models.Taala.objects.fuzzy(t)
except carnatic.models.Taala.DoesNotExist:
try:
carnatic.models.TaalaAlias.objects.fuzzy(t)
except carnatic.models.TaalaAlias.DoesNotExist:
missingt.append(t)
if missingt:
res["missingt"] = missingt
res["gotraaga"] = len(raagas) > 0
res["gottaala"] = len(taalas) > 0
res["raaga"] = raagas
res["taala"] = taalas
if raagas and taalas and not missingt and not missingr:
return (True, res)
else:
return (False, res)
def _create_collectionfile(cd, name):
"""arguments:
cd: a collectiondirectory
name: the name of the file, with no path information
"""
path = os.path.join(cd.full_path, name)
meta = compmusic.file_metadata(path)
recordingid = meta["meta"].get("recordingid")
size = os.path.getsize(path)
cfile, created = models.CollectionFile.objects.get_or_create(
name=name,
directory=cd,
recordingid=recordingid,
defaults={'filesize': size})
if not created:
cfile.filesize = size
cfile.save()
def jingjuRecordingIDreader(folder):
mp3Files = []
recordingIDs = []
for name in listdir(folder):
if not name.startswith('.'):
name = "/Users/gong/Documents/MTG document/Jingju arias/京剧之星/" + name + '/'
files = [ join(name, f) for f in listdir(name) if isfile(join(name,f)) and f.endswith('.mp3')]
mp3Files = mp3Files + files
ii = 1
length = len(mp3Files)
for f in mp3Files:
recordingIDs.append(cm.file_metadata(f)['meta']['recordingid'])
print 'reading recording ID ', ii, 'of total', length
ii += 1
return recordingIDs
def _match_directory_to_release(collectionid, root):
""" Try and match a single directory containing audio files to a release
that exists in the given collection.
collectionid: the ID of the collection we want the directory to be in
root: the root path of the directory containing audio files
"""
coll = models.Collection.objects.get(pk=collectionid)
collectionroot = coll.root_directory
print root
print "======="
# Remove the path to the collection from the path to the files (we only store relative paths)
if not collectionroot.endswith("/"):
collectionroot += "/"
if root.startswith(collectionroot):
# This should always be true since we scan from the collection root
shortpath = root[len(collectionroot):]
else:
shortpath = root
# Try and find the musicbrainz release for the files in the directory
cd, created = models.CollectionDirectory.objects.get_or_create(collection=coll, path=shortpath)
rels = _get_musicbrainz_release_for_dir(root)
if len(rels) == 1:
releaseid = rels[0]
try:
therelease = models.MusicbrainzRelease.objects.get(mbid=releaseid, collection=coll)
cd.musicbrainzrelease = therelease
cd.save()
cd.add_log_message("Successfully matched to a release", None)
mp3files = _get_mp3_files(os.listdir(root))
for f in mp3files:
meta = compmusic.file_metadata(os.path.join(root, f))
recordingid = meta["meta"].get("recordingid")
cfile, created = models.CollectionFile.objects.get_or_create(name=f, directory=cd, recordingid=recordingid)
except models.MusicbrainzRelease.DoesNotExist:
cd.add_log_message("Cannot find this directory's release (%s) in the imported releases" % (releaseid, ), None)
elif len(rels) == 0:
cd.add_log_message("No releases found in ID3 tags", None)
else:
cd.add_log_message("More than one release found in ID3 tags", None)
def _match_directory_to_release(collectionid, root):
""" Try and match a single directory containing audio files to a release
that exists in the given collection.
collectionid: the ID of the collection we want the directory to be in
root: the root path of the directory containing audio files
"""
coll = models.Collection.objects.get(pk=collectionid)
collectionroot = coll.root_directory
print root
print "======="
# Remove the path to the collection from the path to the files (we only store relative paths)
if not collectionroot.endswith("/"):
collectionroot += "/"
if root.startswith(collectionroot):
# This should always be true since we scan from the collection root
shortpath = root[len(collectionroot):]
else:
shortpath = root
# Try and find the musicbrainz release for the files in the directory
cd, created = models.CollectionDirectory.objects.get_or_create(collection=coll, path=shortpath)
rels = _get_musicbrainz_release_for_dir(root)
if len(rels) == 1:
releaseid = rels[0]
try:
therelease = models.MusicbrainzRelease.objects.get(mbid=releaseid, collection=coll)
cd.musicbrainzrelease = therelease
cd.save()
cd.add_log_message("Successfully matched to a release", None)
mp3files = _get_mp3_files(os.listdir(root))
for f in mp3files:
meta = compmusic.file_metadata(os.path.join(root, f))
recordingid = meta["meta"].get("recordingid")
cfile, created = models.CollectionFile.objects.get_or_create(name=f, directory=cd, recordingid=recordingid)
except models.MusicbrainzRelease.DoesNotExist:
cd.add_log_message("Cannot find this directory's release (%s) in the imported releases" % (releaseid, ), None)
elif len(rels) == 0:
cd.add_log_message("No releases found in ID3 tags", None)
else:
cd.add_log_message("More than one release found in ID3 tags", None)
def task(self, collectionfile_id):
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
m = meta["meta"]
recordingid = m["recordingid"]
mbrec = compmusic.mb.get_recording_by_id(recordingid,
includes=["tags", "work-rels"])["recording"]
tags = mbrec.get("tag-list", [])
res = {}
m, u, f = self._get_tags_from_list(tags)
recms = m[:]
recus = u[:]
recfs = f[:]
works = mbrec.get("work-relation-list", [])
wkms = []
wkus = []
wkfs = []
# If there's a work, get its id and the id of its composer
# Also scan for makam/usul tags here too
for w in works:
if w["type"] == "performance":
wid = w["work"]["id"]
mbwork = compmusic.mb.get_work_by_id(wid, includes=["tags"])["work"]
tags = mbwork.get("tag-list", [])
m, u, f = self._get_tags_from_list(tags)
wkms.extend(m)
wkus.extend(u)
wkfs.extend(f)
wkms = set(wkms)
wkus = set(wkus)
wkfs = set(wkfs)
recms = set(recms)
recus = set(recus)
recfs = set(recfs)
res["makamerr"] = []
res["usulerr"] = []
res["formerr"] = []
if wkms and recms and wkms != recms:
res["makamerr"].append("Work and Recording makams don't match")
if wkus and recus and wkus != recus:
res["usulerr"].append("Work and Recording usuls don't match")
if wkfs and recfs and wkfs != recfs:
res["formerr"].append("Work and Recording forms don't match")
# de-duplicate the lists
res["makams"] = list(wkms | recms)
res["usuls"] = list(wkus | recus)
res["forms"] = list(wkfs | recfs)
res["recordingid"] = recordingid
missingm = []
for m in res["makams"]:
try:
makam.models.Makam.objects.fuzzy(name=m)
except makam.models.Makam.DoesNotExist:
missingm.append(m)
for u in res["usuls"]:
try:
makam.models.Usul.objects.fuzzy(name=u)
except makam.models.Usul.DoesNotExist:
missingu.append(u)
for f in res["form"]:
try:
makam.models.Form.objects.fuzzy(name=f)
except makam.models.Form.DoesNotExist:
missingf.append(f)
if missingm:
res["missingm"] = missingm
if missingu:
res["missingu"] = missingu
if missingf:
res["missingf"] = missingf
res["gotmakam"] = len(res["makams"]) > 0
res["gotusul"] = len(res["usuls"]) > 0
res["gotform"] = len(res["forms"]) > 0
if res["makams"] and res["usuls"] and res["forms"]:
return (True, res)
else:
return (False, res)
def task(self, collectionfile_id):
thefile = models.CollectionFile.objects.get(pk=collectionfile_id)
fpath = thefile.path
meta = compmusic.file_metadata(fpath)
m = meta["meta"]
recordingid = m["recordingid"]
mbrec = compmusic.mb.get_recording_by_id(recordingid,
includes=["tags", "work-rels"])["recording"]
tags = mbrec.get("tag-list", [])
res = {}
m, u, f = self._get_tags_from_list(tags)
recms = m[:]
recus = u[:]
recfs = f[:]
works = mbrec.get("work-relation-list", [])
wkms = []
wkus = []
wkfs = []
# If there's a work, get its id and the id of its composer
# Also scan for makam/usul tags here too
for w in works:
if w["type"] == "performance":
wid = w["work"]["id"]
mbwork = compmusic.mb.get_work_by_id(wid, includes=["tags"])["work"]
tags = mbwork.get("tag-list", [])
m, u, f = self._get_tags_from_list(tags)
wkms.extend(m)
wkus.extend(u)
wkfs.extend(f)
wkms = set(wkms)
wkus = set(wkus)
wkfs = set(wkfs)
recms = set(recms)
recus = set(recus)
recfs = set(recfs)
res["makamerr"] = []
res["usulerr"] = []
res["formerr"] = []
if wkms and recms and wkms != recms:
res["makamerr"].append("Work and Recording makams don't match")
if wkus and recus and wkus != recus:
res["usulerr"].append("Work and Recording usuls don't match")
if wkfs and recfs and wkfs != recfs:
res["formerr"].append("Work and Recording forms don't match")
# de-duplicate the lists
res["makams"] = list(wkms | recms)
res["usuls"] = list(wkus | recus)
res["forms"] = list(wkfs | recfs)
res["recordingid"] = recordingid
missingm = []
for m in res["makams"]:
try:
makam.models.Makam.objects.fuzzy(name=m)
except makam.models.Makam.DoesNotExist:
missingm.append(m)
for u in res["usuls"]:
try:
makam.models.Usul.objects.fuzzy(name=u)
except makam.models.Usul.DoesNotExist:
missingu.append(u)
for f in res["form"]:
try:
makam.models.Form.objects.fuzzy(name=f)
except makam.models.Form.DoesNotExist:
missingf.append(f)
if missingm:
res["missingm"] = missingm
if missingu:
res["missingu"] = missingu
if missingf:
res["missingf"] = missingf
res["gotmakam"] = len(res["makams"]) > 0
res["gotusul"] = len(res["usuls"]) > 0
res["gotform"] = len(res["forms"]) > 0
if res["makams"] and res["usuls"] and res["forms"]:
return (True, res)
else:
return (False, res)