def getRagaWiseStats(raga_mbid_file, output_dir, music_tradition):
#Hindustani: dbs.getRagaWiseStats('RagaRecognitionDS/hindustani/PerRagaStats/Hindustani30Raga300_FILE_MBID_RAGA.txt', 'RagaRecognitionDS/hindustani/PerRagaStats/', 'hindustani')
#Carnatic: dbs.getRagaWiseStats('RagaRecognitionDS/carnatic/PerRagaStats/Carnatic40Raga480_FILE_MBID_RAGA.txt', 'RagaRecognitionDS/carnatic/PerRagaStats/', 'carnatic')
lines = open(raga_mbid_file, 'r').readlines()
mbid_ragaid = []
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
for line in lines:
lsplit = line.split('\t')
mbid_ragaid.append([lsplit[1].strip(), lsplit[2].strip()])
mbid_ragaid = np.array(mbid_ragaid)
mbids = mbid_ragaid[:, 0]
ragaids = mbid_ragaid[:, 1]
uragaids = np.unique(ragaids)
for ragaid in uragaids:
ind_raga = np.where(ragaids == ragaid)[0]
mbid_selected = mbids[ind_raga]
output_file = os.path.join(output_dir, ragaid + '.json')
getDatasetStats(mbid_selected,
output_file,
music_tradition=music_tradition)
def download_scores(andalusian_description, dunya_token, target_folder='../data/scores'):
"""
Given descriptive dataframe of andalusian corpus, download scores to taget folder
returns: list of score ids downloaded with tab name
"""
dunya.set_token(dunya_token)
scores = []
# Some scores do not have the relevant metadata
for i, row in andalusian_description.iterrows():
try:
tn = row['sections'][0]['tab']['transliterated_name']
except:
tn = ''
scores.append([row['mbid'], tn])
bad_mbid = []
for s in scores: # Download scores from dunya through its mbid
mbid = s[0]
# Some scores aren't available
try:
score_xml = dunya.docserver.file_for_document(mbid, 'symbtrxml')
name = mbid + '.xml'
path = os.path.join(target_folder, name)
open(path, "wb").write(score_xml)
except:
bad_mbid.append(mbid)
scores = [x for x in scores if x[0] not in bad_mbid]
return scores
def download_mbid(mbid):
dunya.set_token(config.dunya_token)
save_path = os.path.join(config.chroma.audio_load_path, mbid + '.mp3')
if not os.path.exists(save_path):
contents = compmusic.dunya.docserver.get_mp3(mbid)
with open(save_path, 'wb') as f:
f.write(contents)
def download_and_pickle_recording_list(filename, token):
"""downloads the list of all carnatic music samples in dunya, and
pickles them to PICKLE_FILE as a python list. Needs to import
compmusic, so it has to be installed (see )
"""
dunya.set_token(token)
recordings = dunya.carnatic.get_recordings()
with open(filename,'w') as f:
pickle.dump(recordings, f)
def fetchHindustaniCollection(out_dir):
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
#fetch all the recordings in the collection
recordings = hn.get_recordings()
for ii, rec in enumerate(recordings):
print "processing %d out of %d\n" % (ii + 1, len(recordings))
recording = rec['mbid']
recInfo = hn.get_recording(recording)
recName = recInfo['title']
releaseName = recInfo['release']['title']
artist = recInfo['artists'][0]['name']
dirName = os.path.join(out_dir, artist, releaseName)
fileName = os.path.join(out_dir, artist, releaseName, recName)
fileName_mp3 = fileName + '.mp3'
fileName_tonic = fileName + '.tonic'
fileName_pitch = fileName + '.pitch'
if not os.path.exists(dirName):
os.makedirs(dirName)
#download mp3 file
if not os.path.isfile(fileName_mp3):
try:
mp3File = ds.get_mp3(recording)
fid = open(fileName_mp3, "w")
fid.write(mp3File)
fid.close()
except:
print "couldn't fetch mp3 file: %s\n" % fileName
#download tonic information
if not os.path.isfile(fileName_tonic):
try:
tonic = json.loads(
ds.file_for_document(recording, 'ctonic', 'tonic'))
np.savetxt(fileName_tonic,
np.array([tonic]),
fmt='%.3e',
delimiter='\t')
except:
print "couldn't fetch tonic data: %s\n" % fileName
#download pitch of the recording
if not os.path.isfile(fileName_pitch):
try:
pitch = json.loads(
ds.file_for_document(recording, 'pitch', 'pitch'))
np.savetxt(fileName_pitch, pitch, fmt='%.3e', delimiter='\t')
except:
print "couldn't fetch pitch data: %s\n" % fileName
def getMBIDsInCollections(collection_id, music_tradition, output_file):
if music_tradition == 'hindustani':
tradition = hi
elif music_tradition == 'carnatic':
tradition = ca
else:
print "Please specify a valid music tradition"
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
tradition.set_collections([collection_id])
recs = tradition.get_recordings()
mbids = [r['mbid'] for r in recs]
json.dump(mbids, open(output_file, 'w'))
return len(mbids)
def getStatsDunyaCorpus(collectionId, output_file, music_tradition=''):
"""
This function will fetch all the stats/numbers for a given collection/corpus/set using function 'getDatasetStats'
collectionId: list of mbids of the collections
music_tradition: 'hindustani' or 'carnatic'
output_file: json output file to dump the stats
"""
if music_tradition == 'hindustani':
tradition = hi
elif music_tradition == 'carnatic':
tradition = ca
else:
print "Please specify a valid music tradition"
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
tradition.set_collections(collectionId)
recs = tradition.get_recordings()
mbids = [r['mbid'] for r in recs]
getDatasetStats(mbids, output_file, music_tradition)
def fillFileTableWithRagaIds(myDatabase, collection='carnatic'):
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
if collection == 'hindustani':
tradition = hn
elif collection == 'carnatic':
tradition = ca
getMBIDs = "select mbid from file"
cmd1 = "update file set raagaId ='%s' where mbid='%s'"
try:
con = psy.connect(database=myDatabase, user=myUser)
cur = con.cursor()
cur.execute(getMBIDs)
mbids = cur.fetchall()
for ii, mbid in enumerate(mbids):
mbid = mbid[0]
print ii
try:
recData = tradition.get_recording(mbid)
except:
print "Api didn't respond to this mbid = %s" % mbid
if len(recData['raaga']) == 1:
ragaId = recData['raaga'][0]['uuid']
cur.execute(cmd1 % (ragaId, mbid))
else:
print "Can't fetch correct raga for mbid = %s" % mbid
con.commit()
print "Successfully updated file table in %s database" % (myDatabase)
except psy.DatabaseError, e:
print 'Error %s' % e
if con:
con.rollback()
con.close()
sys.exit(1)
def getStatsForTonicDataset():
"""
NOTE: there are two issues with this dataset, first files are mixed of hindustani and carnatic, second several files in this dataset are not included in any of the compmusic collection. So Dunya doesn't give anything for such fiels
"""
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_MBID.json'
output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_STATS_Carnatic.json'
output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_Pretty_Carnatic.txt'
music_tradition = 'carnatic'
mbids = json.load(open(mbids_file, 'r'))
getDatasetStats(mbids, output_file, music_tradition)
generatePretyReport(output_file, output_file_pretty)
mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_MBID.json'
output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_STATS_Hindustani.json'
output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_Pretty_Hindustani.txt'
music_tradition = 'hindustani'
mbids = json.load(open(mbids_file, 'r'))
getDatasetStats(mbids, output_file, music_tradition)
generatePretyReport(output_file, output_file_pretty)
mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_MBID.json'
output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_STATS_Carnatic.json'
output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_Pretty_Carnatic.txt'
music_tradition = 'carnatic'
mbids = json.load(open(mbids_file, 'r'))
getDatasetStats(mbids, output_file, music_tradition)
generatePretyReport(output_file, output_file_pretty)
mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_MBID.json'
output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_STATS_Hindustani.json'
output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_Pretty_Hindustani.txt'
music_tradition = 'hindustani'
mbids = json.load(open(mbids_file, 'r'))
getDatasetStats(mbids, output_file, music_tradition)
generatePretyReport(output_file, output_file_pretty)
def __init__(self, tradition_slug, api_token=None):
"""
Parameters
----------
tradition_slug
api_token
use_cached
"""
# setting api token first
if not api_token:
raise Exception("API token not provided")
else:
dn.set_token(api_token)
self.api_token = api_token
if tradition_slug not in meta.info.keys():
raise Exception("Wrong tradition slug specified")
self.tradition_slug = tradition_slug
# setting the info for the tradition
self.info = meta.info[tradition_slug]
# reading metadata info (just a mapping of needed info)
self.__read_metadata_info(tradition_slug)
# reading file info (just a mapping of needed info)
self.__read_file_info(tradition_slug=tradition_slug)
# init some needed variables
self.metadata_stats = None
self.metadata = None
self.files = None
self.recs = None
self.rec_infos = None
def get_db_raga_dunya_raga(outputfile, myDatabase, collection='carnatic'):
"""
Due to changes in raga labels there are cases where the raga labels in the dunya are changed.This function just dumps a file
with mbid, raga in the curent db and in the dunya (from musicbrainz)
"""
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
if collection == 'hindustani':
tradition = hn
elif collection == 'carnatic':
tradition = ca
getMBIDs = "select mbid from file"
cmd1 = "select raagaId from file where mbid='%s'"
try:
con = psy.connect(database=myDatabase, user=myUser)
cur = con.cursor()
cur.execute(getMBIDs)
mbids = cur.fetchall()
for ii, mbid in enumerate(mbids):
mbid = mbid[0]
try:
recData = tradition.get_recording(mbid)
ragaId_dunya = recData['raaga'][0]['uuid']
cur.execute(cmd1 % (mbid))
raaga_id = cur.fetchone()[0]
print mbid, ragaId_dunya, raaga_id
except:
pass
except psy.DatabaseError, e:
print 'Error %s' % e
if con:
con.rollback()
con.close()
sys.exit(1)
def fillFileTableWithRagaIds_withoutAPI(myDatabase, mbid_ragaid_file):
mbid_raga = {}
lines = open(mbid_ragaid_file, 'r').readlines()
for line in lines:
sline = line.split('\t')
mbid = sline[0].strip()
ragaid = sline[1].strip()
mbid_raga[mbid] = ragaid
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
getMBIDs = "select mbid from file"
cmd1 = "update file set raagaId ='%s' where mbid='%s'"
try:
con = psy.connect(database=myDatabase, user=myUser)
cur = con.cursor()
cur.execute(getMBIDs)
mbids = cur.fetchall()
for ii, mbid in enumerate(mbids):
mbid = mbid[0]
if mbid_raga.has_key(mbid):
cur.execute(cmd1 % (mbid_raga[mbid], mbid))
else:
print "problem with mbid = %s" % (mbid)
con.commit()
print "Successfully updated file table in %s database" % (myDatabase)
except psy.DatabaseError, e:
print 'Error %s' % e
if con:
con.rollback()
con.close()
sys.exit(1)
parentDir = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir,
os.path.pardir, os.path.pardir, os.path.pardir))
pathAlignmentDur = os.path.join(parentDir, 'AlignmentDuration')
if pathAlignmentDur not in sys.path:
sys.path.append(pathAlignmentDur)
import compmusic.extractors
# from docserver import util
from compmusic import dunya
ON_SERVER = True
dunya.set_token("69ed3d824c4c41f59f0bc853f696a7dd80707779")
def getWork(musicbrainzid):
try:
rec_data = dunya.makam.get_recording(musicbrainzid)
if len(rec_data['works']) == 0:
raise Exception('No work on recording %s' % musicbrainzid)
if len(rec_data['works']) > 1:
raise Exception(
'More than one work for recording %s Not implemented!' %
musicbrainzid)
w = rec_data['works'][0]
except Exception:
sys.exit('no recording with this UUID found or no works related')
w = {}
parentDir = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir, os.path.pardir, os.path.pardir,
os.path.pardir))
pathAlignmentDur = os.path.join(parentDir, 'AlignmentDuration')
if pathAlignmentDur not in sys.path:
sys.path.append(pathAlignmentDur)
import compmusic.extractors
# from docserver import util
from compmusic import dunya
ON_SERVER = True
dunya.set_token("69ed3d824c4c41f59f0bc853f696a7dd80707779")
def getWork(musicbrainzid):
try:
rec_data = dunya.makam.get_recording(musicbrainzid)
if len(rec_data['works']) == 0:
raise Exception('No work on recording %s' % musicbrainzid)
if len(rec_data['works']) > 1:
raise Exception('More than one work for recording %s Not implemented!' % musicbrainzid)
w = rec_data['works'][0]
except Exception:
sys.exit('no recording with this UUID found or no works related')
w = {}
w['mbid'] = ''
return w
from compmusic import dunya
from compmusic.dunya import hindustani
from compmusic.dunya import docserver
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
def hz2cents(pitch_hz, tonic=440):
return 1200 * np.log2(pitch_hz / float(tonic))
# SET YOUR DUNYA API KEY HERE
API_KEY = ''
dunya.set_token(API_KEY)
concerts = hindustani.get_recordings()
recording = list(
filter(
lambda recording: '158ecd08-3751-46bc-bd48-e74c8b85dda8' in recording[
'mbid'], concerts))
mbid = recording[0]['mbid']
document_details = docserver.document(recording[0]['mbid'])
for document_type, data in document_details['derivedfiles'].items():
for subtype, metadata in data.items():
if metadata['mimetype'] == 'application/json': \
print('{},{}'.format(document_type, subtype), metadata['versions'])
import codecs
import json, os, sys
import numpy as np
import compmusic
import json
from compmusic import dunya as dn
from compmusic.dunya import hindustani as hn
from compmusic.dunya import carnatic as ca
from compmusic.dunya import docserver as ds
from compmusic import musicbrainz
import fixpath
ISMIR2015_10RAGASCarnatic = ['55', '10', '27', '8', '9', '137', '159', '20', '13', '210']
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
####
#TODO This is a temporary code becasue audio download was not functioning properly
lines = open('CarnaticInfo/carnaticMBIDLocationMapping_3_March_2015.txt','r').readlines()
location = {}
for line in lines:
splitLine = line.split()
location[splitLine[1].strip()] = splitLine[0].strip()
####
def getRagaNameMapping(outFile, ragas = ISMIR2015_10RAGASCarnatic, collection = 'carnatic'):
if collection == 'hindustani':
tradition = hn
def set_dunya_token(self, token):
self.dunya_token = token
dunya.set_token(self.dunya_token)
"""This script manages the downloading of the labels from the Dunya platform
(http://dunya.compmusic.upf.edu). Note that it requires an authorized key to
work.
"""
from __future__ import print_function
import pickle, urllib2, os.path, time
from optparse import OptionParser
from compmusic import dunya
token = "?"
dunya.set_token(token)
c = dunya.carnatic
with open("recordings_FINAL.pickle", "rb") as f:
rec = pickle.load(f)
rec_complete = []
for x in rec:
try:
rec_complete.append(c.get_recording(x["mbid"]))
print(len(rec_complete))
except:
pass
# # from the list returned by get_recordings(), the download_mp3(recID, path) function returns a 404 error for the following 17 "mbid"s.
# # get_recordings() returns 2472 elements that look like this:
# # {u'mbid': u'ffe2f301-1793-4e07-8256-72c8000a0eca', u'title': u'Agaramumaagi'}
# # from them, the download_mp3(recID, path) returns 2455, and a 404 error for the following 17;
# # [u'0b7fe96c-da9f-45fb-b03e-227c0fbc0768',
#
# Atlı, H. S., Uyar, B., Şentürk, S., Bozkurt, B., and Serra, X. (2014). Audio
# feature extraction for exploring Turkish makam music. In Proceedings of 3rd
# International Conference on Audio Technologies for Music and Media, Ankara,
# Turkey.
import os
import tempfile
from docserver import util
from tomato.symbolic.scoreconverter import ScoreConverter
import compmusic.extractors
from compmusic import dunya
from settings import token
dunya.set_token(token)
class Symbtr2Png(compmusic.extractors.ExtractorModule):
_version = "0.2"
_sourcetype = "symbtrtxt"
_slug = "score"
_output = {
"xmlscore": {
"extension": "xml",
"mimetype": "application/xml"
},
"score": {
"extension": "svg",
"mimetype": "image/svg+xml",
"parts": True
import os
import json
import logging
from compmusic import dunya
import musicbrainzngs as mb
from utilities.constants import *
from utilities.recordingcomputation import *
from utilities.generalutilities import *
mb_logger = logging.getLogger('musicbrainzngs')
mb_logger.setLevel(logging.WARNING)
# Dunya token
dunya.set_token(DUNYA_TOKEN)
mb.set_rate_limit()
mb.set_hostname("musicbrainz.org")
# -------------------------------------------------- CHECK --------------------------------------------------
def check_dunya_metadata():
''' Check if all the json file relating to metadata are stored in the data directory
:return: True if all the file exist. Otherwise, return False
'''
flag = True
for fn in DF_LISTS:
file_name = PREFIX_JSON + fn + '.json'
from compmusic import dunya
dunya.set_token('ad57ef18f8c3a2f4962b7883ac6ed38b3578ba38')
a = dunya.carnatic.get_recordings(recording_detail=True)
import json
with open('carnatic.json', 'w') as fp:
json.dump(a, fp)
def download_all_songs():
if config.mode == 'carnatic':
from compmusic.dunya.carnatic import get_recording
with open(path, 'wb'):
recording = get_recording(mbid)
download_release(rel['mbid'], "./temp/")
#to get the manual annotations
#go to pycompmusic
#import pickle
import os
import compmusic
from compmusic import dunya
dunya.set_token("67a11dc1e00e22cb4bc387ee870129a819989212")
#compmusic.dunya.docserver.get_collections()
collection = compmusic.dunya.docserver.get_collection('dunya-carnatic-cc')
#compmusic.dunya.docserver.file_for_document(recordingid, thetype, subtype=None, part=None, version=None)
#recording id = MBID
#type = the computed filetype
os.makedirs("Manual_Annotations")
for document in collection['documents']:
mbid = document['external_identifier']
print mbid
source_list = compmusic.dunya.docserver.document(mbid)['sourcefiles']
mbid_path = 'Manual_Annotations' + '/' + mbid
os.makedirs(mbid_path)
for source in source_list:
annotation = compmusic.dunya.docserver.file_for_document(mbid, source)
print source
file_name = mbid_path + '/' + source + '.txt'
file = open(file_name, 'w')
file.write(annotation)
#pickle.dump(annotation, file)
file.close
import codecs
import json, os, sys
import numpy as np
import compmusic
import json
from compmusic import dunya as dn
from compmusic.dunya import hindustani as hn
from compmusic.dunya import carnatic as ca
from compmusic.dunya import docserver as ds
from compmusic import musicbrainz
dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07")
svaras = ['S', 'r', 'R', 'g', 'G', 'm', 'M', 'P', 'd', 'D', 'n', 'N']
svaras_np = np.array(svaras)
thaats = {1: 'Bilawal', 2: 'Kalyan', 3:'Khamaj', 4: 'Bhairav', 5: 'Purvi', 6: 'Marwa', 7: 'Kafi', 8: 'Asawari', 9: 'Bhairavi', 10: 'Todi'}
raga2color = {u'24a85738-02e8-4b0e-9098-926f57c19a50': '#CCFFCC',
u'2ed9379f-14c9-49af-8e4d-f9b63e96801f': '#FFA9E9',
u'2fe9f79c-7955-4dcd-939a-e0a73173c07f': '#91FF6C',
u'3eb7ba30-4b94-432e-9618-875ee57e01ab': '#9B9BAB',
u'46997b02-f09c-4969-8138-4e1861f61967': '#6699FF',
u'48b37bed-e847-4882-8a01-5c721e07f07d': '#999980',
u'64e5fb9e-5569-4e80-8e6c-f543af9469c7': '#9966FF',
u'6f13484e-6fdd-402d-baf3-3835835454d0': '#B8FFDB',
u'774ae631-eb8d-4b8e-9edd-5285e919821f': '#7C7C89',
u'78f439fa-88ba-480e-be87-4d65311df381': '#B29980',
u'80f3ec4b-d9db-4bc3-8186-9e6b0581f31a': '#70944D',
u'93c73081-bdf8-4eca-b325-d736b71e9b4b': '#669999',
u'9571385f-2743-41a1-9c6d-3db0545a6773': '#FFDB4D',
u'96ef352a-9baf-4ba9-a628-9cabb59b2175': '#99FFCC',
u'a7d98897-e2fe-4d75-b9dc-e5b4dc88e1c6': '#A34775',
u'd3be1a2a-ad82-495b-b738-b5e3dd664a0c': '#CCFFCC',