def getRagaWiseStats(raga_mbid_file, output_dir, music_tradition): #Hindustani: dbs.getRagaWiseStats('RagaRecognitionDS/hindustani/PerRagaStats/Hindustani30Raga300_FILE_MBID_RAGA.txt', 'RagaRecognitionDS/hindustani/PerRagaStats/', 'hindustani') #Carnatic: dbs.getRagaWiseStats('RagaRecognitionDS/carnatic/PerRagaStats/Carnatic40Raga480_FILE_MBID_RAGA.txt', 'RagaRecognitionDS/carnatic/PerRagaStats/', 'carnatic') lines = open(raga_mbid_file, 'r').readlines() mbid_ragaid = [] dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") for line in lines: lsplit = line.split('\t') mbid_ragaid.append([lsplit[1].strip(), lsplit[2].strip()]) mbid_ragaid = np.array(mbid_ragaid) mbids = mbid_ragaid[:, 0] ragaids = mbid_ragaid[:, 1] uragaids = np.unique(ragaids) for ragaid in uragaids: ind_raga = np.where(ragaids == ragaid)[0] mbid_selected = mbids[ind_raga] output_file = os.path.join(output_dir, ragaid + '.json') getDatasetStats(mbid_selected, output_file, music_tradition=music_tradition)
def download_scores(andalusian_description, dunya_token, target_folder='../data/scores'): """ Given descriptive dataframe of andalusian corpus, download scores to taget folder returns: list of score ids downloaded with tab name """ dunya.set_token(dunya_token) scores = [] # Some scores do not have the relevant metadata for i, row in andalusian_description.iterrows(): try: tn = row['sections'][0]['tab']['transliterated_name'] except: tn = '' scores.append([row['mbid'], tn]) bad_mbid = [] for s in scores: # Download scores from dunya through its mbid mbid = s[0] # Some scores aren't available try: score_xml = dunya.docserver.file_for_document(mbid, 'symbtrxml') name = mbid + '.xml' path = os.path.join(target_folder, name) open(path, "wb").write(score_xml) except: bad_mbid.append(mbid) scores = [x for x in scores if x[0] not in bad_mbid] return scores
def download_mbid(mbid): dunya.set_token(config.dunya_token) save_path = os.path.join(config.chroma.audio_load_path, mbid + '.mp3') if not os.path.exists(save_path): contents = compmusic.dunya.docserver.get_mp3(mbid) with open(save_path, 'wb') as f: f.write(contents)
def download_and_pickle_recording_list(filename, token): """downloads the list of all carnatic music samples in dunya, and pickles them to PICKLE_FILE as a python list. Needs to import compmusic, so it has to be installed (see ) """ dunya.set_token(token) recordings = dunya.carnatic.get_recordings() with open(filename,'w') as f: pickle.dump(recordings, f)
def fetchHindustaniCollection(out_dir): dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") #fetch all the recordings in the collection recordings = hn.get_recordings() for ii, rec in enumerate(recordings): print "processing %d out of %d\n" % (ii + 1, len(recordings)) recording = rec['mbid'] recInfo = hn.get_recording(recording) recName = recInfo['title'] releaseName = recInfo['release']['title'] artist = recInfo['artists'][0]['name'] dirName = os.path.join(out_dir, artist, releaseName) fileName = os.path.join(out_dir, artist, releaseName, recName) fileName_mp3 = fileName + '.mp3' fileName_tonic = fileName + '.tonic' fileName_pitch = fileName + '.pitch' if not os.path.exists(dirName): os.makedirs(dirName) #download mp3 file if not os.path.isfile(fileName_mp3): try: mp3File = ds.get_mp3(recording) fid = open(fileName_mp3, "w") fid.write(mp3File) fid.close() except: print "couldn't fetch mp3 file: %s\n" % fileName #download tonic information if not os.path.isfile(fileName_tonic): try: tonic = json.loads( ds.file_for_document(recording, 'ctonic', 'tonic')) np.savetxt(fileName_tonic, np.array([tonic]), fmt='%.3e', delimiter='\t') except: print "couldn't fetch tonic data: %s\n" % fileName #download pitch of the recording if not os.path.isfile(fileName_pitch): try: pitch = json.loads( ds.file_for_document(recording, 'pitch', 'pitch')) np.savetxt(fileName_pitch, pitch, fmt='%.3e', delimiter='\t') except: print "couldn't fetch pitch data: %s\n" % fileName
def getMBIDsInCollections(collection_id, music_tradition, output_file): if music_tradition == 'hindustani': tradition = hi elif music_tradition == 'carnatic': tradition = ca else: print "Please specify a valid music tradition" dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") tradition.set_collections([collection_id]) recs = tradition.get_recordings() mbids = [r['mbid'] for r in recs] json.dump(mbids, open(output_file, 'w')) return len(mbids)
def getStatsDunyaCorpus(collectionId, output_file, music_tradition=''): """ This function will fetch all the stats/numbers for a given collection/corpus/set using function 'getDatasetStats' collectionId: list of mbids of the collections music_tradition: 'hindustani' or 'carnatic' output_file: json output file to dump the stats """ if music_tradition == 'hindustani': tradition = hi elif music_tradition == 'carnatic': tradition = ca else: print "Please specify a valid music tradition" dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") tradition.set_collections(collectionId) recs = tradition.get_recordings() mbids = [r['mbid'] for r in recs] getDatasetStats(mbids, output_file, music_tradition)
def fillFileTableWithRagaIds(myDatabase, collection='carnatic'): dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") if collection == 'hindustani': tradition = hn elif collection == 'carnatic': tradition = ca getMBIDs = "select mbid from file" cmd1 = "update file set raagaId ='%s' where mbid='%s'" try: con = psy.connect(database=myDatabase, user=myUser) cur = con.cursor() cur.execute(getMBIDs) mbids = cur.fetchall() for ii, mbid in enumerate(mbids): mbid = mbid[0] print ii try: recData = tradition.get_recording(mbid) except: print "Api didn't respond to this mbid = %s" % mbid if len(recData['raaga']) == 1: ragaId = recData['raaga'][0]['uuid'] cur.execute(cmd1 % (ragaId, mbid)) else: print "Can't fetch correct raga for mbid = %s" % mbid con.commit() print "Successfully updated file table in %s database" % (myDatabase) except psy.DatabaseError, e: print 'Error %s' % e if con: con.rollback() con.close() sys.exit(1)
def getStatsForTonicDataset(): """ NOTE: there are two issues with this dataset, first files are mixed of hindustani and carnatic, second several files in this dataset are not included in any of the compmusic collection. So Dunya doesn't give anything for such fiels """ dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_MBID.json' output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_STATS_Carnatic.json' output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_Pretty_Carnatic.txt' music_tradition = 'carnatic' mbids = json.load(open(mbids_file, 'r')) getDatasetStats(mbids, output_file, music_tradition) generatePretyReport(output_file, output_file_pretty) mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_MBID.json' output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_STATS_Hindustani.json' output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_vocal_recs_Pretty_Hindustani.txt' music_tradition = 'hindustani' mbids = json.load(open(mbids_file, 'r')) getDatasetStats(mbids, output_file, music_tradition) generatePretyReport(output_file, output_file_pretty) mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_MBID.json' output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_STATS_Carnatic.json' output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_Pretty_Carnatic.txt' music_tradition = 'carnatic' mbids = json.load(open(mbids_file, 'r')) getDatasetStats(mbids, output_file, music_tradition) generatePretyReport(output_file, output_file_pretty) mbids_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_MBID.json' output_file = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_STATS_Hindustani.json' output_file_pretty = '/home/sankalp/Work/Work_PhD/library_pythonnew/dbStats/TonicDS/Tonic_inst_recs_Pretty_Hindustani.txt' music_tradition = 'hindustani' mbids = json.load(open(mbids_file, 'r')) getDatasetStats(mbids, output_file, music_tradition) generatePretyReport(output_file, output_file_pretty)
def __init__(self, tradition_slug, api_token=None): """ Parameters ---------- tradition_slug api_token use_cached """ # setting api token first if not api_token: raise Exception("API token not provided") else: dn.set_token(api_token) self.api_token = api_token if tradition_slug not in meta.info.keys(): raise Exception("Wrong tradition slug specified") self.tradition_slug = tradition_slug # setting the info for the tradition self.info = meta.info[tradition_slug] # reading metadata info (just a mapping of needed info) self.__read_metadata_info(tradition_slug) # reading file info (just a mapping of needed info) self.__read_file_info(tradition_slug=tradition_slug) # init some needed variables self.metadata_stats = None self.metadata = None self.files = None self.recs = None self.rec_infos = None
def get_db_raga_dunya_raga(outputfile, myDatabase, collection='carnatic'): """ Due to changes in raga labels there are cases where the raga labels in the dunya are changed.This function just dumps a file with mbid, raga in the curent db and in the dunya (from musicbrainz) """ dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") if collection == 'hindustani': tradition = hn elif collection == 'carnatic': tradition = ca getMBIDs = "select mbid from file" cmd1 = "select raagaId from file where mbid='%s'" try: con = psy.connect(database=myDatabase, user=myUser) cur = con.cursor() cur.execute(getMBIDs) mbids = cur.fetchall() for ii, mbid in enumerate(mbids): mbid = mbid[0] try: recData = tradition.get_recording(mbid) ragaId_dunya = recData['raaga'][0]['uuid'] cur.execute(cmd1 % (mbid)) raaga_id = cur.fetchone()[0] print mbid, ragaId_dunya, raaga_id except: pass except psy.DatabaseError, e: print 'Error %s' % e if con: con.rollback() con.close() sys.exit(1)
def fillFileTableWithRagaIds_withoutAPI(myDatabase, mbid_ragaid_file): mbid_raga = {} lines = open(mbid_ragaid_file, 'r').readlines() for line in lines: sline = line.split('\t') mbid = sline[0].strip() ragaid = sline[1].strip() mbid_raga[mbid] = ragaid dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") getMBIDs = "select mbid from file" cmd1 = "update file set raagaId ='%s' where mbid='%s'" try: con = psy.connect(database=myDatabase, user=myUser) cur = con.cursor() cur.execute(getMBIDs) mbids = cur.fetchall() for ii, mbid in enumerate(mbids): mbid = mbid[0] if mbid_raga.has_key(mbid): cur.execute(cmd1 % (mbid_raga[mbid], mbid)) else: print "problem with mbid = %s" % (mbid) con.commit() print "Successfully updated file table in %s database" % (myDatabase) except psy.DatabaseError, e: print 'Error %s' % e if con: con.rollback() con.close() sys.exit(1)
parentDir = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir, os.path.pardir, os.path.pardir, os.path.pardir)) pathAlignmentDur = os.path.join(parentDir, 'AlignmentDuration') if pathAlignmentDur not in sys.path: sys.path.append(pathAlignmentDur) import compmusic.extractors # from docserver import util from compmusic import dunya ON_SERVER = True dunya.set_token("69ed3d824c4c41f59f0bc853f696a7dd80707779") def getWork(musicbrainzid): try: rec_data = dunya.makam.get_recording(musicbrainzid) if len(rec_data['works']) == 0: raise Exception('No work on recording %s' % musicbrainzid) if len(rec_data['works']) > 1: raise Exception( 'More than one work for recording %s Not implemented!' % musicbrainzid) w = rec_data['works'][0] except Exception: sys.exit('no recording with this UUID found or no works related') w = {}
parentDir = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir, os.path.pardir, os.path.pardir, os.path.pardir)) pathAlignmentDur = os.path.join(parentDir, 'AlignmentDuration') if pathAlignmentDur not in sys.path: sys.path.append(pathAlignmentDur) import compmusic.extractors # from docserver import util from compmusic import dunya ON_SERVER = True dunya.set_token("69ed3d824c4c41f59f0bc853f696a7dd80707779") def getWork(musicbrainzid): try: rec_data = dunya.makam.get_recording(musicbrainzid) if len(rec_data['works']) == 0: raise Exception('No work on recording %s' % musicbrainzid) if len(rec_data['works']) > 1: raise Exception('More than one work for recording %s Not implemented!' % musicbrainzid) w = rec_data['works'][0] except Exception: sys.exit('no recording with this UUID found or no works related') w = {} w['mbid'] = '' return w
from compmusic import dunya from compmusic.dunya import hindustani from compmusic.dunya import docserver import numpy as np import matplotlib.pyplot as plt import numpy as np def hz2cents(pitch_hz, tonic=440): return 1200 * np.log2(pitch_hz / float(tonic)) # SET YOUR DUNYA API KEY HERE API_KEY = '' dunya.set_token(API_KEY) concerts = hindustani.get_recordings() recording = list( filter( lambda recording: '158ecd08-3751-46bc-bd48-e74c8b85dda8' in recording[ 'mbid'], concerts)) mbid = recording[0]['mbid'] document_details = docserver.document(recording[0]['mbid']) for document_type, data in document_details['derivedfiles'].items(): for subtype, metadata in data.items(): if metadata['mimetype'] == 'application/json': \ print('{},{}'.format(document_type, subtype), metadata['versions'])
import codecs import json, os, sys import numpy as np import compmusic import json from compmusic import dunya as dn from compmusic.dunya import hindustani as hn from compmusic.dunya import carnatic as ca from compmusic.dunya import docserver as ds from compmusic import musicbrainz import fixpath ISMIR2015_10RAGASCarnatic = ['55', '10', '27', '8', '9', '137', '159', '20', '13', '210'] dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") #### #TODO This is a temporary code becasue audio download was not functioning properly lines = open('CarnaticInfo/carnaticMBIDLocationMapping_3_March_2015.txt','r').readlines() location = {} for line in lines: splitLine = line.split() location[splitLine[1].strip()] = splitLine[0].strip() #### def getRagaNameMapping(outFile, ragas = ISMIR2015_10RAGASCarnatic, collection = 'carnatic'): if collection == 'hindustani': tradition = hn
def set_dunya_token(self, token): self.dunya_token = token dunya.set_token(self.dunya_token)
"""This script manages the downloading of the labels from the Dunya platform (http://dunya.compmusic.upf.edu). Note that it requires an authorized key to work. """ from __future__ import print_function import pickle, urllib2, os.path, time from optparse import OptionParser from compmusic import dunya token = "?" dunya.set_token(token) c = dunya.carnatic with open("recordings_FINAL.pickle", "rb") as f: rec = pickle.load(f) rec_complete = [] for x in rec: try: rec_complete.append(c.get_recording(x["mbid"])) print(len(rec_complete)) except: pass # # from the list returned by get_recordings(), the download_mp3(recID, path) function returns a 404 error for the following 17 "mbid"s. # # get_recordings() returns 2472 elements that look like this: # # {u'mbid': u'ffe2f301-1793-4e07-8256-72c8000a0eca', u'title': u'Agaramumaagi'} # # from them, the download_mp3(recID, path) returns 2455, and a 404 error for the following 17; # # [u'0b7fe96c-da9f-45fb-b03e-227c0fbc0768',
# # Atlı, H. S., Uyar, B., Şentürk, S., Bozkurt, B., and Serra, X. (2014). Audio # feature extraction for exploring Turkish makam music. In Proceedings of 3rd # International Conference on Audio Technologies for Music and Media, Ankara, # Turkey. import os import tempfile from docserver import util from tomato.symbolic.scoreconverter import ScoreConverter import compmusic.extractors from compmusic import dunya from settings import token dunya.set_token(token) class Symbtr2Png(compmusic.extractors.ExtractorModule): _version = "0.2" _sourcetype = "symbtrtxt" _slug = "score" _output = { "xmlscore": { "extension": "xml", "mimetype": "application/xml" }, "score": { "extension": "svg", "mimetype": "image/svg+xml", "parts": True
import os import json import logging from compmusic import dunya import musicbrainzngs as mb from utilities.constants import * from utilities.recordingcomputation import * from utilities.generalutilities import * mb_logger = logging.getLogger('musicbrainzngs') mb_logger.setLevel(logging.WARNING) # Dunya token dunya.set_token(DUNYA_TOKEN) mb.set_rate_limit() mb.set_hostname("musicbrainz.org") # -------------------------------------------------- CHECK -------------------------------------------------- def check_dunya_metadata(): ''' Check if all the json file relating to metadata are stored in the data directory :return: True if all the file exist. Otherwise, return False ''' flag = True for fn in DF_LISTS: file_name = PREFIX_JSON + fn + '.json'
from compmusic import dunya dunya.set_token('ad57ef18f8c3a2f4962b7883ac6ed38b3578ba38') a = dunya.carnatic.get_recordings(recording_detail=True) import json with open('carnatic.json', 'w') as fp: json.dump(a, fp) def download_all_songs(): if config.mode == 'carnatic': from compmusic.dunya.carnatic import get_recording with open(path, 'wb'): recording = get_recording(mbid) download_release(rel['mbid'], "./temp/")
#to get the manual annotations #go to pycompmusic #import pickle import os import compmusic from compmusic import dunya dunya.set_token("67a11dc1e00e22cb4bc387ee870129a819989212") #compmusic.dunya.docserver.get_collections() collection = compmusic.dunya.docserver.get_collection('dunya-carnatic-cc') #compmusic.dunya.docserver.file_for_document(recordingid, thetype, subtype=None, part=None, version=None) #recording id = MBID #type = the computed filetype os.makedirs("Manual_Annotations") for document in collection['documents']: mbid = document['external_identifier'] print mbid source_list = compmusic.dunya.docserver.document(mbid)['sourcefiles'] mbid_path = 'Manual_Annotations' + '/' + mbid os.makedirs(mbid_path) for source in source_list: annotation = compmusic.dunya.docserver.file_for_document(mbid, source) print source file_name = mbid_path + '/' + source + '.txt' file = open(file_name, 'w') file.write(annotation) #pickle.dump(annotation, file) file.close
import codecs import json, os, sys import numpy as np import compmusic import json from compmusic import dunya as dn from compmusic.dunya import hindustani as hn from compmusic.dunya import carnatic as ca from compmusic.dunya import docserver as ds from compmusic import musicbrainz dn.set_token("60312f59428916bb854adaa208f55eb35c3f2f07") svaras = ['S', 'r', 'R', 'g', 'G', 'm', 'M', 'P', 'd', 'D', 'n', 'N'] svaras_np = np.array(svaras) thaats = {1: 'Bilawal', 2: 'Kalyan', 3:'Khamaj', 4: 'Bhairav', 5: 'Purvi', 6: 'Marwa', 7: 'Kafi', 8: 'Asawari', 9: 'Bhairavi', 10: 'Todi'} raga2color = {u'24a85738-02e8-4b0e-9098-926f57c19a50': '#CCFFCC', u'2ed9379f-14c9-49af-8e4d-f9b63e96801f': '#FFA9E9', u'2fe9f79c-7955-4dcd-939a-e0a73173c07f': '#91FF6C', u'3eb7ba30-4b94-432e-9618-875ee57e01ab': '#9B9BAB', u'46997b02-f09c-4969-8138-4e1861f61967': '#6699FF', u'48b37bed-e847-4882-8a01-5c721e07f07d': '#999980', u'64e5fb9e-5569-4e80-8e6c-f543af9469c7': '#9966FF', u'6f13484e-6fdd-402d-baf3-3835835454d0': '#B8FFDB', u'774ae631-eb8d-4b8e-9edd-5285e919821f': '#7C7C89', u'78f439fa-88ba-480e-be87-4d65311df381': '#B29980', u'80f3ec4b-d9db-4bc3-8186-9e6b0581f31a': '#70944D', u'93c73081-bdf8-4eca-b325-d736b71e9b4b': '#669999', u'9571385f-2743-41a1-9c6d-3db0545a6773': '#FFDB4D', u'96ef352a-9baf-4ba9-a628-9cabb59b2175': '#99FFCC', u'a7d98897-e2fe-4d75-b9dc-e5b4dc88e1c6': '#A34775', u'd3be1a2a-ad82-495b-b738-b5e3dd664a0c': '#CCFFCC',