def create_remote_index(httpserver, dataset_name): httpserver.serve_content( open(REMOTE_DATASETS[dataset_name]["local_index"], "rb").read()) remote_index = { "index": download_utils.RemoteFileMetadata( filename=REMOTE_DATASETS[dataset_name]["remote_filename"], url=httpserver.url, checksum=REMOTE_DATASETS[dataset_name]["remote_checksum"], destination_dir="", ) } data_remote = core.LargeData(REMOTE_DATASETS[dataset_name]["filename"], remote_index=remote_index) return data_remote.index
def __init__(self, track_id, data_home, remote_index=None, remote_index_name=None): if remote_index is not None and remote_index_name is not None: data = core.LargeData(remote_index_name, remote_index=remote_index) else: data = DATA if track_id not in data.index["tracks"]: raise ValueError( "{} is not a valid track ID in AcousticBrainz genre Dataset".format( track_id ) ) self.track_id = track_id self._data_home = data_home self._track_paths = data.index["tracks"][track_id] self.path = core.none_path_join([self._data_home, self._track_paths["data"][0]])
def __init__( self, data_home=None, remote_index=None, remote_index_name=None, ): if remote_index and remote_index_name: data = core.LargeData(remote_index_name, remote_index=remote_index) index = data.index else: index = None super().__init__( data_home, index=DATA.index if index is None else index, name=NAME, track_class=Track, bibtex=BIBTEX, remotes=REMOTES, license_info=LICENSE_INFO, )
), "keys": download_utils.RemoteFileMetadata( filename="keys.zip", url="https://zenodo.org/record/1095691/files/keys.zip?download=1", checksum="775b7d17e009f5818544cf505b6a96fd", destination_dir=".", ), "metadata": download_utils.RemoteFileMetadata( filename="original_metadata.zip", url="https://zenodo.org/record/1095691/files/original_metadata.zip?download=1", checksum="54181e0f34c35d9720439750d0b08091", destination_dir=".", ), } DATA = core.LargeData("giantsteps_key_index.json") LICENSE_INFO = "Creative Commons Attribution Share Alike 4.0 International." class Track(core.Track): """giantsteps_key track class Args: track_id (str): track id of the track Attributes: audio_path (str): track audio path keys_path (str): key annotation path metadata_path (str): sections annotation path title (str): title of the track
"Pitch": row[7], "Pitch ID": int(row[8]), "Dynamics": row[9], "Dynamics ID": int(row[10]), "Instance ID": int(row[11]), "Resampled": (row[13] == "TRUE"), } if len(row[12]) > 0: metadata_index[key]["String ID"] = int(float(row[12])) metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("tinysol_index.json", _load_metadata) class Track(core.Track): """tinysol Track class Args: track_id (str): track id of the track Attributes: audio_path (str): path of the audio file dynamics (str): dynamics abbreviation. Ex: pp, mf, ff, etc. dynamics_id (int): pp=0, p=1, mf=2, f=3, ff=4 family (str): instrument family encoded by its English name instance_id (int): instance ID. Either equal to 0, 1, 2, or 3. instrument_abbr (str): instrument abbreviation
def _load_metadata(data_home): metadata_path = os.path.join(data_home, "medleydb_melody_metadata.json") if not os.path.exists(metadata_path): logging.info("Metadata file {} not found.".format(metadata_path)) return None with open(metadata_path, "r") as fhandle: metadata = json.load(fhandle) metadata["data_home"] = data_home return metadata DATA = core.LargeData("medleydb_melody_index.json", _load_metadata) class Track(core.Track): """medleydb_melody Track class Args: track_id (str): track id of the track Attributes: artist (str): artist audio_path (str): path to the audio file genre (str): genre is_excerpt (bool): True if the track is an excerpt is_instrumental (bool): True of the track does not contain vocals melody1_path (str): path to the melody1 annotation file
==> https://zenodo.org/record/1324183 Then, locate the downloaded the cante100audio folder like this: > cante100/ > cante100_spectrum/ ... (rest of the annotation folders) > cante100audio/ Remember to locate the cante100 folder to {} """ LICENSE_INFO = """ The provided datasets are offered free of charge for internal non-commercial use. We do not grant any rights for redistribution or modification. All data collections were gathered by the COFLA team. COFLA 2015. All rights reserved. """ DATA = core.LargeData("cante100_index.json") class Track(core.Track): """cante100 track class Args: track_id (str): track id of the track data_home (str): Local path where the dataset is stored. If `None`, looks for the data in the default directory, `~/mir_datasets/cante100` Attributes: track_id (str): track id identifier (str): musicbrainz id of the track artist (str): performing artists title (str): title of the track song
# load metadata however makes sense for your dataset with open(metadata_path, "r") as fhandle: raw_metadata = json.load(fhandle) metadata = {} for mdata in raw_metadata: track_id = mdata["midi_filename"].split(".")[0] metadata[track_id] = mdata metadata["data_home"] = data_home return metadata DATA = core.LargeData("maestro_index.json", _load_metadata) class Track(core.Track): """MAESTRO Track class Args: track_id (str): track id of the track Attributes: audio_path (str): Path to the track's audio file canonical_composer (str): Composer of the piece, standardized on a single spelling for a given name. canonical_title (str): Title of the piece. Not guaranteed to be standardized to a single representation. duration (float): Duration in seconds, based on the MIDI file.
checksum="021ab25a5fd1b020521824e7fce9c775", destination_dir="temp", ), } REMOTE_INDEX = { "REMOTE_INDEX": download_utils.RemoteFileMetadata( filename="acousticbrainz_genre_index.json.zip", url= "https://zenodo.org/record/4298580/files/acousticbrainz_genre_index.json.zip?download=1", checksum="810f1c003f53cbe58002ba96e6d4d138", destination_dir="", ) } DATA = core.LargeData("acousticbrainz_genre_index.json", remote_index=REMOTE_INDEX) LICENSE_INFO = """ This dataset is composed of 4 subdatasets. Three of them are Creative Commons Attribution Non Commercial Share Alike 4.0 International and the other one is non-comercial. Details about which license correspond to each subdataset can be found in the following websites: * https://zenodo.org/record/2553414#.X_nxnOn7RUI * https://zenodo.org/record/2554044#.X_nw2en7RUI """ class Track(core.Track): """AcousticBrainz Genre Dataset track class
checksum="44afcf7f193d7e48a7d99e7a6f3ed39d", destination_dir="annotations", ), } DOWNLOAD_INFO = """ Unfortunately the audio files of the RWC-Jazz dataset are not available for download. If you have the RWC-Jazz dataset, place the contents into a folder called RWC-Jazz with the following structure: > RWC-Jazz/ > annotations/ > audio/rwc-j-m0i with i in [1 .. 4] > metadata-master/ and copy the RWC-Jazz folder to {} """ DATA = core.LargeData("rwc_jazz_index.json") class Track(core.Track): """rwc_jazz Track class Args: track_id (str): track id of the track Attributes: artist (str): Artist name audio_path (str): path of the audio file beats_path (str): path of the beat annotation file duration (float): Duration of the track in seconds instruments (str): list of used instruments. piece_number (str): Piece number of this Track, [1-50]
"piece_number": line[0], "suffix": line[1], "track_number": line[2], "title": line[3], "composer": line[4], "artist": line[5], "duration": _duration_to_sec(line[6]), "category": line[7], } metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("rwc_classical_index.json", _load_metadata) class Track(core.Track): """rwc_classical Track class Args: track_id (str): track id of the track Attributes: artist (str): the track's artist audio_path (str): path of the audio file beats_path (str): path of the beat annotation file category (str): One of 'Symphony', 'Concerto', 'Orchestral', 'Solo', 'Chamber', 'Vocal', or blank. composer (str): Composer of this Track.
) def _load_metadata(data_home): metadata_path = os.path.join(data_home, os.path.join("dali_metadata.json")) if not os.path.exists(metadata_path): logging.info("Metadata file {} not found.".format(metadata_path)) return None with open(metadata_path, "r") as fhandle: metadata_index = json.load(fhandle) metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("dali_index.json", _load_metadata) class Track(core.Track): """DALI melody Track class Args: track_id (str): track id of the track Attributes: album (str): the track's album annotation_path (str): path to the track's annotation file artist (str): the track's artist audio_path (str): path to the track's audio file audio_url (str): youtube ID dataset_version (int): dataset annotation version
next(csv_reader) for row in csv_reader: subset, instrument_str, instrument_id, song_id, track_id = row metadata_index[str(track_id)] = { "subset": str(subset), "instrument": str(instrument_str), "instrument_id": int(instrument_id), "song_id": int(song_id), } metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("medley_solos_db_index.json", _load_metadata) class Track(core.Track): """medley_solos_db Track class Args: track_id (str): track id of the track Attributes: audio_path (str): path to the track's audio file instrument (str): instrument encoded by its English name instrument_id (int): instrument encoded as an integer song_id (int): song encoded as an integer subset (str): either equal to 'train', 'validation', or 'test' track_id (str): track id
If you have the iKala dataset, place the contents into a folder called iKala with the following structure: > iKala/ > Lyrics/ > PitchLabel/ > Wavfile/ and copy the iKala folder to {} """ LICENSE_INFO = """ When it was distributed, Ikala used to have a custom license. Visit http://mac.citi.sinica.edu.tw/ikala/ for more details. """ DATA = core.LargeData("ikala_index.json") class Track(core.Track): """ikala Track class Args: track_id (str): track id of the track Attributes: audio_path (str): path to the track's audio file f0_path (str): path to the track's f0 annotation file lyrics_path (str): path to the track's lyric annotation file section (str): section. Either 'verse' or 'chorus' singer_id (str): singer id song_id (str): song id of the track
}""" DOWNLOAD_INFO = """ To download this dataset, visit: https://zenodo.org/record/2628782#.XKZdABNKh24 and request access. Once downloaded, unzip the file MedleyDB-Melody.zip and copy the result to: {} """ LICENSE_INFO = ( "Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)." ) DATA = core.LargeData("medleydb_melody_index.json") class Track(core.Track): """medleydb_melody Track class Args: track_id (str): track id of the track Attributes: artist (str): artist audio_path (str): path to the audio file genre (str): genre is_excerpt (bool): True if the track is an excerpt is_instrumental (bool): True of the track does not contain vocals melody1_path (str): path to the melody1 annotation file
REMOTES = { "all": download_utils.RemoteFileMetadata( filename="Orchset_dataset_0.zip", url= "https://zenodo.org/record/1289786/files/Orchset_dataset_0.zip?download=1", checksum="cf6fe52d64624f61ee116c752fb318ca", destination_dir=None, ) } LICENSE_INFO = ( "Creative Commons Attribution Non Commercial Share Alike 4.0 International." ) DATA = core.LargeData("orchset_index.json") class Track(core.Track): """orchset Track class Args: track_id (str): track id of the track Attributes: alternating_melody (bool): True if the melody alternates between instruments audio_path_mono (str): path to the mono audio file audio_path_stereo (str): path to the stereo audio file composer (str): the work's composer contains_brass (bool): True if the track contains any brass instrument contains_strings (bool): True if the track contains any string instrument
def _load_metadata(data_home): metadata_path = os.path.join(data_home, "medleydb_pitch_metadata.json") if not os.path.exists(metadata_path): logging.info("Metadata file {} not found.".format(metadata_path)) return None with open(metadata_path, "r") as fhandle: metadata = json.load(fhandle) metadata["data_home"] = data_home return metadata DATA = core.LargeData("medleydb_pitch_index.json", _load_metadata) class Track(core.Track): """medleydb_pitch Track class Args: track_id (str): track id of the track Attributes: artist (str): artist audio_path (str): path to the audio file genre (str): genre instrument (str): instrument of the track pitch_path (str): path to the pitch annotation file title (str): title
for download. If you have the Salami dataset, place the contents into a folder called Salami with the following structure: > Salami/ > salami-data-public-hierarchy-corrections/ > audio/ and copy the Salami folder to {} """ LICENSE_INFO = """ This data is released under a Creative Commons 0 license, effectively dedicating it to the public domain. More information about this dedication and your rights, please see the details here: http://creativecommons.org/publicdomain/zero/1.0/ and http://creativecommons.org/publicdomain/zero/1.0/legalcode. """ DATA = core.LargeData("salami_index.json") class Track(core.Track): """salami Track class Args: track_id (str): track id of the track Attributes: annotator_1_id (str): number that identifies annotator 1 annotator_1_time (str): time that the annotator 1 took to complete the annotation annotator_2_id (str): number that identifies annotator 1 annotator_2_time (str): time that the annotator 1 took to complete the annotation artist (str): song artist audio_path (str): path to the audio file
filename="original_metadata.zip", url= "https://zenodo.org/record/1101082/files/original_metadata.zip?download=1", checksum="bb3e3ac1fe5dee7600ef2814accdf8f8", destination_dir=".", ), "audio": download_utils.RemoteFileMetadata( filename="audio.zip", url="https://zenodo.org/record/1101082/files/audio.zip?download=1", checksum="f490ee6c23578482d6fcfa11b82636a1", destination_dir=".", ), } DATA = core.LargeData("beatport_key_index.json") LICENSE_INFO = "Creative Commons Attribution Share Alike 4.0 International." class Track(core.Track): """beatport_key track class Args: track_id (str): track id of the track data_home (str): Local path where the dataset is stored. Attributes: audio_path (str): track audio path keys_path (str): key annotation path metadata_path (str): sections annotation path
metadata = dict() metadata["data_home"] = data_home for i, j in zip(indexes, range(len(artists))): metadata[i] = { "musicBrainzID": identifiers[j], "artist": artists[j], "title": titles[j], "release": releases[j], "duration": durations[j], } return metadata DATA = core.LargeData("cante100_index.json", _load_metadata) class Track(core.Track): """cante100 track class Args: track_id (str): track id of the track data_home (str): Local path where the dataset is stored. If `None`, looks for the data in the default directory, `~/mir_datasets/cante100` Attributes: track_id (str): track id identifier (str): musicbrainz id of the track artist (str): performing artists title (str): title of the track song
annotations for 179 Beatles songs. Details can be found in http://matthiasmauch.net/_pdf/mauch_omp_2009.pdf and http://isophonics.net/content/reference-annotations-beatles. """ import csv import os import librosa import numpy as np from mirdata import download_utils from mirdata import jams_utils from mirdata import core from mirdata import annotations DATA = core.LargeData("beatles_index.json") BIBTEX = """@inproceedings{mauch2009beatles, title={OMRAS2 metadata project 2009}, author={Mauch, Matthias and Cannam, Chris and Davies, Matthew and Dixon, Simon and Harte, Christopher and Kolozali, Sefki and Tidhar, Dan and Sandler, Mark}, booktitle={12th International Society for Music Information Retrieval Conference}, year={2009}, series = {ISMIR} }""" REMOTES = { "annotations": download_utils.RemoteFileMetadata( filename="The Beatles Annotations.tar.gz", url=
}""" DOWNLOAD_INFO = """ To download this dataset, visit: https://zenodo.org/record/2620624#.XKZc7hNKh24 and request access. Once downloaded, unzip the file MedleyDB-Pitch.zip and copy the result to: {} """ LICENSE_INFO = ( "Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)." ) DATA = core.LargeData("medleydb_pitch_index.json") class Track(core.Track): """medleydb_pitch Track class Args: track_id (str): track id of the track Attributes: artist (str): artist audio_path (str): path to the audio file genre (str): genre instrument (str): instrument of the track pitch_path (str): path to the pitch annotation file title (str): title
"annotator_2_id": line[3], "duration": duration, "title": line[7], "artist": line[8], "annotator_1_time": line[10], "annotator_2_time": line[11], "class": line[14], "genre": line[15], } metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("salami_index.json", _load_metadata) class Track(core.Track): """salami Track class Args: track_id (str): track id of the track Attributes: annotator_1_id (str): number that identifies annotator 1 annotator_1_time (str): time that the annotator 1 took to complete the annotation annotator_2_id (str): number that identifies annotator 1 annotator_2_time (str): time that the annotator 1 took to complete the annotation artist (str): song artist audio_path (str): path to the audio file
"contains_strings": tf_dict[line[4]], "contains_brass": tf_dict[line[5]], "only_strings": tf_dict[line[6]], "only_winds": tf_dict[line[7]], "only_brass": tf_dict[line[8]], "composer": id_split[0], "work": "-".join(id_split[1:-1]), "excerpt": id_split[-1][2:], } metadata_index["data_home"] = data_home return metadata_index DATA = core.LargeData("orchset_index.json", _load_metadata) class Track(core.Track): """orchset Track class Args: track_id (str): track id of the track Attributes: alternating_melody (bool): True if the melody alternates between instruments audio_path_mono (str): path to the mono audio file audio_path_stereo (str): path to the stereo audio file composer (str): the work's composer contains_brass (bool): True if the track contains any brass instrument contains_strings (bool): True if the track contains any string instrument
"General MIDI": "Ride Cymbal 1", "Simplified": "Ride Cymbal (51)", }, 59: { "Roland": "Ride (Edge)", "General MIDI": "Ride Cymbal 2", "Simplified": "Ride Cymbal (51)", }, 53: { "Roland": "Ride (Bell)", "General MIDI": "Ride Bell", "Simplified": "Ride Cymbal (51)", }, } DATA = core.LargeData("groove_midi_index.json") class Track(core.Track): """Groove MIDI Track class Args: track_id (str): track id of the track Attributes: drummer (str): Drummer id of the track (ex. 'drummer1') session (str): Type of session (ex. 'session1', 'eval_session') track_id (str): track id of the track (ex. 'drummer1/eval_session/1') style (str): Style (genre, groove type) of the track (ex. 'funk/groove1') tempo (int): track tempo in beats per minute (ex. 138) beat_type (str): Whether the track is a beat or a fill (ex. 'beat')
automatic transcription}, pages = {181--185}, title = {{Modal analysis and transcription of strokes of the mridangam using non-negative matrix factorization}}, year = {2013} }""" REMOTES = { "remote_data": download_utils.RemoteFileMetadata( filename="mridangam_stroke_1.5.zip", url="https://zenodo.org/record/4068196/files/mridangam_stroke_1.5.zip?download=1", checksum="39af55b2476b94c7946bec24331ec01a", # the md5 checksum destination_dir=None, # relative path for where to unzip the data, or None ), } DATA = core.LargeData("mridangam_stroke_index.json") STROKE_DICT = { "bheem", "cha", "dheem", "dhin", "num", "ta", "tha", "tham", "thi", "thom", }
title={GTZAN genre collection}, author={Tzanetakis, George and Cook, P}, journal={Music Analysis, Retrieval and Synthesis for Audio Signals}, year={2002} }""" REMOTES = { "all": download_utils.RemoteFileMetadata( filename="genres.tar.gz", url="http://opihi.cs.uvic.ca/sound/genres.tar.gz", checksum="5b3d6dddb579ab49814ab86dba69e7c7", destination_dir="gtzan_genre", ) } DATA = core.LargeData("gtzan_genre_index.json") LICENSE_INFO = "Unfortunately we couldn't find the license information for the GTZAN_genre dataset." class Track(core.Track): """gtzan_genre Track class Args: track_id (str): track id of the track Attributes: audio_path (str): path to the audio file genre (str): annotated genre track_id (str): track id
), "metadata": download_utils.RemoteFileMetadata( filename="maestro-v2.0.0.json", url= "https://storage.googleapis.com/magentadata/datasets/maestro/v2.0.0/maestro-v2.0.0.json", checksum="576172af1cdc4efddcf0be7d260d48f7", destination_dir="maestro-v2.0.0", ), } LICENSE_INFO = ( "Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)." ) DATA = core.LargeData("maestro_index.json") class Track(core.Track): """MAESTRO Track class Args: track_id (str): track id of the track Attributes: audio_path (str): Path to the track's audio file canonical_composer (str): Composer of the piece, standardized on a single spelling for a given name. canonical_title (str): Title of the piece. Not guaranteed to be standardized to a single representation. duration (float): Duration in seconds, based on the MIDI file.
checksum="47ded648a496407ef49dba9c8bf80e87", destination_dir="annotations", ), } DOWNLOAD_INFO = """ Unfortunately the audio files of the RWC-Popular dataset are not available for download. If you have the RWC-Popular dataset, place the contents into a folder called RWC-Popular with the following structure: > RWC-Popular/ > annotations/ > audio/rwc-p-m0i with i in [1 .. 7] > metadata-master/ and copy the RWC-Popular folder to {} """ DATA = core.LargeData("rwc_popular_index.json") class Track(core.Track): """rwc_popular Track class Args: track_id (str): track id of the track Attributes: artist (str): artist audio_path (str): path of the audio file beats_path (str): path of the beat annotation file chords_path (str): path of the chord annotation file drum_information (str): If the drum is 'Drum sequences', 'Live drums', or 'Drum loops'
), "annotations": download_utils.RemoteFileMetadata( filename="TinySOL_metadata.csv", url= "https://zenodo.org/record/3685367/files/TinySOL_metadata.csv?download=1", checksum="a86c9bb115f69e61f2f25872e397fc4a", destination_dir="annotation", ), } STRING_ROMAN_NUMERALS = {1: "I", 2: "II", 3: "III", 4: "IV"} LICENSE_INFO = "Creative Commons Attribution 4.0 International Public License." DATA = core.LargeData("tinysol_index.json") class Track(core.Track): """tinysol Track class Args: track_id (str): track id of the track Attributes: audio_path (str): path of the audio file dynamics (str): dynamics abbreviation. Ex: pp, mf, ff, etc. dynamics_id (int): pp=0, p=1, mf=2, f=3, ff=4 family (str): instrument family encoded by its English name instance_id (int): instance ID. Either equal to 0, 1, 2, or 3. instrument_abbr (str): instrument abbreviation