def create_remote_index(httpserver, dataset_name):
httpserver.serve_content(
open(REMOTE_DATASETS[dataset_name]["local_index"], "rb").read())
remote_index = {
"index":
download_utils.RemoteFileMetadata(
filename=REMOTE_DATASETS[dataset_name]["remote_filename"],
url=httpserver.url,
checksum=REMOTE_DATASETS[dataset_name]["remote_checksum"],
destination_dir="",
)
}
data_remote = core.LargeData(REMOTE_DATASETS[dataset_name]["filename"],
remote_index=remote_index)
return data_remote.index
def __init__(self, track_id, data_home, remote_index=None, remote_index_name=None):
if remote_index is not None and remote_index_name is not None:
data = core.LargeData(remote_index_name, remote_index=remote_index)
else:
data = DATA
if track_id not in data.index["tracks"]:
raise ValueError(
"{} is not a valid track ID in AcousticBrainz genre Dataset".format(
track_id
)
)
self.track_id = track_id
self._data_home = data_home
self._track_paths = data.index["tracks"][track_id]
self.path = core.none_path_join([self._data_home, self._track_paths["data"][0]])
def __init__(
self,
data_home=None,
remote_index=None,
remote_index_name=None,
):
if remote_index and remote_index_name:
data = core.LargeData(remote_index_name, remote_index=remote_index)
index = data.index
else:
index = None
super().__init__(
data_home,
index=DATA.index if index is None else index,
name=NAME,
track_class=Track,
bibtex=BIBTEX,
remotes=REMOTES,
license_info=LICENSE_INFO,
)
),
"keys": download_utils.RemoteFileMetadata(
filename="keys.zip",
url="https://zenodo.org/record/1095691/files/keys.zip?download=1",
checksum="775b7d17e009f5818544cf505b6a96fd",
destination_dir=".",
),
"metadata": download_utils.RemoteFileMetadata(
filename="original_metadata.zip",
url="https://zenodo.org/record/1095691/files/original_metadata.zip?download=1",
checksum="54181e0f34c35d9720439750d0b08091",
destination_dir=".",
),
}
DATA = core.LargeData("giantsteps_key_index.json")
LICENSE_INFO = "Creative Commons Attribution Share Alike 4.0 International."
class Track(core.Track):
"""giantsteps_key track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): track audio path
keys_path (str): key annotation path
metadata_path (str): sections annotation path
title (str): title of the track
"Pitch": row[7],
"Pitch ID": int(row[8]),
"Dynamics": row[9],
"Dynamics ID": int(row[10]),
"Instance ID": int(row[11]),
"Resampled": (row[13] == "TRUE"),
}
if len(row[12]) > 0:
metadata_index[key]["String ID"] = int(float(row[12]))
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("tinysol_index.json", _load_metadata)
class Track(core.Track):
"""tinysol Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): path of the audio file
dynamics (str): dynamics abbreviation. Ex: pp, mf, ff, etc.
dynamics_id (int): pp=0, p=1, mf=2, f=3, ff=4
family (str): instrument family encoded by its English name
instance_id (int): instance ID. Either equal to 0, 1, 2, or 3.
instrument_abbr (str): instrument abbreviation
def _load_metadata(data_home):
metadata_path = os.path.join(data_home, "medleydb_melody_metadata.json")
if not os.path.exists(metadata_path):
logging.info("Metadata file {} not found.".format(metadata_path))
return None
with open(metadata_path, "r") as fhandle:
metadata = json.load(fhandle)
metadata["data_home"] = data_home
return metadata
DATA = core.LargeData("medleydb_melody_index.json", _load_metadata)
class Track(core.Track):
"""medleydb_melody Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): artist
audio_path (str): path to the audio file
genre (str): genre
is_excerpt (bool): True if the track is an excerpt
is_instrumental (bool): True of the track does not contain vocals
melody1_path (str): path to the melody1 annotation file
==> https://zenodo.org/record/1324183
Then, locate the downloaded the cante100audio folder like this:
> cante100/
> cante100_spectrum/
... (rest of the annotation folders)
> cante100audio/
Remember to locate the cante100 folder to {}
"""
LICENSE_INFO = """
The provided datasets are offered free of charge for internal non-commercial use.
We do not grant any rights for redistribution or modification. All data collections
were gathered by the COFLA team. COFLA 2015. All rights reserved.
"""
DATA = core.LargeData("cante100_index.json")
class Track(core.Track):
"""cante100 track class
Args:
track_id (str): track id of the track
data_home (str): Local path where the dataset is stored.
If `None`, looks for the data in the default directory, `~/mir_datasets/cante100`
Attributes:
track_id (str): track id
identifier (str): musicbrainz id of the track
artist (str): performing artists
title (str): title of the track song
# load metadata however makes sense for your dataset
with open(metadata_path, "r") as fhandle:
raw_metadata = json.load(fhandle)
metadata = {}
for mdata in raw_metadata:
track_id = mdata["midi_filename"].split(".")[0]
metadata[track_id] = mdata
metadata["data_home"] = data_home
return metadata
DATA = core.LargeData("maestro_index.json", _load_metadata)
class Track(core.Track):
"""MAESTRO Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): Path to the track's audio file
canonical_composer (str): Composer of the piece, standardized on a
single spelling for a given name.
canonical_title (str): Title of the piece. Not guaranteed to be
standardized to a single representation.
duration (float): Duration in seconds, based on the MIDI file.
checksum="021ab25a5fd1b020521824e7fce9c775",
destination_dir="temp",
),
}
REMOTE_INDEX = {
"REMOTE_INDEX":
download_utils.RemoteFileMetadata(
filename="acousticbrainz_genre_index.json.zip",
url=
"https://zenodo.org/record/4298580/files/acousticbrainz_genre_index.json.zip?download=1",
checksum="810f1c003f53cbe58002ba96e6d4d138",
destination_dir="",
)
}
DATA = core.LargeData("acousticbrainz_genre_index.json",
remote_index=REMOTE_INDEX)
LICENSE_INFO = """
This dataset is composed of 4 subdatasets. Three of them are Creative Commons Attribution
Non Commercial Share Alike 4.0 International and the other one is non-comercial. Details
about which license correspond to each subdataset can be found in the following websites:
* https://zenodo.org/record/2553414#.X_nxnOn7RUI
* https://zenodo.org/record/2554044#.X_nw2en7RUI
"""
class Track(core.Track):
"""AcousticBrainz Genre Dataset track class
checksum="44afcf7f193d7e48a7d99e7a6f3ed39d",
destination_dir="annotations",
),
}
DOWNLOAD_INFO = """
Unfortunately the audio files of the RWC-Jazz dataset are not available
for download. If you have the RWC-Jazz dataset, place the contents into a
folder called RWC-Jazz with the following structure:
> RWC-Jazz/
> annotations/
> audio/rwc-j-m0i with i in [1 .. 4]
> metadata-master/
and copy the RWC-Jazz folder to {}
"""
DATA = core.LargeData("rwc_jazz_index.json")
class Track(core.Track):
"""rwc_jazz Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): Artist name
audio_path (str): path of the audio file
beats_path (str): path of the beat annotation file
duration (float): Duration of the track in seconds
instruments (str): list of used instruments.
piece_number (str): Piece number of this Track, [1-50]
"piece_number": line[0],
"suffix": line[1],
"track_number": line[2],
"title": line[3],
"composer": line[4],
"artist": line[5],
"duration": _duration_to_sec(line[6]),
"category": line[7],
}
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("rwc_classical_index.json", _load_metadata)
class Track(core.Track):
"""rwc_classical Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): the track's artist
audio_path (str): path of the audio file
beats_path (str): path of the beat annotation file
category (str): One of 'Symphony', 'Concerto', 'Orchestral',
'Solo', 'Chamber', 'Vocal', or blank.
composer (str): Composer of this Track.
)
def _load_metadata(data_home):
metadata_path = os.path.join(data_home, os.path.join("dali_metadata.json"))
if not os.path.exists(metadata_path):
logging.info("Metadata file {} not found.".format(metadata_path))
return None
with open(metadata_path, "r") as fhandle:
metadata_index = json.load(fhandle)
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("dali_index.json", _load_metadata)
class Track(core.Track):
"""DALI melody Track class
Args:
track_id (str): track id of the track
Attributes:
album (str): the track's album
annotation_path (str): path to the track's annotation file
artist (str): the track's artist
audio_path (str): path to the track's audio file
audio_url (str): youtube ID
dataset_version (int): dataset annotation version
next(csv_reader)
for row in csv_reader:
subset, instrument_str, instrument_id, song_id, track_id = row
metadata_index[str(track_id)] = {
"subset": str(subset),
"instrument": str(instrument_str),
"instrument_id": int(instrument_id),
"song_id": int(song_id),
}
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("medley_solos_db_index.json", _load_metadata)
class Track(core.Track):
"""medley_solos_db Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): path to the track's audio file
instrument (str): instrument encoded by its English name
instrument_id (int): instrument encoded as an integer
song_id (int): song encoded as an integer
subset (str): either equal to 'train', 'validation', or 'test'
track_id (str): track id
If you have the iKala dataset, place the contents into a folder called
iKala with the following structure:
> iKala/
> Lyrics/
> PitchLabel/
> Wavfile/
and copy the iKala folder to {}
"""
LICENSE_INFO = """
When it was distributed, Ikala used to have a custom license.
Visit http://mac.citi.sinica.edu.tw/ikala/ for more details.
"""
DATA = core.LargeData("ikala_index.json")
class Track(core.Track):
"""ikala Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): path to the track's audio file
f0_path (str): path to the track's f0 annotation file
lyrics_path (str): path to the track's lyric annotation file
section (str): section. Either 'verse' or 'chorus'
singer_id (str): singer id
song_id (str): song id of the track
}"""
DOWNLOAD_INFO = """
To download this dataset, visit:
https://zenodo.org/record/2628782#.XKZdABNKh24
and request access.
Once downloaded, unzip the file MedleyDB-Melody.zip
and copy the result to:
{}
"""
LICENSE_INFO = (
"Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)."
)
DATA = core.LargeData("medleydb_melody_index.json")
class Track(core.Track):
"""medleydb_melody Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): artist
audio_path (str): path to the audio file
genre (str): genre
is_excerpt (bool): True if the track is an excerpt
is_instrumental (bool): True of the track does not contain vocals
melody1_path (str): path to the melody1 annotation file
REMOTES = {
"all":
download_utils.RemoteFileMetadata(
filename="Orchset_dataset_0.zip",
url=
"https://zenodo.org/record/1289786/files/Orchset_dataset_0.zip?download=1",
checksum="cf6fe52d64624f61ee116c752fb318ca",
destination_dir=None,
)
}
LICENSE_INFO = (
"Creative Commons Attribution Non Commercial Share Alike 4.0 International."
)
DATA = core.LargeData("orchset_index.json")
class Track(core.Track):
"""orchset Track class
Args:
track_id (str): track id of the track
Attributes:
alternating_melody (bool): True if the melody alternates between instruments
audio_path_mono (str): path to the mono audio file
audio_path_stereo (str): path to the stereo audio file
composer (str): the work's composer
contains_brass (bool): True if the track contains any brass instrument
contains_strings (bool): True if the track contains any string instrument
def _load_metadata(data_home):
metadata_path = os.path.join(data_home, "medleydb_pitch_metadata.json")
if not os.path.exists(metadata_path):
logging.info("Metadata file {} not found.".format(metadata_path))
return None
with open(metadata_path, "r") as fhandle:
metadata = json.load(fhandle)
metadata["data_home"] = data_home
return metadata
DATA = core.LargeData("medleydb_pitch_index.json", _load_metadata)
class Track(core.Track):
"""medleydb_pitch Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): artist
audio_path (str): path to the audio file
genre (str): genre
instrument (str): instrument of the track
pitch_path (str): path to the pitch annotation file
title (str): title
for download. If you have the Salami dataset, place the contents into a
folder called Salami with the following structure:
> Salami/
> salami-data-public-hierarchy-corrections/
> audio/
and copy the Salami folder to {}
"""
LICENSE_INFO = """
This data is released under a Creative Commons 0 license, effectively dedicating it to
the public domain. More information about this dedication and your rights, please see the
details here: http://creativecommons.org/publicdomain/zero/1.0/ and
http://creativecommons.org/publicdomain/zero/1.0/legalcode.
"""
DATA = core.LargeData("salami_index.json")
class Track(core.Track):
"""salami Track class
Args:
track_id (str): track id of the track
Attributes:
annotator_1_id (str): number that identifies annotator 1
annotator_1_time (str): time that the annotator 1 took to complete the annotation
annotator_2_id (str): number that identifies annotator 1
annotator_2_time (str): time that the annotator 1 took to complete the annotation
artist (str): song artist
audio_path (str): path to the audio file
filename="original_metadata.zip",
url=
"https://zenodo.org/record/1101082/files/original_metadata.zip?download=1",
checksum="bb3e3ac1fe5dee7600ef2814accdf8f8",
destination_dir=".",
),
"audio":
download_utils.RemoteFileMetadata(
filename="audio.zip",
url="https://zenodo.org/record/1101082/files/audio.zip?download=1",
checksum="f490ee6c23578482d6fcfa11b82636a1",
destination_dir=".",
),
}
DATA = core.LargeData("beatport_key_index.json")
LICENSE_INFO = "Creative Commons Attribution Share Alike 4.0 International."
class Track(core.Track):
"""beatport_key track class
Args:
track_id (str): track id of the track
data_home (str): Local path where the dataset is stored.
Attributes:
audio_path (str): track audio path
keys_path (str): key annotation path
metadata_path (str): sections annotation path
metadata = dict()
metadata["data_home"] = data_home
for i, j in zip(indexes, range(len(artists))):
metadata[i] = {
"musicBrainzID": identifiers[j],
"artist": artists[j],
"title": titles[j],
"release": releases[j],
"duration": durations[j],
}
return metadata
DATA = core.LargeData("cante100_index.json", _load_metadata)
class Track(core.Track):
"""cante100 track class
Args:
track_id (str): track id of the track
data_home (str): Local path where the dataset is stored.
If `None`, looks for the data in the default directory, `~/mir_datasets/cante100`
Attributes:
track_id (str): track id
identifier (str): musicbrainz id of the track
artist (str): performing artists
title (str): title of the track song
annotations for 179 Beatles songs. Details can be found in http://matthiasmauch.net/_pdf/mauch_omp_2009.pdf and
http://isophonics.net/content/reference-annotations-beatles.
"""
import csv
import os
import librosa
import numpy as np
from mirdata import download_utils
from mirdata import jams_utils
from mirdata import core
from mirdata import annotations
DATA = core.LargeData("beatles_index.json")
BIBTEX = """@inproceedings{mauch2009beatles,
title={OMRAS2 metadata project 2009},
author={Mauch, Matthias and Cannam, Chris and Davies, Matthew and Dixon, Simon and Harte,
Christopher and Kolozali, Sefki and Tidhar, Dan and Sandler, Mark},
booktitle={12th International Society for Music Information Retrieval Conference},
year={2009},
series = {ISMIR}
}"""
REMOTES = {
"annotations":
download_utils.RemoteFileMetadata(
filename="The Beatles Annotations.tar.gz",
url=
}"""
DOWNLOAD_INFO = """
To download this dataset, visit:
https://zenodo.org/record/2620624#.XKZc7hNKh24
and request access.
Once downloaded, unzip the file MedleyDB-Pitch.zip
and copy the result to:
{}
"""
LICENSE_INFO = (
"Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)."
)
DATA = core.LargeData("medleydb_pitch_index.json")
class Track(core.Track):
"""medleydb_pitch Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): artist
audio_path (str): path to the audio file
genre (str): genre
instrument (str): instrument of the track
pitch_path (str): path to the pitch annotation file
title (str): title
"annotator_2_id": line[3],
"duration": duration,
"title": line[7],
"artist": line[8],
"annotator_1_time": line[10],
"annotator_2_time": line[11],
"class": line[14],
"genre": line[15],
}
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("salami_index.json", _load_metadata)
class Track(core.Track):
"""salami Track class
Args:
track_id (str): track id of the track
Attributes:
annotator_1_id (str): number that identifies annotator 1
annotator_1_time (str): time that the annotator 1 took to complete the annotation
annotator_2_id (str): number that identifies annotator 1
annotator_2_time (str): time that the annotator 1 took to complete the annotation
artist (str): song artist
audio_path (str): path to the audio file
"contains_strings": tf_dict[line[4]],
"contains_brass": tf_dict[line[5]],
"only_strings": tf_dict[line[6]],
"only_winds": tf_dict[line[7]],
"only_brass": tf_dict[line[8]],
"composer": id_split[0],
"work": "-".join(id_split[1:-1]),
"excerpt": id_split[-1][2:],
}
metadata_index["data_home"] = data_home
return metadata_index
DATA = core.LargeData("orchset_index.json", _load_metadata)
class Track(core.Track):
"""orchset Track class
Args:
track_id (str): track id of the track
Attributes:
alternating_melody (bool): True if the melody alternates between instruments
audio_path_mono (str): path to the mono audio file
audio_path_stereo (str): path to the stereo audio file
composer (str): the work's composer
contains_brass (bool): True if the track contains any brass instrument
contains_strings (bool): True if the track contains any string instrument
"General MIDI": "Ride Cymbal 1",
"Simplified": "Ride Cymbal (51)",
},
59: {
"Roland": "Ride (Edge)",
"General MIDI": "Ride Cymbal 2",
"Simplified": "Ride Cymbal (51)",
},
53: {
"Roland": "Ride (Bell)",
"General MIDI": "Ride Bell",
"Simplified": "Ride Cymbal (51)",
},
}
DATA = core.LargeData("groove_midi_index.json")
class Track(core.Track):
"""Groove MIDI Track class
Args:
track_id (str): track id of the track
Attributes:
drummer (str): Drummer id of the track (ex. 'drummer1')
session (str): Type of session (ex. 'session1', 'eval_session')
track_id (str): track id of the track (ex. 'drummer1/eval_session/1')
style (str): Style (genre, groove type) of the track (ex. 'funk/groove1')
tempo (int): track tempo in beats per minute (ex. 138)
beat_type (str): Whether the track is a beat or a fill (ex. 'beat')
automatic transcription},
pages = {181--185},
title = {{Modal analysis and transcription of strokes of the mridangam using non-negative matrix factorization}},
year = {2013}
}"""
REMOTES = {
"remote_data": download_utils.RemoteFileMetadata(
filename="mridangam_stroke_1.5.zip",
url="https://zenodo.org/record/4068196/files/mridangam_stroke_1.5.zip?download=1",
checksum="39af55b2476b94c7946bec24331ec01a", # the md5 checksum
destination_dir=None, # relative path for where to unzip the data, or None
),
}
DATA = core.LargeData("mridangam_stroke_index.json")
STROKE_DICT = {
"bheem",
"cha",
"dheem",
"dhin",
"num",
"ta",
"tha",
"tham",
"thi",
"thom",
}
title={GTZAN genre collection},
author={Tzanetakis, George and Cook, P},
journal={Music Analysis, Retrieval and Synthesis for Audio Signals},
year={2002}
}"""
REMOTES = {
"all":
download_utils.RemoteFileMetadata(
filename="genres.tar.gz",
url="http://opihi.cs.uvic.ca/sound/genres.tar.gz",
checksum="5b3d6dddb579ab49814ab86dba69e7c7",
destination_dir="gtzan_genre",
)
}
DATA = core.LargeData("gtzan_genre_index.json")
LICENSE_INFO = "Unfortunately we couldn't find the license information for the GTZAN_genre dataset."
class Track(core.Track):
"""gtzan_genre Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): path to the audio file
genre (str): annotated genre
track_id (str): track id
),
"metadata":
download_utils.RemoteFileMetadata(
filename="maestro-v2.0.0.json",
url=
"https://storage.googleapis.com/magentadata/datasets/maestro/v2.0.0/maestro-v2.0.0.json",
checksum="576172af1cdc4efddcf0be7d260d48f7",
destination_dir="maestro-v2.0.0",
),
}
LICENSE_INFO = (
"Creative Commons Attribution Non-Commercial Share-Alike 4.0 (CC BY-NC-SA 4.0)."
)
DATA = core.LargeData("maestro_index.json")
class Track(core.Track):
"""MAESTRO Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): Path to the track's audio file
canonical_composer (str): Composer of the piece, standardized on a
single spelling for a given name.
canonical_title (str): Title of the piece. Not guaranteed to be
standardized to a single representation.
duration (float): Duration in seconds, based on the MIDI file.
checksum="47ded648a496407ef49dba9c8bf80e87",
destination_dir="annotations",
),
}
DOWNLOAD_INFO = """
Unfortunately the audio files of the RWC-Popular dataset are not available
for download. If you have the RWC-Popular dataset, place the contents into a
folder called RWC-Popular with the following structure:
> RWC-Popular/
> annotations/
> audio/rwc-p-m0i with i in [1 .. 7]
> metadata-master/
and copy the RWC-Popular folder to {}
"""
DATA = core.LargeData("rwc_popular_index.json")
class Track(core.Track):
"""rwc_popular Track class
Args:
track_id (str): track id of the track
Attributes:
artist (str): artist
audio_path (str): path of the audio file
beats_path (str): path of the beat annotation file
chords_path (str): path of the chord annotation file
drum_information (str): If the drum is 'Drum sequences', 'Live drums',
or 'Drum loops'
),
"annotations":
download_utils.RemoteFileMetadata(
filename="TinySOL_metadata.csv",
url=
"https://zenodo.org/record/3685367/files/TinySOL_metadata.csv?download=1",
checksum="a86c9bb115f69e61f2f25872e397fc4a",
destination_dir="annotation",
),
}
STRING_ROMAN_NUMERALS = {1: "I", 2: "II", 3: "III", 4: "IV"}
LICENSE_INFO = "Creative Commons Attribution 4.0 International Public License."
DATA = core.LargeData("tinysol_index.json")
class Track(core.Track):
"""tinysol Track class
Args:
track_id (str): track id of the track
Attributes:
audio_path (str): path of the audio file
dynamics (str): dynamics abbreviation. Ex: pp, mf, ff, etc.
dynamics_id (int): pp=0, p=1, mf=2, f=3, ff=4
family (str): instrument family encoded by its English name
instance_id (int): instance ID. Either equal to 0, 1, 2, or 3.
instrument_abbr (str): instrument abbreviation
