def load_fma(save_path='datasets', size='small'):
"""Download fma:free music archive (https://github.com/mdeff/fma)
Arguments
---------
save_path: absolute/relative path to store the dataset
size: string, 'small', 'medium', 'large', 'huge'
small: 4K tracks, 10 balance genres, 3.4GB, 30s
medium: 14,511 tracks, 20 unbalanced genres, 12.2GB, 30s
(large: 77,643 tracks, 68 unbalanced genres, 90GB, 30s)
(huge: 77,643 tracks, 68 unbalanced genres, 900GB, untrimmed)
"""
assert size in ('small', 'medium', 'large', 'huge')
assert size in ('small', 'medium') # 18 Mar 2017, only small/medium is released yet.
if size == 'small':
zip_filename = 'fma_small.zip'
zip_path = utils_datasets.get_file(zip_filename, 'https://os.unil.cloud.switch.ch/fma/fma_small.zip',
save_path, untar=False, cache_subdir='fma',
md5_hash='1104d67b3c8235bccbeaa11200a4e0b2')
elif size == 'medium':
zip_filename = 'fma_medium.zip'
zip_path = utils_datasets.get_file(zip_filename, 'https://os.unil.cloud.switch.ch/fma/fma_medium.zip',
save_path, untar=False, cache_subdir='fma',
md5_hash='07ec85c99f942765f35d88aaa87dabd3')
print("unzipping...")
os.system('unzip {} -d {}'.format(os.path.join(zip_path, zip_filename), zip_path))
def load_gtzan_genre(save_path='datasets'):
"""Load gtzan muusic dataset from http://opihi.cs.uvic.ca/sound/genres.tar.gz
It downloads gtzan tarball on save_path/gtzan_music.tar.gz .
After untarring, we got files as below:
```
for genre_name in ['blues', ..., 'rock']:
for idx in xrange(100):
"genres/{}/{}.{:05d}.au".format(genre_name, genre_name, idx)
```
Then it creates a helper csv file.
Arguments
---------
save_path: absolute or relative path to store the dataset
"""
datadir = utils_datasets.get_file('gtzan_genre.tar.gz', 'http://opihi.cs.uvic.ca/sound/genres.tar.gz',
save_path, untar=True, cache_subdir='gtzan_genre',
md5_hash='fe37942310e589be16b04b6d631790de')
labels = ['blues', 'classical', 'country', 'disco', 'hiphop', 'jazz', 'metal', 'pop', 'reggae', 'rock']
rows = utils_datasets.get_rows_from_folders(folder_dataset='genres',
folders=labels,
dataroot=datadir)
columns = ['id', 'filepath', 'label']
csv_path = os.path.join(datadir, 'dataset_summary_kapre.csv')
utils_datasets.write_to_csv(rows=rows, column_names=columns,
csv_fname=csv_path)
def load_gtzan_speechmusic(save_path='datasets'):
"""
Download gtzan speech/music dataset, untar it, and create a helper csv file
Arguments
---------
save_path: str,
Absolute or relative path to store the dataset
"""
datadir = utils_datasets.get_file(
'gtzan_speechmusic.tar.gz',
'http://opihi.cs.uvic.ca/sound/music_speech.tar.gz',
save_path,
untar=True,
cache_subdir='gtzan_speechmusic',
md5_hash='b063639094c169062940becacd3108a0')
labels = ['music', 'speech']
rows = utils_datasets.get_rows_from_folders(folder_dataset='music_speech',
folders=labels,
dataroot=datadir)
columns = ['id', 'filepath', 'label']
csv_path = os.path.join(datadir, 'dataset_summary_kapre.csv')
utils_datasets.write_to_csv(rows=rows,
column_names=columns,
csv_fname=csv_path)
def load_musicnet(save_path='datasets', format='hdf'):
"""Download musicnet (https://homes.cs.washington.edu/~thickstn/start.html)
Arguments
---------
save_path: absolute/relative path to store the dataset
format: string, either 'hdf' or 'npz'
"""
assert format in ('hdf', 'npz')
if format == 'hdf':
utils_datasets.get_file('musicnet.h5', 'https://homes.cs.washington.edu/~thickstn/media/musicnet.h5',
save_path, untar=False, cache_subdir='musicnet',
md5_hash='05103753391a8019029b29b790f7e1f7')
else:
utils_datasets.get_file('musicnet.npz', 'https://homes.cs.washington.edu/~thickstn/media/musicnet.npz',
save_path, untar=False, cache_subdir='musicnet',
md5_hash='9303e5338adefd3715c51997553fb45f')
utils_datasets.get_file('musicnet_metadata.csv',
'https://homes.cs.washington.edu/~thickstn/media/musicnet_metadata.csv',
save_path, untar=False, cache_subdir='musicnet',
md5_hash=None)
def load_magnatagatune(save_path='datasets'):
"""Download magnatagatune dataset, concate the zip files, unzip it,
to `save_path`.
Arguments
---------
save_path: absolute or relative path to store the dataset
"""
# 1GB for each
zip_path = utils_datasets.get_file('mp3.zip.001', 'http://mi.soi.city.ac.uk/datasets/magnatagatune/mp3.zip.001',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='179c91c8c2a6e9b3da3d4e69d306fd3b')
utils_datasets.get_file('mp3.zip.002', 'http://mi.soi.city.ac.uk/datasets/magnatagatune/mp3.zip.002',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='acf8265ff2e35c6ff22210e46457a824')
utils_datasets.get_file('mp3.zip.003', 'http://mi.soi.city.ac.uk/datasets/magnatagatune/mp3.zip.003',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='582dc649cabb8cd991f09e14b99349a5')
print("appending zip files...")
os.system('cat {}/mp3.zip.* > {}/mp3s.zip'.format(zip_path, zip_path))
print("unzipping...")
os.system('unzip {} -d {}/mp3s'.format(os.path.join(zip_path, 'mp3s.zip'), zip_path))
# labels
utils_datasets.get_file('clip_info_final.csv',
'http://mi.soi.city.ac.uk/datasets/magnatagatune/clip_info_final.csv',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='03ef3cb8ddcfe53fdcdb8e0cda005be2')
utils_datasets.get_file('annotations_final.csv',
'http://mi.soi.city.ac.uk/datasets/magnatagatune/annotations_final.csv',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='f04fa01752a8cc64f6e1ca142a0fef1d')
utils_datasets.get_file('comparisons_final.csv',
'http://mi.soi.city.ac.uk/datasets/magnatagatune/comparisons_final.csv',
save_path, untar=False, cache_subdir='magnatagatune')
# echonest feature (377.4 MB)
utils_datasets.get_file('mp3_echonest_xml.zip',
'http://mi.soi.city.ac.uk/datasets/magnatagatune/mp3_echonest_xml.zip',
save_path, untar=False, cache_subdir='magnatagatune',
md5_hash='09be4ac8c682a8c182279276fadb37f9')
def load_jamendo(save_path='datasets',
sr=16000,
mono=True,
duration=None,
offset=0.0):
"""Download jamendo http://www.mathieuramona.com/wp/data/jamendo/
It creates `save_path/jamendo` and the sub-directories, `jamendo_lab`, `train`, `valid`, `test`.
It does not remove the downloaded `.tar.gz` files in `save_path/jamendo`.
As it takes quite a while for decoding the audio files, it would be better to store
the returned value as npy/hdf/whatever and use it.
Parameters
----------
save_path: str,
absolute/relative path to store the dataset
sr: int > 0
sampling rate of audio sources.
It is also used to compute label arrays
mono: bool
Whether downmix the audio signals to mono or not.
duration: float [second]
Duration of audio files
offset: float [second]
Offset to load
Returns
-------
srcs: list, length of 3.
| each element is a list of train/valid/test sources
| e.g. ``srcs[0][0].shape = (1, 3959745)`` when ``sr=16000`` and ``mono=True``
ys: list, length of 3.
| each element is a list of train/valid/test groundtruths
| e.g., ``ys[0][0].shape = (3959745, )``
"""
set_names = ['train', 'valid', 'test']
for set in set_names:
datadir = utils_datasets.get_file(
'jam_{}_audio.tar.gz'.format(set),
'http://www.mathieuramona.com/uploads/Main/jam_{}_audio.tar.gz'.
format(set),
save_path,
untar=True,
cache_subdir='jamendo',
tar_folder_name=set)
datadir = utils_datasets.get_file(
'jam_annote.tar.gz',
'http://www.mathieuramona.com/uploads/Main/jam_annote.tar.gz',
save_path,
untar=True,
cache_subdir='jamendo',
tar_folder_name='jamendo_lab')
# load file names in train/valid/test folder
x_filenames = []
y_filenames = []
for set in set_names:
fnames = [f.lstrip('._') for f in os.listdir(os.path.join(save_path, 'jamendo', set)) \
if f.split('.')[-1] in ('ogg', 'mp3')]
x_filenames.append(fnames)
y_filenames.append([f.split('.')[0] + '.lab' for f in fnames])
srcs = []
ys = []
for set, x_fnames, y_fnames in zip(set_names, x_filenames, y_filenames):
srcs_set = []
ys_set = []
for idx, (x_fname, y_fname) in enumerate(zip(x_fnames, y_fnames)):
# process srcs
print('Loading {}/{}: {}...'.format(idx, len(x_fnames), x_fname))
src, _ = librosa.load(os.path.join(save_path, 'jamendo', set,
x_fname),
sr=sr,
mono=mono,
offset=offset,
duration=duration)
if mono:
src = src[np.newaxis, :] # to make it (1, N) instead of (N,)
len_src = src.shape[1]
srcs_set.append(src)
# process ys
y = np.zeros((len_src, ), dtype=np.bool)
with open(
os.path.join(save_path, 'jamendo', 'jamendo_lab',
y_fname)) as f_label:
for line in f_label:
start, end, label = line.rstrip('\n').split(' ')
if label == 'sing':
start, end = int(np.round(float(start) * sr)), int(
np.round(float(end) * sr))
y[start:end] = True
ys_set.append(y)
srcs.append(srcs_set)
ys.append(ys_set)
# return
return srcs, ys
def load_fma(save_path='datasets', size='small'):
"""Download fma:free music archive (https://github.com/mdeff/fma)
It would be better to download directly from the link for large/full..
Parameters
----------
save_path: str,
absolute/relative path to store the dataset
size: str, 'small', 'medium', 'large', 'huge'
| small: 8,000 tracks of 30s, 8 balanced genres (GTZAN-like) (7.2 GiB)
| medium: 25,000 tracks of 30s, 16 unbalanced genres (22 GiB)
| large: 106,574 tracks of 30s, 161 unbalanced genres (93 GiB)
| full: 106,574 untrimmed tracks, 161 unbalanced genres (879 GiB)
"""
assert size in ('small', 'medium', 'large', 'full')
if size == 'small':
zip_filename = 'fma_small.zip'
zip_path = utils_datasets.get_file(
zip_filename,
'https://os.unil.cloud.switch.ch/fma/fma_small.zip',
save_path,
untar=False,
cache_subdir='fma',
md5_hash='4edb51c99a19d31fe01a7d44d5cac19b')
elif size == 'medium':
zip_filename = 'fma_medium.zip'
zip_path = utils_datasets.get_file(
zip_filename,
'https://os.unil.cloud.switch.ch/fma/fma_medium.zip',
save_path,
untar=False,
cache_subdir='fma')
elif size == 'large':
zip_filename = 'fma_large.zip'
zip_path = utils_datasets.get_file(
zip_filename,
'https://os.unil.cloud.switch.ch/fma/fma_large.zip',
save_path,
untar=False,
cache_subdir='fma')
elif size == 'full':
zip_filename = 'fma_full.zip'
zip_path = utils_datasets.get_file(
zip_filename,
'https://os.unil.cloud.switch.ch/fma/fma_full.zip',
save_path,
untar=False,
cache_subdir='fma')
print("unzipping audio files...")
os.system('unzip {} -d {}'.format(os.path.join(zip_path, zip_filename),
zip_path))
metadata_zip_filename = 'fma_metadata.zip'
metadata_zip_path = utils_datasets.get_file(
metadata_zip_filename,
'https://os.unil.cloud.switch.ch/fma/fma_metadata.zip',
save_path,
untar=False,
cache_subdir='fma',
md5_hash='d3ebfd86e283345ee2366a5492495935')
print("unzipping metadata files...")
os.system('unzip {} -d {}'.format(
os.path.join(metadata_zip_path, metadata_zip_filename),
metadata_zip_path))