def augment_key(key, augment_counts, counts, max_count):
key_count = counts[key]
for augment in AUGMENT_ORDER:
if key_count == max_count:
break
diff = max_count - key_count
key_to_augment_from = keys.shift(key, -augment)
count_to_augment = int(min(diff, counts[key_to_augment_from]))
augment_counts[key_to_augment_from, key] = count_to_augment
key_count += count_to_augment
def parse(token):
'''
Given a token representing a key present in the game,
return a function mimicing that key.
'''
out = None
if token == "<<":
out = shift()
elif token == "R":
out = rev()
elif token == "M":
out = mirror()
elif token == "S":
out = paste()
elif token == "I":
out = inv10()
elif token == "<":
out = rotL()
elif token == ">":
out = rotR()
elif token == "-":
out = negate()
elif token == "+":
out = summer()
elif "=>" in token:
old, new = token.split("=>")
out = rep(old, new)
elif "[+]" in token:
_, num = token.split("[+]")
out = inc(int(num))
elif token[0] == "+":
out = add(float(token[1:]))
elif token[0] == "-":
out = sub(float(token[1:]))
elif token[0] == "^":
out = pow(float(token[1:]))
elif token[0] == "*":
out = mult(float(token[1:]))
elif token[0] == "/":
out = div(float(token[1:]))
else:
out = dir(int(token))
if not hasattr(out, 'label'):
out.label = lambda state: token
return out
def augment_from_dataframe_parallel(proc_id, to_augment):
X_aug = np.zeros((0, 144, fileio.LENGTH * 5 + 1))
Y_aug = np.zeros((0, 24))
for idx, (filename, row) in enumerate(to_augment.iterrows()):
if idx % 100 == 0:
print('Process {} augmenting file {}/{}'.format(proc_id, idx, len(to_augment)))
shifts = row[0]
orig_key = row[1]
sox_shifts = ' ' + ' '.join(str(shift * 100) for shift in shifts) + ' '
# Convert from mp3 if needed
delete = False
if filename.endswith('.mp3'):
audio_data = fileio.read_audio_data(filename, fileio.FS)
t = time.time()
tmp_name = '{}/{}_{}.wav'.format(TMP_DATA, proc_id, t)
librosa.output.write_wav(tmp_name, audio_data, fileio.FS)
filename = tmp_name
delete = True
# Shift with sox
subprocess.call(['./augment.sh', TMP_DATA, filename, sox_shifts])
if delete:
os.remove(filename)
basename = os.path.basename(filename)
# Read shifted files
for key_shift in shifts:
shifted_filename = '{}/{}.{}00.wav'.format(TMP_DATA, basename, key_shift)
shifted_key = keys.shift(orig_key, key_shift)
shifted_audio_data = fileio.cut_or_pad_to_length(
fileio.read_audio_data(shifted_filename, fileio.FS), fileio.NUM_SAMPLES)
os.remove(shifted_filename)
cqt = features.get_cqt(shifted_audio_data)
X_aug = np.concatenate((X_aug, cqt.reshape(1, cqt.shape[0], cqt.shape[1])), axis=0)
Y_aug = np.concatenate((Y_aug, keys.get_vector_from_key(shifted_key).reshape(1, 24)), axis=0)
np.savez_compressed('{}/data_aug_{}.npz'.format(fileio.WORKING_PREFIX, proc_id), X=np.log(X_aug + features.EPS), Y=Y_aug)
print('Process {} done!'.format(proc_id))
def load_msd_data(chunk_start_nr,
chunk_prefix,
labels_prefix,
chunk_size=CHUNK_SIZE,
max_records=None):
"""
See load_all_data for more info. Split these into seperate functions
to allow for easier hacking of partial imports (and adding new data)
"""
print('Processing MSD files {}'.format(20 * '='))
chunk_nr = chunk_start_nr
labels = pd.DataFrame()
meta = MSD_META
pattern = os.path.join(meta['DATA_PREFIX'], '**',
'*' + meta['DATA_SUFFIX'])
# sorted important to preserve order
file_list = sorted(glob(pattern, recursive=True))
if max_records is not None:
file_list = file_list[:max_records]
for cc, files in enumerate(chunks(file_list, chunk_size)):
chunk_nr = chunk_start_nr + cc
delete_rows = []
# use len(files) instead of chunk_size as last chunk will be smaller
X_chunk = np.zeros((len(files), NUM_SAMPLES))
chunk_name = 'chunk{:04d}'.format(chunk_nr)
print('Processing chunk {} (size {})'.format(chunk_nr, len(files)))
chunk_idx = 0
for ii, filepath in enumerate(files):
print('File {}/{}'.format(ii + 1, len(files)), end="\r")
file_stub = filepath.split(
'lmd_matched_mp3/')[1][:-len(meta['DATA_SUFFIX'])]
label_path = os.path.join(meta['LABEL_PREFIX'],
file_stub + meta['LABEL_SUFFIX'])
key_map = h5py.File(label_path, 'r')['analysis']['songs']
key = int(key_map['key'][0])
key_confidence = key_map['key_confidence'][0]
mode = int(key_map['mode'][0])
mode_confidence = key_map['mode_confidence'][0]
# Convert from C=0 (how h5 stores it) to A=0 (how we store it)
key = keys.shift(key, 3)
# Convert from maj=1, min=0 (how h5 stores it) to maj=[0-11], min=[12-23] (how we store it)
key += (1 - mode) * 12
if key_confidence < meta[
'MIN_KEY_CONFIDENCE'] or mode_confidence < meta[
'MIN_MODE_CONFIDENCE']:
print('WARNING: key or mode uncertain ({}, {}), skipping {}.'.
format(key_confidence, mode_confidence, filepath))
delete_rows += [ii]
continue
file_labels = dict(filepath=filepath,
genre=file_stub.split('/')[0],
key=key,
key_str=keys.get_string_from_idx(key).replace(
'\t', ' '),
raw=1,
key_shift=0,
time_shift=1.0,
chunk_nr=chunk_nr,
chunk_idx=chunk_idx)
chunk_idx += 1
labels = labels.append(file_labels, ignore_index=True)
audio_data = read_audio_data(filepath, FS)
audio_data = cut_or_pad_to_length(audio_data, NUM_SAMPLES)
X_chunk[ii, :] = audio_data
X_chunk = np.delete(X_chunk, delete_rows, axis=0)
file_name = '{}/{}.npz'.format(chunk_prefix, chunk_name)
np.savez_compressed(file_name, X=X_chunk)
labels['majmin'] = [
'major' if key < 12 else 'minor' for key in labels['key']
]
labels['dataset'] = meta['NAME']
labels.to_pickle('{}/{}.pkl'.format(labels_prefix, meta['OUTFILE']))
return labels, chunk_nr + 1
def augment_train_full(labels_prefix=fileio.WORKING_PREFIX, batch_size=50, batch_start=0, index_start=0, append=False):
file_name = '{}/{}.npz'.format(labels_prefix, 'splits')
train_idx = np.load(file_name)['train_idx'][index_start:]
df = pd.read_pickle("{}/labels_raw.pkl".format(labels_prefix))
filepaths = list(df['filepath'][train_idx])[index_start:]
all_keys = list(df['key'][train_idx])[index_start:]
num_batches = math.ceil(len(train_idx) / batch_size)
for idx in range(batch_start, num_batches):
print("Augmenting batch {}/{}".format(idx, num_batches - 1))
X_aug = np.zeros((0, 144, fileio.LENGTH * 5 + 1))
Y_aug = np.zeros((0, 24))
if os.path.exists(TMP_DATA):
shutil.rmtree(TMP_DATA)
os.makedirs(TMP_DATA)
bottom = idx * batch_size
top = min((idx + 1) * batch_size, len(train_idx))
files = filepaths[bottom : top]
# Convert all mp3s to wavs
print("Converting mp3s to wavs")
for file_idx, file in enumerate(files):
if file.endswith('.mp3'):
audio_data = fileio.read_audio_data(file, fileio.FS)
t = time.time()
tmp_name = '{}/{}.wav'.format(TMP_DATA, t)
librosa.output.write_wav(tmp_name, audio_data, fileio.FS)
files[file_idx] = tmp_name
train_keys = all_keys[bottom : top]
file_list = ' ' + ' '.join(files) + ' '
print("Calling sox")
subprocess.call(['./augment.sh', TMP_DATA, file_list, SOX_PITCH_STRING])
print("Reading outputs and calculating cqts")
for file_idx, file in enumerate(files):
file = os.path.basename(file)
orig_key = train_keys[file_idx]
for key_shift in [-4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7]:
shifted_filename = '{}/{}.{}00.wav'.format(TMP_DATA, file, key_shift)
shifted_key = keys.shift(orig_key, key_shift)
shifted_audio_data = fileio.cut_or_pad_to_length(
fileio.read_audio_data(shifted_filename, fileio.FS), fileio.NUM_SAMPLES)
cqt = features.get_cqt(shifted_audio_data)
X_aug = np.concatenate((X_aug, cqt.reshape(1, cqt.shape[0], cqt.shape[1])), axis=0)
Y_aug = np.concatenate((Y_aug, keys.get_vector_from_key(shifted_key).reshape(1, 24)), axis=0)
np.savez_compressed('{}/data_aug_{}.npz'.format(labels_prefix, idx), X=np.log(X_aug + features.EPS), Y=Y_aug)
if os.path.exists(TMP_DATA):
shutil.rmtree(TMP_DATA)
print('Joining all batch files')
X_aug = np.zeros((0, 144, fileio.LENGTH * 5 + 1))
Y_aug = np.zeros((0, 24))
if append:
with np.load("{}/data_aug.npz".format(DATA_DIR)) as aug:
X_aug = aug['X']
Y_aug = aug['Y']
for idx in range(num_batches):
file = '{}/data_aug_{}.npz'.format(labels_prefix, idx)
if os.path.isfile(file):
with np.load('{}/data_aug_{}.npz'.format(labels_prefix, idx)) as aug:
X_aug = np.vstack((X_aug, aug['X']))
Y_aug = np.vstack((Y_aug, aug['Y']))
else:
print('WARNING: {} not found. Skipping. (This may be normal with --append and --batch).'.format(file))
print('Writing augmented file and cleaning augmented batches')
np.savez_compressed('{}/data_aug.npz'.format(labels_prefix), X=X_aug, Y=Y_aug)
for idx in range(num_batches):
if os.path.isfile('{}/data_aug_{}.npz'.format(labels_prefix, idx)):
os.remove('{}/data_aug_{}.npz'.format(labels_prefix, idx))