def process_record(wav_data,
ns,
example_id,
min_length=5,
max_length=20,
sample_rate=16000,
allow_empty_notesequence=False):
"""Split a record into chunks and create an example proto.
To use the full length audio and notesequence, set min_length=0 and
max_length=-1.
Args:
wav_data: audio data in WAV format.
ns: corresponding NoteSequence.
example_id: id for the example proto
min_length: minimum length in seconds for audio chunks.
max_length: maximum length in seconds for audio chunks.
sample_rate: desired audio sample rate.
allow_empty_notesequence: whether an empty NoteSequence is allowed.
Yields:
Example protos.
"""
samples = audio_io.wav_data_to_samples(wav_data, sample_rate)
samples = librosa.util.normalize(samples, norm=np.inf)
if max_length == min_length:
splits = np.arange(0, ns.total_time, max_length)
elif max_length > 0:
splits = find_split_points(ns, samples, sample_rate, min_length, max_length)
else:
splits = [0, ns.total_time]
velocities = [note.velocity for note in ns.notes]
velocity_max = np.max(velocities) if velocities else 0
velocity_min = np.min(velocities) if velocities else 0
velocity_range = music_pb2.VelocityRange(min=velocity_min, max=velocity_max)
for start, end in zip(splits[:-1], splits[1:]):
if end - start < min_length:
continue
if start == 0 and end == ns.total_time:
new_ns = ns
else:
new_ns = sequences_lib.extract_subsequence(ns, start, end)
if not new_ns.notes and not allow_empty_notesequence:
tf.logging.warning('skipping empty sequence')
continue
if start == 0 and end == ns.total_time:
new_samples = samples
else:
# the resampling that happen in crop_wav_data is really slow
# and we've already done it once, avoid doing it twice
new_samples = audio_io.crop_samples(samples, sample_rate, start,
end - start)
new_wav_data = audio_io.samples_to_wav_data(new_samples, sample_rate)
yield create_example(
example_id, new_ns, new_wav_data, velocity_range=velocity_range)
def generate_test_set():
"""Generate the test TFRecord."""
test_file_pairs = []
for directory in test_dirs:
path = os.path.join(FLAGS.input_dir, directory)
path = os.path.join(path, '*.wav')
wav_files = glob.glob(path)
# find matching mid files
for wav_file in wav_files:
base_name_root, _ = os.path.splitext(wav_file)
mid_file = base_name_root + '.mid'
test_file_pairs.append((wav_file, mid_file))
test_output_name = os.path.join(FLAGS.output_dir,
'maps_config2_test.tfrecord')
with tf.python_io.TFRecordWriter(test_output_name) as writer:
for idx, pair in enumerate(test_file_pairs):
print('{} of {}: {}'.format(idx, len(test_file_pairs), pair[0]))
# load the wav data and resample it.
samples = audio_io.load_audio(pair[0], FLAGS.sample_rate)
wav_data = audio_io.samples_to_wav_data(samples, FLAGS.sample_rate)
# load the midi data and convert to a notesequence
ns = midi_io.midi_file_to_note_sequence(pair[1])
example = split_audio_and_label_data.create_example(pair[0], ns, wav_data)
writer.write(example.SerializeToString())
return [filename_to_id(wav) for wav, _ in test_file_pairs]
def _ValidateProvideBatchMemory(self,
truncated_length,
batch_size,
lengths,
expected_num_inputs):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples, hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data, frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
self._ValidateProvideBatch(
[e.SerializeToString() for e in examples],
truncated_length,
batch_size,
expected_inputs)
def _CreateExamplesAndExpectedInputs(self,
truncated_length,
lengths,
expected_num_inputs):
hparams = copy.deepcopy(configs.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples, hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data, frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
return examples, expected_inputs
def create_example(filename, hparams):
"""Processes an audio file into an Example proto."""
wav_data = librosa.core.load(filename, sr=hparams.sample_rate)[0]
if hparams.normalize_audio:
audio_io.normalize_wav_data(wav_data, hparams.sample_rate)
wav_data = audio_io.samples_to_wav_data(wav_data, hparams.sample_rate)
example = tf.train.Example(features=tf.train.Features(feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[filename.encode('utf-8')]
)),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[music_pb2.NoteSequence().SerializeToString()]
)),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[wav_data]
)),
'velocity_range':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[music_pb2.VelocityRange().SerializeToString()]
)),
}))
return example.SerializeToString()
def create_example(self, filename, hparams):
"""Processes an audio file into an Example proto."""
wav_data = librosa.core.load(filename, sr=hparams.sample_rate)[0]
if hparams.normalize_audio:
audio_io.normalize_wav_data(wav_data, hparams.sample_rate)
wav_data = audio_io.samples_to_wav_data(wav_data, hparams.sample_rate)
example = tf.train.Example(features=tf.train.Features(
feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[filename.encode('utf-8')])),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[music_pb2.NoteSequence().SerializeToString()])),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[wav_data])),
'velocity_range':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[music_pb2.VelocityRange().SerializeToString()])),
}))
return example.SerializeToString()
def _ValidateProvideBatchMemory(self, truncated_length, batch_size,
lengths, expected_num_inputs):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples,
hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data,
frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
self._ValidateProvideBatch([e.SerializeToString() for e in examples],
truncated_length, batch_size,
expected_inputs)
def validateProvideBatch_TFRecord(self, truncated_length, batch_size,
lengths, expected_num_inputs):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
constants.DEFAULT_SAMPLE_RATE), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(
wav_samples, constants.DEFAULT_SAMPLE_RATE)
num_frames = data.wav_to_num_frames(
wav_data,
frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
with tempfile.NamedTemporaryFile() as temp_rio:
with tf.python_io.TFRecordWriter(temp_rio.name) as writer:
for ex in examples:
writer.write(ex.SerializeToString())
self.validateProvideBatch(temp_rio.name, truncated_length,
batch_size, expected_inputs)
def _CreateExamplesAndExpectedInputs(self,
truncated_length,
lengths,
expected_num_inputs):
hparams = copy.deepcopy(configs.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples, hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data, frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
return examples, expected_inputs
def generate_test_set():
"""Generate the test TFRecord."""
test_file_pairs = []
for directory in test_dirs:
path = os.path.join(FLAGS.input_dir, directory)
path = os.path.join(path, '*.wav')
wav_files = glob.glob(path)
# find matching mid files
for wav_file in wav_files:
base_name_root, _ = os.path.splitext(wav_file)
mid_file = base_name_root + '.mid'
test_file_pairs.append((wav_file, mid_file))
test_output_name = os.path.join(FLAGS.output_dir,
'maps_config2_test.tfrecord')
with tf.python_io.TFRecordWriter(test_output_name) as writer:
for idx, pair in enumerate(test_file_pairs):
print('{} of {}: {}'.format(idx, len(test_file_pairs), pair[0]))
# load the wav data and resample it.
samples = audio_io.load_audio(pair[0], FLAGS.sample_rate)
wav_data = audio_io.samples_to_wav_data(samples, FLAGS.sample_rate)
# load the midi data and convert to a notesequence
ns = midi_io.midi_file_to_note_sequence(pair[1])
example = audio_label_data_utils.create_example(
pair[0], ns, wav_data)
writer.write(example.SerializeToString())
return [filename_to_id(wav) for wav, _ in test_file_pairs]
def generate_test_set():
"""Generate the test TFRecord."""
test_file_pairs = []
for directory in test_dirs:
path = os.path.join(FLAGS.input_dir, directory)
path = os.path.join(path, '*.wav')
wav_files = glob.glob(path)
# find matching mid files
for wav_file in wav_files:
base_name_root, _ = os.path.splitext(wav_file)
mid_file = base_name_root + '.mid'
test_file_pairs.append((wav_file, mid_file))
test_output_name = os.path.join(FLAGS.output_dir,
'maps_config2_test.tfrecord')
with tf.python_io.TFRecordWriter(test_output_name) as writer:
for pair in test_file_pairs:
print(pair)
# load the wav data and resample it.
samples = audio_io.load_audio(pair[0], FLAGS.sample_rate)
wav_data = audio_io.samples_to_wav_data(samples, FLAGS.sample_rate)
# load the midi data and convert to a notesequence
midi_data = tf.gfile.Open(pair[1]).read()
ns = midi_io.midi_to_sequence_proto(midi_data)
velocities = [note.velocity for note in ns.notes]
velocity_max = np.max(velocities)
velocity_min = np.min(velocities)
new_velocity_tuple = music_pb2.VelocityRange(
min=velocity_min, max=velocity_max)
example = tf.train.Example(features=tf.train.Features(feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[pair[0]]
)),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[ns.SerializeToString()]
)),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[wav_data]
)),
'velocity_range':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[new_velocity_tuple.SerializeToString()]
)),
}))
writer.write(example.SerializeToString())
return [filename_to_id(wav) for wav, _ in test_file_pairs]
def generate_test_set():
"""Generate the test TFRecord."""
test_file_pairs = []
for directory in test_dirs:
path = os.path.join(FLAGS.input_dir, directory)
path = os.path.join(path, '*.wav')
wav_files = glob.glob(path)
# find matching mid files
for wav_file in wav_files:
base_name_root, _ = os.path.splitext(wav_file)
mid_file = base_name_root + '.mid'
test_file_pairs.append((wav_file, mid_file))
test_output_name = os.path.join(FLAGS.output_dir,
'maps_config2_test.tfrecord')
with tf.python_io.TFRecordWriter(test_output_name) as writer:
for pair in test_file_pairs:
print(pair)
# load the wav data and resample it.
samples = audio_io.load_audio(pair[0], FLAGS.sample_rate)
wav_data = audio_io.samples_to_wav_data(samples, FLAGS.sample_rate)
# load the midi data and convert to a notesequence
midi_data = tf.gfile.Open(pair[1]).read()
ns = midi_io.midi_to_sequence_proto(midi_data)
velocities = [note.velocity for note in ns.notes]
velocity_max = np.max(velocities)
velocity_min = np.min(velocities)
new_velocity_tuple = music_pb2.VelocityRange(
min=velocity_min, max=velocity_max)
example = tf.train.Example(features=tf.train.Features(feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[pair[0]]
)),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[ns.SerializeToString()]
)),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[wav_data]
)),
'velocity_range':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[new_velocity_tuple.SerializeToString()]
)),
}))
writer.write(example.SerializeToString())
return [filename_to_id(wav) for wav, _ in test_file_pairs]
def process(self, paths):
wav_path, midi_path = paths
if midi_path:
if FLAGS.use_midi_stems:
base_ns = note_sequence_from_directory(
os.path.dirname(midi_path))
else:
base_ns = midi_io.midi_file_to_note_sequence(midi_path)
base_ns.filename = midi_path
else:
base_ns = music_pb2.NoteSequence()
logging.info('Creating Example %s:%s', midi_path, wav_path)
if FLAGS.convert_flac:
samples, sr = librosa.load(wav_path, FLAGS.sample_rate)
wav_data = audio_io.samples_to_wav_data(samples, sr)
else:
wav_data = tf.io.gfile.GFile(wav_path, 'rb').read()
ns = copy.deepcopy(base_ns)
# Use base names.
ns.id = '%s:%s' % (wav_path, midi_path)
Metrics.counter('create_example', 'read_midi_wav').inc()
if FLAGS.max_length > 0:
split_examples = audio_label_data_utils.process_record(
wav_data,
ns,
ns.id,
min_length=FLAGS.min_length,
max_length=FLAGS.max_length,
sample_rate=FLAGS.sample_rate,
load_audio_with_librosa=False)
for example in split_examples:
Metrics.counter('split_wav', 'split_example').inc()
yield example
else:
example = audio_label_data_utils.create_example(
ns.id, ns, wav_data)
Metrics.counter('create_example', 'created_example').inc()
yield example
def _ValidateProvideBatchTFRecord(self,
truncated_length,
batch_size,
lengths,
expected_num_inputs,
crop_sequence_secs=0):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples,
hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data,
frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams) -
crop_sequence_secs * 2, # crop from both ends.
i + constants.MIN_MIDI_PITCH,
start_time=crop_sequence_secs)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length,
crop_training_sequence_to_notes=crop_sequence_secs > 0)
self.assertEqual(expected_num_inputs, len(expected_inputs))
with tempfile.NamedTemporaryFile() as temp_tfr:
with tf.python_io.TFRecordWriter(temp_tfr.name) as writer:
for ex in examples:
writer.write(ex.SerializeToString())
self._ValidateProvideBatch(
temp_tfr.name,
truncated_length,
batch_size,
expected_inputs,
crop_training_sequence_to_notes=crop_sequence_secs > 0)
def mix_examples(mixid_exs, sample_rate, load_audio_with_librosa):
"""Mix several Examples together to create a new example."""
mixid, exs = mixid_exs
del mixid
example_samples = []
example_sequences = []
for ex_str in exs:
ex = tf.train.Example.FromString(ex_str)
wav_data = ex.features.feature['audio'].bytes_list.value[0]
if load_audio_with_librosa:
samples = audio_io.wav_data_to_samples_librosa(
wav_data, sample_rate)
else:
samples = audio_io.wav_data_to_samples(wav_data, sample_rate)
example_samples.append(samples)
ns = music_pb2.NoteSequence.FromString(
ex.features.feature['sequence'].bytes_list.value[0])
example_sequences.append(ns)
mixed_samples, mixed_sequence = audio_label_data_utils.mix_sequences(
individual_samples=example_samples,
sample_rate=sample_rate,
individual_sequences=example_sequences)
mixed_wav_data = audio_io.samples_to_wav_data(mixed_samples, sample_rate)
mixed_id = '::'.join(['mixed'] + [ns.id for ns in example_sequences])
mixed_sequence.id = mixed_id
mixed_filename = '::'.join(['mixed'] +
[ns.filename for ns in example_sequences])
mixed_sequence.filename = mixed_filename
examples = list(
audio_label_data_utils.process_record(mixed_wav_data,
mixed_sequence,
mixed_id,
min_length=0,
max_length=-1,
sample_rate=sample_rate))
assert len(examples) == 1
return examples[0]
def _ValidateProvideBatchTFRecord(self,
truncated_length,
batch_size,
lengths,
expected_num_inputs,
crop_sequence_secs=0):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
hparams.sample_rate), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples, hparams.sample_rate)
num_frames = data.wav_to_num_frames(
wav_data, frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams) -
crop_sequence_secs * 2, # crop from both ends.
i + constants.MIN_MIDI_PITCH,
start_time=crop_sequence_secs)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length,
crop_training_sequence_to_notes=crop_sequence_secs > 0)
self.assertEqual(expected_num_inputs, len(expected_inputs))
with tempfile.NamedTemporaryFile() as temp_tfr:
with tf.python_io.TFRecordWriter(temp_tfr.name) as writer:
for ex in examples:
writer.write(ex.SerializeToString())
self._ValidateProvideBatch(
temp_tfr.name,
truncated_length,
batch_size,
expected_inputs,
crop_training_sequence_to_notes=crop_sequence_secs > 0)
def validateProvideBatch_TFRecord(self,
truncated_length,
batch_size,
lengths,
expected_num_inputs):
hparams = copy.deepcopy(constants.DEFAULT_HPARAMS)
examples = []
expected_inputs = []
for i, length in enumerate(lengths):
wav_samples = np.zeros(
(np.int((length / data.hparams_frames_per_second(hparams)) *
constants.DEFAULT_SAMPLE_RATE), 1), np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples,
constants.DEFAULT_SAMPLE_RATE)
num_frames = data.wav_to_num_frames(
wav_data, frames_per_second=data.hparams_frames_per_second(hparams))
seq = self._SyntheticSequence(
num_frames / data.hparams_frames_per_second(hparams),
i + constants.MIN_MIDI_PITCH)
examples.append(self._FillExample(seq, wav_data, 'ex%d' % i))
expected_inputs += self._ExampleToInputs(
examples[-1],
truncated_length)
self.assertEqual(expected_num_inputs, len(expected_inputs))
with tempfile.NamedTemporaryFile() as temp_rio:
with tf.python_io.TFRecordWriter(temp_rio.name) as writer:
for ex in examples:
writer.write(ex.SerializeToString())
self.validateProvideBatch(
temp_rio.name,
truncated_length,
batch_size,
expected_inputs)
def reduce_audio_in_batch(tensor, hparams=None, is_training=True):
instrument_count = hparams.timbre_training_max_instruments
note_croppping_list = []
instrument_family_list = []
samples_list = []
max_length = 0
for i in range(instrument_count):
pitch = tensor['pitch'][i]
# Move the audio so there are different attack times.
start_idx = tf.random.uniform((),
minval=0,
maxval=hparams.timbre_max_start_offset,
dtype='int64')
samples = K.concatenate(
[tf.zeros(start_idx),
tf.sparse.to_dense(tensor['audio'])[i]])
end_idx = (
start_idx +
tf.py_function(_get_approx_note_length,
[tf.sparse.to_dense(tensor['audio'])[i]], tf.int64))
if hparams.timbre_max_len and end_idx > hparams.timbre_max_len:
samples = tf.slice(samples,
begin=[0],
size=[hparams.timbre_max_len])
end_idx = hparams.timbre_max_len
if len(samples) > max_length:
max_length = len(samples)
samples_list.append(samples)
instrument_family = tensor['instrument_family'][i]
note_croppping_list.append(
timbre_dataset_util.NoteCropping(pitch=pitch,
start_idx=start_idx,
end_idx=end_idx))
instrument_family_list.append(
tf.one_hot(tf.cast(instrument_family, tf.int32),
hparams.timbre_num_classes))
# Pad the end of the shorter audio clips.
samples_list = list(
map(lambda x: tf.pad(x, [[0, max_length - len(x)]]), samples_list))
combined_samples = (
tf.reduce_sum(tf.convert_to_tensor(samples_list), axis=0) /
instrument_count)
# Ensure all audios in batches are the same length.
if hparams.timbre_max_len:
pad_length = hparams.timbre_max_len
else:
pad_length = hparams.timbre_max_start_offset + 5 * hparams.sample_rate
combined_samples = tf.pad(
combined_samples, [[0, pad_length - tf.shape(combined_samples)[0]]])
note_croppings = tf.convert_to_tensor(note_croppping_list, dtype=tf.int32)
instrument_families = tf.convert_to_tensor(instrument_family_list,
dtype=tf.int32)
wav_data = tf.py_function(
lambda x: audio_io.samples_to_wav_data(
x.numpy(), sample_rate=hparams.sample_rate), [combined_samples],
tf.string)
return dict(
audio=wav_data,
note_croppings=note_croppings,
instrument_families=instrument_families,
)
def _CreateSyntheticExample(self):
sequence = self._CreateSyntheticSequence()
wav_samples = np.zeros(9 * SAMPLE_RATE, np.float32)
wav_data = audio_io.samples_to_wav_data(wav_samples, SAMPLE_RATE)
return wav_data, sequence
# }))
# to_process.append(example.SerializeToString())
# print('Processing complete ')
filename = 'richard.wav'
file_dir = './data/wav_format/' + filename
content = open(file_dir, 'rb')
uploaded = {file_dir: content.read()} ##todo
to_process = []
for fn in uploaded.keys():
print('User uploaded file "{name}" with length {length} bytes'.format(
name=fn, length=len(uploaded[fn])))
open(fn, 'w').write(uploaded[fn])
wav_data = audio_io.samples_to_wav_data(
librosa.util.normalize(
librosa.core.load(fn, sr=hparams.sample_rate)[0]),
hparams.sample_rate)
example = tf.train.Example(features=tf.train.Features(
feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[fn.encode('utf-8')])),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[music_pb2.NoteSequence().SerializeToString()])),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[wav_data])),
}))
to_process.append(example.SerializeToString())
print('Processing complete for', fn)
def _CreateSyntheticExample(self): sequence = self._CreateSyntheticSequence() wav_samples = np.zeros(2 * SAMPLE_RATE, np.float32) wav_data = audio_io.samples_to_wav_data(wav_samples, SAMPLE_RATE) return wav_data, sequence
def process_record(wav_data,
ns,
example_id,
min_length=5,
max_length=20,
sample_rate=16000,
allow_empty_notesequence=False,
load_audio_with_librosa=False):
"""Split a record into chunks and create an example proto.
To use the full length audio and notesequence, set min_length=0 and
max_length=-1.
Args:
wav_data: audio data in WAV format.
ns: corresponding NoteSequence.
example_id: id for the example proto
min_length: minimum length in seconds for audio chunks.
max_length: maximum length in seconds for audio chunks.
sample_rate: desired audio sample rate.
allow_empty_notesequence: whether an empty NoteSequence is allowed.
load_audio_with_librosa: Use librosa for sampling. Works with 24-bit wavs.
Yields:
Example protos.
"""
try:
if load_audio_with_librosa:
samples = audio_io.wav_data_to_samples_librosa(
wav_data, sample_rate)
else:
samples = audio_io.wav_data_to_samples(wav_data, sample_rate)
except audio_io.AudioIOReadError as e:
print('Exception %s', e)
return
samples = librosa.util.normalize(samples, norm=np.inf)
# Add padding to samples if notesequence is longer.
pad_to_samples = int(math.ceil(ns.total_time * sample_rate))
padding_needed = pad_to_samples - samples.shape[0]
if padding_needed > 5 * sample_rate:
raise ValueError(
'Would have padded {} more than 5 seconds to match note sequence total '
'time. ({} original samples, {} sample rate, {} sample seconds, '
'{} sequence seconds) This likely indicates a problem with the source '
'data.'.format(example_id, samples.shape[0], sample_rate,
samples.shape[0] / sample_rate, ns.total_time))
samples = np.pad(samples, (0, max(0, padding_needed)), 'constant')
if max_length == min_length:
splits = np.arange(0, ns.total_time, max_length)
elif max_length > 0:
splits = find_split_points(ns, samples, sample_rate, min_length,
max_length)
else:
splits = [0, ns.total_time]
velocities = [note.velocity for note in ns.notes]
velocity_max = np.max(velocities) if velocities else 0
velocity_min = np.min(velocities) if velocities else 0
velocity_range = music_pb2.VelocityRange(min=velocity_min,
max=velocity_max)
for start, end in zip(splits[:-1], splits[1:]):
if end - start < min_length:
continue
if start == 0 and end == ns.total_time:
new_ns = ns
else:
new_ns = sequences_lib.extract_subsequence(ns, start, end)
if not new_ns.notes and not allow_empty_notesequence:
tf.logging.warning('skipping empty sequence')
continue
if start == 0 and end == ns.total_time:
new_samples = samples
else:
# the resampling that happen in crop_wav_data is really slow
# and we've already done it once, avoid doing it twice
new_samples = audio_io.crop_samples(samples, sample_rate, start,
end - start)
new_wav_data = audio_io.samples_to_wav_data(new_samples, sample_rate)
yield create_example(example_id,
new_ns,
new_wav_data,
velocity_range=velocity_range)
def generate_train_set(exclude_ids):
"""Generate the train TFRecord."""
train_file_pairs = []
for directory in TRAIN_DIRS:
path = os.path.join(FLAGS.input_dir, directory)
path = os.path.join(path, '*.wav')
wav_files = glob.glob(path)
# find matching mid files
for wav_file in wav_files:
base_name_root, _ = os.path.splitext(wav_file)
mid_file = base_name_root + '.mid'
if filename_to_id(wav_file) not in exclude_ids:
train_file_pairs.append((wav_file, mid_file))
train_output_name = os.path.join(FLAGS.output_dir,
'maps_config2_train.tfrecord')
with tf.python_io.TFRecordWriter(train_output_name) as writer:
for idx, pair in enumerate(train_file_pairs):
print("{} of {}: {}".format(idx, len(train_file_pairs), pair[0]))
# load the wav data
wav_data = tf.gfile.Open(pair[0], 'rb').read()
samples = audio_io.wav_data_to_samples(wav_data, FLAGS.sample_rate)
norm_samples = librosa.util.normalize(samples, norm=np.inf)
# load the midi data and convert to a notesequence
ns = midi_io.midi_file_to_note_sequence(pair[1])
splits = create_dataset_util.find_split_points(
ns, norm_samples, FLAGS.sample_rate, FLAGS.min_length,
FLAGS.max_length)
velocities = [note.velocity for note in ns.notes]
velocity_max = np.max(velocities)
velocity_min = np.min(velocities)
new_velocity_tuple = music_pb2.VelocityRange(min=velocity_min,
max=velocity_max)
for start, end in zip(splits[:-1], splits[1:]):
if end - start < FLAGS.min_length:
continue
new_ns = sequences_lib.extract_subsequence(ns, start, end)
samples_start = int(start * FLAGS.sample_rate)
samples_end = samples_start + int(
(end - start) * FLAGS.sample_rate)
new_samples = samples[samples_start:samples_end]
new_wav_data = audio_io.samples_to_wav_data(
new_samples, FLAGS.sample_rate)
example = tf.train.Example(features=tf.train.Features(
feature={
'id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[pair[0].encode()])),
'sequence':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[new_ns.SerializeToString()])),
'audio':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[new_wav_data])),
'velocity_range':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[new_velocity_tuple.SerializeToString()])),
}))
writer.write(example.SerializeToString())
