def testMelodyInferencePolyphonic(self):
sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(
sequence, 0, [
(36, 100, 0.0, 4.0), (64, 100, 0.0, 1.0), (67, 100, 0.0, 1.0),
(65, 100, 1.0, 2.0), (69, 100, 1.0, 2.0),
(67, 100, 2.0, 4.0), (71, 100, 2.0, 3.0),
(72, 100, 3.0, 4.0)
])
melody_inference.infer_melody_for_sequence(sequence)
expected_sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(
expected_sequence, 0, [
(36, 100, 0.0, 4.0), (64, 100, 0.0, 1.0), (67, 100, 0.0, 1.0),
(65, 100, 1.0, 2.0), (69, 100, 1.0, 2.0),
(67, 100, 2.0, 4.0), (71, 100, 2.0, 3.0),
(72, 100, 3.0, 4.0)
])
testing_lib.add_track_to_sequence(
expected_sequence, 1, [
(67, 127, 0.0, 1.0), (69, 127, 1.0, 2.0),
(71, 127, 2.0, 3.0), (72, 127, 3.0, 4.0)
])
self.assertEqual(expected_sequence, sequence)
def testMelodyInferenceSingleNote(self):
sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(sequence, 0, [(60, 100, 0.5, 1.0)])
melody_inference.infer_melody_for_sequence(sequence)
expected_sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(expected_sequence, 0,
[(60, 100, 0.5, 1.0)])
testing_lib.add_track_to_sequence(expected_sequence, 1,
[(60, 127, 0.5, 1.0)])
self.assertEqual(expected_sequence, sequence)
def testMelodyInferenceSingleNote(self):
sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(
sequence, 0, [(60, 100, 0.5, 1.0)])
melody_inference.infer_melody_for_sequence(sequence)
expected_sequence = music_pb2.NoteSequence()
testing_lib.add_track_to_sequence(
expected_sequence, 0, [(60, 100, 0.5, 1.0)])
testing_lib.add_track_to_sequence(
expected_sequence, 1, [(60, 127, 0.5, 1.0)])
self.assertEqual(expected_sequence, sequence)
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = music_pb2.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
if (self._min_hop_size_seconds and
ns.total_time < self._min_hop_size_seconds):
Metrics.counter('extract_examples', 'sequence_too_short').inc()
return
sequences = []
for _ in range(self._num_replications):
if self._max_hop_size_seconds:
if self._max_hop_size_seconds == self._min_hop_size_seconds:
# Split using fixed hop size.
sequences += sequences_lib.split_note_sequence(
ns, self._max_hop_size_seconds)
else:
# Sample random hop positions such that each segment size is within
# the specified range.
hop_times = [0.0]
while hop_times[-1] <= ns.total_time - self._min_hop_size_seconds:
if hop_times[-1] + self._max_hop_size_seconds < ns.total_time:
# It's important that we get a valid hop size here, since the
# remainder of the sequence is too long.
max_offset = min(
self._max_hop_size_seconds,
ns.total_time - self._min_hop_size_seconds - hop_times[-1])
else:
# It's okay if the next hop time is invalid (in which case we'll
# just stop).
max_offset = self._max_hop_size_seconds
offset = random.uniform(self._min_hop_size_seconds, max_offset)
hop_times.append(hop_times[-1] + offset)
# Split at the chosen hop times (ignoring zero and the final invalid
# time).
sequences += sequences_lib.split_note_sequence(ns, hop_times[1:-1])
else:
sequences += [ns]
for performance_sequence in sequences:
if self._encode_score_fns:
# We need to extract a score.
if not self._absolute_timing:
# Beats are required to extract a score with metric timing.
beats = [
ta for ta in performance_sequence.text_annotations
if (ta.annotation_type ==
music_pb2.NoteSequence.TextAnnotation.BEAT)
and ta.time <= performance_sequence.total_time
]
if len(beats) < 2:
Metrics.counter('extract_examples', 'not_enough_beats').inc()
continue
# Ensure the sequence starts and ends on a beat.
performance_sequence = sequences_lib.extract_subsequence(
performance_sequence,
start_time=min(beat.time for beat in beats),
end_time=max(beat.time for beat in beats)
)
# Infer beat-aligned chords (only for relative timing).
try:
chord_inference.infer_chords_for_sequence(
performance_sequence,
chord_change_prob=0.25,
chord_note_concentration=50.0,
add_key_signatures=True)
except chord_inference.ChordInferenceError:
Metrics.counter('extract_examples', 'chord_inference_failed').inc()
continue
# Infer melody regardless of relative/absolute timing.
try:
melody_instrument = melody_inference.infer_melody_for_sequence(
performance_sequence,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
except melody_inference.MelodyInferenceError:
Metrics.counter('extract_examples', 'melody_inference_failed').inc()
continue
if not self._absolute_timing:
# Now rectify detected beats to occur at fixed tempo.
# TODO(iansimon): also include the alignment
score_sequence, unused_alignment = sequences_lib.rectify_beats(
performance_sequence, beats_per_minute=SCORE_BPM)
else:
# Score uses same timing as performance.
score_sequence = copy.deepcopy(performance_sequence)
# Remove melody notes from performance.
performance_notes = []
for note in performance_sequence.notes:
if note.instrument != melody_instrument:
performance_notes.append(note)
del performance_sequence.notes[:]
performance_sequence.notes.extend(performance_notes)
# Remove non-melody notes from score.
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# Remove key signatures and beat/chord annotations from performance.
del performance_sequence.key_signatures[:]
del performance_sequence.text_annotations[:]
Metrics.counter('extract_examples', 'extracted_score').inc()
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(performance_sequence)
except DataAugmentationError:
Metrics.counter(
'extract_examples', 'augment_performance_failed').inc()
continue
example_dict = {
'targets': self._encode_performance_fn(
augmented_performance_sequence)
}
if not example_dict['targets']:
Metrics.counter('extract_examples', 'skipped_empty_targets').inc()
continue
if self._encode_score_fns:
# Augment the extracted score.
try:
augmented_score_sequence = augment_fn(score_sequence)
except DataAugmentationError:
Metrics.counter('extract_examples', 'augment_score_failed').inc()
continue
# Apply all score encoding functions.
skip = False
for name, encode_score_fn in self._encode_score_fns.items():
example_dict[name] = encode_score_fn(augmented_score_sequence)
if not example_dict[name]:
Metrics.counter('extract_examples',
'skipped_empty_%s' % name).inc()
skip = True
break
if skip:
continue
Metrics.counter('extract_examples', 'encoded_example').inc()
Metrics.distribution(
'extract_examples', 'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def testMelodyInferenceEmptySequence(self): sequence = music_pb2.NoteSequence() melody_inference.infer_melody_for_sequence(sequence) expected_sequence = music_pb2.NoteSequence() self.assertEqual(expected_sequence, sequence)
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = music_pb2.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
for _ in range(self._num_replications):
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(ns)
except DataAugmentationError:
Metrics.counter('extract_examples',
'augment_performance_failed').inc()
continue
seq = self._encode_performance_fn(
augmented_performance_sequence)
# feed in performance as both input/output to music transformer
# chopping sequence into length 2048 (throw out shorter sequences)
if len(seq) >= 2048:
max_offset = len(seq) - 2048
offset = random.randrange(max_offset + 1)
cropped_seq = seq[offset:offset + 2048]
example_dict = {
'inputs': cropped_seq,
'targets': cropped_seq
}
if self._melody:
# decode truncated performance sequence for melody inference
decoded_midi = self._decode_performance_fn(cropped_seq)
decoded_ns = mm.midi_io.midi_file_to_note_sequence(
decoded_midi)
# extract melody from cropped performance sequence
melody_instrument = melody_inference.infer_melody_for_sequence(
decoded_ns,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
# remove non-melody notes from score
score_sequence = copy.deepcopy(decoded_ns)
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# encode melody
encode_score_fn = self._encode_score_fns['melody']
example_dict['melody'] = encode_score_fn(
score_sequence)
# make sure performance input also matches targets; needed for
# compatibility of both perf and (mel & perf) autoencoders
if self._noisy:
# randomly sample a pitch shift to construct noisy performance
all_pitches = [x.pitch for x in decoded_ns.notes]
min_val = min(all_pitches)
max_val = max(all_pitches)
transpose_range = range(-(min_val - 21),
108 - max_val + 1)
try:
transpose_range.remove(
0) # make sure you transpose
except ValueError:
pass
transpose_amount = random.choice(transpose_range)
augmented_ns, _ = sequences_lib.transpose_note_sequence(
decoded_ns,
transpose_amount,
min_allowed_pitch=21,
max_allowed_pitch=108,
in_place=False)
aug_seq = self._encode_performance_fn(augmented_ns)
example_dict['performance'] = aug_seq
else:
example_dict['performance'] = example_dict[
'targets']
del example_dict['inputs']
Metrics.counter('extract_examples',
'encoded_example').inc()
Metrics.distribution(
'extract_examples',
'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def process(self, kv):
# Seed random number generator based on key so that hop times are
# deterministic.
key, ns_str = kv
m = hashlib.md5(key)
random.seed(int(m.hexdigest(), 16))
# Deserialize NoteSequence proto.
ns = music_pb2.NoteSequence.FromString(ns_str)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
if (self._min_hop_size_seconds and
ns.total_time < self._min_hop_size_seconds):
Metrics.counter('extract_examples', 'sequence_too_short').inc()
return
sequences = []
for _ in range(self._num_replications):
if self._max_hop_size_seconds:
if self._max_hop_size_seconds == self._min_hop_size_seconds:
# Split using fixed hop size.
sequences += sequences_lib.split_note_sequence(
ns, self._max_hop_size_seconds)
else:
# Sample random hop positions such that each segment size is within
# the specified range.
hop_times = [0.0]
while hop_times[-1] <= ns.total_time - self._min_hop_size_seconds:
if hop_times[-1] + self._max_hop_size_seconds < ns.total_time:
# It's important that we get a valid hop size here, since the
# remainder of the sequence is too long.
max_offset = min(
self._max_hop_size_seconds,
ns.total_time - self._min_hop_size_seconds - hop_times[-1])
else:
# It's okay if the next hop time is invalid (in which case we'll
# just stop).
max_offset = self._max_hop_size_seconds
offset = random.uniform(self._min_hop_size_seconds, max_offset)
hop_times.append(hop_times[-1] + offset)
# Split at the chosen hop times (ignoring zero and the final invalid
# time).
sequences += sequences_lib.split_note_sequence(ns, hop_times[1:-1])
else:
sequences += [ns]
for performance_sequence in sequences:
if self._encode_score_fns:
# We need to extract a score.
if not self._absolute_timing:
# Beats are required to extract a score with metric timing.
beats = [
ta for ta in performance_sequence.text_annotations
if (ta.annotation_type ==
music_pb2.NoteSequence.TextAnnotation.BEAT)
and ta.time <= performance_sequence.total_time
]
if len(beats) < 2:
Metrics.counter('extract_examples', 'not_enough_beats').inc()
continue
# Ensure the sequence starts and ends on a beat.
performance_sequence = sequences_lib.extract_subsequence(
performance_sequence,
start_time=min(beat.time for beat in beats),
end_time=max(beat.time for beat in beats)
)
# Infer beat-aligned chords (only for relative timing).
try:
chord_inference.infer_chords_for_sequence(
performance_sequence,
chord_change_prob=0.25,
chord_note_concentration=50.0,
add_key_signatures=True)
except chord_inference.ChordInferenceError:
Metrics.counter('extract_examples', 'chord_inference_failed').inc()
continue
# Infer melody regardless of relative/absolute timing.
try:
melody_instrument = melody_inference.infer_melody_for_sequence(
performance_sequence,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
except melody_inference.MelodyInferenceError:
Metrics.counter('extract_examples', 'melody_inference_failed').inc()
continue
if not self._absolute_timing:
# Now rectify detected beats to occur at fixed tempo.
# TODO(iansimon): also include the alignment
score_sequence, unused_alignment = sequences_lib.rectify_beats(
performance_sequence, beats_per_minute=SCORE_BPM)
else:
# Score uses same timing as performance.
score_sequence = copy.deepcopy(performance_sequence)
# Remove melody notes from performance.
performance_notes = []
for note in performance_sequence.notes:
if note.instrument != melody_instrument:
performance_notes.append(note)
del performance_sequence.notes[:]
performance_sequence.notes.extend(performance_notes)
# Remove non-melody notes from score.
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# Remove key signatures and beat/chord annotations from performance.
del performance_sequence.key_signatures[:]
del performance_sequence.text_annotations[:]
Metrics.counter('extract_examples', 'extracted_score').inc()
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(performance_sequence)
except DataAugmentationError:
Metrics.counter(
'extract_examples', 'augment_performance_failed').inc()
continue
example_dict = {
'targets': self._encode_performance_fn(
augmented_performance_sequence)
}
if not example_dict['targets']:
Metrics.counter('extract_examples', 'skipped_empty_targets').inc()
continue
if self._encode_score_fns:
# Augment the extracted score.
try:
augmented_score_sequence = augment_fn(score_sequence)
except DataAugmentationError:
Metrics.counter('extract_examples', 'augment_score_failed').inc()
continue
# Apply all score encoding functions.
skip = False
for name, encode_score_fn in self._encode_score_fns.items():
example_dict[name] = encode_score_fn(augmented_score_sequence)
if not example_dict[name]:
Metrics.counter('extract_examples',
'skipped_empty_%s' % name).inc()
skip = True
break
if skip:
continue
Metrics.counter('extract_examples', 'encoded_example').inc()
Metrics.distribution(
'extract_examples', 'performance_length_in_seconds').update(
int(augmented_performance_sequence.total_time))
yield generator_utils.to_example(example_dict)
def process_midi(self, f):
def augment_note_sequence(ns, stretch_factor, transpose_amount):
"""Augment a NoteSequence by time stretch and pitch transposition."""
augmented_ns = sequences_lib.stretch_note_sequence(ns,
stretch_factor,
in_place=False)
try:
_, num_deleted_notes = sequences_lib.transpose_note_sequence(
augmented_ns,
transpose_amount,
min_allowed_pitch=MIN_PITCH,
max_allowed_pitch=MAX_PITCH,
in_place=True)
except chord_symbols_lib.ChordSymbolError:
raise datagen_beam.DataAugmentationError(
'Transposition of chord symbol(s) failed.')
if num_deleted_notes:
raise datagen_beam.DataAugmentationError(
'Transposition caused out-of-range pitch(es).')
return augmented_ns
self._min_hop_size_seconds = 0.0
self._max_hop_size_seconds = 0.0
self._num_replications = 1
self._encode_performance_fn = self.performance_encoder(
).encode_note_sequence
self._encode_score_fns = dict(
(name, encoder.encode_note_sequence)
for name, encoder in self.score_encoders())
augment_params = itertools.product(self.stretch_factors,
self.transpose_amounts)
augment_fns = [
functools.partial(augment_note_sequence,
stretch_factor=s,
transpose_amount=t) for s, t in augment_params
]
self._augment_fns = augment_fns
self._absolute_timing = self.absolute_timing
self._random_crop_length = self.random_crop_length_in_datagen
if self._random_crop_length is not None:
self._augment_fns = self._augment_fns
rets = []
ns = magenta.music.midi_file_to_sequence_proto(f)
# Apply sustain pedal.
ns = sequences_lib.apply_sustain_control_changes(ns)
# Remove control changes as there are potentially a lot of them and they are
# no longer needed.
del ns.control_changes[:]
if (self._min_hop_size_seconds
and ns.total_time < self._min_hop_size_seconds):
print("sequence_too_short")
return []
sequences = []
for _ in range(self._num_replications):
if self._max_hop_size_seconds:
if self._max_hop_size_seconds == self._min_hop_size_seconds:
# Split using fixed hop size.
sequences += sequences_lib.split_note_sequence(
ns, self._max_hop_size_seconds)
else:
# Sample random hop positions such that each segment size is within
# the specified range.
hop_times = [0.0]
while hop_times[
-1] <= ns.total_time - self._min_hop_size_seconds:
if hop_times[
-1] + self._max_hop_size_seconds < ns.total_time:
# It's important that we get a valid hop size here, since the
# remainder of the sequence is too long.
max_offset = min(
self._max_hop_size_seconds, ns.total_time -
self._min_hop_size_seconds - hop_times[-1])
else:
# It's okay if the next hop time is invalid (in which case we'll
# just stop).
max_offset = self._max_hop_size_seconds
offset = random.uniform(self._min_hop_size_seconds,
max_offset)
hop_times.append(hop_times[-1] + offset)
# Split at the chosen hop times (ignoring zero and the final invalid
# time).
sequences += sequences_lib.split_note_sequence(
ns, hop_times[1:-1])
else:
sequences += [ns]
for performance_sequence in sequences:
if self._encode_score_fns:
# We need to extract a score.
if not self._absolute_timing:
# Beats are required to extract a score with metric timing.
beats = [
ta for ta in performance_sequence.text_annotations
if (ta.annotation_type ==
music_pb2.NoteSequence.TextAnnotation.BEAT)
and ta.time <= performance_sequence.total_time
]
if len(beats) < 2:
print('not_enough_beats')
continue
# Ensure the sequence starts and ends on a beat.
performance_sequence = sequences_lib.extract_subsequence(
performance_sequence,
start_time=min(beat.time for beat in beats),
end_time=max(beat.time for beat in beats))
# Infer beat-aligned chords (only for relative timing).
try:
chord_inference.infer_chords_for_sequence(
performance_sequence,
chord_change_prob=0.25,
chord_note_concentration=50.0,
add_key_signatures=True)
except chord_inference.ChordInferenceError:
print("chord_inference_failed")
continue
# Infer melody regardless of relative/absolute timing.
try:
melody_instrument = melody_inference.infer_melody_for_sequence(
performance_sequence,
melody_interval_scale=2.0,
rest_prob=0.1,
instantaneous_non_max_pitch_prob=1e-15,
instantaneous_non_empty_rest_prob=0.0,
instantaneous_missing_pitch_prob=1e-15)
except melody_inference.MelodyInferenceError:
print('melody_inference_failed')
continue
if not self._absolute_timing:
# Now rectify detected beats to occur at fixed tempo.
# TODO(iansimon): also include the alignment
score_sequence, unused_alignment = sequences_lib.rectify_beats(
performance_sequence, beats_per_minute=SCORE_BPM)
else:
# Score uses same timing as performance.
score_sequence = copy.deepcopy(performance_sequence)
# Remove melody notes from performance.
performance_notes = []
for note in performance_sequence.notes:
if note.instrument != melody_instrument:
performance_notes.append(note)
del performance_sequence.notes[:]
performance_sequence.notes.extend(performance_notes)
# Remove non-melody notes from score.
score_notes = []
for note in score_sequence.notes:
if note.instrument == melody_instrument:
score_notes.append(note)
del score_sequence.notes[:]
score_sequence.notes.extend(score_notes)
# Remove key signatures and beat/chord annotations from performance.
del performance_sequence.key_signatures[:]
del performance_sequence.text_annotations[:]
for augment_fn in self._augment_fns:
# Augment and encode the performance.
try:
augmented_performance_sequence = augment_fn(
performance_sequence)
except DataAugmentationError as e:
print("augment_performance_failed", e)
continue
example_dict = {
'targets':
self._encode_performance_fn(augmented_performance_sequence)
}
if not example_dict['targets']:
print('skipped_empty_targets')
continue
if (self._random_crop_length and len(example_dict['targets']) >
self._random_crop_length):
# Take a random crop of the encoded performance.
max_offset = len(
example_dict['targets']) - self._random_crop_length
offset = random.randrange(max_offset + 1)
example_dict['targets'] = example_dict['targets'][
offset:offset + self._random_crop_length]
if self._encode_score_fns:
# Augment the extracted score.
try:
augmented_score_sequence = augment_fn(score_sequence)
except DataAugmentationError:
print('augment_score_failed')
continue
# Apply all score encoding functions.
skip = False
for name, encode_score_fn in self._encode_score_fns.items(
):
example_dict[name] = encode_score_fn(
augmented_score_sequence)
if not example_dict[name]:
print('skipped_empty_%s' % name)
skip = True
break
if skip:
continue
rets.append(example_dict)
return rets
