def generateNewSequence(input_sequence, temperature, write_to_file):
input_sequence = mm.quantize_note_sequence(input_sequence, 8)
bundle = sequence_generator_bundle.read_bundle_file(
'/Library/Application Support/Quin Scacheri/Magenta Beats/drum_kit_rnn.mag'
)
generator_map = drums_rnn_sequence_generator.get_generator_map()
drum_rnn = generator_map['drum_kit'](checkpoint=None, bundle=bundle)
drum_rnn.initialize()
qpm = input_sequence.tempos[0].qpm
last_end_time = (max(
n.end_time
for n in input_sequence.notes) if input_sequence.notes else 0)
# total_seconds = num_steps * input_sequence.quantization_info.steps_per_quarter;
generator_options = generator_pb2.GeneratorOptions()
generator_options.args['temperature'].float_value = temperature
generate_section = generator_options.generate_sections.add(
start_time=last_end_time, end_time=8.0)
new_sequence = drum_rnn.generate(input_sequence, generator_options)
new_sequence = mm.trim_note_sequence(new_sequence, 2.0, 4.0)
new_sequence = mm.quantize_note_sequence(new_sequence, 4)
#
# new_sequence.quantization_info.steps_per_quarter = 8
if (True):
mm.sequence_proto_to_midi_file(input_sequence, 'primer.mid')
mm.sequence_proto_to_midi_file(new_sequence, 'new_sequence.mid')
return new_sequence
def generateNewSequence(input_sequence, num_steps, temperature, write_to_file = False):
input_sequence = mm.quantize_note_sequence(input_sequence, 8)
bundle = sequence_generator_bundle.read_bundle_file('drum_kit_rnn.mag')
generator_map = drums_rnn_sequence_generator.get_generator_map()
drum_rnn = generator_map['drum_kit'](checkpoint=None, bundle=bundle)
drum_rnn.initialize()
qpm = input_sequence.tempos[0].qpm
last_end_time = (max(n.end_time for n in input_sequence.notes)
if input_sequence.notes else 0)
total_seconds = num_steps * input_sequence.quantization_info.steps_per_quarter;
generator_options = generator_pb2.GeneratorOptions()
generator_options.args['temperature'].float_value = temperature
generate_section = generator_options.generate_sections.add(start_time=last_end_time, end_time=total_seconds)
new_sequence = drum_rnn.generate(input_sequence, generator_options)
new_sequence.quantization_info.steps_per_quarter = 8
mm.quantize_note_sequence(new_sequence, 8)
if (write_to_file == True):
mm.sequence_proto_to_midi_file(input_sequence, 'oldSequence.mid')
mm.sequence_proto_to_midi_file(new_sequence, 'newSequence.mid')
return new_sequence
def createDrums():
drums = music_pb2.NoteSequence()
drums.notes.add(pitch=36, start_time=0, end_time=0.125, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=0, end_time=0.125, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=0.125, end_time=0.25, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=.25, end_time=0.375, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=0.375, end_time=0.5, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=38, start_time=0.5, end_time=0.625, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=.625, end_time=0.75, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=0.75, end_time=0.875, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=.875, end_time=1.0, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=36, start_time=1.0, end_time=1.125, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=1, end_time=1.125, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=1.125, end_time=1.25, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=1.25, end_time=1.375, is_drum=True, instrument=10, velocity=80)
drums.notes.add(pitch=42, start_time=1.375, end_time=1.5, is_drum=True, instrument=10, velocity=80)
drums.time_signatures.add(numerator = 4, denominator = 4)
drums.time_signatures[0].numerator = 4
drums.time_signatures[0].denominator = 4
drums.tempos.add(qpm=60)
drums.quantization_info.steps_per_quarter = 4
drums.total_time = 1.375
drums.tempos.add(qpm=60)
drums = mm.quantize_note_sequence(drums, 8)
return drums
def magent_melody_extractor(midi_path, max_bars=100):
data_converter = music_vae_data.OneHotMelodyConverter(
valid_programs=music_vae_data.MEL_PROGRAMS,
skip_polyphony=False,
max_bars=max_bars, # Truncate long melodies before slicing.
slice_bars=16,
steps_per_quarter=4)
ns = convert_midi("", '', midi_path)
all_melodies = []
try:
note_sequences = sequences_lib.split_note_sequence_on_time_changes(ns)
for ns in note_sequences:
# filter notes --------------------------------------------------------
def filter(note):
if (data_converter._valid_programs is not None and note.program
not in data_converter._valid_programs):
return False
return data_converter._min_pitch <= note.pitch <= data_converter._max_pitch
notes = list(ns.notes)
del ns.notes[:]
ns.notes.extend([n for n in notes if filter(n)])
# quantize sequence ---------------------------------------------------
quantized_sequence = mm.quantize_note_sequence(
ns, data_converter._steps_per_quarter)
# extract melodies ----------------------------------------------------
melodies, _ = data_converter._event_extractor_fn(
quantized_sequence)
all_melodies.extend(melodies)
except:
pass
return all_melodies
def generate(self,
num_steps=128,
temperature=1.0,
steps_per_second_avail=False,
empty=False):
"""
generates a song.
"""
if hasattr(self, 'num_steps'):
num_steps = self.num_steps
if hasattr(self, 'temperature'):
temperature = self.temperature
input_sequence = self.sequence
if empty:
input_sequence = music_pb2.NoteSequence()
input_sequence.tempos.add(qpm=80)
qpm = input_sequence.tempos[0].qpm
if steps_per_second_avail:
steps_per_quarter = int(self.model.steps_per_second * (1 /
(qpm / 60)))
seconds_per_step = 1 / self.model.steps_per_second
else:
seconds_per_step = 60.0 / qpm / self.model.steps_per_quarter
steps_per_quarter = self.model.steps_per_quarter
quantized_sequence = mm.quantize_note_sequence(input_sequence,
steps_per_quarter)
last_end_time = (max(
n.end_time
for n in input_sequence.notes) if input_sequence.notes else 0)
primer_sequence_steps = quantized_sequence.total_quantized_steps
if primer_sequence_steps > num_steps:
# easier to make num_steps bigger to accommodate for sizes
# 4 times the size of original sequence..
num_steps = primer_sequence_steps * 4
total_seconds = num_steps * seconds_per_step
input_sequence.total_time = min(total_seconds,
input_sequence.total_time)
generator_options = generator_pb2.GeneratorOptions()
generator_options.args['temperature'].float_value = temperature
generate_section = generator_options.generate_sections.add(
start_time=last_end_time + seconds_per_step,
end_time=total_seconds)
self.output_sequence = self.model.generate(input_sequence,
generator_options)
request_dict = self.put_request_dict
utils.generated_sequence_2_mp3(self.output_sequence,
f"{self.unique_id}",
use_salamander=True,
request_dict=request_dict)
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique, one-hot tensor sequences."""
try:
if self._steps_per_quarter:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence)
!= self._steps_per_bar):
return [], []
else:
quantized_sequence = mm.quantize_note_sequence_absolute(
note_sequence, self._steps_per_second)
except (mm.BadTimeSignatureException,
mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return [], []
event_lists, unused_stats = self._event_extractor_fn(
quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_tuples = []
for l in event_lists:
for i in range(self._slice_steps,
len(l) + 1, self._steps_per_bar):
sliced_event_tuples.append(
tuple(l[i - self._slice_steps:i]))
else:
sliced_event_tuples = [tuple(l) for l in event_lists]
# TODO(adarob): Consider handling the fact that different event lists can
# be mapped to identical tensors by the encoder_decoder (e.g., Drums).
unique_event_tuples = list(set(sliced_event_tuples))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
seqs = []
for t in unique_event_tuples:
seqs.append(
np_onehot(
[self._legacy_encoder_decoder.encode_event(e) for e in t] +
([] if self.end_token is None else [self.end_token]),
self.output_depth, self.output_dtype))
return seqs, seqs
def test_split_and_quantize():
for midi_path in common.midi_big_paths:
# print(midi_path)
ns = convert_midi("", '', midi_path)
note_sequences = sequences_lib.split_note_sequence_on_time_changes(ns)
pm = pretty_midi.PrettyMIDI(midi_path)
nc = NoteContainer.from_pretty_midi(pm)
split_ncs = nc_lib.split_on_time_signature_tempo_change(nc)
for nc, ns in zip(split_ncs, note_sequences):
assert_nc_ns(nc, ns)
qnc = nc_lib.quantize_note_container(nc, 4)
qns = mm.quantize_note_sequence(ns, 4)
assert_nc_ns(qnc, qns, quantized=True)
def midi_to_polyphony_sequence(steps_per_quarter, qpm, midi_dir, midi_name):
primer_midi = os.path.join(midi_dir, midi_name)
primer_midi = os.path.expanduser(primer_midi)
primer_sequence = magenta.music.midi_file_to_sequence_proto(primer_midi)
if primer_sequence.tempos and primer_sequence.tempos[0].qpm:
qpm = primer_sequence.tempos[0].qpm
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, steps_per_quarter)
extracted_seqs, _ = polyphony_lib.extract_polyphonic_sequences(
quantized_primer_sequence, start_step=0)
poly_seq = extracted_seqs[0]
return poly_seq
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique, one-hot tensor sequences."""
try:
if self._steps_per_quarter:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return [], []
else:
quantized_sequence = mm.quantize_note_sequence_absolute(
note_sequence, self._steps_per_second)
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return [], []
event_lists, unused_stats = self._event_extractor_fn(quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_tuples = []
for l in event_lists:
for i in range(self._slice_steps, len(l) + 1, self._steps_per_bar):
sliced_event_tuples.append(tuple(l[i - self._slice_steps: i]))
else:
sliced_event_tuples = [tuple(l) for l in event_lists]
# TODO(adarob): Consider handling the fact that different event lists can
# be mapped to identical tensors by the encoder_decoder (e.g., Drums).
unique_event_tuples = list(set(sliced_event_tuples))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
seqs = []
for t in unique_event_tuples:
seqs.append(np_onehot(
[self._legacy_encoder_decoder.encode_event(e) for e in t] +
([] if self.end_token is None else [self.end_token]),
self.output_depth, self.output_dtype))
return seqs, seqs
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException('This model supports at most one input_sections message, but got %s' %len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException('This model supports only 1 generate_sections message, but got %s' %len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm if input_sequence and input_sequence.tempos else mm.DEFAULT_QUARTERS_PER_MINUTE)
steps_per_second = mm.steps_per_quarter_to_steps_per_second(self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence, input_section.start_time, input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time, steps_per_second, quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(n.end_time for n in primer_sequence.notes)
if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException('start time: %s, Final note end time: %s' % (generate_section.start_time, last_end_time))
quantized_sequence = mm.quantize_note_sequence(primer_sequence, self.steps_per_quarter)
extracted_melodies, _ = mm.extract_melodies(quantized_sequence, search_start_step=input_start_step, min_bars=0,min_unique_pitches=1, gap_bars=float('inf'),ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = mm.quantize_to_step(
generate_section.start_time, steps_per_second, quantize_cutoff=0)
end_step = mm.quantize_to_step(generate_section.end_time, steps_per_second, quantize_cutoff=1.0)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
steps_per_bar = int(mm.steps_per_bar_in_quantized_sequence(quantized_sequence))
melody = mm.Melody([],start_step=max(0, start_step - 1),steps_per_bar=steps_per_bar,steps_per_quarter=self.steps_per_quarter)
melody.set_length(start_step - melody.start_step)
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name])) for name, value_fn in arg_types.items() if name in generator_options.args)
generated_melody = self._model.generate_melody(end_step - melody.start_step, melody, **args)
generated_sequence = generated_melody.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def testMelodyChords(self):
melodies = [
mm.Melody([60, -2, -2, -1],
start_step=0,
steps_per_quarter=1,
steps_per_bar=4),
mm.Melody([62, -2, -2, -1],
start_step=4,
steps_per_quarter=1,
steps_per_bar=4),
]
quantized_sequence = mm.quantize_note_sequence(self.note_sequence,
steps_per_quarter=1)
chords = chord_utils.event_list_chords(quantized_sequence, melodies)
expected_chords = [[mm.NO_CHORD, mm.NO_CHORD, 'C', 'C'],
['C', 'C', 'G7', 'G7']]
self.assertEqual(expected_chords, chords)
def generate(self, empty=False,
num_bars=64, temperature=0.5, backup_seq=None,
chord_arr=None):
# Interpolation, Repeating Chord Progression
if chord_arr is None:
if backup_seq is not None:
self.sequence = copy.deepcopy(backup_seq)
if hasattr(self, 'temperature'):
temperature = self.temperature
copy_sequence = copy.deepcopy(self.sequence)
quantized_sequence = mm.quantize_note_sequence(copy_sequence, 8)
# infer chords for sequence is a bit more natural
mm.infer_chords_for_sequence(quantized_sequence)
chords = []
for annotation in quantized_sequence.text_annotations:
if annotation.annotation_type == CHORD_SYMBOL:
chord_name = annotation.text
chords.append(chord_name)
else:
# follow a user defined chord progression
chords = chord_arr
mod_val = len(chords)
z1 = np.random.normal(size=[Z_SIZE])
z2 = np.random.normal(size=[Z_SIZE])
z = np.array([self.slerp(z1, z2, t)
for t in np.linspace(0, 1, num_bars)])
seqs = [
self.model.decode(length=TOTAL_STEPS, z=z[i:i+1, :], temperature=temperature,
c_input=self.chord_encoding(chords[i % mod_val]))[0]
for i in range(num_bars)
]
self.fix_instruments_for_concatenation(seqs)
prog_ns = concatenate_sequences(seqs)
request_dict = self.put_request_dict
generated_sequence_2_mp3(prog_ns, f"{self.unique_id}",
request_dict=request_dict)
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.extract_subsequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = self.seconds_to_steps(input_section.start_time,
qpm)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(
n.end_time
for n in primer_sequence.notes) if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = polyphony_lib.extract_polyphonic_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
generate_start_step = self.seconds_to_steps(
generate_section.start_time, qpm)
generate_end_step = self.seconds_to_steps(generate_section.end_time,
qpm)
if extracted_seqs and extracted_seqs[0]:
poly_seq = extracted_seqs[0]
else:
# If no track could be extracted, create an empty track that starts at the
# requested generate_start_step. This will result in a sequence that
# contains only the START token.
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(quantized_primer_sequence.quantization_info.
steps_per_quarter),
start_step=generate_start_step)
# Ensure that the track extends up to the step we want to start generating.
poly_seq.set_length(generate_start_step - poly_seq.start_step)
# Trim any trailing end events to prepare the sequence for more events to be
# appended during generation.
poly_seq.trim_trailing_end_events()
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
# Inject the priming sequence as melody in the output of the generator.
# Note that start_step is 0 because we overwrite poly_seq below. If we
# included the priming sequence in poly_seq, it would be poly_seq.num_steps.
melody_to_inject = copy.deepcopy(poly_seq)
args['modify_events_callback'] = partial(_inject_melody,
melody_to_inject, 0)
# Overwrite poly_seq with a blank sequence to feed into the generator so it
# is conditioned only on the melody events that are injected as the sequence
# is created. Otherwise, the generator would have to determine the most
# likely sequence to follow a monophonic line, which is something not
# present in the current training data (Bach Chorales).
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(
quantized_primer_sequence.quantization_info.steps_per_quarter),
start_step=generate_start_step)
poly_seq.trim_trailing_end_events()
# If we wanted to include the priming sequence and didn't clear poly_seq
# above, this is how we would calculate total_steps.
# total_steps = poly_seq.num_steps + (
# generate_end_step - generate_start_step)
total_steps = generate_end_step - generate_start_step
while poly_seq.num_steps < total_steps:
# Assume it takes ~5 rnn steps to generate one quantized step.
# Can't know for sure until generation is finished because the number of
# notes per quantized step is variable.
steps_to_gen = total_steps - poly_seq.num_steps
rnn_steps_to_gen = 5 * steps_to_gen
tf.logging.info(
'Need to generate %d more steps for this sequence, will try asking '
'for %d RNN steps' % (steps_to_gen, rnn_steps_to_gen))
poly_seq = self._model.generate_polyphonic_sequence(
len(poly_seq) + rnn_steps_to_gen, poly_seq, **args)
poly_seq.set_length(total_steps)
# Specify a base_note_sequence because the priming sequence is not included
# in poly_seq. If we did not clear poly_seq above, then we would not want to
# specify a base_note_sequence.
generated_sequence = poly_seq.to_sequence(
qpm=qpm, base_note_sequence=copy.deepcopy(primer_sequence))
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _to_tensors(self, note_sequence):
# Performance sequences require sustain to be correctly interpreted.
note_sequence = sequences_lib.apply_sustain_control_changes(note_sequence)
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in note_sequence.text_annotations):
try:
# Quantize just for the purpose of chord inference.
# TODO(iansimon): Allow chord inference in unquantized sequences.
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return data.ConverterTensors()
# Infer chords in quantized sequence.
mm.infer_chords_for_sequence(quantized_sequence)
except (mm.BadTimeSignatureError, mm.NonIntegerStepsPerBarError,
mm.NegativeTimeError, mm.ChordInferenceError):
return data.ConverterTensors()
# Copy inferred chords back to original sequence.
for qta in quantized_sequence.text_annotations:
if qta.annotation_type == CHORD_SYMBOL:
ta = note_sequence.text_annotations.add()
ta.annotation_type = CHORD_SYMBOL
ta.time = qta.time
ta.text = qta.text
if note_sequence.tempos:
quarters_per_minute = note_sequence.tempos[0].qpm
else:
quarters_per_minute = mm.DEFAULT_QUARTERS_PER_MINUTE
quarters_per_bar = self._steps_per_bar / self._steps_per_quarter
hop_size_quarters = quarters_per_bar * self._hop_size_bars
hop_size_seconds = 60.0 * hop_size_quarters / quarters_per_minute
# Split note sequence by bar hop size (in seconds).
subsequences = sequences_lib.split_note_sequence(
note_sequence, hop_size_seconds)
if self._first_subsequence_only and len(subsequences) > 1:
return data.ConverterTensors()
sequence_tensors = []
sequence_chord_tensors = []
for subsequence in subsequences:
# Quantize this subsequence.
try:
quantized_subsequence = mm.quantize_note_sequence(
subsequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_subsequence) !=
self._steps_per_bar):
return data.ConverterTensors()
except (mm.BadTimeSignatureError, mm.NonIntegerStepsPerBarError,
mm.NegativeTimeError):
return data.ConverterTensors()
# Convert the quantized subsequence to tensors.
tensors, chord_tensors = self._quantized_subsequence_to_tensors(
quantized_subsequence)
if tensors:
sequence_tensors.append(tensors)
if self._chord_encoding:
sequence_chord_tensors.append(chord_tensors)
return data.ConverterTensors(
inputs=sequence_tensors, outputs=sequence_tensors,
controls=sequence_chord_tensors)
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise sequence_generator.SequenceGeneratorError(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise sequence_generator.SequenceGeneratorError(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time,
steps_per_second,
quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
if primer_sequence.notes:
last_end_time = max(n.end_time for n in primer_sequence.notes)
else:
last_end_time = 0
if last_end_time > generate_section.start_time:
raise sequence_generator.SequenceGeneratorError(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = polyphony_lib.extract_polyphonic_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
generate_start_step = mm.quantize_to_step(generate_section.start_time,
steps_per_second,
quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
generate_end_step = mm.quantize_to_step(generate_section.end_time,
steps_per_second,
quantize_cutoff=1.0)
if extracted_seqs and extracted_seqs[0]:
poly_seq = extracted_seqs[0]
else:
# If no track could be extracted, create an empty track that starts at the
# requested generate_start_step. This will result in a sequence that
# contains only the START token.
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(quantized_primer_sequence.quantization_info.
steps_per_quarter),
start_step=generate_start_step)
# Ensure that the track extends up to the step we want to start generating.
poly_seq.set_length(generate_start_step - poly_seq.start_step)
# Trim any trailing end events to prepare the sequence for more events to be
# appended during generation.
poly_seq.trim_trailing_end_events()
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
# Inject the priming sequence as melody in the output of the generator, if
# requested.
# This option starts with no_ so that if it is unspecified (as will be the
# case when used with the midi interface), the default will be to inject the
# primer.
if not (generator_options.args['no_inject_primer_during_generation'].
bool_value):
melody_to_inject = copy.deepcopy(poly_seq)
if generator_options.args['condition_on_primer'].bool_value:
inject_start_step = poly_seq.num_steps
else:
# 0 steps because we'll overwrite poly_seq with a blank sequence below.
inject_start_step = 0
args['modify_events_callback'] = functools.partial(
_inject_melody, melody_to_inject, inject_start_step)
# If we don't want to condition on the priming sequence, then overwrite
# poly_seq with a blank sequence to feed into the generator.
if not generator_options.args['condition_on_primer'].bool_value:
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(quantized_primer_sequence.quantization_info.
steps_per_quarter),
start_step=generate_start_step)
poly_seq.trim_trailing_end_events()
total_steps = poly_seq.num_steps + (generate_end_step -
generate_start_step)
while poly_seq.num_steps < total_steps:
# Assume it takes ~5 rnn steps to generate one quantized step.
# Can't know for sure until generation is finished because the number of
# notes per quantized step is variable.
steps_to_gen = total_steps - poly_seq.num_steps
rnn_steps_to_gen = 5 * steps_to_gen
tf.logging.info(
'Need to generate %d more steps for this sequence, will try asking '
'for %d RNN steps' % (steps_to_gen, rnn_steps_to_gen))
poly_seq = self._model.generate_polyphonic_sequence(
len(poly_seq) + rnn_steps_to_gen, poly_seq, **args)
poly_seq.set_length(total_steps)
if generator_options.args['condition_on_primer'].bool_value:
generated_sequence = poly_seq.to_sequence(qpm=qpm)
else:
# Specify a base_note_sequence because the priming sequence was not
# included in poly_seq.
generated_sequence = poly_seq.to_sequence(
qpm=qpm, base_note_sequence=copy.deepcopy(primer_sequence))
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _to_tensors(self, note_sequence):
# Performance sequences require sustain to be correctly interpreted.
note_sequence = sequences_lib.apply_sustain_control_changes(
note_sequence)
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in note_sequence.text_annotations):
try:
# Quantize just for the purpose of chord inference.
# TODO(iansimon): Allow chord inference in unquantized sequences.
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence)
!= self._steps_per_bar):
return data.ConverterTensors()
# Infer chords in quantized sequence.
mm.infer_chords_for_sequence(quantized_sequence)
except (mm.BadTimeSignatureException,
mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException, mm.ChordInferenceException):
return data.ConverterTensors()
# Copy inferred chords back to original sequence.
for qta in quantized_sequence.text_annotations:
if qta.annotation_type == CHORD_SYMBOL:
ta = note_sequence.text_annotations.add()
ta.annotation_type = CHORD_SYMBOL
ta.time = qta.time
ta.text = qta.text
quarters_per_minute = (note_sequence.tempos[0].qpm
if note_sequence.tempos else
mm.DEFAULT_QUARTERS_PER_MINUTE)
quarters_per_bar = self._steps_per_bar / self._steps_per_quarter
hop_size_quarters = quarters_per_bar * self._hop_size_bars
hop_size_seconds = 60.0 * hop_size_quarters / quarters_per_minute
# Split note sequence by bar hop size (in seconds).
subsequences = sequences_lib.split_note_sequence(
note_sequence, hop_size_seconds)
if self._first_subsequence_only and len(subsequences) > 1:
return data.ConverterTensors()
sequence_tensors = []
sequence_chord_tensors = []
for subsequence in subsequences:
# Quantize this subsequence.
try:
quantized_subsequence = mm.quantize_note_sequence(
subsequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(
quantized_subsequence) != self._steps_per_bar):
return data.ConverterTensors()
except (mm.BadTimeSignatureException,
mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException):
return data.ConverterTensors()
# Convert the quantized subsequence to tensors.
tensors, chord_tensors = self._quantized_subsequence_to_tensors(
quantized_subsequence)
if tensors:
sequence_tensors.append(tensors)
if self._chord_encoding:
sequence_chord_tensors.append(chord_tensors)
return data.ConverterTensors(inputs=sequence_tensors,
outputs=sequence_tensors,
controls=sequence_chord_tensors)
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique sequences."""
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return [], []
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return [], []
new_notes = []
for n in quantized_sequence.notes:
if not n.is_drum:
continue
if n.pitch not in self._pitch_class_map:
continue
n.pitch = self._pitch_class_map[n.pitch]
new_notes.append(n)
del quantized_sequence.notes[:]
quantized_sequence.notes.extend(new_notes)
event_lists, unused_stats = self._drums_extractor_fn(quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_tuples = []
for l in event_lists:
for i in range(self._slice_steps, len(l) + 1, self._steps_per_bar):
sliced_event_tuples.append(tuple(l[i - self._slice_steps: i]))
else:
sliced_event_tuples = [tuple(l) for l in event_lists]
unique_event_tuples = list(set(sliced_event_tuples))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
rolls = []
oh_vecs = []
for t in unique_event_tuples:
if self._roll_input or self._roll_output:
if self.end_token is not None:
t_roll = list(t) + [(self._pr_encoder_decoder.input_size - 1,)]
else:
t_roll = t
rolls.append(np.vstack([
self._pr_encoder_decoder.events_to_input(t_roll, i).astype(np.bool)
for i in range(len(t_roll))]))
if not (self._roll_input and self._roll_output):
labels = [self._oh_encoder_decoder.encode_event(e) for e in t]
if self.end_token is not None:
labels += [self._oh_encoder_decoder.num_classes]
oh_vecs.append(np_onehot(
labels,
self._oh_encoder_decoder.num_classes + (self.end_token is not None),
np.bool))
if self._roll_input:
input_seqs = [
np.append(roll, np.expand_dims(np.all(roll == 0, axis=1), axis=1),
axis=1) for roll in rolls]
else:
input_seqs = oh_vecs
output_seqs = rolls if self._roll_output else oh_vecs
return input_seqs, output_seqs
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm if input_sequence
and input_sequence.tempos else mm.DEFAULT_QUARTERS_PER_MINUTE)
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time,
steps_per_second)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(
n.end_time
for n in primer_sequence.notes) if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(primer_sequence,
self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_melodies, _ = mm.extract_melodies(
quantized_sequence,
search_start_step=input_start_step,
min_bars=0,
min_unique_pitches=1,
gap_bars=float('inf'),
ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = mm.quantize_to_step(generate_section.start_time,
steps_per_second)
end_step = mm.quantize_to_step(generate_section.end_time,
steps_per_second)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
# If no melody could be extracted, create an empty melody that starts 1
# step before the request start_step. This will result in 1 step of
# silence when the melody is extended below.
steps_per_bar = int(
mm.steps_per_bar_in_quantized_sequence(quantized_sequence))
melody = mm.Melody([],
start_step=max(0, start_step - 1),
steps_per_bar=steps_per_bar,
steps_per_quarter=self.steps_per_quarter)
# Ensure that the melody extends up to the step we want to start generating.
melody.set_length(start_step - melody.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_melody = self._model.generate_melody(
end_step - melody.start_step, melody, **args)
generated_sequence = generated_melody.to_sequence(qpm=qpm)
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise sequence_generator.SequenceGeneratorError(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise sequence_generator.SequenceGeneratorError(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
if input_sequence and input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
else:
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time,
steps_per_second,
quantize_cutoff=0.0)
else:
primer_sequence = input_sequence
input_start_step = 0
if primer_sequence.notes:
last_end_time = max(n.end_time for n in primer_sequence.notes)
else:
last_end_time = 0
if last_end_time > generate_section.start_time:
raise sequence_generator.SequenceGeneratorError(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(primer_sequence,
self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_drum_tracks, _ = drum_pipelines.extract_drum_tracks(
quantized_sequence,
search_start_step=input_start_step,
min_bars=0,
gap_bars=float('inf'),
ignore_is_drum=True)
assert len(extracted_drum_tracks) <= 1
start_step = mm.quantize_to_step(generate_section.start_time,
steps_per_second,
quantize_cutoff=0.0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
end_step = mm.quantize_to_step(generate_section.end_time,
steps_per_second,
quantize_cutoff=1.0)
if extracted_drum_tracks and extracted_drum_tracks[0]:
drums = extracted_drum_tracks[0]
else:
# If no drum track could be extracted, create an empty drum track that
# starts 1 step before the request start_step. This will result in 1 step
# of silence when the drum track is extended below.
steps_per_bar = int(
mm.steps_per_bar_in_quantized_sequence(quantized_sequence))
drums = mm.DrumTrack([],
start_step=max(0, start_step - 1),
steps_per_bar=steps_per_bar,
steps_per_quarter=self.steps_per_quarter)
# Ensure that the drum track extends up to the step we want to start
# generating.
drums.set_length(start_step - drums.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_drums = self._model.generate_drum_track(
end_step - drums.start_step, drums, **args)
generated_sequence = generated_drums.to_sequence(qpm=qpm)
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorError(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorError(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
if input_sequence and input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
else:
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
# Use primer melody from input section only. Take backing chords from
# beginning of input section through end of generate section.
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, input_section.end_time)
backing_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, generate_section.end_time)
input_start_step = mm.quantize_to_step(
input_section.start_time, steps_per_second, quantize_cutoff=0.0)
else:
# No input section. Take primer melody from the beginning of the sequence
# up until the start of the generate section.
primer_sequence = mm.trim_note_sequence(
input_sequence, 0.0, generate_section.start_time)
backing_sequence = mm.trim_note_sequence(
input_sequence, 0.0, generate_section.end_time)
input_start_step = 0
if primer_sequence.notes:
last_end_time = max(n.end_time for n in primer_sequence.notes)
else:
last_end_time = 0
if last_end_time >= generate_section.start_time:
raise mm.SequenceGeneratorError(
'Got GenerateSection request for section that is before or equal to '
'the end of the input section. This model can only extend melodies. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming and backing sequences.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
quantized_backing_sequence = mm.quantize_note_sequence(
backing_sequence, self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_melodies, _ = mm.extract_melodies(
quantized_primer_sequence, search_start_step=input_start_step,
min_bars=0, min_unique_pitches=1, gap_bars=float('inf'),
ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = mm.quantize_to_step(
generate_section.start_time, steps_per_second, quantize_cutoff=0.0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
end_step = mm.quantize_to_step(
generate_section.end_time, steps_per_second, quantize_cutoff=1.0)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
# If no melody could be extracted, create an empty melody that starts 1
# step before the request start_step. This will result in 1 step of
# silence when the melody is extended below.
steps_per_bar = int(
mm.steps_per_bar_in_quantized_sequence(quantized_primer_sequence))
melody = mm.Melody([],
start_step=max(0, start_step - 1),
steps_per_bar=steps_per_bar,
steps_per_quarter=self.steps_per_quarter)
extracted_chords, _ = mm.extract_chords(quantized_backing_sequence)
chords = extracted_chords[0]
# Make sure that chords and melody start on the same step.
if chords.start_step < melody.start_step:
chords.set_length(len(chords) - melody.start_step + chords.start_step)
assert chords.end_step == end_step
# Ensure that the melody extends up to the step we want to start generating.
melody.set_length(start_step - melody.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_melody = self._model.generate_melody(melody, chords, **args)
generated_lead_sheet = mm.LeadSheet(generated_melody, chords)
generated_sequence = generated_lead_sheet.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, input_section.end_time)
input_start_step = mm.quantize_to_step(
input_section.start_time, steps_per_second, quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(n.end_time for n in primer_sequence.notes)
if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = polyphony_lib.extract_polyphonic_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
generate_start_step = mm.quantize_to_step(
generate_section.start_time, steps_per_second, quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
generate_end_step = mm.quantize_to_step(
generate_section.end_time, steps_per_second, quantize_cutoff=1.0)
if extracted_seqs and extracted_seqs[0]:
poly_seq = extracted_seqs[0]
else:
# If no track could be extracted, create an empty track that starts at the
# requested generate_start_step. This will result in a sequence that
# contains only the START token.
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(
quantized_primer_sequence.quantization_info.steps_per_quarter),
start_step=generate_start_step)
# Ensure that the track extends up to the step we want to start generating.
poly_seq.set_length(generate_start_step - poly_seq.start_step)
# Trim any trailing end events to prepare the sequence for more events to be
# appended during generation.
poly_seq.trim_trailing_end_events()
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
# Inject the priming sequence as melody in the output of the generator, if
# requested.
# This option starts with no_ so that if it is unspecified (as will be the
# case when used with the midi interface), the default will be to inject the
# primer.
if not (generator_options.args[
'no_inject_primer_during_generation'].bool_value):
melody_to_inject = copy.deepcopy(poly_seq)
if generator_options.args['condition_on_primer'].bool_value:
inject_start_step = poly_seq.num_steps
else:
# 0 steps because we'll overwrite poly_seq with a blank sequence below.
inject_start_step = 0
args['modify_events_callback'] = partial(
_inject_melody, melody_to_inject, inject_start_step)
# If we don't want to condition on the priming sequence, then overwrite
# poly_seq with a blank sequence to feed into the generator.
if not generator_options.args['condition_on_primer'].bool_value:
poly_seq = polyphony_lib.PolyphonicSequence(
steps_per_quarter=(
quantized_primer_sequence.quantization_info.steps_per_quarter),
start_step=generate_start_step)
poly_seq.trim_trailing_end_events()
total_steps = poly_seq.num_steps + (
generate_end_step - generate_start_step)
while poly_seq.num_steps < total_steps:
# Assume it takes ~5 rnn steps to generate one quantized step.
# Can't know for sure until generation is finished because the number of
# notes per quantized step is variable.
steps_to_gen = total_steps - poly_seq.num_steps
rnn_steps_to_gen = 5 * steps_to_gen
tf.logging.info(
'Need to generate %d more steps for this sequence, will try asking '
'for %d RNN steps' % (steps_to_gen, rnn_steps_to_gen))
poly_seq = self._model.generate_polyphonic_sequence(
len(poly_seq) + rnn_steps_to_gen, poly_seq, **args)
poly_seq.set_length(total_steps)
if generator_options.args['condition_on_primer'].bool_value:
generated_sequence = poly_seq.to_sequence(qpm=qpm)
else:
# Specify a base_note_sequence because the priming sequence was not
# included in poly_seq.
generated_sequence = poly_seq.to_sequence(
qpm=qpm, base_note_sequence=copy.deepcopy(primer_sequence))
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def _to_tensors(self, note_sequence):
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return []
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException):
return []
total_bars = int(
np.ceil(quantized_sequence.total_quantized_steps / self._steps_per_bar))
total_bars = min(total_bars, self._max_bars)
# Assign an instrument class for each instrument, and compute its coverage.
# If an instrument has multiple classes, it is considered INVALID.
instrument_type = np.zeros(MAX_INSTRUMENT_NUMBER + 1, np.uint8)
coverage = np.zeros((total_bars, MAX_INSTRUMENT_NUMBER + 1), np.bool)
for note in quantized_sequence.notes:
i = note.instrument
if i > MAX_INSTRUMENT_NUMBER:
tf.logging.warning('Skipping invalid instrument number: %d', i)
continue
inferred_type = (
self.InstrumentType.DRUMS if note.is_drum else
self._program_map.get(note.program, self.InstrumentType.INVALID))
if not instrument_type[i]:
instrument_type[i] = inferred_type
elif instrument_type[i] != inferred_type:
instrument_type[i] = self.InstrumentType.INVALID
start_bar = note.quantized_start_step // self._steps_per_bar
end_bar = int(np.ceil(note.quantized_end_step / self._steps_per_bar))
if start_bar >= total_bars:
continue
coverage[start_bar:min(end_bar, total_bars), i] = True
# Group instruments by type.
instruments_by_type = collections.defaultdict(list)
for i, type_ in enumerate(instrument_type):
if type_ not in (self.InstrumentType.UNK, self.InstrumentType.INVALID):
instruments_by_type[type_].append(i)
if len(instruments_by_type) < 3:
# This NoteSequence doesn't have all 3 types.
return [], []
# Encode individual instruments.
# Set total time so that instruments will be padded correctly.
note_sequence.total_time = (
total_bars * self._steps_per_bar *
60 / note_sequence.tempos[0].qpm / self._steps_per_quarter)
encoded_instruments = {}
for i in (instruments_by_type[self.InstrumentType.MEL] +
instruments_by_type[self.InstrumentType.BASS]):
_, t = self._melody_converter.to_tensors(
_extract_instrument(note_sequence, i))
if t:
encoded_instruments[i] = t[0]
else:
coverage[:, i] = False
for i in instruments_by_type[self.InstrumentType.DRUMS]:
_, t = self._drums_converter.to_tensors(
_extract_instrument(note_sequence, i))
if t:
encoded_instruments[i] = t[0]
else:
coverage[:, i] = False
# Fill in coverage gaps up to self._gap_bars.
og_coverage = coverage.copy()
for j in range(total_bars):
coverage[j] = np.any(
og_coverage[
max(0, j-self._gap_bars):min(total_bars, j+self._gap_bars) + 1],
axis=0)
# Take cross product of instruments from each class and compute combined
# encodings where they overlap.
seqs = []
for grp in itertools.product(
instruments_by_type[self.InstrumentType.MEL],
instruments_by_type[self.InstrumentType.BASS],
instruments_by_type[self.InstrumentType.DRUMS]):
# Consider an instrument covered within gap_bars from the end if any of
# the other instruments are. This allows more leniency when re-encoding
# slices.
grp_coverage = np.all(coverage[:, grp], axis=1)
grp_coverage[:self._gap_bars] = np.any(coverage[:self._gap_bars, grp])
grp_coverage[-self._gap_bars:] = np.any(coverage[-self._gap_bars:, grp])
for j in range(total_bars - self._slice_bars + 1):
if np.all(grp_coverage[j:j + self._slice_bars]):
start_step = j * self._steps_per_bar
end_step = (j + self._slice_bars) * self._steps_per_bar
seqs.append(np.concatenate(
[encoded_instruments[i][start_step:end_step] for i in grp],
axis=-1))
return seqs, seqs
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique sequences."""
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return [], []
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return [], []
new_notes = []
for n in quantized_sequence.notes:
if not n.is_drum:
continue
if n.pitch not in self._pitch_class_map:
continue
n.pitch = self._pitch_class_map[n.pitch]
new_notes.append(n)
del quantized_sequence.notes[:]
quantized_sequence.notes.extend(new_notes)
event_lists, unused_stats = self._drums_extractor_fn(quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_tuples = []
for l in event_lists:
for i in range(self._slice_steps, len(l) + 1, self._steps_per_bar):
sliced_event_tuples.append(tuple(l[i - self._slice_steps: i]))
else:
sliced_event_tuples = [tuple(l) for l in event_lists]
unique_event_tuples = list(set(sliced_event_tuples))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
rolls = []
oh_vecs = []
for t in unique_event_tuples:
if self._roll_input or self._roll_output:
if self.end_token is not None:
t_roll = list(t) + [(self._pr_encoder_decoder.input_size - 1,)]
else:
t_roll = t
rolls.append(np.vstack([
self._pr_encoder_decoder.events_to_input(t_roll, i).astype(np.bool)
for i in range(len(t_roll))]))
if not (self._roll_input and self._roll_output):
labels = [self._oh_encoder_decoder.encode_event(e) for e in t]
if self.end_token is not None:
labels += [self._oh_encoder_decoder.num_classes]
oh_vecs.append(np_onehot(
labels,
self._oh_encoder_decoder.num_classes + (self.end_token is not None),
np.bool))
if self._roll_input:
input_seqs = [
np.append(roll, np.expand_dims(np.all(roll == 0, axis=1), axis=1),
axis=1) for roll in rolls]
else:
input_seqs = oh_vecs
output_seqs = rolls if self._roll_output else oh_vecs
return input_seqs, output_seqs
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique, one-hot tensor sequences."""
try:
if self._steps_per_quarter:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return ConverterTensors()
else:
quantized_sequence = mm.quantize_note_sequence_absolute(
note_sequence, self._steps_per_second)
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return ConverterTensors()
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in quantized_sequence.text_annotations):
# We are conditioning on chords but sequence does not have chords. Try to
# infer them.
try:
mm.infer_chords_for_sequence(quantized_sequence)
except mm.ChordInferenceException:
return ConverterTensors()
event_lists, unused_stats = self._event_extractor_fn(quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_lists = []
for l in event_lists:
for i in range(self._slice_steps, len(l) + 1, self._steps_per_bar):
sliced_event_lists.append(l[i - self._slice_steps: i])
else:
sliced_event_lists = event_lists
if self._chord_encoding:
try:
sliced_chord_lists = chords_lib.event_list_chords(
quantized_sequence, sliced_event_lists)
except chords_lib.CoincidentChordsException:
return ConverterTensors()
sliced_event_lists = [zip(el, cl) for el, cl in zip(sliced_event_lists,
sliced_chord_lists)]
# TODO(adarob): Consider handling the fact that different event lists can
# be mapped to identical tensors by the encoder_decoder (e.g., Drums).
unique_event_tuples = list(set(tuple(l) for l in sliced_event_lists))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
if not unique_event_tuples:
return ConverterTensors()
control_seqs = []
if self._chord_encoding:
unique_event_tuples, unique_chord_tuples = zip(
*[zip(*t) for t in unique_event_tuples if t])
for t in unique_chord_tuples:
try:
chord_tokens = [self._chord_encoding.encode_event(e) for e in t]
if self.end_token:
# Repeat the last chord instead of using a special token; otherwise
# the model may learn to rely on the special token to detect
# endings.
chord_tokens.append(chord_tokens[-1] if chord_tokens else
self._chord_encoding.encode_event(mm.NO_CHORD))
except (mm.ChordSymbolException, mm.ChordEncodingException):
return ConverterTensors()
control_seqs.append(
np_onehot(chord_tokens, self.control_depth, self.control_dtype))
seqs = []
for t in unique_event_tuples:
seqs.append(np_onehot(
[self._legacy_encoder_decoder.encode_event(e) for e in t] +
([] if self.end_token is None else [self.end_token]),
self.output_depth, self.output_dtype))
return ConverterTensors(inputs=seqs, outputs=seqs, controls=control_seqs)
def _to_tensors(self, note_sequence):
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return ConverterTensors()
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException):
return ConverterTensors()
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in quantized_sequence.text_annotations):
# We are conditioning on chords but sequence does not have chords. Try to
# infer them.
try:
mm.infer_chords_for_sequence(quantized_sequence)
except mm.ChordInferenceException:
return ConverterTensors()
# The trio parts get extracted from the original NoteSequence, so copy the
# inferred chords back to that one.
for qta in quantized_sequence.text_annotations:
if qta.annotation_type == CHORD_SYMBOL:
ta = note_sequence.text_annotations.add()
ta.annotation_type = CHORD_SYMBOL
ta.time = qta.time
ta.text = qta.text
total_bars = int(
np.ceil(quantized_sequence.total_quantized_steps / self._steps_per_bar))
total_bars = min(total_bars, self._max_bars)
# Assign an instrument class for each instrument, and compute its coverage.
# If an instrument has multiple classes, it is considered INVALID.
instrument_type = np.zeros(MAX_INSTRUMENT_NUMBER + 1, np.uint8)
coverage = np.zeros((total_bars, MAX_INSTRUMENT_NUMBER + 1), np.bool)
for note in quantized_sequence.notes:
i = note.instrument
if i > MAX_INSTRUMENT_NUMBER:
tf.logging.warning('Skipping invalid instrument number: %d', i)
continue
inferred_type = (
self.InstrumentType.DRUMS if note.is_drum else
self._program_map.get(note.program, self.InstrumentType.INVALID))
if not instrument_type[i]:
instrument_type[i] = inferred_type
elif instrument_type[i] != inferred_type:
instrument_type[i] = self.InstrumentType.INVALID
start_bar = note.quantized_start_step // self._steps_per_bar
end_bar = int(np.ceil(note.quantized_end_step / self._steps_per_bar))
if start_bar >= total_bars:
continue
coverage[start_bar:min(end_bar, total_bars), i] = True
# Group instruments by type.
instruments_by_type = collections.defaultdict(list)
for i, type_ in enumerate(instrument_type):
if type_ not in (self.InstrumentType.UNK, self.InstrumentType.INVALID):
instruments_by_type[type_].append(i)
if len(instruments_by_type) < 3:
# This NoteSequence doesn't have all 3 types.
return ConverterTensors()
# Encode individual instruments.
# Set total time so that instruments will be padded correctly.
note_sequence.total_time = (
total_bars * self._steps_per_bar *
60 / note_sequence.tempos[0].qpm / self._steps_per_quarter)
encoded_instruments = {}
encoded_chords = None
for i in (instruments_by_type[self.InstrumentType.MEL] +
instruments_by_type[self.InstrumentType.BASS]):
tensors = self._melody_converter.to_tensors(
_extract_instrument(note_sequence, i))
if tensors.outputs:
encoded_instruments[i] = tensors.outputs[0]
if encoded_chords is None:
encoded_chords = tensors.controls[0]
elif not np.array_equal(encoded_chords, tensors.controls[0]):
tf.logging.warning('Trio chords disagreement between instruments.')
else:
coverage[:, i] = False
for i in instruments_by_type[self.InstrumentType.DRUMS]:
tensors = self._drums_converter.to_tensors(
_extract_instrument(note_sequence, i))
if tensors.outputs:
encoded_instruments[i] = tensors.outputs[0]
else:
coverage[:, i] = False
# Fill in coverage gaps up to self._gap_bars.
og_coverage = coverage.copy()
for j in range(total_bars):
coverage[j] = np.any(
og_coverage[
max(0, j-self._gap_bars):min(total_bars, j+self._gap_bars) + 1],
axis=0)
# Take cross product of instruments from each class and compute combined
# encodings where they overlap.
seqs = []
control_seqs = []
for grp in itertools.product(
instruments_by_type[self.InstrumentType.MEL],
instruments_by_type[self.InstrumentType.BASS],
instruments_by_type[self.InstrumentType.DRUMS]):
# Consider an instrument covered within gap_bars from the end if any of
# the other instruments are. This allows more leniency when re-encoding
# slices.
grp_coverage = np.all(coverage[:, grp], axis=1)
grp_coverage[:self._gap_bars] = np.any(coverage[:self._gap_bars, grp])
grp_coverage[-self._gap_bars:] = np.any(coverage[-self._gap_bars:, grp])
for j in range(total_bars - self._slice_bars + 1):
if (np.all(grp_coverage[j:j + self._slice_bars]) and
all(i in encoded_instruments for i in grp)):
start_step = j * self._steps_per_bar
end_step = (j + self._slice_bars) * self._steps_per_bar
seqs.append(np.concatenate(
[encoded_instruments[i][start_step:end_step] for i in grp],
axis=-1))
if encoded_chords is not None:
control_seqs.append(encoded_chords[start_step:end_step])
return ConverterTensors(inputs=seqs, outputs=seqs, controls=control_seqs)
def _primer_melody_to_event_sequence(self, input_sequence,
generator_options, config):
qpm = (input_sequence.tempos[0].qpm if input_sequence
and input_sequence.tempos else mm.DEFAULT_QUARTERS_PER_MINUTE)
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time,
steps_per_second,
quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(
n.end_time
for n in primer_sequence.notes) if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(primer_sequence,
self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_melodies, _ = mm.extract_melodies(
quantized_sequence,
search_start_step=input_start_step,
min_bars=0,
min_unique_pitches=1,
gap_bars=float('inf'),
ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = mm.quantize_to_step(generate_section.start_time,
steps_per_second,
quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
end_step = mm.quantize_to_step(generate_section.end_time,
steps_per_second,
quantize_cutoff=1.0)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
# If no melody could be extracted, create an empty melody that starts 1
# step before the request start_step. This will result in 1 step of
# silence when the melody is extended below.
steps_per_bar = int(
mm.steps_per_bar_in_quantized_sequence(quantized_sequence))
melody = mm.Melody([],
start_step=max(0, start_step - 1),
steps_per_bar=steps_per_bar,
steps_per_quarter=self.steps_per_quarter)
# Ensure that the melody extends up to the step we want to start generating.
melody.set_length(start_step - melody.start_step - 2)
now_encoding = config.encoder_decoder._one_hot_encoding
# Extract generation arguments from generator options.
primer_events = self._model.primer_melody_to_events(
end_step - melody.start_step, melody)
for i, event in enumerate(primer_events):
primer_events[i] = now_encoding.encode_event(event)
return primer_events
def createDrums():
drums = music_pb2.NoteSequence()
drums.notes.add(pitch=36,
start_time=0,
end_time=0.125,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=0,
end_time=0.125,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=0.125,
end_time=0.25,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=.25,
end_time=0.375,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=0.375,
end_time=0.5,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=38,
start_time=0.5,
end_time=0.625,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=.625,
end_time=0.75,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=0.75,
end_time=0.875,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=.875,
end_time=1.0,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=36,
start_time=1.0,
end_time=1.125,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=1,
end_time=1.125,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=1.125,
end_time=1.25,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=1.25,
end_time=1.375,
is_drum=True,
instrument=10,
velocity=80)
drums.notes.add(pitch=42,
start_time=1.375,
end_time=1.5,
is_drum=True,
instrument=10,
velocity=80)
drums.time_signatures.add(numerator=4, denominator=4)
drums.time_signatures[0].numerator = 4
drums.time_signatures[0].denominator = 4
drums.total_time = 1.375
drums.tempos.add(qpm=120)
drums = mm.quantize_note_sequence(drums, 4)
mm.sequence_proto_to_midi_file(drums, "primer.mid")
return drums
def generate(self, empty=False, backup_seq=None):
"""
different implementation is needed for improv rnn's
generation function.
"""
if backup_seq is not None:
self.sequence = copy.deepcopy(backup_seq)
input_sequence = copy.deepcopy(self.sequence)
num_steps = self.num_steps # change this for shorter/longer sequences
temperature = self.temperature
# Set the start time to begin on the next step after the last note ends.
last_end_time = (max(n.end_time for n in input_sequence.notes)
if input_sequence.notes else 0)
qpm = input_sequence.tempos[0].qpm
input_sequence = mm.quantize_note_sequence(input_sequence, self.model.steps_per_quarter)
primer_sequence_steps = input_sequence.total_quantized_steps
if primer_sequence_steps > num_steps:
# easier to make num_steps bigger to accommodate for sizes
# 4 times the size of original sequence..
num_steps = primer_sequence_steps * 4
mm.infer_chords_for_sequence(input_sequence)
raw_chord_string = ""
for annotation in input_sequence.text_annotations:
if annotation.annotation_type == CHORD_SYMBOL:
chord_name = annotation.text
raw_chord_string += f'{chord_name} '
raw_chord_string = raw_chord_string[:-1]
raw_chords = raw_chord_string.split()
repeated_chords = [chord for chord in raw_chords
for _ in range(16)] * self.phrase_num
self.backing_chords = mm.ChordProgression(repeated_chords)
chord_sequence = self.backing_chords.to_sequence(sequence_start_time=0.0, qpm=qpm)
for text_annotation in chord_sequence.text_annotations:
if text_annotation.annotation_type == CHORD_SYMBOL:
chord = self.sequence.text_annotations.add()
chord.CopyFrom(text_annotation)
seconds_per_step = 60.0 / qpm / self.model.steps_per_quarter
total_seconds = len(self.backing_chords) * seconds_per_step
self.sequence.total_time = total_seconds
generator_options = generator_pb2.GeneratorOptions()
generator_options.args['temperature'].float_value = temperature
generate_section = generator_options.generate_sections.add(
start_time=last_end_time + seconds_per_step,
end_time=total_seconds)
sequence = self.model.generate(self.sequence, generator_options)
renderer = mm.BasicChordRenderer(velocity=CHORD_VELOCITY)
renderer.render(sequence)
request_dict = self.put_request_dict
generated_sequence_2_mp3(sequence, f"{self.unique_id}",
request_dict=request_dict)
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm
if input_sequence and input_sequence.tempos
else mm.DEFAULT_QUARTERS_PER_MINUTE)
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, input_section.end_time)
input_start_step = mm.quantize_to_step(
input_section.start_time, steps_per_second, quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(n.end_time for n in primer_sequence.notes)
if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_melodies, _ = mm.extract_melodies(
quantized_sequence, search_start_step=input_start_step, min_bars=0,
min_unique_pitches=1, gap_bars=float('inf'),
ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = mm.quantize_to_step(
generate_section.start_time, steps_per_second, quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
end_step = mm.quantize_to_step(
generate_section.end_time, steps_per_second, quantize_cutoff=1.0)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
# If no melody could be extracted, create an empty melody that starts 1
# step before the request start_step. This will result in 1 step of
# silence when the melody is extended below.
steps_per_bar = int(
mm.steps_per_bar_in_quantized_sequence(quantized_sequence))
melody = mm.Melody([],
start_step=max(0, start_step - 1),
steps_per_bar=steps_per_bar,
steps_per_quarter=self.steps_per_quarter)
# Ensure that the melody extends up to the step we want to start generating.
melody.set_length(start_step - melody.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_melody = self._model.generate_melody(
end_step - melody.start_step, melody, **args)
generated_sequence = generated_melody.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm if input_sequence
and input_sequence.tempos else mm.DEFAULT_QUARTERS_PER_MINUTE)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = self.seconds_to_steps(input_section.start_time,
qpm)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(
n.end_time
for n in primer_sequence.notes) if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(primer_sequence,
self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_drum_tracks, _ = mm.extract_drum_tracks(
quantized_sequence,
search_start_step=input_start_step,
min_bars=0,
gap_bars=float('inf'))
assert len(extracted_drum_tracks) <= 1
start_step = self.seconds_to_steps(generate_section.start_time, qpm)
end_step = self.seconds_to_steps(generate_section.end_time, qpm)
if extracted_drum_tracks and extracted_drum_tracks[0]:
drums = extracted_drum_tracks[0]
else:
# If no drum track could be extracted, create an empty drum track that
# starts 1 step before the request start_step. This will result in 1 step
# of silence when the drum track is extended below.
drums = mm.DrumTrack([], start_step=max(0, start_step - 1))
# Ensure that the drum track extends up to the step we want to start
# generating.
drums.set_length(start_step - drums.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_drums = self._model.generate_drum_track(
end_step - drums.start_step, drums, **args)
generated_sequence = generated_drums.to_sequence(qpm=qpm)
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm
if input_sequence and input_sequence.tempos
else mm.DEFAULT_QUARTERS_PER_MINUTE)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, input_section.end_time)
input_start_step = self.seconds_to_steps(input_section.start_time, qpm)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(n.end_time for n in primer_sequence.notes)
if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before the end of '
'the NoteSequence. This model can only extend sequences. Requested '
'start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_drum_tracks, _ = mm.extract_drum_tracks(
quantized_sequence, search_start_step=input_start_step, min_bars=0,
gap_bars=float('inf'))
assert len(extracted_drum_tracks) <= 1
start_step = self.seconds_to_steps(
generate_section.start_time, qpm)
end_step = self.seconds_to_steps(generate_section.end_time, qpm)
if extracted_drum_tracks and extracted_drum_tracks[0]:
drums = extracted_drum_tracks[0]
else:
# If no drum track could be extracted, create an empty drum track that
# starts 1 step before the request start_step. This will result in 1 step
# of silence when the drum track is extended below.
drums = mm.DrumTrack([], start_step=max(0, start_step - 1))
# Ensure that the drum track extends up to the step we want to start
# generating.
drums.set_length(start_step - drums.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_drums = self._model.generate_drum_track(
end_step - drums.start_step, drums, **args)
generated_sequence = generated_drums.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def _to_tensors(self, note_sequence):
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return ConverterTensors()
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException):
return ConverterTensors()
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in quantized_sequence.text_annotations):
# We are conditioning on chords but sequence does not have chords. Try to
# infer them.
try:
mm.infer_chords_for_sequence(quantized_sequence)
except mm.ChordInferenceException:
return ConverterTensors()
# The trio parts get extracted from the original NoteSequence, so copy the
# inferred chords back to that one.
for qta in quantized_sequence.text_annotations:
if qta.annotation_type == CHORD_SYMBOL:
ta = note_sequence.text_annotations.add()
ta.annotation_type = CHORD_SYMBOL
ta.time = qta.time
ta.text = qta.text
total_bars = int(
np.ceil(quantized_sequence.total_quantized_steps / self._steps_per_bar))
total_bars = min(total_bars, self._max_bars)
# Assign an instrument class for each instrument, and compute its coverage.
# If an instrument has multiple classes, it is considered INVALID.
instrument_type = np.zeros(MAX_INSTRUMENT_NUMBER + 1, np.uint8)
coverage = np.zeros((total_bars, MAX_INSTRUMENT_NUMBER + 1), np.bool)
for note in quantized_sequence.notes:
i = note.instrument
if i > MAX_INSTRUMENT_NUMBER:
tf.logging.warning('Skipping invalid instrument number: %d', i)
continue
inferred_type = (
self.InstrumentType.DRUMS if note.is_drum else
self._program_map.get(note.program, self.InstrumentType.INVALID))
if not instrument_type[i]:
instrument_type[i] = inferred_type
elif instrument_type[i] != inferred_type:
instrument_type[i] = self.InstrumentType.INVALID
start_bar = note.quantized_start_step // self._steps_per_bar
end_bar = int(np.ceil(note.quantized_end_step / self._steps_per_bar))
if start_bar >= total_bars:
continue
coverage[start_bar:min(end_bar, total_bars), i] = True
# Group instruments by type.
instruments_by_type = collections.defaultdict(list)
for i, type_ in enumerate(instrument_type):
if type_ not in (self.InstrumentType.UNK, self.InstrumentType.INVALID):
instruments_by_type[type_].append(i)
if len(instruments_by_type) < 3:
# This NoteSequence doesn't have all 3 types.
return ConverterTensors()
# Encode individual instruments.
# Set total time so that instruments will be padded correctly.
note_sequence.total_time = (
total_bars * self._steps_per_bar *
60 / note_sequence.tempos[0].qpm / self._steps_per_quarter)
encoded_instruments = {}
encoded_chords = None
for i in (instruments_by_type[self.InstrumentType.MEL] +
instruments_by_type[self.InstrumentType.BASS]):
tensors = self._melody_converter.to_tensors(
_extract_instrument(note_sequence, i))
if tensors.outputs:
encoded_instruments[i] = tensors.outputs[0]
if encoded_chords is None:
encoded_chords = tensors.controls[0]
elif not np.array_equal(encoded_chords, tensors.controls[0]):
tf.logging.warning('Trio chords disagreement between instruments.')
else:
coverage[:, i] = False
for i in instruments_by_type[self.InstrumentType.DRUMS]:
tensors = self._drums_converter.to_tensors(
_extract_instrument(note_sequence, i))
if tensors.outputs:
encoded_instruments[i] = tensors.outputs[0]
else:
coverage[:, i] = False
# Fill in coverage gaps up to self._gap_bars.
og_coverage = coverage.copy()
for j in range(total_bars):
coverage[j] = np.any(
og_coverage[
max(0, j-self._gap_bars):min(total_bars, j+self._gap_bars) + 1],
axis=0)
# Take cross product of instruments from each class and compute combined
# encodings where they overlap.
seqs = []
control_seqs = []
for grp in itertools.product(
instruments_by_type[self.InstrumentType.MEL],
instruments_by_type[self.InstrumentType.BASS],
instruments_by_type[self.InstrumentType.DRUMS]):
# Consider an instrument covered within gap_bars from the end if any of
# the other instruments are. This allows more leniency when re-encoding
# slices.
grp_coverage = np.all(coverage[:, grp], axis=1)
grp_coverage[:self._gap_bars] = np.any(coverage[:self._gap_bars, grp])
grp_coverage[-self._gap_bars:] = np.any(coverage[-self._gap_bars:, grp])
for j in range(total_bars - self._slice_bars + 1):
if (np.all(grp_coverage[j:j + self._slice_bars]) and
all(i in encoded_instruments for i in grp)):
start_step = j * self._steps_per_bar
end_step = (j + self._slice_bars) * self._steps_per_bar
seqs.append(np.concatenate(
[encoded_instruments[i][start_step:end_step] for i in grp],
axis=-1))
if encoded_chords is not None:
control_seqs.append(encoded_chords[start_step:end_step])
return ConverterTensors(inputs=seqs, outputs=seqs, controls=control_seqs)
def _to_tensors(self, note_sequence):
try:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return []
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException):
return []
total_bars = int(
np.ceil(quantized_sequence.total_quantized_steps / self._steps_per_bar))
total_bars = min(total_bars, self._max_bars)
# Assign an instrument class for each instrument, and compute its coverage.
# If an instrument has multiple classes, it is considered INVALID.
instrument_type = np.zeros(MAX_INSTRUMENT_NUMBER + 1, np.uint8)
coverage = np.zeros((total_bars, MAX_INSTRUMENT_NUMBER + 1), np.bool)
for note in quantized_sequence.notes:
i = note.instrument
if i > MAX_INSTRUMENT_NUMBER:
tf.logging.warning('Skipping invalid instrument number: %d', i)
continue
inferred_type = (
self.InstrumentType.DRUMS if note.is_drum else
self._program_map.get(note.program, self.InstrumentType.INVALID))
if not instrument_type[i]:
instrument_type[i] = inferred_type
elif instrument_type[i] != inferred_type:
instrument_type[i] = self.InstrumentType.INVALID
start_bar = note.quantized_start_step // self._steps_per_bar
end_bar = int(np.ceil(note.quantized_end_step / self._steps_per_bar))
if start_bar >= total_bars:
continue
coverage[start_bar:min(end_bar, total_bars), i] = True
# Group instruments by type.
instruments_by_type = collections.defaultdict(list)
for i, type_ in enumerate(instrument_type):
if type_ not in (self.InstrumentType.UNK, self.InstrumentType.INVALID):
instruments_by_type[type_].append(i)
if len(instruments_by_type) < 3:
# This NoteSequence doesn't have all 3 types.
return [], []
# Encode individual instruments.
# Set total time so that instruments will be padded correctly.
note_sequence.total_time = (
total_bars * self._steps_per_bar *
60 / note_sequence.tempos[0].qpm / self._steps_per_quarter)
encoded_instruments = {}
for i in (instruments_by_type[self.InstrumentType.MEL] +
instruments_by_type[self.InstrumentType.BASS]):
_, t = self._melody_converter.to_tensors(
_extract_instrument(note_sequence, i))
if t:
encoded_instruments[i] = t[0]
else:
coverage[:, i] = False
for i in instruments_by_type[self.InstrumentType.DRUMS]:
_, t = self._drums_converter.to_tensors(
_extract_instrument(note_sequence, i))
if t:
encoded_instruments[i] = t[0]
else:
coverage[:, i] = False
# Fill in coverage gaps up to self._gap_bars.
og_coverage = coverage.copy()
for j in range(total_bars):
coverage[j] = np.any(
og_coverage[
max(0, j-self._gap_bars):min(total_bars, j+self._gap_bars) + 1],
axis=0)
# Take cross product of instruments from each class and compute combined
# encodings where they overlap.
seqs = []
for grp in itertools.product(
instruments_by_type[self.InstrumentType.MEL],
instruments_by_type[self.InstrumentType.BASS],
instruments_by_type[self.InstrumentType.DRUMS]):
# Consider an instrument covered within gap_bars from the end if any of
# the other instruments are. This allows more leniency when re-encoding
# slices.
grp_coverage = np.all(coverage[:, grp], axis=1)
grp_coverage[:self._gap_bars] = np.any(coverage[:self._gap_bars, grp])
grp_coverage[-self._gap_bars:] = np.any(coverage[-self._gap_bars:, grp])
for j in range(total_bars - self._slice_bars + 1):
if np.all(grp_coverage[j:j + self._slice_bars]):
start_step = j * self._steps_per_bar
end_step = (j + self._slice_bars) * self._steps_per_bar
seqs.append(np.concatenate(
[encoded_instruments[i][start_step:end_step] for i in grp],
axis=-1))
return seqs, seqs
def _to_tensors(self, note_sequence):
"""Converts NoteSequence to unique, one-hot tensor sequences."""
try:
if self._steps_per_quarter:
quantized_sequence = mm.quantize_note_sequence(
note_sequence, self._steps_per_quarter)
if (mm.steps_per_bar_in_quantized_sequence(quantized_sequence) !=
self._steps_per_bar):
return ConverterTensors()
else:
quantized_sequence = mm.quantize_note_sequence_absolute(
note_sequence, self._steps_per_second)
except (mm.BadTimeSignatureException, mm.NonIntegerStepsPerBarException,
mm.NegativeTimeException) as e:
return ConverterTensors()
if self._chord_encoding and not any(
ta.annotation_type == CHORD_SYMBOL
for ta in quantized_sequence.text_annotations):
# We are conditioning on chords but sequence does not have chords. Try to
# infer them.
try:
mm.infer_chords_for_sequence(quantized_sequence)
except mm.ChordInferenceException:
return ConverterTensors()
event_lists, unused_stats = self._event_extractor_fn(quantized_sequence)
if self._pad_to_total_time:
for e in event_lists:
e.set_length(len(e) + e.start_step, from_left=True)
e.set_length(quantized_sequence.total_quantized_steps)
if self._slice_steps:
sliced_event_lists = []
for l in event_lists:
for i in range(self._slice_steps, len(l) + 1, self._steps_per_bar):
sliced_event_lists.append(l[i - self._slice_steps: i])
else:
sliced_event_lists = event_lists
if self._chord_encoding:
try:
sliced_chord_lists = chords_lib.event_list_chords(
quantized_sequence, sliced_event_lists)
except chords_lib.CoincidentChordsException:
return ConverterTensors()
sliced_event_lists = [zip(el, cl) for el, cl in zip(sliced_event_lists,
sliced_chord_lists)]
# TODO(adarob): Consider handling the fact that different event lists can
# be mapped to identical tensors by the encoder_decoder (e.g., Drums).
unique_event_tuples = list(set(tuple(l) for l in sliced_event_lists))
unique_event_tuples = self._maybe_sample_outputs(unique_event_tuples)
if not unique_event_tuples:
return ConverterTensors()
control_seqs = []
if self._chord_encoding:
unique_event_tuples, unique_chord_tuples = zip(
*[zip(*t) for t in unique_event_tuples if t])
for t in unique_chord_tuples:
try:
chord_tokens = [self._chord_encoding.encode_event(e) for e in t]
if self.end_token:
# Repeat the last chord instead of using a special token; otherwise
# the model may learn to rely on the special token to detect
# endings.
chord_tokens.append(chord_tokens[-1] if chord_tokens else
self._chord_encoding.encode_event(mm.NO_CHORD))
except (mm.ChordSymbolException, mm.ChordEncodingException):
return ConverterTensors()
control_seqs.append(
np_onehot(chord_tokens, self.control_depth, self.control_dtype))
seqs = []
for t in unique_event_tuples:
seqs.append(np_onehot(
[self._legacy_encoder_decoder.encode_event(e) for e in t] +
([] if self.end_token is None else [self.end_token]),
self.output_depth, self.output_dtype))
return ConverterTensors(inputs=seqs, outputs=seqs, controls=control_seqs)
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
qpm = (input_sequence.tempos[0].qpm
if input_sequence and input_sequence.tempos
else mm.DEFAULT_QUARTERS_PER_MINUTE)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
# Use primer melody from input section only. Take backing chords from
# beginning of input section through end of generate section.
input_section = generator_options.input_sections[0]
primer_sequence = mm.extract_subsequence(
input_sequence, input_section.start_time, input_section.end_time)
backing_sequence = mm.extract_subsequence(
input_sequence, input_section.start_time, generate_section.end_time)
input_start_step = self.seconds_to_steps(input_section.start_time, qpm)
else:
# No input section. Take primer melody from the beginning of the sequence
# up until the start of the generate section.
primer_sequence = mm.extract_subsequence(
input_sequence, 0.0, generate_section.start_time)
backing_sequence = mm.extract_subsequence(
input_sequence, 0.0, generate_section.end_time)
input_start_step = 0
last_end_time = (max(n.end_time for n in primer_sequence.notes)
if primer_sequence.notes else 0)
if last_end_time >= generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before or equal to '
'the end of the input section. This model can only extend melodies. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming and backing sequences.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self._steps_per_quarter)
quantized_backing_sequence = mm.quantize_note_sequence(
backing_sequence, self._steps_per_quarter)
# Setting gap_bars to infinite ensures that the entire input will be used.
extracted_melodies, _ = mm.extract_melodies(
quantized_primer_sequence, search_start_step=input_start_step,
min_bars=0, min_unique_pitches=1, gap_bars=float('inf'),
ignore_polyphonic_notes=True)
assert len(extracted_melodies) <= 1
start_step = self.seconds_to_steps(
generate_section.start_time, qpm)
end_step = self.seconds_to_steps(generate_section.end_time, qpm)
if extracted_melodies and extracted_melodies[0]:
melody = extracted_melodies[0]
else:
# If no melody could be extracted, create an empty melody that starts 1
# step before the request start_step. This will result in 1 step of
# silence when the melody is extended below.
melody = mm.Melody([], start_step=max(0, start_step - 1))
extracted_chords, _ = mm.extract_chords(quantized_backing_sequence)
chords = extracted_chords[0]
# Make sure that chords and melody start on the same step.
if chords.start_step < melody.start_step:
chords.set_length(len(chords) - melody.start_step + chords.start_step)
assert chords.end_step == end_step
# Ensure that the melody extends up to the step we want to start generating.
melody.set_length(start_step - melody.start_step)
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
generated_melody = self._model.generate_melody(melody, chords, **args)
generated_lead_sheet = mm.LeadSheet(generated_melody, chords)
generated_sequence = generated_lead_sheet.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorException(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorException(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.extract_subsequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = self.seconds_to_steps(input_section.start_time,
qpm)
else:
primer_sequence = input_sequence
input_start_step = 0
last_end_time = (max(
n.end_time
for n in primer_sequence.notes) if primer_sequence.notes else 0)
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorException(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = polyphony_lib.extract_polyphonic_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
start_step = self.seconds_to_steps(generate_section.start_time, qpm)
end_step = self.seconds_to_steps(generate_section.end_time, qpm)
if extracted_seqs and extracted_seqs[0]:
poly_seq = extracted_seqs[0]
else:
# If no track could be extracted, create an empty track that starts 1 step
# before the request start_step. This will result in 1 step of silence
# when the track is extended below.
poly_seq = polyphony_lib.PolyphonicSequence(steps_per_quarter=(
quantized_primer_sequence.quantization_info.steps_per_quarter),
start_step=start_step)
# Ensure that the track extends up to the step we want to start generating.
poly_seq.set_length(start_step - poly_seq.start_step)
poly_seq.trim_trailing_end_and_step_end_events()
# Extract generation arguments from generator options.
arg_types = {
'temperature': lambda arg: arg.float_value,
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
'steps_per_iteration': lambda arg: arg.int_value
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
total_steps = end_step - start_step
while poly_seq.num_steps < total_steps:
# Assume it takes ~5 rnn steps to generate one quantized step.
# Can't know for sure until generation is finished because the number of
# notes per quantized step is variable.
steps_to_gen = total_steps - poly_seq.num_steps
rnn_steps_to_gen = 5 * steps_to_gen
tf.logging.info(
'Need to generate %d more steps for this sequence, will try asking '
'for %d RNN steps' % (steps_to_gen, rnn_steps_to_gen))
poly_seq = self._model.generate_polyphonic_sequence(
len(poly_seq) + rnn_steps_to_gen, poly_seq, **args)
poly_seq.set_length(total_steps)
generated_sequence = poly_seq.to_sequence(qpm=qpm)
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorError(
'This model supports at most one input_sections message, but got %s'
% len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorError(
'This model supports only 1 generate_sections message, but got %s'
% len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(input_sequence,
input_section.start_time,
input_section.end_time)
input_start_step = mm.quantize_to_step(input_section.start_time,
steps_per_second,
quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
if primer_sequence.notes:
last_end_time = max(n.end_time for n in primer_sequence.notes)
else:
last_end_time = 0
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorError(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = pianoroll_pipeline.extract_pianoroll_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
generate_start_step = mm.quantize_to_step(generate_section.start_time,
steps_per_second,
quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
generate_end_step = mm.quantize_to_step(generate_section.end_time,
steps_per_second,
quantize_cutoff=1.0)
if extracted_seqs and extracted_seqs[0]:
pianoroll_seq = extracted_seqs[0]
else:
raise ValueError('No priming pianoroll could be extracted.')
# Ensure that the track extends up to the step we want to start generating.
pianoroll_seq.set_length(generate_start_step -
pianoroll_seq.start_step)
# Extract generation arguments from generator options.
arg_types = {
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
total_steps = pianoroll_seq.num_steps + (generate_end_step -
generate_start_step)
pianoroll_seq = self._model.generate_pianoroll_sequence(
total_steps, pianoroll_seq, **args)
pianoroll_seq.set_length(total_steps)
generated_sequence = pianoroll_seq.to_sequence(qpm=qpm)
assert (generated_sequence.total_time -
generate_section.end_time) <= 1e-5
return generated_sequence
def _generate(self, input_sequence, generator_options):
if len(generator_options.input_sections) > 1:
raise mm.SequenceGeneratorError(
'This model supports at most one input_sections message, but got %s' %
len(generator_options.input_sections))
if len(generator_options.generate_sections) != 1:
raise mm.SequenceGeneratorError(
'This model supports only 1 generate_sections message, but got %s' %
len(generator_options.generate_sections))
# This sequence will be quantized later, so it is guaranteed to have only 1
# tempo.
qpm = mm.DEFAULT_QUARTERS_PER_MINUTE
if input_sequence.tempos:
qpm = input_sequence.tempos[0].qpm
steps_per_second = mm.steps_per_quarter_to_steps_per_second(
self.steps_per_quarter, qpm)
generate_section = generator_options.generate_sections[0]
if generator_options.input_sections:
input_section = generator_options.input_sections[0]
primer_sequence = mm.trim_note_sequence(
input_sequence, input_section.start_time, input_section.end_time)
input_start_step = mm.quantize_to_step(
input_section.start_time, steps_per_second, quantize_cutoff=0)
else:
primer_sequence = input_sequence
input_start_step = 0
if primer_sequence.notes:
last_end_time = max(n.end_time for n in primer_sequence.notes)
else:
last_end_time = 0
if last_end_time > generate_section.start_time:
raise mm.SequenceGeneratorError(
'Got GenerateSection request for section that is before or equal to '
'the end of the NoteSequence. This model can only extend sequences. '
'Requested start time: %s, Final note end time: %s' %
(generate_section.start_time, last_end_time))
# Quantize the priming sequence.
quantized_primer_sequence = mm.quantize_note_sequence(
primer_sequence, self.steps_per_quarter)
extracted_seqs, _ = mm.extract_pianoroll_sequences(
quantized_primer_sequence, start_step=input_start_step)
assert len(extracted_seqs) <= 1
generate_start_step = mm.quantize_to_step(
generate_section.start_time, steps_per_second, quantize_cutoff=0)
# Note that when quantizing end_step, we set quantize_cutoff to 1.0 so it
# always rounds down. This avoids generating a sequence that ends at 5.0
# seconds when the requested end time is 4.99.
generate_end_step = mm.quantize_to_step(
generate_section.end_time, steps_per_second, quantize_cutoff=1.0)
if extracted_seqs and extracted_seqs[0]:
pianoroll_seq = extracted_seqs[0]
else:
raise ValueError('No priming pianoroll could be extracted.')
# Ensure that the track extends up to the step we want to start generating.
pianoroll_seq.set_length(generate_start_step - pianoroll_seq.start_step)
# Extract generation arguments from generator options.
arg_types = {
'beam_size': lambda arg: arg.int_value,
'branch_factor': lambda arg: arg.int_value,
}
args = dict((name, value_fn(generator_options.args[name]))
for name, value_fn in arg_types.items()
if name in generator_options.args)
total_steps = pianoroll_seq.num_steps + (
generate_end_step - generate_start_step)
pianoroll_seq = self._model.generate_pianoroll_sequence(
total_steps, pianoroll_seq, **args)
pianoroll_seq.set_length(total_steps)
generated_sequence = pianoroll_seq.to_sequence(qpm=qpm)
assert (generated_sequence.total_time - generate_section.end_time) <= 1e-5
return generated_sequence
