def _plot():
j = 0
for edits_batch, pitch_batch in zip(edits_batches, pitch_batches):
waves = model.generate_samples_from_edits(pitch_batch, edits_batch,
pca)
for i, edits in enumerate(edits_batch):
if j >= len(edits_list):
return
row = j // FLAGS.steps
col = j % FLAGS.steps
wave = waves[i]
# wave = np.random.rand(64000)
x = edits[0]
y = edits[1]
# plot wave
subplot = ax[col][row]
subplot.title.set_text("({}, {})".format(
format_float(x), format_float(y)))
plotstft(subplot, wave, 16000, binsize=2**8, colormap="magma")
subplot.set_axis_off()
# save wave
gu.save_wav(
wave,
os.path.join(output_dir, "wave_{},{}.wav".format(x, y)))
j += 1
def main(unused_argv):
absl.flags.FLAGS.alsologtostderr = True
# Load the model
flags = lib_flags.Flags({'batch_size_schedule': [FLAGS.batch_size]})
model = lib_model.Model.load_from_path(FLAGS.ckpt_dir, flags)
# Make an output directory if it doesn't exist
output_dir = util.expand_path(FLAGS.output_dir)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
# generate 4 random latent vectors
z_instruments = model.generate_z(4)
instrument_names = list(
gen_instrument_name(random.randint(3, 8)) for _ in range(4))
# interpolate
res = FLAGS.resolution
pitches = parse_pitches(FLAGS.pitches)
xy_grid = make_grid(res)
print()
print("resolution =", res)
print("pitches =", pitches)
print("z_instruments.shape =", z_instruments.shape)
print("z_instruments =", z_instruments)
print("instrument_names =", instrument_names)
z_notes, note_metas = get_z_notes(z_instruments, instrument_names, xy_grid)
print("z_notes.shape =", z_notes.shape)
z_notes_rep = np.repeat(z_notes, len(pitches), axis=0)
print("z_notes_rep.shape =", z_notes_rep.shape)
pitches_rep = pitches * z_notes.shape[0]
print("len(pitches_rep) =", len(pitches_rep))
print("generating {} samples,,".format(len(z_notes_rep)))
audio_notes = model.generate_samples_from_z(z_notes_rep, pitches_rep)
audio_metas = []
for note_meta in note_metas:
for pitch in pitches:
meta = dict(note_meta)
meta["pitch"] = pitch
audio_metas.append(meta)
print("audio_notes.shape =", audio_notes.shape)
print("len(audio_metas) =", len(audio_metas))
for i, (wave, meta) in enumerate(zip(audio_notes, audio_metas)):
name = meta_to_name(meta)
fn = os.path.join(output_dir, "gen_{}.wav".format(name))
gu.save_wav(wave, fn)
def main(unused_argv):
absl.flags.FLAGS.alsologtostderr = True
# Load the model
flags = lib_flags.Flags(
{
'batch_size_schedule': [FLAGS.batch_size],
'tfds_data_dir': FLAGS.tfds_data_dir
})
model = lib_model.Model.load_from_path(FLAGS.ckpt_dir, flags)
# Make an output directory if it doesn't exist
output_dir = util.expand_path(FLAGS.output_dir)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
if FLAGS.midi_file:
# If a MIDI file is provided, synthesize interpolations across the clip
unused_ns, notes = gu.load_midi(FLAGS.midi_file)
# Distribute latent vectors linearly in time
z_instruments, t_instruments = gu.get_random_instruments(
model,
notes['end_times'][-1],
secs_per_instrument=FLAGS.secs_per_instrument)
# Get latent vectors for each note
z_notes = gu.get_z_notes(notes['start_times'], z_instruments, t_instruments)
# Generate audio for each note
print('Generating {} samples...'.format(len(z_notes)))
audio_notes = model.generate_samples_from_z(z_notes, notes['pitches'])
# Make a single audio clip
audio_clip = gu.combine_notes(audio_notes,
notes['start_times'],
notes['end_times'],
notes['velocities'])
# Write the wave files
fname = os.path.join(output_dir, 'generated_clip.wav')
gu.save_wav(audio_clip, fname)
else:
# Otherwise, just generate a batch of random sounds
waves = model.generate_samples(FLAGS.batch_size)
# Write the wave files
for i in range(len(waves)):
fname = os.path.join(output_dir, 'generated_{}.wav'.format(i))
gu.save_wav(waves[i], fname)
def save_audio(audio_clip: np.ndarray) -> None:
"""
Writes the audio clip to disk as a spectogram plot (constant Q transform)
and wav file. See audio_utils.save_spectogram_plot.
:param audio_clip: the audio clip to save
"""
# Saves the CQT Spectrogram on disk
save_spectrogram_plot(audio_clip,
output_dir=os.path.join("output", "gansynth"))
# Saves the wav file on disk
date_and_time = time.strftime("%Y-%m-%d_%H%M%S")
filename = f"{date_and_time}.wav"
path = os.path.join(os.path.join("output", "gansynth"), filename)
save_wav(audio_clip, path)
def main(unused_argv):
absl.flags.FLAGS.alsologtostderr = True
# Load the model
flags = lib_flags.Flags({'batch_size_schedule': [FLAGS.batch_size]})
model = lib_model.Model.load_from_path(FLAGS.ckpt_dir, flags)
# Make an output directory if it doesn't exist
output_dir = util.expand_path(FLAGS.output_dir)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
if FLAGS.midi_file:
# If a MIDI file is provided, synthesize interpolations across the clip
unused_ns, notes = gu.load_midi(FLAGS.midi_file)
# Distribute latent vectors linearly in time
z_instruments, t_instruments = gu.get_random_instruments(
model,
notes['end_times'][-1],
secs_per_instrument=FLAGS.secs_per_instrument)
# Get latent vectors for each note
z_notes = gu.get_z_notes(notes['start_times'], z_instruments, t_instruments)
# Generate audio for each note
print('Generating {} samples...'.format(len(z_notes)))
audio_notes = model.generate_samples_from_z(z_notes, notes['pitches'])
# Make a single audio clip
audio_clip = gu.combine_notes(audio_notes,
notes['start_times'],
notes['end_times'],
notes['velocities'])
# Write the wave files
fname = os.path.join(output_dir, 'generated_clip.wav')
gu.save_wav(audio_clip, fname)
else:
# Otherwise, just generate a batch of random sounds
waves = model.generate_samples(FLAGS.batch_size)
# Write the wave files
for i in range(len(waves)):
fname = os.path.join(output_dir, 'generated_{}.wav'.format(i))
gu.save_wav(waves[i], fname)
def handle_hallucinate(model, stdin, stdout):
max_note_length = model.config['audio_length']
sample_rate = model.config['sample_rate']
hallucinate_msg = read_msg(stdin, gss.hallucinate_struct.size)
args = gss.from_hallucinate_msg(hallucinate_msg)
note_count, interpolation_steps, spacing, start_trim, attack, sustain, release = args
print(
"note_count = {} interpolation_steps = {}, spacing = {}s, start_trim = {}s, attack = {}s, sustain = {}s, release = {}s"
.format(*args),
file=sys.stderr)
initial_notes = model.generate_z(note_count)
initial_piches = np.array(
[32] * len(initial_notes)
) # np.floor(30 + np.random.rand(len(initial_notes)) * 30)
final_notes, final_pitches = interpolate_notes(initial_notes,
initial_piches,
interpolation_steps)
audios = synthesize(model, final_notes, final_pitches)
final_audio = combine_notes(audios,
spacing=spacing,
start_trim=start_trim,
attack=attack,
sustain=sustain,
release=release,
max_note_length=max_note_length,
sr=sample_rate)
# TODO: Remove!
try:
gu.save_wav(final_audio, "/Users/oskar.koli/Desktop/hallucination.wav")
except Exception:
pass
final_audio = final_audio.astype('float32')
stdout.write(gss.to_tag_msg(gss.OUT_TAG_AUDIO))
stdout.write(gss.to_audio_size_msg(final_audio.size *
final_audio.itemsize))
stdout.write(gss.to_audio_msg(final_audio))
notes['start_times'],
notes['end_times'],
notes['velocities'])
# Play the audio
print('\nAudio:')
play(audio_clip, sample_rate=SR)
print('CQT Spectrogram:')
specplot(audio_clip)
#@title Download
#@markdown Get the .wav file (optional)
# Write the file
fname = os.path.join(output_dir, 'generated_clip.wav')
gu.save_wav(audio_clip, fname)
download(fname)
"""## 2(b): You Choose the Interpolation
These cells allow you to choose two latent vectors and interpolate between them over a MIDI clip.
"""
#@title Choose a MIDI file
#@markdown Upload a MIDI file _(.mid, single instrument)_ for audio synthesis, or use the provided default.
midi_file = "Arpeggio (Default)" #@param ["Arpeggio (Default)", "Upload your own"]
midi_path = MIDI_RIFF_DEFAULT
if midi_file == "Upload your own":
def main(unused_argv):
absl.flags.FLAGS.alsologtostderr = True
# Load the model
flags = lib_flags.Flags({
'batch_size_schedule': [FLAGS.batch_size],
**({
'tfds_data_dir': FLAGS.tfds_data_dir
} if FLAGS.tfds_data_dir else {})
})
model = lib_model.Model.load_from_path(FLAGS.ckpt_dir, flags)
# Make an output directory if it doesn't exist
output_dir = util.expand_path(FLAGS.output_dir)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
if FLAGS.seed != None:
np.random.seed(seed=FLAGS.seed)
tf.random.set_random_seed(FLAGS.seed)
layer_offsets = {}
if FLAGS.edits_file:
with open(FLAGS.edits_file, "rb") as fp:
edits_dict = pickle.load(fp)
assert "layer" in edits_dict
assert "comp" in edits_dict
directions = edits_dict["comp"]
amounts = np.zeros(edits_dict["comp"].shape[:1], dtype=np.float32)
amounts[:len(list(map(float, FLAGS.edits)))] = FLAGS.edits
scaled_directions = amounts.reshape(-1, 1, 1, 1) * directions
linear_combination = np.sum(scaled_directions, axis=0)
linear_combination_batch = np.repeat(linear_combination.reshape(
1, *linear_combination.shape),
FLAGS.batch_size,
axis=0)
layer_offsets[edits_dict["layer"]] = linear_combination_batch
if FLAGS.midi_file:
# If a MIDI file is provided, synthesize interpolations across the clip
unused_ns, notes = gu.load_midi(FLAGS.midi_file)
# Distribute latent vectors linearly in time
z_instruments, t_instruments = gu.get_random_instruments(
model,
notes['end_times'][-1],
secs_per_instrument=FLAGS.secs_per_instrument)
# Get latent vectors for each note
z_notes = gu.get_z_notes(notes['start_times'], z_instruments,
t_instruments)
# Generate audio for each note
print('Generating {} samples...'.format(len(z_notes)))
audio_notes = model.generate_samples_from_z(
z_notes, notes['pitches'], layer_offsets=layer_offsets)
# Make a single audio clip
audio_clip = gu.combine_notes(audio_notes, notes['start_times'],
notes['end_times'], notes['velocities'])
# Write the wave files
fname = os.path.join(output_dir, 'generated_clip.wav')
gu.save_wav(audio_clip, fname)
else:
# Otherwise, just generate a batch of random sounds
waves = model.generate_samples(FLAGS.batch_size,
pitch=FLAGS.pitch,
layer_offsets=layer_offsets)
# Write the wave files
for i in range(len(waves)):
fname = os.path.join(output_dir, 'generated_{}.wav'.format(i))
gu.save_wav(waves[i], fname)
def main(unused_argv):
absl.flags.FLAGS.alsologtostderr = True
# Load the model
flags = lib_flags.Flags({'batch_size_schedule': [FLAGS.batch_size]})
model = lib_model.Model.load_from_path(FLAGS.ckpt_dir, flags)
# Make an output directory if it doesn't exist
output_dir = util.expand_path(FLAGS.output_dir)
if not tf.gfile.Exists(output_dir):
tf.gfile.MakeDirs(output_dir)
if FLAGS.midi_file:
# If a MIDI file is provided, synthesize interpolations across the clip
unused_ns, notes = gu.load_midi(FLAGS.midi_file)
# Distribute latent vectors linearly in time
z_instruments, t_instruments = gu.get_random_instruments(
model,
notes['end_times'][-1],
secs_per_instrument=FLAGS.secs_per_instrument)
# Get latent vectors for each note
z_notes = gu.get_z_notes(notes['start_times'], z_instruments,
t_instruments)
# Generate audio for each note
print('Generating {} samples...'.format(len(z_notes)))
audio_notes = model.generate_samples_from_z(z_notes, notes['pitches'])
# Make a single audio clip
audio_clip = gu.combine_notes(audio_notes, notes['start_times'],
notes['end_times'], notes['velocities'])
# Write the wave files
fname = os.path.join(output_dir, 'generated_clip.wav')
gu.save_wav(audio_clip, fname)
else:
# Otherwise, just generate a batch of random sounds
# waves = model.generate_samples(FLAGS.batch_size) # original
waves, z = model.generate_samples(
FLAGS.batch_size, pitch=44
) #DEBUG: generate on singular pitch (range: 24-84), return latent vectors
# Write the wave files
for i in range(len(waves)):
fname = os.path.join(output_dir, 'generated_{}.wav'.format(i))
gu.save_wav(waves[i], fname)
# DEBUG: write z to file for later analysis
fname = os.path.join(output_dir, 'z.p')
pickle.dump(z, open(fname, 'wb'))
# DEBUG: flag samples based on similar latent variables
flagged = get_flagged_latents(z, n=10)
print("\nflagged (z):")
for i in flagged:
print(i)
# DEBUG: flag samples based on similar waveforms
flagged = get_flagged_waves_par(waves, n=10, frac=0.01)
print("\nflagged (waves):")
for i in flagged:
print(i)
fname = os.path.join(output_dir,
'_sim_{}-{}.wav'.format(i[0][0], i[0][1]))
gu.save_wav(np.array(list(waves[i[0][0]]) + list(waves[i[0][1]])),
fname)