def visualize(args, test_loader, encoder, decoder, epoch, n_classes, curr_task_labels, device):
plotter = plot_utils.plot_samples(args.results_path, args.n_img_x, args.n_img_y, args.img_size, args.img_size)
# plot samples of the reconstructed images from the first batch of the test set of the current task
for test_batch_idx, (test_data, test_target) in enumerate(test_loader):
test_data, test_target = test_data.to(device), test_target.to(device)
x = test_data[0:plotter.n_total_imgs, :]
x_id = test_target[0:plotter.n_total_imgs]
x_id_onehot = get_categorical(x_id,n_classes).to(device)
encoder.eval()
decoder.eval()
with torch.no_grad():
z,_,_ = encoder(x)
reconstructed_x = decoder(torch.cat([z, x_id_onehot], dim=1))
reconstructed_x = reconstructed_x.reshape(plotter.n_total_imgs, args.img_size, args.img_size)
plotter.save_images(x.cpu().data, name="/x_epoch_%02d" %(epoch) + ".jpg")
plotter.save_images(reconstructed_x.cpu().data, name="/reconstructed_x_epoch_%02d" %(epoch) + ".jpg")
break
#plot pseudo random samples from the previous learned tasks
z = Variable(Tensor(np.random.normal(0, 1, (plotter.n_total_imgs, args.latent_dim))))
z_id = np.random.randint(0, curr_task_labels[-1]+1, size=[plotter.n_total_imgs])
z_id_one_hot = get_categorical(z_id, n_classes).to(device)
decoder.eval()
with torch.no_grad():
pseudo_samples = decoder(torch.cat([z,Variable(Tensor(z_id_one_hot))],1))
pseudo_samples = pseudo_samples.reshape(plotter.n_total_imgs, args.img_size, args.img_size)
plotter.save_images(pseudo_samples.cpu().data, name="/pseudo_sample_epoch_%02d" % (epoch) + ".jpg")
def visualize(args, test_loader, encoder, decoder, epoch, n_classes,
curr_task_labels, device, task_id):
plotter = plot_utils.plot_samples(args.results_path, args.n_img_x,
args.n_img_y, args.img_size,
args.img_size, args.channels)
# plot samples of the reconstructed images from the first batch of the test set of the current task
tsne = TSNE()
for test_batch_idx, (test_data, test_target) in enumerate(test_loader):
test_data, test_target = test_data.to(device), test_target.to(device)
x = test_data[0:plotter.n_total_imgs, :]
x_id = test_target[0:plotter.n_total_imgs]
encoder.eval()
decoder.eval()
with torch.no_grad():
z, _, _ = encoder(x)
x_id_onehot = get_categorical(x_id, n_classes).to(device)
reconstructed_x = decoder(torch.cat([z, x_id_onehot], dim=1))
reconstructed_x = reconstructed_x.permute(
0, 2, 3, 1
) #reshape(plotter.n_total_imgs, args.img_size, args.img_size,args.channels)
plotter.save_images(x.permute(0, 2, 3, 1).cpu().data,
name="/%02d_x_epoch_%02d" % (task_id, epoch) +
".jpg")
plotter.save_images(reconstructed_x.cpu().data,
name="/%02d_reconstructed_x_epoch_%02d" %
(task_id, epoch) + ".jpg")
z_new = tsne.fit_transform(z.cpu().numpy())
plotter.scatter(z_new[:, 0],
z_new[:, 1],
x_id.cpu().numpy().astype(int),
name="/%02d_scatter_epoch_%02d" %
(task_id, epoch) + ".jpg")
break
#plot pseudo random samples from the previous learned tasks
z = Variable(
Tensor(np.random.normal(0, 1,
(plotter.n_total_imgs, args.latent_dim))))
z_id = np.random.randint(0,
curr_task_labels[-1] + 1,
size=[plotter.n_total_imgs])
z_id_one_hot = get_categorical(z_id, n_classes).to(device)
decoder.eval()
with torch.no_grad():
pseudo_samples = decoder(
torch.cat([z, Variable(Tensor(z_id_one_hot))], 1))
z_, _, _ = encoder(pseudo_samples)
pseudo_samples = pseudo_samples.permute(
0, 2, 3, 1
) #.reshape(plotter.n_total_imgs, args.img_size, args.img_size, args.channels)
plotter.save_images(pseudo_samples.cpu().data,
name="/%02d_pseudo_sample_epoch_%02d" %
(task_id, epoch) + ".jpg")
z_new = tsne.fit_transform(z_.cpu().numpy())
plotter.scatter(z_new[:, 0],
z_new[:, 1],
z_id.astype(int),
name="/%02d_scatter_pseudo_epoch_%02d" %
(task_id, epoch) + ".jpg")
def train(samples_path='data/interim/samples.npy',
lengths_path='data/interim/lengths.npy',
epochs_qty=EPOCHS_QTY,
learning_rate=LEARNING_RATE):
"""
Train model.
:return:
"""
# Create folders to save models into
if not os.path.exists('results'):
os.makedirs('results')
if WRITE_HISTORY and not os.path.exists('results/history'):
os.makedirs('results/history')
# Load dataset into memory
print("Loading Data...")
if not os.path.exists(samples_path) or not os.path.exists(lengths_path):
print('No input data found, run preprocess_songs.py first.')
exit(1)
y_samples = np.load(samples_path)
y_lengths = np.load(lengths_path)
samples_qty = y_samples.shape[0]
songs_qty = y_lengths.shape[0]
print("Loaded " + str(samples_qty) + " samples from " + str(songs_qty) +
" songs.")
print(np.sum(y_lengths))
assert (np.sum(y_lengths) == samples_qty)
print("Preparing song samples, padding songs...")
x_shape = (songs_qty * NUM_OFFSETS, 1) # for embedding
x_orig = np.expand_dims(np.arange(x_shape[0]), axis=-1)
y_shape = (songs_qty * NUM_OFFSETS, MAX_WINDOWS) + y_samples.shape[
1:] # (songs_qty, max number of windows, window pitch qty, window beats per measure)
y_orig = np.zeros(y_shape, dtype=np.float32) # prepare dataset array
# fill in measure of songs into input windows for network
song_start_ix = 0
song_end_ix = y_lengths[0]
for song_ix in range(songs_qty):
for offset in range(NUM_OFFSETS):
ix = song_ix * NUM_OFFSETS + offset # calculate the index of the song with its offset
song_end_ix = song_start_ix + y_lengths[song_ix] # get song end ix
for window_ix in range(
MAX_WINDOWS
): # get a maximum number of measures from a song
song_measure_ix = (window_ix + offset) % y_lengths[
song_ix] # chosen measure of song to be placed in window (modulo song length)
y_orig[ix, window_ix] = y_samples[
song_start_ix +
song_measure_ix] # move measure into window
song_start_ix = song_end_ix # new song start index is previous song end index
assert (song_end_ix == samples_qty)
x_train = np.copy(x_orig)
y_train = np.copy(y_orig)
# copy some song from the samples and write it to midi again
test_ix = 0
y_test_song = np.copy(y_train[test_ix:test_ix + 1])
x_test_song = np.copy(x_train[test_ix:test_ix + 1])
midi_utils.samples_to_midi(y_test_song[0], 'data/interim/gt.mid')
# create model
if CONTINUE_TRAIN or GENERATE_ONLY:
print("Loading model...")
model = load_model('results/history/model.h5')
else:
print("Building model...")
model = models.create_autoencoder_model(
input_shape=y_shape[1:],
latent_space_size=LATENT_SPACE_SIZE,
dropout_rate=DROPOUT_RATE,
max_windows=MAX_WINDOWS,
batchnorm_momentum=BATCHNORM_MOMENTUM,
use_vae=USE_VAE,
vae_b1=VAE_B1,
use_embedding=USE_EMBEDDING,
embedding_input_shape=x_shape[1:],
embedding_shape=x_train.shape[0])
if USE_VAE:
model.compile(optimizer=Adam(lr=learning_rate), loss=vae_loss)
#elif params.encode_volume:
#model.compile(optimizer=RMSprop(lr=learning_rate), loss='mean_squared_logarithmic_error')
else:
model.compile(optimizer=RMSprop(lr=learning_rate),
loss='binary_crossentropy')
#model.compile(optimizer=RMSprop(lr=learning_rate), loss='mean_squared_error')
# plot model with graphvis if installed
#try:
# plot_model(model, to_file='results/model.png', show_shapes=True)
#except OSError as e:
# print(e)
# train
print("Referencing sub-models...")
decoder = K.function(
[model.get_layer('decoder').input,
K.learning_phase()], [model.layers[-1].output])
encoder = Model(inputs=model.input,
outputs=model.get_layer('encoder').output)
random_vectors = np.random.normal(0.0, 1.0,
(NUM_RAND_SONGS, LATENT_SPACE_SIZE))
np.save('data/interim/random_vectors.npy', random_vectors)
if GENERATE_ONLY:
print("Generating songs...")
generate_normalized_random_songs(x_orig, y_orig, encoder, decoder,
random_vectors, 'results/')
for save_epoch in range(20):
x_test_song = x_train[save_epoch:save_epoch + 1]
y_song = model.predict(x_test_song, batch_size=BATCH_SIZE)[0]
midi_utils.samples_to_midi(y_song,
'results/gt' + str(save_epoch) + '.mid')
exit(0)
save_training_config(songs_qty, model, learning_rate)
print("Training model...")
train_loss = []
offset = 0
for epoch in range(epochs_qty):
print("Training epoch: ", epoch, "of", epochs_qty)
if USE_EMBEDDING:
history = model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=1)
else:
# produce songs from its samples with a different starting point of the song each time
song_start_ix = 0
for song_ix in range(songs_qty):
song_end_ix = song_start_ix + y_lengths[song_ix]
for window_ix in range(MAX_WINDOWS):
song_measure_ix = (window_ix + offset) % y_lengths[song_ix]
y_train[song_ix, window_ix] = y_samples[song_start_ix +
song_measure_ix]
#if params.encode_volume:
#y_train[song_ix, window_ix] /= 100.0
song_start_ix = song_end_ix
assert (song_end_ix == samples_qty)
offset += 1
history = model.fit(y_train,
y_train,
batch_size=BATCH_SIZE,
epochs=1) # train model on reconstruction loss
# store last loss
loss = history.history["loss"][-1]
train_loss.append(loss)
print("Train loss: " + str(train_loss[-1]))
if WRITE_HISTORY:
plot_losses(train_loss, 'results/history/losses.png', True)
else:
plot_losses(train_loss, 'results/losses.png', True)
# save model periodically
save_epoch = epoch + 1
if save_epoch in EPOCHS_TO_SAVE or (save_epoch % 100
== 0) or save_epoch == epochs_qty:
write_dir = ''
if WRITE_HISTORY:
# Create folder to save models into
write_dir += 'results/history/e' + str(save_epoch)
if not os.path.exists(write_dir):
os.makedirs(write_dir)
write_dir += '/'
model.save('results/history/model.h5')
else:
model.save('results/model.h5')
print("...Saved.")
if USE_EMBEDDING:
y_song = model.predict(x_test_song, batch_size=BATCH_SIZE)[0]
else:
y_song = model.predict(y_test_song, batch_size=BATCH_SIZE)[0]
plot_utils.plot_samples(write_dir + 'test', y_song)
midi_utils.samples_to_midi(y_song, write_dir + 'test.mid')
generate_normalized_random_songs(x_orig, y_orig, encoder, decoder,
random_vectors, write_dir)
print("...Done.")