def do_experiment(model_config): tf.reset_default_graph() experiment_id = ex.current_run._id print('Experiment ID: {eid}'.format(eid=experiment_id)) # Prepare data print('Preparing dataset') train_data, val_data, test_data = Dataset.prepare_datasets(model_config) # Start session tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True sess = tf.Session(config=tf_config) # Create iterators handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle(handle, train_data.output_types, train_data.output_shapes) mixed_spec, voice_spec, mixed_audio, voice_audio = iterator.get_next() training_iterator = train_data.make_initializable_iterator() validation_iterator = val_data.make_initializable_iterator() testing_iterator = test_data.make_initializable_iterator() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) testing_handle = sess.run(testing_iterator.string_handle()) # Create variable placeholders is_training = tf.placeholder(shape=(), dtype=bool) mixed_mag = tf.expand_dims(mixed_spec[:, :, 1:, 0], 3) mixed_phase = tf.expand_dims(mixed_spec[:, :, 1:, 1], 3) voice_mag = tf.expand_dims(voice_spec[:, :, 1:, 0], 3) # Build U-Net model print('Creating model') model = UNet.UNetModel(mixed_mag, voice_mag, mixed_phase, mixed_audio, voice_audio, 'unet', is_training, name='U_Net_Model') if model_config['loading']: # TODO - Think this works now but needs proper testing print('Loading checkpoint') checkpoint = os.path.join(model_config['model_base_dir'], model_config['checkpoint_to_load']) restorer = tf.train.Saver() restorer.restore(sess, checkpoint) # Summaries model_folder = str(experiment_id) writer = tf.summary.FileWriter(os.path.join(model_config["log_dir"], model_folder), graph=sess.graph) # Get baseline metrics at initialisation sess.run(tf.global_variables_initializer()) test_count = 0 if model_config['INITIALISATION_TEST']: print('Running initialisation test') initial_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count) # Train the model model = train(sess, model, model_config, model_folder, handle, training_iterator, training_handle, validation_iterator, validation_handle, writer) # Test trained model mean_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count) print('{ts}:\n\tAll done!'.format(ts=datetime.datetime.now())) if model_config['INITIALISATION_TEST']: print('\tInitial test loss: {init}'.format(init=initial_test_loss)) print('\tFinal test loss: {final}'.format(final=mean_test_loss))
def do_experiment(model_config): tf.reset_default_graph() experiment_id = ex.current_run._id print('Experiment ID: {eid}'.format(eid=experiment_id)) # Prepare data print('Preparing dataset') train_data, val_data, test_data = Dataset.prepare_datasets(model_config) # Start session sess = tf.Session() # Create iterators handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle(handle, train_data.output_types, train_data.output_shapes) mixed, voice = iterator.get_next() training_iterator = train_data.make_initializable_iterator() validation_iterator = val_data.make_initializable_iterator() testing_iterator = test_data.make_initializable_iterator() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) testing_handle = sess.run(testing_iterator.string_handle()) # Create variable placeholders is_training = tf.placeholder(shape=(), dtype=bool) mixed_mag = tf.expand_dims(mixed[0][:, :, 1:, 0], 3) # Yet more hacking to get around this tuple problem mixed_phase = tf.expand_dims(mixed[0][:, :, 1:, 1], 3) voice_mag = tf.expand_dims(voice[0][:, :, 1:, 0], 3) # Build U-Net model print('Creating model') model = UNet.UNetModel(mixed_mag, voice_mag, mixed_phase, is_training, name='U_Net_Model') if model_config['loading']: # TODO - Think this works now but needs proper testing print('Loading checkpoint') checkpoint = os.path.join(model_config['model_base_dir'], model_config['checkpoint_to_load']) restorer = tf.train.Saver() restorer.restore(sess, checkpoint) # Summaries model_folder = str(experiment_id) writer = tf.summary.FileWriter(os.path.join(model_config["log_dir"], model_folder), graph=sess.graph) # Get baseline metrics at initialisation sess.run(tf.global_variables_initializer()) print('Running initialisation test') test_count = 1 initial_test_loss = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count) test_count += 1 # Train the model model = train(sess, model, model_config, model_folder, handle, training_iterator, training_handle, validation_iterator, validation_handle, writer) # Test trained model mean_test_loss = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count) print('All done!\nInitial test loss: {init}\nFinal test loss: {final}' .format(init=initial_test_loss, final=mean_test_loss))