def do_experiment(model_config): if not model_config['training']: checkpoint_to_load = model_config['checkpoint_to_load'] experiment_to_load = checkpoint_to_load.split('/')[0] print(experiment_to_load) config_file_loc = "my_runs/{experiment_to_load}/config.json".format( experiment_to_load=experiment_to_load) with open(config_file_loc) as config_file: model_config = json.load(config_file)['model_config'] print('Config file for experiment {experiment_to_load} loaded'. format(experiment_to_load=experiment_to_load)) model_config['checkpoint_to_load'] = checkpoint_to_load model_config['training'] = False model_config['loading'] = True model_config['completion_test'] = True model_config['model_base_dir'] = 'checkpoints' tf.reset_default_graph() experiment_id = ex.current_run._id print('Experiment ID: {eid}'.format(eid=experiment_id)) # Prepare data print('Preparing dataset') train_data, val_data, test_data = dataset.prepare_datasets(model_config) print('Dataset ready') # Start session tf_config = tf.ConfigProto() #tf_config.gpu_options.allow_growth = True tf_config.gpu_options.visible_device_list = str(model_config['GPU']) sess = tf.Session(config=tf_config) #sess = tf.Session() #sess = tf_debug.LocalCLIDebugWrapperSession(sess, ui_type="readline") print('Session started') # Create data iterators handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle(handle, train_data.output_types, train_data.output_shapes) mixed_spec, voice_spec, background_spec, mixed_audio, voice_audio, background_audio = iterator.get_next( ) training_iterator = train_data.make_initializable_iterator() validation_iterator = val_data.make_initializable_iterator() testing_iterator = test_data.make_initializable_iterator() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) testing_handle = sess.run(testing_iterator.string_handle()) print('Iterators created') # Create variable placeholders and model is_training = tf.placeholder(shape=(), dtype=bool) mixed_phase = tf.expand_dims(mixed_spec[:, :, :-1, 3], 3) print('Creating model') # Restructure data from pipeline based on data type required if model_config['data_type'] == 'mag': mixed_input = tf.expand_dims(mixed_spec[:, :, :-1, 2], 3) voice_input = tf.expand_dims(voice_spec[:, :, :-1, 2], 3) elif model_config['data_type'] in ['mag_phase', 'mag_phase_diff']: mixed_input = mixed_spec[:, :, :-1, 2:4] voice_input = voice_spec[:, :, :-1, 2:4] elif model_config['data_type'] == 'real_imag': mixed_input = mixed_spec[:, :, :-1, 0:2] voice_input = voice_spec[:, :, :-1, 0:2] elif model_config['data_type'] in ['mag_real_imag', 'mag_phase2']: mixed_input = tf.concat([ tf.expand_dims(mixed_spec[:, :, :-1, 2], 3), mixed_spec[:, :, :-1, 0:2] ], 3) voice_input = tf.concat([ tf.expand_dims(voice_spec[:, :, :-1, 2], 3), voice_spec[:, :, :-1, 0:2] ], 3) elif model_config['data_type'] in [ 'mag_phase_real_imag', 'complex_to_mag_phase' ]: mixed_input = mixed_spec[:, :, :-1, :] voice_input = voice_spec[:, :, :-1, :] model = audio_models.MagnitudeModel( mixed_input, voice_input, mixed_phase, mixed_audio, voice_audio, background_audio, is_training, model_config['learning_rate'], model_config['data_type'], model_config['phase_weight'], model_config['phase_loss_masking'], model_config['phase_loss_approximation'], 'Magnitude_Model', model_config['loss_function']) sess.run(tf.global_variables_initializer()) if model_config['loading']: print('Loading checkpoint') checkpoint = os.path.join(model_config['model_base_dir'], model_config['checkpoint_to_load']) restorer = tf.train.Saver() restorer.restore(sess, checkpoint) # Summaries model_folder = str(experiment_id) writer = tf.summary.FileWriter(os.path.join(model_config["log_dir"], model_folder), graph=sess.graph) # Get baseline metrics at initialisation test_count = 0 if model_config['initialisation_test']: print('Running initialisation test') initial_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, test_count, experiment_id) if model_config['training']: # Train the model model = train(sess, model, model_config, model_folder, handle, training_iterator, training_handle, validation_iterator, validation_handle, writer) if model_config['completion_test']: # Test the trained model mean_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, test_count, experiment_id) if model_config['initialisation_test']: print('\tInitial test loss: {init}'.format(init=initial_test_loss)) print('\tFinal test loss: {final}'.format(final=mean_test_loss)) print('{ts}:\n\tAll done with experiment {exid}!'.format( ts=datetime.datetime.now(), exid=experiment_id))
def do_experiment(model_config): tf.reset_default_graph() experiment_id = ex.current_run._id print('Experiment ID: {eid}'.format(eid=experiment_id)) # Prepare data print('Preparing dataset') train_data, val_data, test_data = dataset.prepare_datasets(model_config) print('Dataset ready') # Start session tf_config = tf.ConfigProto() #tf_config.gpu_options.allow_growth = True tf_config.gpu_options.visible_device_list = "1" sess = tf.Session(config=tf_config) #sess = tf.Session() print('Session started') # Create iterators handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle(handle, train_data.output_types, train_data.output_shapes) mixed_spec, voice_spec, mixed_audio, voice_audio = iterator.get_next() training_iterator = train_data.make_initializable_iterator() validation_iterator = val_data.make_initializable_iterator() testing_iterator = test_data.make_initializable_iterator() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) testing_handle = sess.run(testing_iterator.string_handle()) print('Iterators created') # Create variable placeholders is_training = tf.placeholder(shape=(), dtype=bool) mixed_mag = tf.expand_dims(mixed_spec[:, :, :-1, 0], 3) mixed_phase = tf.expand_dims(mixed_spec[:, :, :-1, 1], 3) voice_mag = tf.expand_dims(voice_spec[:, :, :-1, 0], 3) # Build U-Net model print('Creating model') model = audio_models.AudioModel(mixed_mag, voice_mag, mixed_phase, mixed_audio, voice_audio, model_config['model_variant'], is_training, name='U_Net_Model') sess.run(tf.global_variables_initializer()) if model_config['loading']: # TODO - Think this works now but needs proper testing print('Loading checkpoint') checkpoint = os.path.join(model_config['model_base_dir'], model_config['checkpoint_to_load']) restorer = tf.train.Saver() restorer.restore(sess, checkpoint) # Summaries model_folder = str(experiment_id) writer = tf.summary.FileWriter(os.path.join(model_config["log_dir"], model_folder), graph=sess.graph) # Get baseline metrics at initialisation test_count = 0 if model_config['initialisation_test']: print('Running initialisation test') initial_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count, experiment_id) # Train the model model = train(sess, model, model_config, model_folder, handle, training_iterator, training_handle, validation_iterator, validation_handle, writer) # Test trained model mean_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, writer, test_count, experiment_id) print('{ts}:\n\tAll done!'.format(ts=datetime.datetime.now())) if model_config['initialisation_test']: print('\tInitial test loss: {init}'.format(init=initial_test_loss)) print('\tFinal test loss: {final}'.format(final=mean_test_loss))
def do_experiment(model_config): tf.reset_default_graph() experiment_id = ex.current_run._id print('Experiment ID: {eid}'.format(eid=experiment_id)) # Prepare data print('Preparing dataset') train_data, val_data, test_data = dataset.prepare_datasets(model_config) print('Dataset ready') # Start session tf_config = tf.ConfigProto() #tf_config.gpu_options.allow_growth = True tf_config.gpu_options.visible_device_list = str(model_config['GPU']) sess = tf.Session(config=tf_config) #sess = tf.Session() #sess = tf_debug.LocalCLIDebugWrapperSession(sess, ui_type="readline") print('Session started') # Create iterators handle = tf.placeholder(tf.string, shape=[]) iterator = tf.data.Iterator.from_string_handle(handle, train_data.output_types, train_data.output_shapes) mixed_spec, voice_spec, background_spec, mixed_audio, voice_audio, background_audio = iterator.get_next( ) training_iterator = train_data.make_initializable_iterator() validation_iterator = val_data.make_initializable_iterator() testing_iterator = test_data.make_initializable_iterator() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) testing_handle = sess.run(testing_iterator.string_handle()) print('Iterators created') # Create variable placeholders and model is_training = tf.placeholder(shape=(), dtype=bool) mixed_phase = tf.expand_dims(mixed_spec[:, :, :-1, 3], 3) print('Creating model') if model_config['data_type'] == 'mag': mixed_input = tf.expand_dims(mixed_spec[:, :, :-1, 2], 3) voice_input = tf.expand_dims(voice_spec[:, :, :-1, 2], 3) elif model_config['data_type'] in ['mag_phase', 'mag_phase_diff']: mixed_input = mixed_spec[:, :, :-1, 2:4] voice_input = voice_spec[:, :, :-1, 2:4] elif model_config['data_type'] == 'real_imag': mixed_input = mixed_spec[:, :, :-1, 0:2] voice_input = voice_spec[:, :, :-1, 0:2] #elif data_type == 'mag_real_imag': # mixed_input = mixed_spec[:, :, ] model = audio_models.MagnitudeModel(mixed_input, voice_input, mixed_phase, mixed_audio, voice_audio, background_audio, model_config['model_variant'], is_training, model_config['learning_rate'], model_config['data_type'], name='Magnitude_Model') sess.run(tf.global_variables_initializer()) if model_config['loading']: print('Loading checkpoint') checkpoint = os.path.join(model_config['model_base_dir'], model_config['checkpoint_to_load']) restorer = tf.train.Saver() restorer.restore(sess, checkpoint) # Summaries model_folder = str(experiment_id) writer = tf.summary.FileWriter(os.path.join(model_config["log_dir"], model_folder), graph=sess.graph) # Get baseline metrics at initialisation test_count = 0 if model_config['initialisation_test']: print('Running initialisation test') initial_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, test_count, experiment_id) # Train the model model = train(sess, model, model_config, model_folder, handle, training_iterator, training_handle, validation_iterator, validation_handle, writer) # Test trained model mean_test_loss, test_count = test(sess, model, model_config, handle, testing_iterator, testing_handle, test_count, experiment_id) print('{ts}:\n\tAll done with experiment {exid}!'.format( ts=datetime.datetime.now(), exid=experiment_id)) if model_config['initialisation_test']: print('\tInitial test loss: {init}'.format(init=initial_test_loss)) print('\tFinal test loss: {final}'.format(final=mean_test_loss))