def tune_train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names patch_filenames = pd.read_csv(args.train_csv, dtype=object, keep_default_na=False, na_values=[]).as_matrix() subj_filenames = pd.read_csv(args.val_csv, dtype=object, keep_default_na=False, na_values=[]).as_matrix() app_json = get_config_for_app() val_filenames = [] for row in subj_filenames: if row[4] == '1': val_filenames.append(row) # Set up a data reader to handle the file i/o. reader_params = { 'n_examples': 16, 'example_size': [1, 64, 64], 'extract_examples': True } num_channels = app_json['num_channels'] reader_example_shapes = { 'features': { 'x': reader_params['example_size'] + [ num_channels, ] }, 'labels': { 'y': reader_params['example_size'] } } # module_name = 'contributions.applications.AL_framework.applications.app' + str(app_json['id']) + '.readers.' # # if app_json['reader_type'] == "Patch": # module_name = module_name + 'patch_reader' # elif app_json['reader_type'] == "Slice": # module_name = module_name + 'slice_reader' # elif app_json['reader_type'] == "Stack": # module_name = module_name + 'stack_reader' # else: # print("Unsupported reader type: please specify a new one") # return # mod = import_module(module_name) mod = import_module('readers.tune_reader') read_fn = vars(mod)['read_fn'] reader = Reader(read_fn, { 'features': { 'x': tf.float32 }, 'labels': { 'y': tf.int32 } }) # Get input functions and queue initialisation hooks for training and # validation data batch_size = app_json['batch_size'] train_input_fn, train_qinit_hook = reader.get_inputs( file_references=patch_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=batch_size, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=batch_size, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator nn = tf.estimator.Estimator(model_fn=model_fn, model_dir=args.model_path, params={"learning_rate": 0.001}, config=tf.estimator.RunConfig()) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting tuning...') max_steps = app_json['max_steps'] try: for _ in range(max_steps // EVAL_EVERY_N_STEPS): nn.train(input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f};'.format( results_val['global_step'], results_val['loss'])) except KeyboardInterrupt: pass print('Stopping now.') export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn( reader_example_shapes)) print('Model saved to {}.'.format(export_dir)) app_json['model_status'] = 2 write_app_config(app_json) print('Updated model status in model config')
def train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names all_filenames = pd.read_csv(args.train_csv, dtype=object, keep_default_na=False, na_values=[]).as_matrix() train_filenames = all_filenames[1:10] val_filenames = all_filenames[10:12] # Set up a data reader to handle the file i/o. reader_params = { 'n_examples': 16, 'example_size': [1, 64, 64], 'extract_examples': True } reader_example_shapes = { 'features': { 'x': reader_params['example_size'] + [ NUM_CHANNELS, ] }, 'labels': { 'y': reader_params['example_size'] } } reader = Reader(read_fn, { 'features': { 'x': tf.float32 }, 'labels': { 'y': tf.int32 } }) # Get input functions and queue initialisation hooks for training and # validation data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator nn = tf.estimator.Estimator(model_fn=model_fn, model_dir=args.model_path, params={"learning_rate": 0.001}, config=tf.estimator.RunConfig()) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting training...') try: for _ in range(MAX_STEPS // EVAL_EVERY_N_STEPS): nn.train(input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f};'.format( results_val['global_step'], results_val['loss'])) except KeyboardInterrupt: pass print('Stopping now.') export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn( reader_example_shapes)) print('Model saved to {}.'.format(export_dir))
def train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names all_filenames = pd.read_csv( args.data_csv, dtype=object, keep_default_na=False, na_values=[]).as_matrix() train_filenames = all_filenames[:100] val_filenames = all_filenames[100:] # Set up a data reader to handle the file i/o. reader_params = {'n_examples': 8, 'example_size': [32, 128, 128], 'extract_examples': True} reader_example_shapes = {'features': {'x': reader_params['example_size'] + [NUM_CHANNELS, ]}} reader = Reader(read_fn, {'features': {'x': tf.float32}}) # Get input functions and queue initialisation hooks for training and # validation data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator nn = tf.estimator.Estimator( model_fn=model_fn, model_dir=args.model_path, params={'learning_rate': 0.01, 'upsampling_factor': UPSAMPLING_FACTOR}, config=tf.estimator.RunConfig()) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook( every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting training...') try: for _ in range(MAX_STEPS // EVAL_EVERY_N_STEPS): nn.train( input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f};'.format( results_val['global_step'], results_val['loss'])) except KeyboardInterrupt: pass print('Stopping now.') export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn(reader_example_shapes)) print('Model saved to {}.'.format(export_dir))
def train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names target_df = pd.read_csv('/data1/users/adoyle/IBIS/ibis_t1_qc.csv', dtype=object, keep_default_na=False) all_filenames = target_df.iloc[:, 0].tolist() all_labels = target_df.iloc[:, 1].tolist() skf = StratifiedKFold(n_splits=10) for train_filenames, val_filenames in skf.split(all_filenames, all_labels): # Set up a data reader to handle the file i/o. reader_params = {'n_examples': 2, 'example_size': [160, 256, 224], 'extract_examples': True} reader_example_shapes = {'features': {'x': reader_params['example_size'] + [NUM_CHANNELS]}, 'labels': {'y': [1]}} reader = Reader(read_fn, {'features': {'x': tf.float32}, 'labels': {'y': tf.int32}}) # Get input functions and queue initialisation hooks for training and # validation data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator nn = tf.estimator.Estimator( model_fn=model_fn, model_dir=args.model_path, params={"learning_rate": 0.001}, config=tf.estimator.RunConfig()) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting training...') try: for _ in range(MAX_STEPS // EVAL_EVERY_N_STEPS): nn.train( input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f};'.format( results_val['global_step'], results_val['loss'])) except KeyboardInterrupt: pass # When exporting we set the expected input shape to be arbitrary. export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn( {'features': {'x': [None, None, None, NUM_CHANNELS]}, 'labels': {'y': [1]}})) print('Model saved to {}.'.format(export_dir))
def train(args): np.random.seed(8) tf.set_random_seed(8) print('Setting Up...') # Read Training-Fold.csv train_filenames = pd.read_csv(args.train_csv, dtype=object, keep_default_na=False, na_values=[]).values # Read Validation-Fold.csv val_filenames = pd.read_csv(args.val_csv, dtype=object, keep_default_na=False, na_values=[]).values # Set DLTK Reader Parameters (No. of Patches, Patch Size) reader_params = { 'n_patches': NUM_PATCHES, 'patch_size': [PATCH_Z, PATCH_XY, PATCH_XY], # Target Patch Size 'extract_patches': True } # Enable Training Mode Patch Extraction # Set Patch Dimensions reader_patch_shapes = { 'features': { 'x': reader_params['patch_size'] + [ NUM_CHANNELS, ] }, 'labels': { 'y': reader_params['patch_size'] } } # Initiate Data Reader + Patch Extraction reader = Reader(read_fn, { 'features': { 'x': tf.float32 }, 'labels': { 'y': tf.int32 } }) # Create Input Functions + Queue Initialisation Hooks for Training/Validation Data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_patch_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, prefetch_cache_size=PREFETCH_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_patch_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, prefetch_cache_size=PREFETCH_CACHE_SIZE, params=reader_params) # Instantiate Neural Network Estimator nn = tf.estimator.Estimator(model_fn=model_fn, model_dir=args.model_path, config=tf.estimator.RunConfig()) # Hooks for Validation Summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Begin Training...') try: for _ in range(MAX_STEPS // EVAL_EVERY_N_STEPS): nn.train(input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) EPOCH_DISPLAY = int( int(results_val['global_step']) / (TRAIN_SIZE / BATCH_SIZE)) print('Epoch = {}; Step = {} / ValLoss = {:.5f};'.format( EPOCH_DISPLAY, results_val['global_step'], results_val['loss'])) dim = args.model_path + 'Step{}ValLoss{:.5f}'.format( results_val['global_step'], results_val['loss']) export_dir = nn.export_savedmodel( export_dir_base=dim, serving_input_receiver_fn=reader.serving_input_receiver_fn( reader_patch_shapes)) print('Model saved to {}.'.format(export_dir)) count_steps.append(results_val['global_step']) count_loss.append(results_val['loss']) except KeyboardInterrupt: pass # Arbitrary Input Shape during Export export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn( reader_patch_shapes)) print('Model saved to {}.'.format(export_dir)) step_Loss = pd.DataFrame(list(zip(count_steps, count_loss)), columns=['steps', 'val_loss']) step_Loss.to_csv("RU_Validation_Loss.csv", encoding='utf-8', index=False)
def train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names all_filenames = pd.read_csv(args.data_csv, dtype=object, keep_default_na=False, na_values=[]).as_matrix() # 3300 train_filenames = all_filenames[:3300] val_filenames = all_filenames[3300:] # Set up a data reader to handle the file i/o. reader_params = { 'n_examples': 1, 'example_size': RESIZE_SIZE, 'extract_examples': True } reader_example_shapes = { 'features': { 'x': reader_params['example_size'] + [NUM_CHANNELS] }, 'labels': { 'y': [] } } reader = Reader(read_fn, { 'features': { 'x': tf.float32 }, 'labels': { 'y': tf.int32 } }) # Get input functions and queue initialisation hooks for training and # validation data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_filenames, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_filenames, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator config = tf.ConfigProto() config.gpu_options.allow_growth = True nn = tf.estimator.Estimator( model_fn=model_fn, model_dir=args.model_path, params={"learning_rate": 1e-3}, config=tf.estimator.RunConfig(session_config=config)) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) train_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'train')) # train_summary_hook = tf.train.SummarySaverHook(save_steps=1, # output_dir=os.path.join(args.model_path, 'train'), # summary_op=tf.summary.) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting training...') try: for _ in tqdm(range(MAX_STEPS // EVAL_EVERY_N_STEPS)): nn.train( input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook, train_summary_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f}; val acc = {:.5f}'.format( results_val['global_step'], results_val['loss'], results_val['accuracy_val'])) except KeyboardInterrupt: pass # When exporting we set the expected input shape to be arbitrary. export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn({ 'features': { 'x': [None, None, None, NUM_CHANNELS] }, 'labels': { 'y': [1] } })) print('Model saved to {}.'.format(export_dir))
def train(args): np.random.seed(42) tf.set_random_seed(42) print('Setting up...') # Parse csv files for file names train_df = pd.read_csv(args.train_csv) val_df = pd.read_csv(args.val_csv) # Set up a data reader to handle the file i/o. reader_params = { 'n_examples': 2, 'example_size': [121, 145, 121], 'extract_examples': False } reader_example_shapes = { 'features': { 'x': reader_params['example_size'] + [NUM_CHANNELS] }, 'labels': { 'y': [1] } } reader = Reader(read_fn, { 'features': { 'x': tf.float32 }, 'labels': { 'y': tf.float32 } }) # Get input functions and queue initialisation hooks for training and # validation data train_input_fn, train_qinit_hook = reader.get_inputs( file_references=train_df, mode=tf.estimator.ModeKeys.TRAIN, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) val_input_fn, val_qinit_hook = reader.get_inputs( file_references=val_df, mode=tf.estimator.ModeKeys.EVAL, example_shapes=reader_example_shapes, batch_size=BATCH_SIZE, shuffle_cache_size=SHUFFLE_CACHE_SIZE, params=reader_params) # Instantiate the neural network estimator nn = tf.estimator.Estimator(model_fn=model_fn, model_dir=args.model_path, params={"learning_rate": 0.001}, config=tf.estimator.RunConfig()) # Hooks for validation summaries val_summary_hook = tf.contrib.training.SummaryAtEndHook( os.path.join(args.model_path, 'eval')) step_cnt_hook = tf.train.StepCounterHook(every_n_steps=EVAL_EVERY_N_STEPS, output_dir=args.model_path) print('Starting training...') best_model_path = os.path.join(args.model_path, 'best') best_val_loss = None try: for _ in range(MAX_STEPS // EVAL_EVERY_N_STEPS): print('Training epoch {}/{}'.format( _, MAX_STEPS // EVAL_EVERY_N_STEPS)) nn.train(input_fn=train_input_fn, hooks=[train_qinit_hook, step_cnt_hook], steps=EVAL_EVERY_N_STEPS) if args.run_validation: results_val = nn.evaluate( input_fn=val_input_fn, hooks=[val_qinit_hook, val_summary_hook], steps=EVAL_STEPS) print('Step = {}; val loss = {:.5f};'.format( results_val['global_step'], results_val['loss'])) if best_val_loss is None or results_val['loss'] < best_val_loss: os.system('rm -rf {}/{}'.format(best_model_path, '*')) export_dir = nn.export_savedmodel( export_dir_base=os.path.join(args.model_path, 'best'), serving_input_receiver_fn=reader. serving_input_receiver_fn({ 'features': { 'x': [None, None, None, NUM_CHANNELS] }, 'labels': { 'y': [1] } })) print('Best Model saved to {}.'.format(export_dir)) best_val_loss = results_val['loss'] except KeyboardInterrupt: pass # When exporting we set the expected input shape to be arbitrary. export_dir = nn.export_savedmodel( export_dir_base=args.model_path, serving_input_receiver_fn=reader.serving_input_receiver_fn({ 'features': { 'x': [None, None, None, NUM_CHANNELS] }, 'labels': { 'y': [1] } })) print('Model saved to {}.'.format(export_dir))