def main(experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, config_file=None, ckpt_file=None, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params config = np.load(config_file).item() dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) config.epochs = 1 config.shuffle = False with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=['resize_and_crop'], num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Normalize labels if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] log.info('Mean-centering labels.') # Training model if len(dataset_module.output_size) == 2: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): train_grad_images = tf.gradients( train_scores[0] * tf.cast(train_labels, tf.float32), train_images)[0] log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() if len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) elif len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels)[0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if len(train_labels.get_shape()) > 2: tf_fun.image_summaries(train_labels, tag='Training_targets') tf_fun.image_summaries(train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax(train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op(tag='training_pr', predictions=tf.cast( tf.argmax(train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, # False, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): val_grad_images = tf.gradients( val_scores[0] * tf.cast(val_labels, tf.float32), val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(val_scores.get_shape()) != len(val_labels.get_shape()): val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_labels = tf.expand_dims(val_labels, axis=-1) if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_scores = tf.expand_dims(val_scores, axis=-1) val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels)[0] # TODO: Fix for multiloss # Prepare tensorboard summaries if len(val_images.get_shape()) == 4: tf_fun.image_summaries(val_images, tag='Validation') if len(val_labels.get_shape()) > 2: tf_fun.image_summaries(val_labels, tag='Validation_targets') tf_fun.image_summaries(val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast(tf.argmax(val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op(tag='validation_pr', predictions=tf.cast( tf.argmax(val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores, 'train_grad_images': train_grad_images } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, 'val_grad_images': val_grad_images } if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop checkpoint_dir = dir_list['checkpoints'] step = 0 train_losses, train_accs, train_aux, timesteps = {}, {}, {}, {} val_scores, val_aux, val_labels, val_grads = {}, {}, {}, {} train_images, val_images = {}, {} train_scores, train_labels = {}, {} train_aux_check = np.any(['aux_score' in k for k in train_dict.keys()]) val_aux_check = np.any(['aux_score' in k for k in val_dict.keys()]) # Restore model saver.restore(sess, ckpt_file) # Start evaluation try: while not coord.should_stop(): start_time = time.time() train_vars = sess.run(train_dict.values()) it_train_dict = { k: v for k, v in zip(train_dict.keys(), train_vars) } duration = time.time() - start_time train_losses[step] = it_train_dict['train_loss'] train_accs[step] = it_train_dict['train_accuracy_0'] train_images[step] = it_train_dict['train_images'] train_labels[step] = it_train_dict['train_labels'] train_scores[step] = it_train_dict['train_scores'] timesteps[step] = duration if train_aux_check: # Loop through to find aux scores it_train_aux = { itk: itv for itk, itv in it_train_dict.iteritems() if 'aux_score' in itk } train_aux[step] = it_train_aux assert not np.isnan(it_train_dict['train_loss']).any( ), 'Model diverged with loss = NaN' if step % config.validation_iters == 0: it_val_scores, it_val_labels, it_val_aux, it_val_grads, it_val_ims = [], [], [], [], [] for num_vals in range(config.num_validation_evals): # Validation accuracy as the average of n batches val_vars = sess.run(val_dict.values()) it_val_dict = { k: v for k, v in zip(val_dict.keys(), val_vars) } it_val_labels += [it_val_dict['val_labels']] it_val_scores += [it_val_dict['val_scores']] it_val_grads += [it_val_dict['val_grad_images']] it_val_ims += [it_val_dict['val_images']] if val_aux_check: iva = { itk: itv for itk, itv in it_val_dict.iteritems() if 'aux_score' in itk } it_val_aux += [iva] val_scores[step] = it_val_scores val_labels[step] = it_val_labels val_aux[step] = it_val_aux val_images[step] = it_val_grads val_grads[step] = it_val_ims # End iteration step += 1 except tf.errors.OutOfRangeError: print 'Done with evaluation for %d epochs, %d steps.' % (config.epochs, step) print 'Saved to: %s' % checkpoint_dir finally: coord.request_stop() coord.join(threads) sess.close() import ipdb ipdb.set_trace() np.savez( 'val_imgs_grads', val_images=val_images, # it_val_dict['val_images'], val_grads=val_grads, # it_val_dict['val_grad_images'], val_labels=val_labels, # it_val_dict['val_labels'], val_scores=val_scores) # it_val_dict['val_scores'][0])
def main(experiment_name, list_experiments=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means = get_data_pointers(dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s' % experiment_name # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model if len(dataset_module.output_size) > 1: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, labels=train_labels, loss_type=config.loss_function, dataset_module=dataset_module) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=train_scores, labels=train_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('train images', train_images) tf.summary.scalar('training loss', train_loss) tf.summary.scalar('training accuracy', train_accuracy) log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=val_means, training=True, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built validation model.') val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, dataset_module=dataset_module) val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=val_scores, labels=val_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('val images', val_images) tf.summary.scalar('validation loss', val_loss) tf.summary.scalar('validation accuracy', val_accuracy) log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_accuracy': train_accuracy, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_accuracy': val_accuracy, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } # Start training loop np.save( os.path.join(dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') py_utils.save_npys(data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def main( experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, placeholder_data=None, grad_images=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) assert experiment_name is not None, 'Empty experiment name.' experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer from ops import evaluation config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key, log=log) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=val_key, log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join( config.checkpoints, condition_label), 'summaries': os.path.join( config.summaries, condition_label), 'condition_evaluations': os.path.join( config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join( config.visualizations, condition_label), 'weights': os.path.join( config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) if load_and_evaluate_ckpt is not None: config.epochs = 1 config.train_shuffle = False config.val_shuffle = False with tf.device('/cpu:0'): if placeholder_data: placeholder_shape = placeholder_data['train_image_shape'] placeholder_dtype = placeholder_data['train_image_dtype'] original_train_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_images') placeholder_shape = placeholder_data['train_label_shape'] placeholder_dtype = placeholder_data['train_label_dtype'] original_train_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_labels') placeholder_shape = placeholder_data['val_image_shape'] placeholder_dtype = placeholder_data['val_image_dtype'] original_val_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_images') placeholder_shape = placeholder_data['val_label_shape'] placeholder_dtype = placeholder_data['val_label_dtype'] original_val_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_labels') # Apply augmentations ( train_images, train_labels ) = data_loader.placeholder_image_augmentations( images=original_train_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_train_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) ( val_images, val_labels ) = data_loader.placeholder_image_augmentations( images=original_val_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_val_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) # Store in the placeholder dict placeholder_data['train_images'] = original_train_images placeholder_data['train_labels'] = original_train_labels placeholder_data['val_images'] = original_val_images placeholder_data['val_labels'] = original_val_labels else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join( 'models', 'structs', experiment_name).replace(os.path.sep, '.') ) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare variables for the models if len(dataset_module.output_size) == 2: log.warning( 'Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None # Correct number of output neurons if needed if config.dataloader_override and\ 'weights' in output_structure[-1].keys(): output_neurons = output_structure[-1]['weights'][0] size_check = output_neurons != dataset_module.output_size fc_check = output_structure[-1]['layers'][0] == 'fc' if size_check and fc_check: output_structure[-1]['weights'][0] = dataset_module.output_size log.warning('Adjusted output neurons from %s to %s.' % ( output_neurons, dataset_module.output_size)) # Prepare model on GPU if not hasattr(dataset_module, 'input_normalization'): dataset_module.input_normalization = None with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) train_scores, model_summary, _ = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(train_scores.get_shape()[-1]) target_scores = tf.one_hot(train_labels, oh_dims) * train_scores train_gradients = tf.gradients(target_scores, train_images)[0] log.info('Built training model.') log.debug( json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Normalize labels on GPU if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] val_labels -= train_means_label['mean'] val_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] val_labels -= val_means_label['mean'] log.info('Mean-centering labels.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len( train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) val_labels = tf.expand_dims(val_labels, axis=-1) elif len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) val_scores = tf.expand_dims(val_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) assert config.lr is not None, 'No learning rate.' # TODO: Make a QC function if config.lr > 1: old_lr = config.lr config.lr = loss_utils.create_lr_schedule( train_batch=config.batch_size, num_training=config.lr) config.optimizer = 'momentum' log.info('Forcing momentum classifier.') else: old_lr = None train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels) # [0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if config.tensorboard_images: if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if (np.asarray( train_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( train_labels, tag='Training_targets') tf_fun.image_summaries( train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax( train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op( tag='training_pr', predictions=tf.cast( tf.argmax( train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') with tf.variable_scope('cnn', tf.AUTO_REUSE) as scope: # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) val_scores, _, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(val_scores.get_shape()[-1]) target_scores = tf.one_hot(val_labels, oh_dims) * val_scores val_gradients = tf.gradients(target_scores, val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels) # [0] # TODO: Fix for multiloss # Prepare tensorboard summaries if config.tensorboard_images: if len(val_images.get_shape()) == 4: tf_fun.image_summaries( val_images, tag='Validation') if (np.asarray( val_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( val_labels, tag='Validation_targets') tf_fun.image_summaries( val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast( tf.argmax( val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op( tag='validation_pr', predictions=tf.cast( tf.argmax( val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver if not hasattr(config, 'max_to_keep'): config.max_to_keep = None saver = tf.train.Saver( var_list=tf.global_variables(), max_to_keep=config.max_to_keep) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading if placeholder_data: coord, threads = None, None else: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } if grad_images: train_dict['train_gradients'] = train_gradients val_dict['val_gradients'] = val_gradients if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop if old_lr is not None: config.lr = old_lr np.save( os.path.join( dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') if load_and_evaluate_ckpt is not None: return evaluation.evaluation_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params, placeholder_data=placeholder_data) else: output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') if output_dict is not None: py_utils.save_npys( data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def main( experiment_name, im_ext='.pdf', transform_loss=None, # 'log', colors='Paired', flip_axis=False, exclude=None): """Plot results of provided experiment name.""" config = Config() pl_creds = credentials.plotly_credentials() py.sign_in( pl_creds['username'], pl_creds['api_key']) # Get experiment data perf = db.get_performance(experiment_name=experiment_name) if len(perf) == 0: raise RuntimeError('Could not find any results.') structure_names = [x['model_struct'].split('/')[-1] for x in perf] optimizers = [x['optimizer'] for x in perf] lrs = [x['lr'] for x in perf] datasets = [x['dataset'] for x in perf] loss_funs = [x['loss_function'] for x in perf] optimizers = [x['optimizer'] for x in perf] wd_types = [x['regularization_type'] for x in perf] wd_penalties = [x['regularization_strength'] for x in perf] steps = [float(x['training_step']) for x in perf] training_loss = [float(x['training_loss']) for x in perf] validation_loss = [float(x['validation_loss']) for x in perf] timesteps = [0. if x['timesteps'] is None else float(x['timesteps']) for x in perf] u_t = [0. if x['u_t'] is None else float(x['u_t']) for x in perf] q_t = [0. if x['q_t'] is None else float(x['q_t']) for x in perf] p_t = [0. if x['p_t'] is None else float(x['p_t']) for x in perf] t_t = [0. if x['t_t'] is None else float(x['t_t']) for x in perf] # Pass data into a pandas DF model_params = [ '%s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s' % ( ipa, ipb, ipc, ipd, ipe, ipf, ipg, iph, ipi, ipj, ipk, ipl, ipm) for ipa, ipb, ipc, ipd, ipe, ipf, ipg, iph, ipi, ipj, ipk, ipl, ipm in zip( structure_names, optimizers, lrs, loss_funs, optimizers, wd_types, wd_penalties, datasets, timesteps, u_t, q_t, p_t, t_t)] # DF and plot df = pd.DataFrame( np.vstack( ( model_params, steps, training_loss, validation_loss ) ).transpose(), columns=[ 'model parameters', 'training iteration', 'training loss', 'validation loss' ] ) df['training iteration'] = pd.to_numeric( df['training iteration'], errors='coerce') df['training loss'] = pd.to_numeric(df['training loss'], errors='coerce') if exclude is not None: exclusion_search = df['model parameters'].str.contains(exclude) df = df[exclusion_search == False] print 'Removed %s rows.' % exclusion_search.sum() # Start plotting experiment_dict = experiments.experiments()[experiment_name]() print 'Plotting results for dataset: %s.' % experiment_dict['dataset'][0] dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=experiment_dict['dataset'][0]) dataset_module = dataset_module.data_processing() # hardcoded class name if transform_loss is None: loss_label = '' elif transform_loss == 'log': loss_label = ' log loss' df['training loss'] = np.log(df['training loss']) elif transform_loss == 'max': loss_label = ' normalized (x / max(x)) ' df['training loss'] /= df.groupby( 'model parameters')['training loss'].transform(max) if ['loss_function'] in experiment_dict.keys(): loss_metric = experiment_dict['loss_function'][0] else: loss_metric = dataset_module.default_loss_function df['validation loss'] = pd.to_numeric(df['validation loss']) if loss_metric == 'pearson': loss_label = 'Pearson correlation' + loss_label elif loss_metric == 'l2': loss_label = 'L2' + loss_label else: loss_label = 'Classification accuracy (%)' df['validation loss'] *= 100. if ['score_metric'] in experiment_dict.keys(): score_metric = experiment_dict['score_metric'] else: score_metric = dataset_module.score_metric if score_metric == 'pearson': y_lab = 'Pearson correlation' matplotlib.style.use('ggplot') plt.rc('font', size=6) plt.rc('legend', fontsize=8, labelspacing=3) f, axs = plt.subplots(2, figsize=(20, 30)) ax = axs[1] NUM_COLORS = len(df['model parameters'].unique()) cm = plt.get_cmap('gist_rainbow') ax.set_color_cycle([cm(1.*i/NUM_COLORS) for i in range(NUM_COLORS)]) for k in df['model parameters'].unique(): tmp = df[df['model parameters'] == k] tmp = tmp.sort('training iteration') ax = tmp.plot( x='training iteration', y='training loss', label=k, kind='line', ax=ax, logy=False) plt.setp(ax.xaxis.get_majorticklabels(), rotation=30) ax.xaxis.set_major_locator(MaxNLocator(integer=True)) ax.set_title('Training') ax.set_ylabel(loss_label) # ax.legend_.remove() ax = axs[0] ax.set_color_cycle([cm(1.*i/NUM_COLORS) for i in range(NUM_COLORS)]) for k in df['model parameters'].unique(): tmp = df[df['model parameters'] == k] tmp = tmp.sort('training iteration') ax = tmp.plot( x='training iteration', y='validation loss', label=k, kind='line', ax=ax, logy=False) plt.setp(ax.xaxis.get_majorticklabels(), rotation=30) ax.xaxis.set_major_locator(MaxNLocator(integer=True)) ax.set_title('Validation') # TODO: Mine the experiment declarations for the appropos metric name. ax.set_ylabel(y_lab) # ax.legend_.remove() out_name = os.path.join( config.plots, '%s_%s%s' % ( experiment_name, py_utils.get_dt_stamp(), im_ext)) plt.savefig(out_name) print 'Saved to: %s' % out_name plotly_fig = tls.mpl_to_plotly(f) plotly_fig['layout']['autosize'] = True # plotly_fig['layout']['showlegend'] = True plot_with_plotly(plotly_fig, 'line') plt.close(f) # Plot max performance bar graph f = plt.figure() max_perf = df.groupby( ['model parameters'], as_index=False)['validation loss'].max() plt.rc('xtick', labelsize=2) ax = max_perf.plot.bar( x='model parameters', y='validation loss', legend=False) plt.tight_layout() ax.set_title('Max validation value') ax.set_ylabel(y_lab) out_name = os.path.join( config.plots, '%s_%s_bar%s' % ( experiment_name, py_utils.get_dt_stamp(), im_ext)) plt.savefig(out_name) print 'Saved to: %s' % out_name try: plotly_fig = tls.mpl_to_plotly(f) plot_with_plotly(plotly_fig, chart='bar') except Exception as e: print 'Failed to plot bar chart in plotly: %s' % e plt.close(f)
def model_builder( params, config, model_spec, gpu_device, cpu_device, placeholders=False, tensorboard_images=False): """Standard model building routines.""" config = py_utils.add_to_config( d=params, config=config) exp_label = '%s_%s' % (params['exp_name'], py_utils.get_dt_stamp()) directories = py_utils.prepare_directories(config, exp_label) dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] ( train_data, train_means_image, train_means_label) = py_utils.get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] if hasattr(config, 'val_dataset'): val_dataset = config.val_dataset else: val_dataset = config.dataset val_data, val_means_image, val_means_label = py_utils.get_data_pointers( dataset=val_dataset, base_dir=config.tf_records, cv=val_key) # Create data tensors with tf.device(cpu_device): if placeholders: ( train_images, train_labels, val_images, val_labels) = get_placeholders(dataset_module, config) placeholders = dataset_module.get_data() else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train) val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val) # Build training and val models with tf.device(gpu_device): train_logits, train_hgru_act = model_spec( data_tensor=train_images, reuse=None, training=True) val_logits, val_hgru_act = model_spec( data_tensor=val_images, reuse=tf.AUTO_REUSE, training=False) # Derive loss loss_type = None if hasattr(config, 'loss_type'): loss_type = config.loss_type train_loss = losses.derive_loss( labels=train_labels, logits=train_logits, loss_type=loss_type) val_loss = losses.derive_loss( labels=val_labels, logits=val_logits, loss_type=loss_type) if hasattr(config, 'metric_type'): metric_type = config.metric_type else: metric_type = 'accuracy' if metric_type == 'pearson': train_accuracy = metrics.pearson_score( labels=train_labels, pred=train_logits, REDUCTION=tf.reduce_mean) val_accuracy = metrics.pearson_score( labels=val_labels, pred=val_logits, REDUCTION=tf.reduce_mean) else: train_accuracy = metrics.class_accuracy( labels=train_labels, logits=train_logits) val_accuracy = metrics.class_accuracy( labels=val_labels, logits=val_logits) tf.summary.scalar('train_accuracy', train_accuracy) tf.summary.scalar('val_accuracy', val_accuracy) if tensorboard_images: tf.summary.image('train_images', train_images) tf.summary.image('val_images', val_images) # Build optimizer train_op = optimizers.get_optimizer( train_loss, config['lr'], config['optimizer']) # Initialize tf variables saver = tf.train.Saver( var_list=tf.global_variables()) summary_op = tf.summary.merge_all() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer())) summary_writer = tf.summary.FileWriter( directories['summaries'], sess.graph) if not placeholders: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) else: coord, threads = None, None # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_accuracy': train_accuracy } if isinstance(train_hgru_act, dict): for k, v in train_hgru_act.iteritems(): train_dict[k] = v else: train_dict['activity'] = train_hgru_act val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_accuracy': val_accuracy, } if isinstance(val_hgru_act, dict): for k, v in val_hgru_act.iteritems(): val_dict[k] = v else: val_dict['activity'] = val_hgru_act # Count parameters num_params = np.sum( [np.prod(x.get_shape().as_list()) for x in tf.trainable_variables()]) print 'Model has approximately %s trainable params.' % num_params # Create datastructure for saving data ds = data_structure.data( batch_size=config.batch_size, validation_iters=config.validation_iters, num_validation_evals=config.num_validation_evals, shuffle_val=config.shuffle_val, lr=config.lr, loss_function=config.loss_function, optimizer=config.optimizer, model_name=config.model_name, dataset=config.dataset, num_params=num_params, output_directory=config.results) # Start training loop training.training_loop( config=config, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, directories=directories, train_dict=train_dict, val_dict=val_dict, exp_label=exp_label, data_structure=ds, placeholders=placeholders)
def build_model(exp_params, config, log, dt_string, gpu_device, cpu_device, use_db=True, add_config=None, placeholders=False, checkpoint=None, test=False, map_out=None, num_batches=None, tensorboard_images=False): """Standard model building routines.""" config = py_utils.add_to_config(d=exp_params, config=config) if not hasattr(config, 'force_path'): config.force_path = False exp_label = '%s_%s_%s' % (exp_params['model'], exp_params['experiment'], py_utils.get_dt_stamp()) directories = py_utils.prepare_directories(config, exp_label) dataset_module = py_utils.import_module(pre_path=config.dataset_classes, module=config.train_dataset) train_dataset_module = dataset_module.data_processing() if not config.force_path: (train_data, _, _) = py_utils.get_data_pointers( dataset=train_dataset_module.output_name, base_dir=config.tf_records, local_dir=config.local_tf_records, cv='train') else: train_data = train_dataset_module.train_path dataset_module = py_utils.import_module(pre_path=config.dataset_classes, module=config.val_dataset) val_dataset_module = dataset_module.data_processing() if not config.force_path: val_data, _, _ = py_utils.get_data_pointers( dataset=val_dataset_module.output_name, base_dir=config.tf_records, local_dir=config.local_tf_records, cv='val') else: val_data = train_dataset_module.val_path # val_means_image, val_means_label = None, None # Create data tensors if hasattr(train_dataset_module, 'aux_loss'): train_aux_loss = train_dataset_module.aux_loss else: train_aux_loss = None with tf.device(cpu_device): if placeholders and not test: # Train with placeholders (pl_train_images, pl_train_labels, pl_val_images, pl_val_labels, train_images, train_labels, val_images, val_labels) = get_placeholders(train_dataset=train_dataset_module, val_dataset=val_dataset_module, config=config) train_module_data = train_dataset_module.get_data() val_module_data = val_dataset_module.get_data() placeholders = { 'train': { 'images': train_module_data[0]['train'], 'labels': train_module_data[1]['train'] }, 'val': { 'images': val_module_data[0]['val'], 'labels': val_module_data[1]['val'] }, } train_aux, val_aux = None, None elif placeholders and test: test_dataset_module = train_dataset_module # Test with placeholders (pl_test_images, pl_test_labels, test_images, test_labels) = get_placeholders_test( test_dataset=test_dataset_module, config=config) test_module_data = test_dataset_module.get_data() placeholders = { 'test': { 'images': test_module_data[0]['test'], 'labels': test_module_data[1]['test'] }, } train_aux, val_aux = None, None else: train_images, train_labels, train_aux = data_loader.inputs( dataset=train_data, batch_size=config.train_batch_size, model_input_image_size=train_dataset_module. model_input_image_size, tf_dict=train_dataset_module.tf_dict, data_augmentations=config.train_augmentations, num_epochs=config.epochs, aux=train_aux_loss, tf_reader_settings=train_dataset_module.tf_reader, shuffle=config.shuffle_train) if hasattr(val_dataset_module, 'val_model_input_image_size'): val_dataset_module.model_input_image_size = val_dataset_module.val_model_input_image_size val_images, val_labels, val_aux = data_loader.inputs( dataset=val_data, batch_size=config.val_batch_size, model_input_image_size=val_dataset_module. model_input_image_size, tf_dict=val_dataset_module.tf_dict, data_augmentations=config.val_augmentations, num_epochs=None, tf_reader_settings=val_dataset_module.tf_reader, shuffle=config.shuffle_val) # Build training and val models model_spec = py_utils.import_module(module=config.model, pre_path=config.model_classes) if hasattr(train_dataset_module, 'force_output_size'): train_dataset_module.output_size = train_dataset_module.force_output_size if hasattr(val_dataset_module, 'force_output_size'): val_dataset_module.output_size = val_dataset_module.force_output_size if hasattr(config, 'loss_function'): train_loss_function = config.loss_function val_loss_function = config.loss_function else: train_loss_function = config.train_loss_function val_loss_function = config.val_loss_function # Route test vs train/val h_check = [ x for x in tf.trainable_variables() if 'homunculus' in x.name or 'humonculus' in x.name ] if not hasattr(config, 'default_restore'): config.default_restore = False if test: assert len(gpu_device) == 1, 'Testing only works with 1 gpu.' gpu_device = gpu_device[0] with tf.device(gpu_device): if not placeholders: test_images = val_images test_labels = val_labels test_dataset_module = val_dataset_module test_logits, test_vars = model_spec.build_model( data_tensor=test_images, reuse=None, training=False, output_shape=test_dataset_module.output_size) if test_logits.dtype is not tf.float32: test_logits = tf.cast(test_logits, tf.float32) # Derive loss if not hasattr(config, 'test_loss_function'): test_loss_function = val_loss_function else: test_loss_function = config.test_loss_function test_loss = losses.derive_loss(labels=test_labels, logits=test_logits, loss_type=test_loss_function) # Derive score test_score = losses.derive_score(labels=test_labels, logits=test_logits, loss_type=test_loss_function, score_type=config.score_function) # Initialize model (sess, saver, summary_op, summary_writer, coord, threads, restore_saver) = initialize_tf(config=config, placeholders=placeholders, ckpt=checkpoint, default_restore=config.default_restore, directories=directories) if placeholders: proc_images = test_images proc_labels = test_labels test_images = pl_test_images test_labels = pl_test_labels _, H, W, _ = test_vars['model_output_y'].shape jacobian = tf.gradients(test_logits, test_vars['model_output_x'])[ 0] # g.batch_jacobian(test_vars['model_output_x'], test_images) test_dict = { 'test_loss': test_loss, 'test_score': test_score, 'test_images': test_images, 'test_labels': test_labels, 'test_logits': test_logits, 'test_jacobian': jacobian } if placeholders: test_dict['test_proc_images'] = proc_images test_dict['test_proc_labels'] = proc_labels if len(h_check): test_dict['homunculus'] = h_check[0] if isinstance(test_vars, dict): for k, v in test_vars.iteritems(): test_dict[k] = v else: test_dict['activity'] = test_vars else: train_losses, val_losses, tower_grads, norm_updates = [], [], [], [] train_scores, val_scores = [], [] train_image_list, train_label_list = [], [] val_image_list, val_label_list = [], [] train_reuse = None if not hasattr(config, 'lr_schedule'): config.lr_schedule = None if hasattr(config, 'loss_function'): train_loss_function = config.loss_function val_loss_function = config.loss_function else: train_loss_function = config.train_loss_function val_loss_function = config.val_loss_function # Prepare loop if not placeholders: train_batch_queue = tf_fun.get_batch_queues(images=train_images, labels=train_labels, gpu_device=gpu_device) val_batch_queue = tf_fun.get_batch_queues(images=val_images, labels=val_labels, gpu_device=gpu_device) config.lr = optimizers.get_lr_schedule(lr=config.lr, lr_schedule=config.lr_schedule) opt = optimizers.get_optimizers(optimizer=config.optimizer, lr=config.lr, dtype=train_images.dtype) with tf.device(cpu_device): global_step = tf.train.get_or_create_global_step() for i, gpu in enumerate(gpu_device): # rs = tf.AUTO_REUSE if i > 0 else None with tf.device(gpu): with tf.name_scope('tower_%d' % i) as scope: # Prepare tower data if placeholders: # Multi-gpu: will have to split # train_images per gpu by hand train_image_batch = train_images val_image_batch = val_images train_label_batch = train_labels val_label_batch = val_labels else: (train_image_batch, train_label_batch) = train_batch_queue.dequeue() (val_image_batch, val_label_batch) = val_batch_queue.dequeue() train_image_list += [train_image_batch] train_label_list += [train_label_batch] val_image_list += [val_image_batch] val_label_list += [val_label_batch] # Build models train_logits, train_vars = model_spec.build_model( data_tensor=train_image_batch, reuse=train_reuse, training=True, output_shape=train_dataset_module.output_size) num_training_vars = len(tf.trainable_variables()) val_logits, val_vars = model_spec.build_model( data_tensor=val_image_batch, reuse=True, training=False, output_shape=val_dataset_module.output_size) num_validation_vars = len(tf.trainable_variables()) assert num_training_vars == num_validation_vars, \ 'Found a different # of train and val variables.' train_reuse = True # Derive losses if train_logits.dtype is not tf.float32: train_logits = tf.cast(train_logits, tf.float32) if val_logits.dtype is not tf.float32: val_logits = tf.cast(val_logits, tf.float32) train_loss = losses.derive_loss( labels=train_label_batch, logits=train_logits, images=train_image_batch, loss_type=train_loss_function) val_loss = losses.derive_loss( labels=val_label_batch, logits=val_logits, images=val_image_batch, loss_type=val_loss_function) # Derive score train_score = losses.derive_score( labels=train_labels, logits=train_logits, loss_type=train_loss_function, score_type=config.score_function) val_score = losses.derive_score( labels=val_labels, logits=val_logits, loss_type=val_loss_function, score_type=config.score_function) # Add aux losses if requested if hasattr(model_spec, 'weight_decay'): wd = (model_spec.weight_decay() * tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'batch_normalization' not in v.name and 'horizontal' not in v.name and 'mu' not in v.name and 'beta' not in v.name and 'intercept' not in v.name ])) tf.summary.scalar('weight_decay', wd) train_loss += wd if hasattr(model_spec, 'bsds_weight_decay'): wd = (model_spec.bsds_weight_decay()['l2'] * tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'horizontal' not in v.name and 'norm' not in v.name ])) tf.summary.scalar('weight_decay_readout', wd) train_loss += wd wd = (model_spec.bsds_weight_decay()['l1'] * tf.add_n([ tf.reduce_sum(tf.abs(v)) for v in tf.trainable_variables() if 'horizontal' in v.name ])) tf.summary.scalar('weight_decay_horizontal', wd) train_loss += wd if hasattr(model_spec, 'orthogonal'): weights = [ v for v in tf.trainable_variables() if 'horizontal' in v.name ] assert len(weights) is not None, \ 'No horizontal weights for laplace.' wd = model_spec.orthogonal() * tf.add_n( [tf_fun.orthogonal(w) for w in weights]) tf.summary.scalar('weight_decay', wd) train_loss += wd if hasattr(model_spec, 'laplace'): weights = [ v for v in tf.trainable_variables() if 'horizontal' in v.name ] assert len(weights) is not None, \ 'No horizontal weights for laplace.' wd = model_spec.laplace() * tf.add_n( [tf_fun.laplace(w) for w in weights]) tf.summary.scalar('weight_decay', wd) train_loss += wd # Derive auxilary losses if hasattr(config, 'aux_loss'): aux_loss_type, scale = config.aux_loss.items()[0] for k, v in train_vars.iteritems(): # if k in train_dataset_module.aux_loss.keys(): # ( # aux_loss_type, # scale # ) = train_dataset_module.aux_loss[k] train_loss += (losses.derive_loss( labels=train_labels, logits=v, loss_type=aux_loss_type) * scale) # Gather everything train_losses += [train_loss] val_losses += [val_loss] train_scores += [train_score] val_scores += [val_score] # Compute and store gradients with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): grads = opt.compute_gradients(train_loss) optimizers.check_grads(grads) tower_grads += [grads] # Gather normalization variables norm_updates += [ tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=scope) ] # Recompute and optimize gradients grads = optimizers.average_gradients(tower_grads) if hasattr(config, 'clip_gradients') and config.clip_gradients: grads = optimizers.apply_grad_clip(grads, config.clip_gradients) op_vars = [] if hasattr(config, 'exclusion_lr') and hasattr(config, 'exclusion_scope'): grads_0 = [ x for x in grads if config.exclusion_scope not in x[1].name ] grads_1 = [x for x in grads if config.exclusion_scope in x[1].name] op_vars_0 = optimizers.apply_gradients(opt=opt, grads=grads_0, global_step=global_step) opt_1 = optimizers.get_optimizers(optimizer=config.optimizer, lr=config.exclusion_lr, dtype=train_images.dtype) op_vars_1 = optimizers.apply_gradients(opt=opt_1, grads=grads_1, global_step=global_step) op_vars += [op_vars_0] op_vars += [op_vars_1] else: op_vars += [ optimizers.apply_gradients(opt=opt, grads=grads, global_step=global_step) ] if not hasattr(config, 'variable_moving_average'): config.variable_moving_average = False if config.variable_moving_average: variable_averages = tf.train.ExponentialMovingAverage( config.variable_moving_average, global_step) op_vars += [variable_averages.apply(tf.trainable_variables())] if len(norm_updates): op_vars += [tf.group(*norm_updates)] train_op = tf.group(*op_vars) # Summarize losses and scores train_loss = tf.reduce_mean(train_losses) val_loss = tf.reduce_mean(val_losses) train_score = tf.reduce_mean(train_scores) val_score = tf.reduce_mean(val_scores) if len(train_image_list) > 1: train_image_list = tf.stack(train_image_list, axis=0) train_label_list = tf.stack(train_label_list, axis=0) else: train_image_list = train_image_list[0] train_label_list = train_label_list[0] if len(val_image_list) > 1: val_image_list = tf.stack(val_image_list, axis=0) val_label_list = tf.stack(val_label_list, axis=0) else: val_image_list = val_image_list[0] val_label_list = val_label_list[0] tf.summary.scalar('train_loss', train_loss) tf.summary.scalar('val_loss', val_loss) if tensorboard_images: tf.summary.image('train_images', train_images) tf.summary.image('val_images', val_images) # Initialize model (sess, saver, summary_op, summary_writer, coord, threads, restore_saver) = initialize_tf(config=config, placeholders=placeholders, ckpt=checkpoint, default_restore=config.default_restore, directories=directories) # Create dictionaries of important training and validation information if placeholders: proc_train_images = train_images proc_train_labels = train_labels proc_val_images = val_images proc_val_labels = val_labels train_images = pl_train_images train_labels = pl_train_labels val_images = pl_val_images val_labels = pl_val_labels train_dict = { 'train_loss': train_loss, 'train_score': train_score, 'train_images': train_image_list, 'train_labels': train_label_list, 'train_logits': train_logits, 'train_op': train_op } if placeholders: train_dict['proc_train_images'] = proc_train_images train_dict['proc_train_labels'] = proc_train_labels if train_aux is not None: train_dict['train_aux'] = train_aux if tf.contrib.framework.is_tensor(config.lr): train_dict['lr'] = config.lr else: train_dict['lr'] = tf.constant(config.lr) if isinstance(train_vars, dict): for k, v in train_vars.iteritems(): train_dict[k] = v else: train_dict['activity'] = train_vars if hasattr(config, 'save_gradients') and config.save_gradients: grad = tf.gradients(train_logits, train_images)[0] if grad is not None: train_dict['gradients'] = grad else: log.warning('Could not calculate val gradients.') val_dict = { 'val_loss': val_loss, 'val_score': val_score, 'val_images': val_image_list, 'val_logits': val_logits, 'val_labels': val_label_list, } if placeholders: val_dict['proc_val_images'] = proc_val_images val_dict['proc_val_labels'] = proc_val_labels if val_aux is not None: val_dict['aux'] = val_aux if isinstance(val_vars, dict): for k, v in val_vars.iteritems(): val_dict[k] = v else: val_dict['activity'] = val_vars if hasattr(config, 'save_gradients') and config.save_gradients: grad = tf.gradients(val_logits, val_images)[0] if grad is not None: val_dict['gradients'] = grad else: log.warning('Could not calculate val gradients.') if len(h_check): val_dict['homunculus'] = h_check[0] # Add optional info to the config if add_config is not None: extra_list = add_config.split(',') for eidx, extra in enumerate(extra_list): setattr(config, 'extra_%s' % eidx, extra) # Count parameters num_params = tf_fun.count_parameters(var_list=tf.trainable_variables()) print 'Model has approximately %s trainable params.' % num_params if test: return training.test_loop(log=log, config=config, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, restore_saver=restore_saver, directories=directories, test_dict=test_dict, exp_label=exp_label, num_params=num_params, checkpoint=checkpoint, num_batches=num_batches, save_weights=config.save_weights, save_checkpoints=config.save_checkpoints, save_activities=config.save_activities, save_gradients=config.save_gradients, map_out=map_out, placeholders=placeholders) else: # Start training loop training.training_loop(log=log, config=config, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, restore_saver=restore_saver, threads=threads, directories=directories, train_dict=train_dict, val_dict=val_dict, exp_label=exp_label, num_params=num_params, checkpoint=checkpoint, use_db=use_db, save_weights=config.save_weights, save_checkpoints=config.save_checkpoints, save_activities=config.save_activities, save_gradients=config.save_gradients, placeholders=placeholders)
def main(experiment, model, train, val, checkpoint, use_db=True, test=False, reduction=0, random=True, add_config=None, gpu_device=['/gpu:0'], cpu_device='/cpu:0', num_gpus=False, transfer=False, placeholders=False, save_test_npz=True, num_batches=None, map_out='test_maps', out_dir=None): """Interpret and run a model.""" main_config = Config() dt_string = py_utils.get_dt_stamp() log = logger.get( os.path.join(main_config.log_dir, '%s_%s' % (experiment, dt_string))) if num_gpus: gpu_device = ['/gpu:%d' % i for i in range(num_gpus)] if test and save_test_npz and out_dir is None: raise RuntimeError('You must specify an out_dir.') if use_db: exp_params = db.get_parameters(log=log, experiment=experiment, random=random)[0] else: exp = py_utils.import_module(experiment, pre_path='experiments') exp_params = exp.experiment_params() exp_params['_id'] = -1 exp_params['experiment'] = experiment if model is not None: exp_params['model'] = model else: assert len(exp_params['model']) > 1, 'No model name supplied.' exp_params['model'] = exp_params['model'][0] if train is not None: exp_params['train_dataset'] = train if val is not None: exp_params['val_dataset'] = val # if reduction or out_dir is not None or transfer: # fine_tune = get_fine_tune_params( # out_dir=out_dir, reduction=reduction) # else: # pass results = model_tools.build_model(exp_params=exp_params, dt_string=dt_string, log=log, test=test, config=main_config, use_db=use_db, num_batches=num_batches, map_out=map_out, placeholders=placeholders, add_config=add_config, gpu_device=gpu_device, cpu_device=cpu_device, checkpoint=checkpoint) if test and save_test_npz: # Save results somewhere safe py_utils.make_dir(out_dir) results['checkpoint'] = checkpoint results['model'] = model results['experiment'] = experiment np.savez(os.path.join(out_dir, results['exp_label']), **results) log.info('Finished.')