def encode_dataset(dataset): config = Config() data_class = py_utils.import_module(dataset) data_proc = data_class.data_processing() files, labels = data_proc.get_data() targets = data_proc.targets im_size = data_proc.im_size preproc_list = data_proc.preprocess if hasattr(data_proc, 'label_size'): label_size = data_proc.label_size else: label_size = None if hasattr(data_proc, 'label_size'): store_z = data_proc.store_z else: store_z = False if hasattr(data_proc, 'normalize_im'): normalize_im = data_proc.normalize_im else: normalize_im = False ds_name = os.path.join(config.tf_records, data_proc.output_name) data_to_tfrecords(files=files, labels=labels, targets=targets, ds_name=ds_name, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im)
def encode_dataset(dataset): config = Config() data_class = py_utils.import_module(dataset) data_proc = data_class.data_processing() files, labels = data_proc.get_data() targets = data_proc.targets im_size = data_proc.im_size preproc_list = data_proc.preprocess ds_name = os.path.join(config.tf_records, data_proc.name) data_to_tfrecords(files=files, labels=labels, targets=targets, ds_name=ds_name, im_size=im_size, preprocess=preproc_list)
def extract_dataset(dataset, config, cv, out_dir): """Save dataset npys into a directory""" dataset_module = py_utils.import_module(pre_path=config.dataset_classes, module=dataset) dataset_module = dataset_module.data_processing() (train_data, _, _) = py_utils.get_data_pointers( dataset=dataset_module.output_name, base_dir= "/media/data_cifs/cluttered_nist_experiments/tf_records", # config.tf_records, local_dir= "/media/data_cifs/cluttered_nist_experiments/tf_records", # config.local_tf_records, cv=cv) train_images, train_labels, train_aux = data_loader.inputs( dataset=train_data, batch_size=1000, # config.train_batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=[], # config.train_augmentations, num_epochs=1, # config.epochs, aux=None, # train_aux_loss, tf_reader_settings=dataset_module.tf_reader, shuffle=False) # config.shuffle_train) sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) count = 0 try: while not coord.should_stop(): images, labels = sess.run([train_images, train_labels]) np.savez(os.path.join(out_dir, "{}".format(count)), images=images, labels=labels) count += 1 except tf.errors.OutOfRangeError: print("Finished loop") finally: coord.request_stop() coord.join(threads) sess.close()
def main(experiment_name, list_experiments=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means = get_data_pointers(dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s' % experiment_name # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model if len(dataset_module.output_size) > 1: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, labels=train_labels, loss_type=config.loss_function, dataset_module=dataset_module) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=train_scores, labels=train_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('train images', train_images) tf.summary.scalar('training loss', train_loss) tf.summary.scalar('training accuracy', train_accuracy) log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=val_means, training=True, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built validation model.') val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, dataset_module=dataset_module) val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=val_scores, labels=val_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('val images', val_images) tf.summary.scalar('validation loss', val_loss) tf.summary.scalar('validation accuracy', val_accuracy) log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_accuracy': train_accuracy, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_accuracy': val_accuracy, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } # Start training loop np.save( os.path.join(dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') py_utils.save_npys(data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def main(experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, config_file=None, ckpt_file=None, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params config = np.load(config_file).item() dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) config.epochs = 1 config.shuffle = False with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=['resize_and_crop'], num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Normalize labels if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] log.info('Mean-centering labels.') # Training model if len(dataset_module.output_size) == 2: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): train_grad_images = tf.gradients( train_scores[0] * tf.cast(train_labels, tf.float32), train_images)[0] log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() if len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) elif len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels)[0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if len(train_labels.get_shape()) > 2: tf_fun.image_summaries(train_labels, tag='Training_targets') tf_fun.image_summaries(train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax(train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op(tag='training_pr', predictions=tf.cast( tf.argmax(train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, # False, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): val_grad_images = tf.gradients( val_scores[0] * tf.cast(val_labels, tf.float32), val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(val_scores.get_shape()) != len(val_labels.get_shape()): val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_labels = tf.expand_dims(val_labels, axis=-1) if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_scores = tf.expand_dims(val_scores, axis=-1) val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels)[0] # TODO: Fix for multiloss # Prepare tensorboard summaries if len(val_images.get_shape()) == 4: tf_fun.image_summaries(val_images, tag='Validation') if len(val_labels.get_shape()) > 2: tf_fun.image_summaries(val_labels, tag='Validation_targets') tf_fun.image_summaries(val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast(tf.argmax(val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op(tag='validation_pr', predictions=tf.cast( tf.argmax(val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores, 'train_grad_images': train_grad_images } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, 'val_grad_images': val_grad_images } if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop checkpoint_dir = dir_list['checkpoints'] step = 0 train_losses, train_accs, train_aux, timesteps = {}, {}, {}, {} val_scores, val_aux, val_labels, val_grads = {}, {}, {}, {} train_images, val_images = {}, {} train_scores, train_labels = {}, {} train_aux_check = np.any(['aux_score' in k for k in train_dict.keys()]) val_aux_check = np.any(['aux_score' in k for k in val_dict.keys()]) # Restore model saver.restore(sess, ckpt_file) # Start evaluation try: while not coord.should_stop(): start_time = time.time() train_vars = sess.run(train_dict.values()) it_train_dict = { k: v for k, v in zip(train_dict.keys(), train_vars) } duration = time.time() - start_time train_losses[step] = it_train_dict['train_loss'] train_accs[step] = it_train_dict['train_accuracy_0'] train_images[step] = it_train_dict['train_images'] train_labels[step] = it_train_dict['train_labels'] train_scores[step] = it_train_dict['train_scores'] timesteps[step] = duration if train_aux_check: # Loop through to find aux scores it_train_aux = { itk: itv for itk, itv in it_train_dict.iteritems() if 'aux_score' in itk } train_aux[step] = it_train_aux assert not np.isnan(it_train_dict['train_loss']).any( ), 'Model diverged with loss = NaN' if step % config.validation_iters == 0: it_val_scores, it_val_labels, it_val_aux, it_val_grads, it_val_ims = [], [], [], [], [] for num_vals in range(config.num_validation_evals): # Validation accuracy as the average of n batches val_vars = sess.run(val_dict.values()) it_val_dict = { k: v for k, v in zip(val_dict.keys(), val_vars) } it_val_labels += [it_val_dict['val_labels']] it_val_scores += [it_val_dict['val_scores']] it_val_grads += [it_val_dict['val_grad_images']] it_val_ims += [it_val_dict['val_images']] if val_aux_check: iva = { itk: itv for itk, itv in it_val_dict.iteritems() if 'aux_score' in itk } it_val_aux += [iva] val_scores[step] = it_val_scores val_labels[step] = it_val_labels val_aux[step] = it_val_aux val_images[step] = it_val_grads val_grads[step] = it_val_ims # End iteration step += 1 except tf.errors.OutOfRangeError: print 'Done with evaluation for %d epochs, %d steps.' % (config.epochs, step) print 'Saved to: %s' % checkpoint_dir finally: coord.request_stop() coord.join(threads) sess.close() import ipdb ipdb.set_trace() np.savez( 'val_imgs_grads', val_images=val_images, # it_val_dict['val_images'], val_grads=val_grads, # it_val_dict['val_grad_images'], val_labels=val_labels, # it_val_dict['val_labels'], val_scores=val_scores) # it_val_dict['val_scores'][0])
def main( experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, placeholder_data=None, grad_images=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) assert experiment_name is not None, 'Empty experiment name.' experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer from ops import evaluation config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key, log=log) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=val_key, log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join( config.checkpoints, condition_label), 'summaries': os.path.join( config.summaries, condition_label), 'condition_evaluations': os.path.join( config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join( config.visualizations, condition_label), 'weights': os.path.join( config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) if load_and_evaluate_ckpt is not None: config.epochs = 1 config.train_shuffle = False config.val_shuffle = False with tf.device('/cpu:0'): if placeholder_data: placeholder_shape = placeholder_data['train_image_shape'] placeholder_dtype = placeholder_data['train_image_dtype'] original_train_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_images') placeholder_shape = placeholder_data['train_label_shape'] placeholder_dtype = placeholder_data['train_label_dtype'] original_train_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_labels') placeholder_shape = placeholder_data['val_image_shape'] placeholder_dtype = placeholder_data['val_image_dtype'] original_val_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_images') placeholder_shape = placeholder_data['val_label_shape'] placeholder_dtype = placeholder_data['val_label_dtype'] original_val_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_labels') # Apply augmentations ( train_images, train_labels ) = data_loader.placeholder_image_augmentations( images=original_train_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_train_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) ( val_images, val_labels ) = data_loader.placeholder_image_augmentations( images=original_val_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_val_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) # Store in the placeholder dict placeholder_data['train_images'] = original_train_images placeholder_data['train_labels'] = original_train_labels placeholder_data['val_images'] = original_val_images placeholder_data['val_labels'] = original_val_labels else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join( 'models', 'structs', experiment_name).replace(os.path.sep, '.') ) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare variables for the models if len(dataset_module.output_size) == 2: log.warning( 'Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None # Correct number of output neurons if needed if config.dataloader_override and\ 'weights' in output_structure[-1].keys(): output_neurons = output_structure[-1]['weights'][0] size_check = output_neurons != dataset_module.output_size fc_check = output_structure[-1]['layers'][0] == 'fc' if size_check and fc_check: output_structure[-1]['weights'][0] = dataset_module.output_size log.warning('Adjusted output neurons from %s to %s.' % ( output_neurons, dataset_module.output_size)) # Prepare model on GPU if not hasattr(dataset_module, 'input_normalization'): dataset_module.input_normalization = None with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) train_scores, model_summary, _ = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(train_scores.get_shape()[-1]) target_scores = tf.one_hot(train_labels, oh_dims) * train_scores train_gradients = tf.gradients(target_scores, train_images)[0] log.info('Built training model.') log.debug( json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Normalize labels on GPU if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] val_labels -= train_means_label['mean'] val_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] val_labels -= val_means_label['mean'] log.info('Mean-centering labels.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len( train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) val_labels = tf.expand_dims(val_labels, axis=-1) elif len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) val_scores = tf.expand_dims(val_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) assert config.lr is not None, 'No learning rate.' # TODO: Make a QC function if config.lr > 1: old_lr = config.lr config.lr = loss_utils.create_lr_schedule( train_batch=config.batch_size, num_training=config.lr) config.optimizer = 'momentum' log.info('Forcing momentum classifier.') else: old_lr = None train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels) # [0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if config.tensorboard_images: if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if (np.asarray( train_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( train_labels, tag='Training_targets') tf_fun.image_summaries( train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax( train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op( tag='training_pr', predictions=tf.cast( tf.argmax( train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') with tf.variable_scope('cnn', tf.AUTO_REUSE) as scope: # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) val_scores, _, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(val_scores.get_shape()[-1]) target_scores = tf.one_hot(val_labels, oh_dims) * val_scores val_gradients = tf.gradients(target_scores, val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels) # [0] # TODO: Fix for multiloss # Prepare tensorboard summaries if config.tensorboard_images: if len(val_images.get_shape()) == 4: tf_fun.image_summaries( val_images, tag='Validation') if (np.asarray( val_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( val_labels, tag='Validation_targets') tf_fun.image_summaries( val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast( tf.argmax( val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op( tag='validation_pr', predictions=tf.cast( tf.argmax( val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver if not hasattr(config, 'max_to_keep'): config.max_to_keep = None saver = tf.train.Saver( var_list=tf.global_variables(), max_to_keep=config.max_to_keep) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading if placeholder_data: coord, threads = None, None else: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } if grad_images: train_dict['train_gradients'] = train_gradients val_dict['val_gradients'] = val_gradients if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop if old_lr is not None: config.lr = old_lr np.save( os.path.join( dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') if load_and_evaluate_ckpt is not None: return evaluation.evaluation_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params, placeholder_data=placeholder_data) else: output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') if output_dict is not None: py_utils.save_npys( data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def main( experiment_name, im_ext='.pdf', transform_loss=None, # 'log', colors='Paired', flip_axis=False, exclude=None): """Plot results of provided experiment name.""" config = Config() pl_creds = credentials.plotly_credentials() py.sign_in( pl_creds['username'], pl_creds['api_key']) # Get experiment data perf = db.get_performance(experiment_name=experiment_name) if len(perf) == 0: raise RuntimeError('Could not find any results.') structure_names = [x['model_struct'].split('/')[-1] for x in perf] optimizers = [x['optimizer'] for x in perf] lrs = [x['lr'] for x in perf] datasets = [x['dataset'] for x in perf] loss_funs = [x['loss_function'] for x in perf] optimizers = [x['optimizer'] for x in perf] wd_types = [x['regularization_type'] for x in perf] wd_penalties = [x['regularization_strength'] for x in perf] steps = [float(x['training_step']) for x in perf] training_loss = [float(x['training_loss']) for x in perf] validation_loss = [float(x['validation_loss']) for x in perf] timesteps = [0. if x['timesteps'] is None else float(x['timesteps']) for x in perf] u_t = [0. if x['u_t'] is None else float(x['u_t']) for x in perf] q_t = [0. if x['q_t'] is None else float(x['q_t']) for x in perf] p_t = [0. if x['p_t'] is None else float(x['p_t']) for x in perf] t_t = [0. if x['t_t'] is None else float(x['t_t']) for x in perf] # Pass data into a pandas DF model_params = [ '%s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s | %s' % ( ipa, ipb, ipc, ipd, ipe, ipf, ipg, iph, ipi, ipj, ipk, ipl, ipm) for ipa, ipb, ipc, ipd, ipe, ipf, ipg, iph, ipi, ipj, ipk, ipl, ipm in zip( structure_names, optimizers, lrs, loss_funs, optimizers, wd_types, wd_penalties, datasets, timesteps, u_t, q_t, p_t, t_t)] # DF and plot df = pd.DataFrame( np.vstack( ( model_params, steps, training_loss, validation_loss ) ).transpose(), columns=[ 'model parameters', 'training iteration', 'training loss', 'validation loss' ] ) df['training iteration'] = pd.to_numeric( df['training iteration'], errors='coerce') df['training loss'] = pd.to_numeric(df['training loss'], errors='coerce') if exclude is not None: exclusion_search = df['model parameters'].str.contains(exclude) df = df[exclusion_search == False] print 'Removed %s rows.' % exclusion_search.sum() # Start plotting experiment_dict = experiments.experiments()[experiment_name]() print 'Plotting results for dataset: %s.' % experiment_dict['dataset'][0] dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=experiment_dict['dataset'][0]) dataset_module = dataset_module.data_processing() # hardcoded class name if transform_loss is None: loss_label = '' elif transform_loss == 'log': loss_label = ' log loss' df['training loss'] = np.log(df['training loss']) elif transform_loss == 'max': loss_label = ' normalized (x / max(x)) ' df['training loss'] /= df.groupby( 'model parameters')['training loss'].transform(max) if ['loss_function'] in experiment_dict.keys(): loss_metric = experiment_dict['loss_function'][0] else: loss_metric = dataset_module.default_loss_function df['validation loss'] = pd.to_numeric(df['validation loss']) if loss_metric == 'pearson': loss_label = 'Pearson correlation' + loss_label elif loss_metric == 'l2': loss_label = 'L2' + loss_label else: loss_label = 'Classification accuracy (%)' df['validation loss'] *= 100. if ['score_metric'] in experiment_dict.keys(): score_metric = experiment_dict['score_metric'] else: score_metric = dataset_module.score_metric if score_metric == 'pearson': y_lab = 'Pearson correlation' matplotlib.style.use('ggplot') plt.rc('font', size=6) plt.rc('legend', fontsize=8, labelspacing=3) f, axs = plt.subplots(2, figsize=(20, 30)) ax = axs[1] NUM_COLORS = len(df['model parameters'].unique()) cm = plt.get_cmap('gist_rainbow') ax.set_color_cycle([cm(1.*i/NUM_COLORS) for i in range(NUM_COLORS)]) for k in df['model parameters'].unique(): tmp = df[df['model parameters'] == k] tmp = tmp.sort('training iteration') ax = tmp.plot( x='training iteration', y='training loss', label=k, kind='line', ax=ax, logy=False) plt.setp(ax.xaxis.get_majorticklabels(), rotation=30) ax.xaxis.set_major_locator(MaxNLocator(integer=True)) ax.set_title('Training') ax.set_ylabel(loss_label) # ax.legend_.remove() ax = axs[0] ax.set_color_cycle([cm(1.*i/NUM_COLORS) for i in range(NUM_COLORS)]) for k in df['model parameters'].unique(): tmp = df[df['model parameters'] == k] tmp = tmp.sort('training iteration') ax = tmp.plot( x='training iteration', y='validation loss', label=k, kind='line', ax=ax, logy=False) plt.setp(ax.xaxis.get_majorticklabels(), rotation=30) ax.xaxis.set_major_locator(MaxNLocator(integer=True)) ax.set_title('Validation') # TODO: Mine the experiment declarations for the appropos metric name. ax.set_ylabel(y_lab) # ax.legend_.remove() out_name = os.path.join( config.plots, '%s_%s%s' % ( experiment_name, py_utils.get_dt_stamp(), im_ext)) plt.savefig(out_name) print 'Saved to: %s' % out_name plotly_fig = tls.mpl_to_plotly(f) plotly_fig['layout']['autosize'] = True # plotly_fig['layout']['showlegend'] = True plot_with_plotly(plotly_fig, 'line') plt.close(f) # Plot max performance bar graph f = plt.figure() max_perf = df.groupby( ['model parameters'], as_index=False)['validation loss'].max() plt.rc('xtick', labelsize=2) ax = max_perf.plot.bar( x='model parameters', y='validation loss', legend=False) plt.tight_layout() ax.set_title('Max validation value') ax.set_ylabel(y_lab) out_name = os.path.join( config.plots, '%s_%s_bar%s' % ( experiment_name, py_utils.get_dt_stamp(), im_ext)) plt.savefig(out_name) print 'Saved to: %s' % out_name try: plotly_fig = tls.mpl_to_plotly(f) plot_with_plotly(plotly_fig, chart='bar') except Exception as e: print 'Failed to plot bar chart in plotly: %s' % e plt.close(f)
def model_builder( params, config, model_spec, gpu_device, cpu_device, placeholders=False, tensorboard_images=False): """Standard model building routines.""" config = py_utils.add_to_config( d=params, config=config) exp_label = '%s_%s' % (params['exp_name'], py_utils.get_dt_stamp()) directories = py_utils.prepare_directories(config, exp_label) dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] ( train_data, train_means_image, train_means_label) = py_utils.get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] if hasattr(config, 'val_dataset'): val_dataset = config.val_dataset else: val_dataset = config.dataset val_data, val_means_image, val_means_label = py_utils.get_data_pointers( dataset=val_dataset, base_dir=config.tf_records, cv=val_key) # Create data tensors with tf.device(cpu_device): if placeholders: ( train_images, train_labels, val_images, val_labels) = get_placeholders(dataset_module, config) placeholders = dataset_module.get_data() else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train) val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val) # Build training and val models with tf.device(gpu_device): train_logits, train_hgru_act = model_spec( data_tensor=train_images, reuse=None, training=True) val_logits, val_hgru_act = model_spec( data_tensor=val_images, reuse=tf.AUTO_REUSE, training=False) # Derive loss loss_type = None if hasattr(config, 'loss_type'): loss_type = config.loss_type train_loss = losses.derive_loss( labels=train_labels, logits=train_logits, loss_type=loss_type) val_loss = losses.derive_loss( labels=val_labels, logits=val_logits, loss_type=loss_type) if hasattr(config, 'metric_type'): metric_type = config.metric_type else: metric_type = 'accuracy' if metric_type == 'pearson': train_accuracy = metrics.pearson_score( labels=train_labels, pred=train_logits, REDUCTION=tf.reduce_mean) val_accuracy = metrics.pearson_score( labels=val_labels, pred=val_logits, REDUCTION=tf.reduce_mean) else: train_accuracy = metrics.class_accuracy( labels=train_labels, logits=train_logits) val_accuracy = metrics.class_accuracy( labels=val_labels, logits=val_logits) tf.summary.scalar('train_accuracy', train_accuracy) tf.summary.scalar('val_accuracy', val_accuracy) if tensorboard_images: tf.summary.image('train_images', train_images) tf.summary.image('val_images', val_images) # Build optimizer train_op = optimizers.get_optimizer( train_loss, config['lr'], config['optimizer']) # Initialize tf variables saver = tf.train.Saver( var_list=tf.global_variables()) summary_op = tf.summary.merge_all() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer())) summary_writer = tf.summary.FileWriter( directories['summaries'], sess.graph) if not placeholders: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) else: coord, threads = None, None # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_accuracy': train_accuracy } if isinstance(train_hgru_act, dict): for k, v in train_hgru_act.iteritems(): train_dict[k] = v else: train_dict['activity'] = train_hgru_act val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_accuracy': val_accuracy, } if isinstance(val_hgru_act, dict): for k, v in val_hgru_act.iteritems(): val_dict[k] = v else: val_dict['activity'] = val_hgru_act # Count parameters num_params = np.sum( [np.prod(x.get_shape().as_list()) for x in tf.trainable_variables()]) print 'Model has approximately %s trainable params.' % num_params # Create datastructure for saving data ds = data_structure.data( batch_size=config.batch_size, validation_iters=config.validation_iters, num_validation_evals=config.num_validation_evals, shuffle_val=config.shuffle_val, lr=config.lr, loss_function=config.loss_function, optimizer=config.optimizer, model_name=config.model_name, dataset=config.dataset, num_params=num_params, output_directory=config.results) # Start training loop training.training_loop( config=config, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, directories=directories, train_dict=train_dict, val_dict=val_dict, exp_label=exp_label, data_structure=ds, placeholders=placeholders)
def build_model(exp_params, config, log, dt_string, gpu_device, cpu_device, use_db=True, add_config=None, placeholders=False, checkpoint=None, test=False, map_out=None, num_batches=None, tensorboard_images=False): """Standard model building routines.""" config = py_utils.add_to_config(d=exp_params, config=config) if not hasattr(config, 'force_path'): config.force_path = False exp_label = '%s_%s_%s' % (exp_params['model'], exp_params['experiment'], py_utils.get_dt_stamp()) directories = py_utils.prepare_directories(config, exp_label) dataset_module = py_utils.import_module(pre_path=config.dataset_classes, module=config.train_dataset) train_dataset_module = dataset_module.data_processing() if not config.force_path: (train_data, _, _) = py_utils.get_data_pointers( dataset=train_dataset_module.output_name, base_dir=config.tf_records, local_dir=config.local_tf_records, cv='train') else: train_data = train_dataset_module.train_path dataset_module = py_utils.import_module(pre_path=config.dataset_classes, module=config.val_dataset) val_dataset_module = dataset_module.data_processing() if not config.force_path: val_data, _, _ = py_utils.get_data_pointers( dataset=val_dataset_module.output_name, base_dir=config.tf_records, local_dir=config.local_tf_records, cv='val') else: val_data = train_dataset_module.val_path # val_means_image, val_means_label = None, None # Create data tensors if hasattr(train_dataset_module, 'aux_loss'): train_aux_loss = train_dataset_module.aux_loss else: train_aux_loss = None with tf.device(cpu_device): if placeholders and not test: # Train with placeholders (pl_train_images, pl_train_labels, pl_val_images, pl_val_labels, train_images, train_labels, val_images, val_labels) = get_placeholders(train_dataset=train_dataset_module, val_dataset=val_dataset_module, config=config) train_module_data = train_dataset_module.get_data() val_module_data = val_dataset_module.get_data() placeholders = { 'train': { 'images': train_module_data[0]['train'], 'labels': train_module_data[1]['train'] }, 'val': { 'images': val_module_data[0]['val'], 'labels': val_module_data[1]['val'] }, } train_aux, val_aux = None, None elif placeholders and test: test_dataset_module = train_dataset_module # Test with placeholders (pl_test_images, pl_test_labels, test_images, test_labels) = get_placeholders_test( test_dataset=test_dataset_module, config=config) test_module_data = test_dataset_module.get_data() placeholders = { 'test': { 'images': test_module_data[0]['test'], 'labels': test_module_data[1]['test'] }, } train_aux, val_aux = None, None else: train_images, train_labels, train_aux = data_loader.inputs( dataset=train_data, batch_size=config.train_batch_size, model_input_image_size=train_dataset_module. model_input_image_size, tf_dict=train_dataset_module.tf_dict, data_augmentations=config.train_augmentations, num_epochs=config.epochs, aux=train_aux_loss, tf_reader_settings=train_dataset_module.tf_reader, shuffle=config.shuffle_train) if hasattr(val_dataset_module, 'val_model_input_image_size'): val_dataset_module.model_input_image_size = val_dataset_module.val_model_input_image_size val_images, val_labels, val_aux = data_loader.inputs( dataset=val_data, batch_size=config.val_batch_size, model_input_image_size=val_dataset_module. model_input_image_size, tf_dict=val_dataset_module.tf_dict, data_augmentations=config.val_augmentations, num_epochs=None, tf_reader_settings=val_dataset_module.tf_reader, shuffle=config.shuffle_val) # Build training and val models model_spec = py_utils.import_module(module=config.model, pre_path=config.model_classes) if hasattr(train_dataset_module, 'force_output_size'): train_dataset_module.output_size = train_dataset_module.force_output_size if hasattr(val_dataset_module, 'force_output_size'): val_dataset_module.output_size = val_dataset_module.force_output_size if hasattr(config, 'loss_function'): train_loss_function = config.loss_function val_loss_function = config.loss_function else: train_loss_function = config.train_loss_function val_loss_function = config.val_loss_function # Route test vs train/val h_check = [ x for x in tf.trainable_variables() if 'homunculus' in x.name or 'humonculus' in x.name ] if not hasattr(config, 'default_restore'): config.default_restore = False if test: assert len(gpu_device) == 1, 'Testing only works with 1 gpu.' gpu_device = gpu_device[0] with tf.device(gpu_device): if not placeholders: test_images = val_images test_labels = val_labels test_dataset_module = val_dataset_module test_logits, test_vars = model_spec.build_model( data_tensor=test_images, reuse=None, training=False, output_shape=test_dataset_module.output_size) if test_logits.dtype is not tf.float32: test_logits = tf.cast(test_logits, tf.float32) # Derive loss if not hasattr(config, 'test_loss_function'): test_loss_function = val_loss_function else: test_loss_function = config.test_loss_function test_loss = losses.derive_loss(labels=test_labels, logits=test_logits, loss_type=test_loss_function) # Derive score test_score = losses.derive_score(labels=test_labels, logits=test_logits, loss_type=test_loss_function, score_type=config.score_function) # Initialize model (sess, saver, summary_op, summary_writer, coord, threads, restore_saver) = initialize_tf(config=config, placeholders=placeholders, ckpt=checkpoint, default_restore=config.default_restore, directories=directories) if placeholders: proc_images = test_images proc_labels = test_labels test_images = pl_test_images test_labels = pl_test_labels _, H, W, _ = test_vars['model_output_y'].shape jacobian = tf.gradients(test_logits, test_vars['model_output_x'])[ 0] # g.batch_jacobian(test_vars['model_output_x'], test_images) test_dict = { 'test_loss': test_loss, 'test_score': test_score, 'test_images': test_images, 'test_labels': test_labels, 'test_logits': test_logits, 'test_jacobian': jacobian } if placeholders: test_dict['test_proc_images'] = proc_images test_dict['test_proc_labels'] = proc_labels if len(h_check): test_dict['homunculus'] = h_check[0] if isinstance(test_vars, dict): for k, v in test_vars.iteritems(): test_dict[k] = v else: test_dict['activity'] = test_vars else: train_losses, val_losses, tower_grads, norm_updates = [], [], [], [] train_scores, val_scores = [], [] train_image_list, train_label_list = [], [] val_image_list, val_label_list = [], [] train_reuse = None if not hasattr(config, 'lr_schedule'): config.lr_schedule = None if hasattr(config, 'loss_function'): train_loss_function = config.loss_function val_loss_function = config.loss_function else: train_loss_function = config.train_loss_function val_loss_function = config.val_loss_function # Prepare loop if not placeholders: train_batch_queue = tf_fun.get_batch_queues(images=train_images, labels=train_labels, gpu_device=gpu_device) val_batch_queue = tf_fun.get_batch_queues(images=val_images, labels=val_labels, gpu_device=gpu_device) config.lr = optimizers.get_lr_schedule(lr=config.lr, lr_schedule=config.lr_schedule) opt = optimizers.get_optimizers(optimizer=config.optimizer, lr=config.lr, dtype=train_images.dtype) with tf.device(cpu_device): global_step = tf.train.get_or_create_global_step() for i, gpu in enumerate(gpu_device): # rs = tf.AUTO_REUSE if i > 0 else None with tf.device(gpu): with tf.name_scope('tower_%d' % i) as scope: # Prepare tower data if placeholders: # Multi-gpu: will have to split # train_images per gpu by hand train_image_batch = train_images val_image_batch = val_images train_label_batch = train_labels val_label_batch = val_labels else: (train_image_batch, train_label_batch) = train_batch_queue.dequeue() (val_image_batch, val_label_batch) = val_batch_queue.dequeue() train_image_list += [train_image_batch] train_label_list += [train_label_batch] val_image_list += [val_image_batch] val_label_list += [val_label_batch] # Build models train_logits, train_vars = model_spec.build_model( data_tensor=train_image_batch, reuse=train_reuse, training=True, output_shape=train_dataset_module.output_size) num_training_vars = len(tf.trainable_variables()) val_logits, val_vars = model_spec.build_model( data_tensor=val_image_batch, reuse=True, training=False, output_shape=val_dataset_module.output_size) num_validation_vars = len(tf.trainable_variables()) assert num_training_vars == num_validation_vars, \ 'Found a different # of train and val variables.' train_reuse = True # Derive losses if train_logits.dtype is not tf.float32: train_logits = tf.cast(train_logits, tf.float32) if val_logits.dtype is not tf.float32: val_logits = tf.cast(val_logits, tf.float32) train_loss = losses.derive_loss( labels=train_label_batch, logits=train_logits, images=train_image_batch, loss_type=train_loss_function) val_loss = losses.derive_loss( labels=val_label_batch, logits=val_logits, images=val_image_batch, loss_type=val_loss_function) # Derive score train_score = losses.derive_score( labels=train_labels, logits=train_logits, loss_type=train_loss_function, score_type=config.score_function) val_score = losses.derive_score( labels=val_labels, logits=val_logits, loss_type=val_loss_function, score_type=config.score_function) # Add aux losses if requested if hasattr(model_spec, 'weight_decay'): wd = (model_spec.weight_decay() * tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'batch_normalization' not in v.name and 'horizontal' not in v.name and 'mu' not in v.name and 'beta' not in v.name and 'intercept' not in v.name ])) tf.summary.scalar('weight_decay', wd) train_loss += wd if hasattr(model_spec, 'bsds_weight_decay'): wd = (model_spec.bsds_weight_decay()['l2'] * tf.add_n([ tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'horizontal' not in v.name and 'norm' not in v.name ])) tf.summary.scalar('weight_decay_readout', wd) train_loss += wd wd = (model_spec.bsds_weight_decay()['l1'] * tf.add_n([ tf.reduce_sum(tf.abs(v)) for v in tf.trainable_variables() if 'horizontal' in v.name ])) tf.summary.scalar('weight_decay_horizontal', wd) train_loss += wd if hasattr(model_spec, 'orthogonal'): weights = [ v for v in tf.trainable_variables() if 'horizontal' in v.name ] assert len(weights) is not None, \ 'No horizontal weights for laplace.' wd = model_spec.orthogonal() * tf.add_n( [tf_fun.orthogonal(w) for w in weights]) tf.summary.scalar('weight_decay', wd) train_loss += wd if hasattr(model_spec, 'laplace'): weights = [ v for v in tf.trainable_variables() if 'horizontal' in v.name ] assert len(weights) is not None, \ 'No horizontal weights for laplace.' wd = model_spec.laplace() * tf.add_n( [tf_fun.laplace(w) for w in weights]) tf.summary.scalar('weight_decay', wd) train_loss += wd # Derive auxilary losses if hasattr(config, 'aux_loss'): aux_loss_type, scale = config.aux_loss.items()[0] for k, v in train_vars.iteritems(): # if k in train_dataset_module.aux_loss.keys(): # ( # aux_loss_type, # scale # ) = train_dataset_module.aux_loss[k] train_loss += (losses.derive_loss( labels=train_labels, logits=v, loss_type=aux_loss_type) * scale) # Gather everything train_losses += [train_loss] val_losses += [val_loss] train_scores += [train_score] val_scores += [val_score] # Compute and store gradients with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): grads = opt.compute_gradients(train_loss) optimizers.check_grads(grads) tower_grads += [grads] # Gather normalization variables norm_updates += [ tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=scope) ] # Recompute and optimize gradients grads = optimizers.average_gradients(tower_grads) if hasattr(config, 'clip_gradients') and config.clip_gradients: grads = optimizers.apply_grad_clip(grads, config.clip_gradients) op_vars = [] if hasattr(config, 'exclusion_lr') and hasattr(config, 'exclusion_scope'): grads_0 = [ x for x in grads if config.exclusion_scope not in x[1].name ] grads_1 = [x for x in grads if config.exclusion_scope in x[1].name] op_vars_0 = optimizers.apply_gradients(opt=opt, grads=grads_0, global_step=global_step) opt_1 = optimizers.get_optimizers(optimizer=config.optimizer, lr=config.exclusion_lr, dtype=train_images.dtype) op_vars_1 = optimizers.apply_gradients(opt=opt_1, grads=grads_1, global_step=global_step) op_vars += [op_vars_0] op_vars += [op_vars_1] else: op_vars += [ optimizers.apply_gradients(opt=opt, grads=grads, global_step=global_step) ] if not hasattr(config, 'variable_moving_average'): config.variable_moving_average = False if config.variable_moving_average: variable_averages = tf.train.ExponentialMovingAverage( config.variable_moving_average, global_step) op_vars += [variable_averages.apply(tf.trainable_variables())] if len(norm_updates): op_vars += [tf.group(*norm_updates)] train_op = tf.group(*op_vars) # Summarize losses and scores train_loss = tf.reduce_mean(train_losses) val_loss = tf.reduce_mean(val_losses) train_score = tf.reduce_mean(train_scores) val_score = tf.reduce_mean(val_scores) if len(train_image_list) > 1: train_image_list = tf.stack(train_image_list, axis=0) train_label_list = tf.stack(train_label_list, axis=0) else: train_image_list = train_image_list[0] train_label_list = train_label_list[0] if len(val_image_list) > 1: val_image_list = tf.stack(val_image_list, axis=0) val_label_list = tf.stack(val_label_list, axis=0) else: val_image_list = val_image_list[0] val_label_list = val_label_list[0] tf.summary.scalar('train_loss', train_loss) tf.summary.scalar('val_loss', val_loss) if tensorboard_images: tf.summary.image('train_images', train_images) tf.summary.image('val_images', val_images) # Initialize model (sess, saver, summary_op, summary_writer, coord, threads, restore_saver) = initialize_tf(config=config, placeholders=placeholders, ckpt=checkpoint, default_restore=config.default_restore, directories=directories) # Create dictionaries of important training and validation information if placeholders: proc_train_images = train_images proc_train_labels = train_labels proc_val_images = val_images proc_val_labels = val_labels train_images = pl_train_images train_labels = pl_train_labels val_images = pl_val_images val_labels = pl_val_labels train_dict = { 'train_loss': train_loss, 'train_score': train_score, 'train_images': train_image_list, 'train_labels': train_label_list, 'train_logits': train_logits, 'train_op': train_op } if placeholders: train_dict['proc_train_images'] = proc_train_images train_dict['proc_train_labels'] = proc_train_labels if train_aux is not None: train_dict['train_aux'] = train_aux if tf.contrib.framework.is_tensor(config.lr): train_dict['lr'] = config.lr else: train_dict['lr'] = tf.constant(config.lr) if isinstance(train_vars, dict): for k, v in train_vars.iteritems(): train_dict[k] = v else: train_dict['activity'] = train_vars if hasattr(config, 'save_gradients') and config.save_gradients: grad = tf.gradients(train_logits, train_images)[0] if grad is not None: train_dict['gradients'] = grad else: log.warning('Could not calculate val gradients.') val_dict = { 'val_loss': val_loss, 'val_score': val_score, 'val_images': val_image_list, 'val_logits': val_logits, 'val_labels': val_label_list, } if placeholders: val_dict['proc_val_images'] = proc_val_images val_dict['proc_val_labels'] = proc_val_labels if val_aux is not None: val_dict['aux'] = val_aux if isinstance(val_vars, dict): for k, v in val_vars.iteritems(): val_dict[k] = v else: val_dict['activity'] = val_vars if hasattr(config, 'save_gradients') and config.save_gradients: grad = tf.gradients(val_logits, val_images)[0] if grad is not None: val_dict['gradients'] = grad else: log.warning('Could not calculate val gradients.') if len(h_check): val_dict['homunculus'] = h_check[0] # Add optional info to the config if add_config is not None: extra_list = add_config.split(',') for eidx, extra in enumerate(extra_list): setattr(config, 'extra_%s' % eidx, extra) # Count parameters num_params = tf_fun.count_parameters(var_list=tf.trainable_variables()) print 'Model has approximately %s trainable params.' % num_params if test: return training.test_loop(log=log, config=config, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, restore_saver=restore_saver, directories=directories, test_dict=test_dict, exp_label=exp_label, num_params=num_params, checkpoint=checkpoint, num_batches=num_batches, save_weights=config.save_weights, save_checkpoints=config.save_checkpoints, save_activities=config.save_activities, save_gradients=config.save_gradients, map_out=map_out, placeholders=placeholders) else: # Start training loop training.training_loop(log=log, config=config, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, restore_saver=restore_saver, threads=threads, directories=directories, train_dict=train_dict, val_dict=val_dict, exp_label=exp_label, num_params=num_params, checkpoint=checkpoint, use_db=use_db, save_weights=config.save_weights, save_checkpoints=config.save_checkpoints, save_activities=config.save_activities, save_gradients=config.save_gradients, placeholders=placeholders)
def main(reset_process, initialize_db, experiment_name, remove=None, force_repeat=None): """Populate db with experiments to run.""" main_config = config.Config() log = logger.get(os.path.join(main_config.log_dir, 'prepare_experiments')) if reset_process: db.reset_in_process() log.info('Reset experiment progress counter in DB.') if initialize_db: db.initialize_database() log.info('Initialized DB.') if remove is not None: db_config = credentials.postgresql_connection() with db.db(db_config) as db_conn: db_conn.remove_experiment(remove) log.info('Removed %s.' % remove) if experiment_name is not None: # TODO: add capability for bayesian opt. if ',' in experiment_name: # Parse a comma-delimeted string of experiments experiment_name = experiment_name.split(',') else: experiment_name = [experiment_name] db_config = credentials.postgresql_connection() for exp in experiment_name: experiment_dict = py_utils.import_module( module=exp, pre_path=main_config.experiment_classes) experiment_dict = experiment_dict.experiment_params() if not hasattr(experiment_dict, 'get_map'): experiment_dict['get_map'] = [False] train_loss_function = experiment_dict.get('train_loss_function', None) if train_loss_function is None: experiment_dict['train_loss_function'] = experiment_dict[ 'loss_function'] experiment_dict['val_loss_function'] = experiment_dict[ 'loss_function'] experiment_dict.pop('loss_function', None) exp_combos = package_parameters(experiment_dict, log) log.info('Preparing experiment.') assert exp_combos is not None, 'Experiment is empty.' # Repeat if requested repeats = experiment_dict.get('repeat', 0) if force_repeat is not None: repeats = force_repeat if repeats: dcs = [] for copy in range(repeats): # Need to make deep copies dcs += deepcopy(exp_combos) exp_combos = dcs log.info('Expanded %sx to %s combinations.' % (experiment_dict['repeat'], len(exp_combos))) # Convert augmentations to json json_combos = [] for combo in exp_combos: combo['train_augmentations'] = json.dumps( deepcopy(combo['train_augmentations'])) combo['val_augmentations'] = json.dumps( deepcopy(combo['val_augmentations'])) json_combos += [combo] # Add data to the DB with db.db(db_config) as db_conn: db_conn.populate_db(json_combos) db_conn.return_status('CREATE') log.info('Added new experiments.')
def main(experiment, model, train, val, checkpoint, use_db=True, test=False, reduction=0, random=True, add_config=None, gpu_device=['/gpu:0'], cpu_device='/cpu:0', num_gpus=False, transfer=False, placeholders=False, save_test_npz=True, num_batches=None, map_out='test_maps', out_dir=None): """Interpret and run a model.""" main_config = Config() dt_string = py_utils.get_dt_stamp() log = logger.get( os.path.join(main_config.log_dir, '%s_%s' % (experiment, dt_string))) if num_gpus: gpu_device = ['/gpu:%d' % i for i in range(num_gpus)] if test and save_test_npz and out_dir is None: raise RuntimeError('You must specify an out_dir.') if use_db: exp_params = db.get_parameters(log=log, experiment=experiment, random=random)[0] else: exp = py_utils.import_module(experiment, pre_path='experiments') exp_params = exp.experiment_params() exp_params['_id'] = -1 exp_params['experiment'] = experiment if model is not None: exp_params['model'] = model else: assert len(exp_params['model']) > 1, 'No model name supplied.' exp_params['model'] = exp_params['model'][0] if train is not None: exp_params['train_dataset'] = train if val is not None: exp_params['val_dataset'] = val # if reduction or out_dir is not None or transfer: # fine_tune = get_fine_tune_params( # out_dir=out_dir, reduction=reduction) # else: # pass results = model_tools.build_model(exp_params=exp_params, dt_string=dt_string, log=log, test=test, config=main_config, use_db=use_db, num_batches=num_batches, map_out=map_out, placeholders=placeholders, add_config=add_config, gpu_device=gpu_device, cpu_device=cpu_device, checkpoint=checkpoint) if test and save_test_npz: # Save results somewhere safe py_utils.make_dir(out_dir) results['checkpoint'] = checkpoint results['model'] = model results['experiment'] = experiment np.savez(os.path.join(out_dir, results['exp_label']), **results) log.info('Finished.')
def encode_dataset(dataset, train_shards=0, val_shards=0, force_val=False): config = Config() data_class = py_utils.import_module(module=dataset, pre_path=config.dataset_classes) data_proc = data_class.data_processing() data = data_proc.get_data() if len(data) == 2: files, labels = data nhot = None elif len(data) == 3: files, labels, nhot = data else: raise NotImplementedError targets = data_proc.targets im_size = data_proc.im_size if hasattr(data_proc, 'preprocess'): preproc_list = data_proc.preprocess else: preproc_list = [] if hasattr(data_proc, 'label_size'): label_size = data_proc.label_size else: label_size = None if hasattr(data_proc, 'label_size'): store_z = data_proc.store_z else: store_z = False if hasattr(data_proc, 'normalize_im'): normalize_im = data_proc.normalize_im else: normalize_im = False if not train_shards: ds_name = os.path.join(config.tf_records, data_proc.output_name) data_to_tfrecords(files=files, labels=labels, targets=targets, nhot=nhot, ds_name=ds_name, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im) else: assert val_shards > 0, 'Choose the number of val shards.' raise NotImplementedError('Needs support for nhot.') shard_dir = os.path.join(config.tf_records, data_proc.output_name) py_utils.make_dir(shard_dir) if not force_val: create_shards(it_shards=train_shards, shard_dir=shard_dir, key='train', files=files, labels=labels, targets=targets, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im) create_shards(it_shards=val_shards, shard_dir=shard_dir, key='val', files=files, labels=labels, targets=targets, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im)
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, num_models=3, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_type='tfrecord', top_n=1, grad='lrp', target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join( allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()[ 'experiment_name']} if query_db: perf = query_hp_hist( exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT for gd in data_files: mt = gd.split( os.path.sep)[-1].split( template_exp + '_')[-1].split('_' + 'val')[0] it_data = np.load(gd).item() sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view( np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] scores = [] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] # Only get top conv/sep spots sperfs = sperfs[sel_mts != 'dog'] sperfs = sperfs[sel_mts != 'DoG'] scores += [sperfs.mean() / sperfs.std()] best_fits = np.argmax(np.asarray(scores)) xs = np.asarray([uxys[best_fits][0]]) ys = np.asarray([uxys[best_fits][1]]) sel_idx = (xys == uxys[best_fits]).sum(axis=-1) == 2 perfs = np.asarray(perfs[sel_idx]) exps = np.asarray(exps[sel_idx]) model_types = np.asarray(model_types[sel_idx]) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] for idx in sorted_perfs: perf = sel_exp_query( experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % ( model_file, model_file.split(os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull stimuli stim_dir = os.path.join( main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] # stim_val_data = [x for x in stim_files if 'val.tfrecords' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Store sparse noise for reference sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): if stimulus_type == 'sparse_noise': pass elif stimulus_type == 'drifting_grating': pass elif stimulus_type == 'tfrecord': val_images, val_labels = data_loader.inputs( dataset=stim_val_data, batch_size=1, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=[None], # dataset_module.preprocess, num_epochs=1, tf_reader_settings=dataset_module.tf_reader, shuffle=False ) # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append(os.path.join('models', 'structs', sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad == 'vanilla': grad_image = tf.gradients(model.output, val_images)[0] elif grad == 'lrp': eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map( {'Relu': 'GradLRP'}): grad_image = tf.gradients(model.output, val_images)[0] elif grad == 'cam': eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map( {'Relu': 'GuidedRelu'}): grad_image = tf.gradients(model.output, val_images)[0] else: raise NotImplementedError print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) saver.restore(sess, model_ckpt) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name] elif target_model == 'dog' or target_model == 'DoG': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name] else: raise NotImplementedError val_tensors = { 'images': val_images, 'labels': val_labels, 'filts': fname, 'responses': model.output, # model[target_layer], 'grads': grad_image } all_images, all_preds, all_grads, all_responses = [], [], [], [] step = 0 try: while not coord.should_stop(): val_vals = sess.run(val_tensors.values()) val_dict = {k: v for k, v in zip(val_tensors.keys(), val_vals)} all_images += [val_dict['images']] all_responses += [val_dict['responses']] all_preds += [val_dict['labels'].squeeze()] all_grads += [val_dict['grads'].squeeze()] print 'Finished step %s' % step step += 1 except: print 'Finished tfrecords' finally: coord.request_stop() coord.join(threads) sess.close() # Process and save data # if target_model != 'dog': # filters = val_dict['filts'][0].squeeze().transpose(2, 0, 1) all_images = np.concatenate(all_images).squeeze() all_grads = np.asarray(all_grads) all_preds = np.asarray(all_preds).reshape(-1, 1) all_responses = np.asarray(all_responses).squeeze() np.savez( os.path.join(data_dir, 'data'), images=all_images, pred=all_preds, # filters=filters, grads=all_grads) # if target_model != 'dog': # save_mosaic( # maps=filters, # [0].squeeze().transpose(2, 0, 1), # output=os.path.join(data_dir, '%s_filters' % target_layer), # rc=8, # cc=4, # title='%s filters' % ( # target_layer)) print 'Complete.'
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_dir='/media/data_cifs/AllenData/DataForTrain/all_stimulus_template', stimulus_type='tfrecord', top_n=100, recalc=False, preload_stim=False, target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join(allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()['experiment_name'] } if query_db: perf = query_hp_hist(exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT score_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_scores.npy')) # Scores has preds, labels has GT lab_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_labels.npy')) # Scores has preds, labels has GT for gd, sd, ld in zip(data_files, score_files, lab_files): mt = gd.split(os.path.sep)[-1].split(template_exp + '_')[-1].split('_' + 'val')[0] if not recalc: it_data = np.load(gd).item() else: lds = np.load(ld).item() sds = np.load(sd).item() it_data = { k: np.corrcoef(lds[k], sds[k])[0, 1] for k in sds.keys() } sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view(np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] bp = np.argmax(sperfs) fxs += [xy[0]] fys += [xy[1]] fperfs += [sperfs[bp]] fargs += [sargs[bp]] fmodel_types += [sel_mts[bp]] fexps += [sexps[bp]] xs = np.asarray(fxs) ys = np.asarray(fys) perfs = np.asarray(fperfs) arg_perf = np.asarray(fargs) exps = np.asarray(fexps) model_types = np.asarray(fmodel_types) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] perf = sel_exp_query(experiment_name=it_exps[sorted_perfs[0]], model=target_model, db_config=db_config) dummy_sel_model = perf[-1] print 'Using %s' % dummy_sel_model model_file = dummy_sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=dummy_sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): val_images = tf.placeholder(tf.float32, shape=[1] + [x for x in dataset_module.im_size]) # Pull stimuli stim_dir = os.path.join(main_config.tf_record_output, dummy_sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append( os.path.join('models', 'structs', dummy_sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') grad_image = tf.gradients(model.output, val_images)[0] print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) all_filters = [] all_rfs = [] max_scores = [] for idx in sorted_perfs: perf = sel_exp_query(experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Store sparse noise for reference stim_dir = os.path.join(main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] rf_stim_meta_data = rf_stim_meta_data.values()[0][0] sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Set up exemplar threading if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name ] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name ] elif target_model == 'dog' or target_model == 'DoG': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name ] else: raise NotImplementedError print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) saver.restore(sess, model_ckpt) all_filters += [sess.run(fname)] all_rfs += [sparse_rf] max_scores += [max_score] np.savez('tests/ALLEN_files/filters/%s_%s_recalc_%s' % (experiment, target_model, recalc), rfs=all_rfs, perf=max_scores, filters=all_filters) print 'SAVED'
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, num_models=3, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_dir='/media/data_cifs/AllenData/DataForTrain/all_stimulus_template', stimulus_type='tfrecord', top_n=0, preload_stim=False, target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join( allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()[ 'experiment_name']} if query_db: perf = query_hp_hist( exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT for gd in data_files: mt = gd.split( os.path.sep)[-1].split( template_exp + '_')[-1].split('_' + 'val')[0] it_data = np.load(gd).item() sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view( np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] bp = np.argmax(sperfs) fxs += [xy[0]] fys += [xy[1]] fperfs += [sperfs[bp]] fargs += [sargs[bp]] fmodel_types += [sel_mts[bp]] fexps += [sexps[bp]] xs = np.asarray(fxs) ys = np.asarray(fys) perfs = np.asarray(fperfs) arg_perf = np.asarray(fargs) exps = np.asarray(fexps) model_types = np.asarray(fmodel_types) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] # sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] sorted_perfs = [np.argsort(it_perfs)[::-1][top_n]] for idx in sorted_perfs: perf = sel_exp_query( experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % ( model_file, model_file.split(os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull stimuli stim_dir = os.path.join( main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] # stim_val_data = [x for x in stim_files if 'val.tfrecords' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Store sparse noise for reference sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): val_images = tf.placeholder( tf.float32, shape=[1] + [x for x in dataset_module.im_size]) # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append(os.path.join('models', 'structs', sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') grad_image = tf.gradients(model.output, val_images)[0] print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) saver.restore(sess, model_ckpt) # Set up exemplar threading if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name] elif target_model == 'dog': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name] else: raise NotImplementedError val_tensors = { 'images': val_images, # 'labels': val_labels, 'filts': fname, 'responses': model[target_layer], 'labels': model['output'], 'grads': grad_image } all_images, all_preds, all_grads, all_responses = [], [], [], [] num_steps = 10000 stimuli = os.path.join( stimulus_dir, 'locally_sparse_noise_8deg_template.pkl') stimuli = pickle.load(open(stimuli, 'rb'))[:num_steps] ih, iw = 304, 608 ns, sh, sw = stimuli.shape sh, sw = ih, iw sh = int(sh) sw = int(sw) tb = int((ih - sh) // 2) lr = int((iw - sw) // 2) gns = 32 cnst = 127.5 for step in range(num_steps): # step, stim in enumerate(stimuli): # step in range(num_steps): chosen_stim = np.random.permutation(ns)[0] if preload_stim: it_stim = stimuli[chosen_stim].astype(np.float32) it_stim = it_stim.astype(np.float32) noise_im = (misc.imresize(it_stim, [sh, sw], interp='nearest')) noise_im = cv2.copyMakeBorder( noise_im.squeeze(), tb, tb, lr, lr, cv2.BORDER_CONSTANT, value=cnst) else: stim_noise = scipy.sparse.csr_matrix(scipy.sparse.random(ih // gns, iw // gns, density=0.05)).todense() stim_noise_mask = stim_noise == 0 stim_noise[stim_noise > 0.5] = 255. stim_noise[stim_noise < 0.5] = 0. stim_noise[stim_noise_mask] = cnst noise_im = (misc.imresize(stim_noise.squeeze(), [sh, sw], interp='nearest')) if np.random.rand() < 0.5: noise_im = np.fliplr(noise_im) if np.random.rand() < 0.5: noise_im = np.flipud(noise_im) # noise_im = (misc.imresize(it_stim, [ih, iw], interp='nearest'))[None, :, :, None] noise_im = noise_im / 255. noise_im = noise_im[None, :, :, None] assert noise_im.max() <= 1 val_vals = sess.run(val_tensors.values(), feed_dict={val_images: noise_im}) val_dict = {k: v for k, v in zip(val_tensors.keys(), val_vals)} all_images += [noise_im] # val_dict['images']] # all_responses += [val_dict['responses']] all_preds += [val_dict['labels'].squeeze()] # all_grads += [val_dict['grads'].squeeze()] print 'Finished step %s' % step # Process and save data all_images = np.concatenate(all_images).squeeze() ev, vals = peakdet(all_preds, np.median(all_preds)) sp = np.zeros_like(all_preds) sp[ev[:, 0].astype(int)] = 1 plt.imshow(np.matmul(all_images.reshape(all_images.shape[0], -1).transpose(), all_preds).reshape(ih, iw));plt.show() plt.imshow(np.matmul(all_images.reshape(all_images.shape[0], -1).transpose(), sp).reshape(ih, iw));plt.show() filters = val_dict['filts'][0].squeeze().transpose(2, 0, 1) import ipdb;ipdb.set_trace() all_grads = np.asarray(all_grads) all_preds = np.asarray(all_preds).reshape(-1, 1) import ipdb;ipdb.set_trace() # res_f = all_responses.reshape(ne * h * w, k) # res_g = res_grads.reshape(ne, rh * rw) # i_cov = np.cov(res_i.transpose()) # f_cov = np.cov(res_f.transpose()) # g_cov = np.cov(res_g.transpose()) # sp = (all_preds > all_preds.mean()).astype(np.float32) # res_i = res_g # ev, vals = peakdet(all_preds, 0.5) # sp = np.zeros_like(all_preds) # sp[ev[:, 0].astype(int)] = 1 # slen = ne # nsp = np.sum(sp) # number of spikes # swid = rh * rw # Msz = np.dot(np.dot(slen, swid), ne) # Size of full stimulus matrix # rowlen = 1830 # np.dot(swid, ne) # Length of a single row of stimulus matrix # Compute raw mean and covariance # RawMu = np.mean(res_i, 0).T # RawCov = np.dot(res_i.T, res_i) / (slen-1.) - (RawMu*np.vstack(RawMu)*slen) / (slen-1.) # Compute spike-triggered mean and covariance # iisp = np.nonzero((sp > 0.)) # spvec = sp[iisp] # STA = np.divide(np.dot(spvec.T, res_i[iisp[0],:]).T, nsp) # STC = np.dot(res_i[iisp[0],:].T, np.multiply(res_i[iisp[0],:], ml.repmat(spvec, rowlen, 1).T))/(nsp-1.) - (STA*np.vstack(STA)*nsp)/(nsp-1.) # res_i_cov = np.matmul(res_i.transpose(), res_i) # inv_res_i = np.linalg.pinv(res_i_cov) # sta = inv_res_i * np.matmul(res_i.transpose(), all_preds) # sta = inv_res_i * np.matmul(res_i.transpose(), spike_preds) # sti = (1. / float(ne)) * (np.linalg.pinv(i_cov) * np.matmul(res_i, all_preds)) # sta = (1. / float(ne)) * (np.linalg.pinv(f_cov) * np.matmul(res_f, all_preds)) # sta = sta.reshape(h, w) # stg = (1. / float(ne)) * np.matmul(all_grads.reshape(h * w, ne), all_preds) # stg = stg.reshape(h, w) np.savez( os.path.join(data_dir, 'data'), images=all_images, pred=all_preds, filters=filters, STA=STA, fits=fits, grads=all_grads) if target_model != 'dog': save_mosaic( maps=filters, # [0].squeeze().transpose(2, 0, 1), output=os.path.join(data_dir, '%s_filters' % target_layer), rc=8, cc=4, title='%s filters' % ( target_layer)) else: import ipdb;ipdb.set_trace() print 'Complete.'
def main(experiment_name, list_experiments=False): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 return # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config = process_DB_exps(experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name # Prepare data loaders on the cpu with tf.device('/cpu:0'): # Test issues with data loading? Try placeholders instead. train_images = tf.placeholder(tf.float32, name='train_images', shape=[config.batch_size] + dataset_module.im_size) train_labels = tf.placeholder(tf.int64, name='train_labels', shape=[config.batch_size]) val_images = tf.placeholder(tf.float32, name='val_images', shape=[config.batch_size] + dataset_module.im_size) val_labels = tf.placeholder(tf.int64, name='val_labels', shape=[config.batch_size]) log.info('Created tfrecord dataloader tensors.') # Prepare model on GPU with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: # Training model if len(dataset_module.output_size) > 1: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) # Click weighting flat_ims = tf.reshape( train_images, [config.batch_size, np.prod(dataset_module.im_size)]) W = tf.get_variable( name='W', initializer=tf.truncated_normal_initializer(stddev=0.1), shape=[ np.prod(dataset_module.im_size), dataset_module.output_size ]) b = tf.get_variable( name='b', initializer=tf.truncated_normal_initializer(stddev=0.1), shape=[dataset_module.output_size]) output_scores = tf.matmul(flat_ims, W) + b # Prepare the loss function train_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=train_labels, logits=output_scores)) train_op = tf.train.GradientDescentOptimizer( config.lr).minimize(train_loss) log.info('Built training loss function.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # Start training loop step, time_elapsed = 0, 0 train_losses, train_accs, val_losses, val_accs, timesteps = {}, {}, {}, {}, {} files, labels = dataset_module.get_data() combined_files = files['train'] combined_labels = labels['train'] batch_size = config.batch_size num_batches = len(combined_files) // batch_size log.info('Finished training.') for image_batch, label_batch, _ in tqdm(image_batcher( start=0, num_batches=num_batches, images=combined_files, labels=combined_labels, batch_size=batch_size), total=num_batches): feed_dict = { train_images: image_batch.astype(np.float32), train_labels: np.asarray(label_batch).astype(int) } import ipdb ipdb.set_trace() start_time = time.time() _, loss_value = sess.run([ train_op, train_loss, ], feed_dict=feed_dict) files_to_save = { 'training_loss': tr_loss, 'validation_loss': val_loss, 'training_acc': tr_accs, 'validation_acc': val_accs, 'timesteps': timesteps } model_name = config.model_struct.replace('/', '_') py_utils.save_npys(data=files_to_save, model_name=model_name, output_string=dir_list['experiment_evaluations']) # Compare this condition w/ all others. plotting.plot_data(train_loss=tr_loss, val_loss=val_loss, model_name=model_name, timesteps=timesteps, config=config, output=os.path.join(dir_list['condition_evaluations'], 'loss'), output_ext='.pdf', data_type='loss') plotting.plot_data(tr_accs=tr_accs, val_accs=val_accs, model_name=model_name, timesteps=timesteps, config=config, output=os.path.join(dir_list['condition_evaluations'], 'acc'), output_ext='.pdf', data_type='acc') log.info('Completed plots.')