def plot_wrapper(data_files, keep_models, out_name, max_samples, baseline_file, out_data, create_subdir=True): """Wrapper for running routines.""" print('Working on %s' % out_name) it_data_files = [x for x in data_files if out_name not in x] if create_subdir: py_utils.make_dir(os.path.join(out_data, out_name)) out_name = '%s%s%s' % (out_name, os.path.sep, out_name) df, dm = gather_data(data_files=it_data_files, keep_models=keep_models) baseline_file = pd.read_csv(baseline_file) df, diff_df = process_data(df=df, out_name=out_name, out_data=out_data, baseline_file=baseline_file, max_samples=max_samples) # Create color maps colors = keep_models.values() color_pal = sns.xkcd_palette(colors) hue_order = keep_models.keys() create_plots(df=df, out_data=out_data, colors=color_pal, hue_order=hue_order, out_plot='%s.pdf' % out_name) print('Finished on %s' % out_name)
def main(f, tag): """Plot BSDS figs.""" fn = f.split(os.path.sep)[-1].strip('.npz') dir_name = f.split(os.path.sep)[0] fig_dir = os.path.join(dir_name, 'trip_%s_%s' % (tag, fn)) res_dir = os.path.join(dir_name, 'preds_%s_%s' % (tag, fn)) py_utils.make_dir(fig_dir) py_utils.make_dir(res_dir) d = np.load(f) test_dict = d['test_dict'] if 'portrait' in f: from datasets.BSDS500_test_portrait import data_processing as dp elif 'landscape' in f: from datasets.BSDS500_test_landscape import data_processing as dp elif 'multicue_edges' in f: from datasets.multicue_100_edges_jk_test import data_processing as dp else: raise NotImplementedError(f) dP = dp() files = dP.get_test_files() print('Saving files to %s and %s' % (fig_dir, res_dir)) for idx, (td, im) in tqdm(enumerate(zip(test_dict, files)), total=len(test_dict)): f = plt.figure() im_name = im.split(os.path.sep)[-1].split('.')[0] score = tf_fun.sigmoid_fun(td['logits'].squeeze()) proc_im = (td['images'].squeeze()[..., [2, 1, 0]] + PASCAL).astype( np.uint8) proc_lab = td['labels'].squeeze() sc_shape = score.shape # if sc_shape[0] > sc_shape[1]: # diff_h = sc_shape[0] - 481 # diff_w = sc_shape[1] - 321 # score = score[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # proc_im = proc_im[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # [:481, :321] # proc_lab = proc_lab[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # [:481, :321] # elif sc_shape[1] > sc_shape[0]: # diff_h = sc_shape[1] - 481 # diff_w = sc_shape[0] - 321 # score = score[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # proc_im = proc_im[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # [:481, :321] # proc_lab = proc_lab[diff_h // 2:-(diff_h - diff_h // 2), diff_w // 2: (-diff_w - diff_w // 2)] # [:481, :321] # else: # raise RuntimeError(sc_shape) plt.subplot(131) plt.imshow(proc_im) plt.axis('off') plt.subplot(132) plt.imshow(proc_lab) plt.axis('off') plt.subplot(133) plt.imshow(score, cmap='Greys_r') # io.imsave( # os.path.join( # res_dir, # '%s.tiff' % im_name), # score) np.save(os.path.join(res_dir, '%s' % im_name), score) plt.savefig(os.path.join(fig_dir, '%s.pdf' % im_name), dpi=150) plt.close(f)
def visualize_recurrence(idx, image, label, logits, ff, h2s, config, debug=False): """Visualize norm of diffs of timesteps of activity.""" f, axarr = plt.subplots(2, len(h2s), figsize=(30, 15)) image = image.squeeze() post_decoded_final = logits.squeeze() if image.shape[-1] == 3: axarr[0, 2].imshow((image).astype(np.uint8)) else: axarr[0, 2].imshow((image).astype(np.uint8), cmap='Greys_r') axarr[0, 3].imshow(label.squeeze(), cmap='Greys', vmin=0, vmax=0.5) axarr[0, 4].imshow(sigmoid_fun(post_decoded_final), vmin=0.0, vmax=1.0, cmap='Greys_r') for its, post in enumerate(h2s): axarr[1, its].imshow(sigmoid_fun(post.squeeze()), vmin=0, vmax=1, cmap='Greys_r') [axi.set_xticks([]) for axi in axarr.ravel()] [axi.set_yticks([]) for axi in axarr.ravel()] py_utils.make_dir(config.model) plt.savefig(os.path.join(config.model, '%d.pdf' % idx)) if debug: plt.show() plt.close(f)
def plot_wrapper(data_files, keep_models, out_name, max_samples, out_data, model_csv, create_subdir=True): """Wrapper for running routines.""" print('Working on %s' % out_name) it_data_files = [x for x in data_files if out_name not in x] if create_subdir: py_utils.make_dir(os.path.join(out_data, out_name)) out_name = '%s%s%s' % (out_name, os.path.sep, out_name) dm, df = gather_data(data_files=it_data_files, keep_models=keep_models, model_csv=model_csv) sel_df = process_data(df=df, out_name=out_name, out_data=out_data, max_samples=max_samples) py_utils.make_dir(os.path.join(out_data, out_name)) # Do the plotting for idx in range(len(dm)): sel_data = dm[idx] sel_info = df.iloc[idx] it_train = sel_info.train_dataset it_val = sel_info.val_dataset it_model = sel_info.model # Plot images f = plt.figure(dpi=450) im = dm[idx][0] lab = dm[idx][1] log = dm[idx][2] plt.subplot(131) plt.imshow(im, cmap='Greys_r') plt.axis('off') plt.subplot(132) plt.imshow(lab, cmap='Greys_r') plt.axis('off') plt.subplot(133) plt.imshow((log > 0.5).astype(np.float32), cmap='Greys_r', vmin=0, vmax=1) plt.axis('off') plt.savefig( os.path.join(out_data, out_name, '%s_%s_%s' % (it_model, it_train, it_val))) print( 'Saved to: %s' % os.path.join(out_data, out_name, '%s_%s_%s' % (it_model, it_train, it_val))) plt.close(f)
def get_labels(self, files): """Process and save label images.""" labels = {} new_files = {} for k, images in files.iteritems(): # Replace extension and path with labels label_vec = [] file_vec = [] fold = k # New label dir proc_dir = os.path.join(images[0].split(fold)[0], fold, self.processed_labels) py_utils.make_dir(proc_dir) # New image dir proc_image_dir = os.path.join(self.config.data_root, self.name, self.images_dir, fold, self.processed_images) py_utils.make_dir(proc_image_dir) all_images = [] for im in tqdm(images, total=len(images), desc='Storing %s labels and images for %s' % (self.name, k)): it_label = im.split(os.path.sep)[-1] it_label_path = '%s%s' % (im.split('.')[0], self.lab_extension) it_label_path = it_label_path.replace(self.images_dir, self.labels_dir) # Process every label and duplicate images for each #Obtain image and contour label in im_size shape im_data, ip_lab = get_label_image(im, it_label_path, output_size=self.lab_size) transpose_labels = False if not np.all(self.im_size == list(im_data.shape)): im_data = np.swapaxes(im_data, 0, 1) transpose_labels = True assert np.all(self.im_size == list( im_data.shape)), 'Mismatched dimensions.' if transpose_labels: ip_lab = np.swapaxes(ip_lab, 0, 1) it_im_name = it_label #Copying image name from a previously assigned variable it_lab_name = '%s.npy' % it_im_name.split('.')[0] out_lab = os.path.join(proc_dir, it_lab_name) np.save(out_lab, ip_lab) label_vec += [out_lab] # Process images proc_im = os.path.join(proc_image_dir, it_im_name) #misc.imsave(proc_im, im_data) np.save(proc_im + '.npy', im_data.astype(np.float32)) file_vec += [proc_im + '.npy'] #Cannot compute z-score for SBD, too many images labels[k] = label_vec new_files[k] = file_vec return labels, new_files
def create_label_image_dirs(self, sample_img_path, fold): """Create directories for labels and images""" proc_dir = os.path.join(self.config.data_root, self.name, self.labels_dir, fold, self.processed_labels) #sample_img_path.split(fold)[0], #fold, #self.processed_labels) py_utils.make_dir(proc_dir) proc_image_dir = os.path.join(self.config.data_root, self.name, self.images_dir, fold, self.processed_images) py_utils.make_dir(proc_image_dir) return proc_dir, proc_image_dir
def main(): # datasets = ["curv_contour_length_6_full", "curv_contour_length_9_full", "curv_contour_length_14_full"] datasets = [ "curv_contour_length_14_full" ] # "curv_contour_length_6_full", "curv_contour_length_9_full", "curv_contour_length_14_full"] cvs = ["val", "train"] config = Config() for ds in datasets: for cv in cvs: py_utils.make_dir(ds) out_dir = os.path.join(ds, cv) py_utils.make_dir(out_dir) extract_dataset(dataset=ds, config=config, cv=cv, out_dir=out_dir)
def get_files(self): files = {} all_ims = np.load(os.path.join( self.config.data_root, self.name, self.data_file))[self.file_key] all_ims = np.asarray([x for x in all_ims if x not in self.ignore_ims]) # Create folders for training/validation splits self.rand_order = np.random.permutation(len(all_ims)) self.test_split = np.round(len(all_ims) * self.crossval_split).astype(int) shuffled_ims = all_ims[self.rand_order] test_ims = shuffled_ims[:self.test_split] train_ims = shuffled_ims[self.test_split:] target_test_ims = [os.path.join( self.config.data_root, self.name, self.folds['test'], f) for f in test_ims] target_train_ims = [os.path.join( self.config.data_root, self.name, self.folds['train'], f) for f in train_ims] test_ims = [os.path.join( self.config.data_root, self.name, f) for f in test_ims] train_ims = [os.path.join( self.config.data_root, self.name, f) for f in train_ims] py_utils.make_dir(os.path.join( self.config.data_root, self.name, self.folds['test'])) py_utils.make_dir(os.path.join( self.config.data_root, self.name, self.folds['train'])) [shutil.copyfile(s, t) for s, t in zip(test_ims, target_test_ims)] [shutil.copyfile(s, t) for s, t in zip(train_ims, target_train_ims)] files = { self.folds['test']: test_ims, self.folds['train']: train_ims } return files
def __init__(self, **kwargs): """Global config file for normalization experiments.""" self.data_root = '/media/data_cifs/image_datasets/' self.project_directory = '/media/data_cifs/contextual_circuit/' self.tf_records = os.path.join(self.project_directory, 'tf_records') self.checkpoints = os.path.join(self.project_directory, 'checkpoints') self.summaries = os.path.join(self.project_directory, 'summaries') self.experiment_evaluations = os.path.join(self.project_directory, 'experiment_evaluations') self.condition_evaluations = os.path.join(self.project_directory, 'condition_evaluations') self.visualizations = os.path.join(self.project_directory, 'visualizations') self.plots = os.path.join(self.project_directory, 'plots') self.log_dir = os.path.join(self.project_directory, 'logs') self.dataset_info = 'dataset_processing' # local dataset classes # DB self.db_ssh_forward = False machine_name = os.uname()[1] if len(machine_name) == 12 or ('serre' in machine_name and machine_name != 'serrep3'): # Docker container or master p-node self.db_ssh_forward = True # Create directories if they do not exist check_dirs = [ self.tf_records, self.checkpoints, self.experiment_evaluations, self.condition_evaluations, self.visualizations, self.plots, self.log_dir, self.dataset_info ] [py_utils.make_dir(x) for x in check_dirs]
def get_labels(self, n_counts, n_img_per_count): """Process and save label images.""" labels = {} new_files = {} counts = range(1, n_counts + 1) total_n_imgs = n_counts * n_img_per_count self.n_imgs_fold['train'] = int(0.7 * total_n_imgs) self.n_imgs_fold['val'] = int(0.3 * total_n_imgs) for k in self.folds.keys(): n_imgs = self.n_imgs_fold[k] label_vec = [] file_vec = [] fold = k # New label dir proc_dir = os.path.join(self.config.data_root, self.name, self.labels_dir, fold, self.processed_labels) py_utils.make_dir(proc_dir) # New image dir proc_image_dir = os.path.join(self.config.data_root, self.name, self.images_dir, fold, self.processed_images) py_utils.make_dir(proc_image_dir) all_images = [] for i in tqdm(range(n_imgs), desc='Generating images for %s..' % (fold)): # Replace extension and path with labels it_label = 'gt_%06d' % (i) + '.npy' it_label_path = proc_dir + '/' + it_label label_data = sample( counts, 1)[0] #Randomly sampling a number of objects it_im_name = 'img_%06d_%s.npy' % (i, label_data) im_data = psst.generate_image(label_data) label_vec += [ label_data ] #Since labels are just counts, append the label value to label_vec proc_im = os.path.join(proc_image_dir, it_im_name) np.save(proc_im, im_data) file_vec += [proc_im] labels[k] = label_vec new_files[k] = file_vec return labels, new_files
def plot_wrapper( data_files, keep_models, out_name, max_samples, out_data, generalization=False, create_subdir=True): """Wrapper for running routines.""" print('Working on %s' % out_name) it_data_files = [x for x in data_files if out_name not in x] if create_subdir: py_utils.make_dir(os.path.join(out_data, out_name)) out_name = '%s%s%s' % (out_name, os.path.sep, out_name) df, dm = gather_data( data_files=it_data_files, keep_models=keep_models, generalization=generalization) df = process_data( df=df, out_name=out_name, out_data=out_data, generalization=generalization, max_samples=max_samples) colors = keep_models.values() color_pal = sns.xkcd_palette(colors) hue_order = keep_models.keys() if generalization: create_gen_plots( df=df, out_data=out_data, out_plot='%s.pdf' % out_name) else: create_plots( df=df, out_data=out_data, colors=color_pal, hue_order=hue_order, out_plot='%s.pdf' % out_name) print('Finished on %s' % out_name)
def main(experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, config_file=None, ckpt_file=None, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params config = np.load(config_file).item() dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) config.epochs = 1 config.shuffle = False with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=['resize_and_crop'], num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Normalize labels if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] log.info('Mean-centering labels.') # Training model if len(dataset_module.output_size) == 2: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): train_grad_images = tf.gradients( train_scores[0] * tf.cast(train_labels, tf.float32), train_images)[0] log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() if len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) elif len(train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels)[0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if len(train_labels.get_shape()) > 2: tf_fun.image_summaries(train_labels, tag='Training_targets') tf_fun.image_summaries(train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax(train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op(tag='training_pr', predictions=tf.cast( tf.argmax(train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, # False, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map({'selu': 'GradLRP'}): val_grad_images = tf.gradients( val_scores[0] * tf.cast(val_labels, tf.float32), val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len(val_scores.get_shape()) != len(val_labels.get_shape()): val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_labels = tf.expand_dims(val_labels, axis=-1) if len(val_shape) == 2 and len( val_label_shape) == 1 and val_shape[-1] == 1: val_scores = tf.expand_dims(val_scores, axis=-1) val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels)[0] # TODO: Fix for multiloss # Prepare tensorboard summaries if len(val_images.get_shape()) == 4: tf_fun.image_summaries(val_images, tag='Validation') if len(val_labels.get_shape()) > 2: tf_fun.image_summaries(val_labels, tag='Validation_targets') tf_fun.image_summaries(val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast(tf.argmax(val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op(tag='validation_pr', predictions=tf.cast( tf.argmax(val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores, 'train_grad_images': train_grad_images } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, 'val_grad_images': val_grad_images } if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop checkpoint_dir = dir_list['checkpoints'] step = 0 train_losses, train_accs, train_aux, timesteps = {}, {}, {}, {} val_scores, val_aux, val_labels, val_grads = {}, {}, {}, {} train_images, val_images = {}, {} train_scores, train_labels = {}, {} train_aux_check = np.any(['aux_score' in k for k in train_dict.keys()]) val_aux_check = np.any(['aux_score' in k for k in val_dict.keys()]) # Restore model saver.restore(sess, ckpt_file) # Start evaluation try: while not coord.should_stop(): start_time = time.time() train_vars = sess.run(train_dict.values()) it_train_dict = { k: v for k, v in zip(train_dict.keys(), train_vars) } duration = time.time() - start_time train_losses[step] = it_train_dict['train_loss'] train_accs[step] = it_train_dict['train_accuracy_0'] train_images[step] = it_train_dict['train_images'] train_labels[step] = it_train_dict['train_labels'] train_scores[step] = it_train_dict['train_scores'] timesteps[step] = duration if train_aux_check: # Loop through to find aux scores it_train_aux = { itk: itv for itk, itv in it_train_dict.iteritems() if 'aux_score' in itk } train_aux[step] = it_train_aux assert not np.isnan(it_train_dict['train_loss']).any( ), 'Model diverged with loss = NaN' if step % config.validation_iters == 0: it_val_scores, it_val_labels, it_val_aux, it_val_grads, it_val_ims = [], [], [], [], [] for num_vals in range(config.num_validation_evals): # Validation accuracy as the average of n batches val_vars = sess.run(val_dict.values()) it_val_dict = { k: v for k, v in zip(val_dict.keys(), val_vars) } it_val_labels += [it_val_dict['val_labels']] it_val_scores += [it_val_dict['val_scores']] it_val_grads += [it_val_dict['val_grad_images']] it_val_ims += [it_val_dict['val_images']] if val_aux_check: iva = { itk: itv for itk, itv in it_val_dict.iteritems() if 'aux_score' in itk } it_val_aux += [iva] val_scores[step] = it_val_scores val_labels[step] = it_val_labels val_aux[step] = it_val_aux val_images[step] = it_val_grads val_grads[step] = it_val_ims # End iteration step += 1 except tf.errors.OutOfRangeError: print 'Done with evaluation for %d epochs, %d steps.' % (config.epochs, step) print 'Saved to: %s' % checkpoint_dir finally: coord.request_stop() coord.join(threads) sess.close() import ipdb ipdb.set_trace() np.savez( 'val_imgs_grads', val_images=val_images, # it_val_dict['val_images'], val_grads=val_grads, # it_val_dict['val_grad_images'], val_labels=val_labels, # it_val_dict['val_labels'], val_scores=val_scores) # it_val_dict['val_scores'][0])
def get_labels(self, files): """Process and save label images.""" labels = {} new_files = {} for k, images in files.iteritems(): # Replace extension and path with labels label_vec = [] file_vec = [] fold = images[0].split(os.path.sep)[-2] # New label dir proc_dir = os.path.join( images[0].split(fold)[0], fold, self.processed_labels) py_utils.make_dir(proc_dir) # New image dir proc_image_dir = os.path.join( self.config.data_root, self.name, self.images_dir, fold, self.processed_images) py_utils.make_dir(proc_image_dir) ims = [] for im in tqdm(images, total=len(images), desc=k): it_label = im.split(os.path.sep)[-1] it_label_path = '%s%s' % (im.split('.')[0], self.lab_extension) it_label_path = it_label_path.replace( self.images_dir, self.labels_dir) # Process every label and duplicate images for each label_data = io.loadmat( it_label_path)['groundTruth'].reshape(-1) im_data = misc.imread(im) transpose_labels = False if not np.all(self.im_size == list(im_data.shape)): im_data = np.swapaxes(im_data, 0, 1) # im_data = im_proc.pad_square(im_data) transpose_labels = True assert np.all( self.im_size == list(im_data.shape)), \ 'Mismatched dimensions.' ims += [im_data] if self.fold_options[k] == 'duplicate': # Loop through all labels for idx, lab in enumerate(label_data): # Process labels ip_lab = lab.item()[1].astype(np.float32) if transpose_labels: ip_lab = np.swapaxes(ip_lab, 0, 1) # ip_lab = im_proc.pad_square(ip_lab) it_im_name = 'image_%s_%s.npy' % (idx, it_label) it_lab_name = 'label_%s.npy' % it_im_name.split('.')[0] out_lab = os.path.join(proc_dir, it_lab_name) np.save(out_lab, ip_lab) label_vec += [out_lab] # Process images proc_im = os.path.join(proc_image_dir, it_im_name) np.save(proc_im, im_data) file_vec += [proc_im] elif self.fold_options[k] == 'mean': mean_labs = [] for idx, lab in enumerate(label_data): # Process labels ip_lab = lab.item()[1].astype(np.float32) if transpose_labels: ip_lab = np.swapaxes(ip_lab, 0, 1) # ip_lab = im_proc.pad_square(ip_lab) mean_labs += [ip_lab] mean_lab = np.asarray(mean_labs).mean(0) out_lab = os.path.join( proc_dir, 'label_%s.npy' % it_label.split('.')[0]) np.save(out_lab, mean_lab) label_vec += [out_lab] # Process images proc_im = os.path.join(proc_image_dir, it_label) np.save(proc_im, im_data) # misc.imsave(proc_im, im_data) file_vec += ['%s.npy' % proc_im] else: raise NotImplementedError if k == 'train': label_vec = label_vec[:self.train_size] file_vec = file_vec[:self.train_size] ims = ims[:self.train_size] labels[k] = label_vec new_files[k] = file_vec ims = np.array(ims) np.savez( os.path.join( self.config.data_root, self.name, self.images_dir, fold, 'file_paths' ), labels=labels, files=new_files, mean=np.mean(ims, axis=(0, 1, 2), keepdims=True), stds=np.std(ims, axis=(0, 1, 2), keepdims=True)) return labels, new_files
def main( experiment_name, list_experiments=False, load_and_evaluate_ckpt=None, placeholder_data=None, grad_images=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 if len(exps) == 0: print 'No experiments found.' else: print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) assert experiment_name is not None, 'Empty experiment name.' experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config.load_and_evaluate_ckpt = load_and_evaluate_ckpt if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer from ops import evaluation config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_key = [k for k in dataset_module.folds.keys() if 'train' in k] if not len(train_key): train_key = 'train' else: train_key = train_key[0] train_data, train_means_image, train_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=train_key, log=log) val_key = [k for k in dataset_module.folds.keys() if 'val' in k] if not len(val_key): val_key = 'train' else: val_key = val_key[0] val_data, val_means_image, val_means_label = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=val_key, log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join( config.checkpoints, condition_label), 'summaries': os.path.join( config.summaries, condition_label), 'condition_evaluations': os.path.join( config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join( config.visualizations, condition_label), 'weights': os.path.join( config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu if all(isinstance(i, list) for i in config.data_augmentations): if config.data_augmentations: config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) if load_and_evaluate_ckpt is not None: config.epochs = 1 config.train_shuffle = False config.val_shuffle = False with tf.device('/cpu:0'): if placeholder_data: placeholder_shape = placeholder_data['train_image_shape'] placeholder_dtype = placeholder_data['train_image_dtype'] original_train_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_images') placeholder_shape = placeholder_data['train_label_shape'] placeholder_dtype = placeholder_data['train_label_dtype'] original_train_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='train_labels') placeholder_shape = placeholder_data['val_image_shape'] placeholder_dtype = placeholder_data['val_image_dtype'] original_val_images = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_images') placeholder_shape = placeholder_data['val_label_shape'] placeholder_dtype = placeholder_data['val_label_dtype'] original_val_labels = tf.placeholder( dtype=placeholder_dtype, shape=placeholder_shape, name='val_labels') # Apply augmentations ( train_images, train_labels ) = data_loader.placeholder_image_augmentations( images=original_train_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_train_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) ( val_images, val_labels ) = data_loader.placeholder_image_augmentations( images=original_val_images, model_input_image_size=dataset_module.model_input_image_size, labels=original_val_labels, data_augmentations=config.data_augmentations, batch_size=config.batch_size) # Store in the placeholder dict placeholder_data['train_images'] = original_train_images placeholder_data['train_labels'] = original_train_labels placeholder_data['val_images'] = original_val_images placeholder_data['val_labels'] = original_val_labels else: train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_train, resize_output=config.resize_output) if hasattr(config, 'val_augmentations'): val_augmentations = config.val_augmentations else: val_augmentations = config.data_augmentations val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=val_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle_val, resize_output=config.resize_output) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join( 'models', 'structs', experiment_name).replace(os.path.sep, '.') ) except IOError: print 'Could not find the model structure: %s in folder %s' % ( struct_name, experiment_name) # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare variables for the models if len(dataset_module.output_size) == 2: log.warning( 'Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None # Correct number of output neurons if needed if config.dataloader_override and\ 'weights' in output_structure[-1].keys(): output_neurons = output_structure[-1]['weights'][0] size_check = output_neurons != dataset_module.output_size fc_check = output_structure[-1]['layers'][0] == 'fc' if size_check and fc_check: output_structure[-1]['weights'][0] = dataset_module.output_size log.warning('Adjusted output neurons from %s to %s.' % ( output_neurons, dataset_module.output_size)) # Prepare model on GPU if not hasattr(dataset_module, 'input_normalization'): dataset_module.input_normalization = None with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model model = model_utils.model_class( mean=train_means_image, training=True, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) train_scores, model_summary, _ = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(train_scores.get_shape()[-1]) target_scores = tf.one_hot(train_labels, oh_dims) * train_scores train_gradients = tf.gradients(target_scores, train_images)[0] log.info('Built training model.') log.debug( json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Normalize labels on GPU if needed if 'normalize_labels' in exp_params.keys(): if exp_params['normalize_labels'] == 'zscore': train_labels -= train_means_label['mean'] train_labels /= train_means_label['std'] val_labels -= train_means_label['mean'] val_labels /= train_means_label['std'] log.info('Z-scoring labels.') elif exp_params['normalize_labels'] == 'mean': train_labels -= train_means_label['mean'] val_labels -= val_means_label['mean'] log.info('Mean-centering labels.') # Check the shapes of labels and scores if not isinstance(train_scores, list): if len( train_scores.get_shape()) != len( train_labels.get_shape()): train_shape = train_scores.get_shape().as_list() label_shape = train_labels.get_shape().as_list() val_shape = val_scores.get_shape().as_list() val_label_shape = val_labels.get_shape().as_list() if len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_labels = tf.expand_dims(train_labels, axis=-1) val_labels = tf.expand_dims(val_labels, axis=-1) elif len( train_shape) == 2 and len( label_shape) == 1 and train_shape[-1] == 1: train_scores = tf.expand_dims(train_scores, axis=-1) val_scores = tf.expand_dims(val_scores, axis=-1) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, # TODO labels=train_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(train_loss, list): for lidx, tl in enumerate(train_loss): tf.summary.scalar('training_loss_%s' % lidx, tl) train_loss = tf.add_n(train_loss) else: tf.summary.scalar('training_loss', train_loss) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) assert config.lr is not None, 'No learning rate.' # TODO: Make a QC function if config.lr > 1: old_lr = config.lr config.lr = loss_utils.create_lr_schedule( train_batch=config.batch_size, num_training=config.lr) config.optimizer = 'momentum' log.info('Forcing momentum classifier.') else: old_lr = None train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') # Add a score for the training set train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO: Attach to exp cnfg pred=train_scores, # TODO labels=train_labels) # Add aux scores if requested train_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: train_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=train_scores, labels=train_labels) # [0] # TODO: Fix for multiloss # Prepare remaining tensorboard summaries if config.tensorboard_images: if len(train_images.get_shape()) == 4: tf_fun.image_summaries(train_images, tag='Training images') if (np.asarray( train_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( train_labels, tag='Training_targets') tf_fun.image_summaries( train_scores, tag='Training_predictions') if isinstance(train_accuracy, list): for tidx, ta in enumerate(train_accuracy): tf.summary.scalar('training_accuracy_%s' % tidx, ta) else: tf.summary.scalar('training_accuracy', train_accuracy) if config.pr_curve: if isinstance(train_scores, list): for pidx, train_score in enumerate(train_scores): train_label = train_labels[:, pidx] pr_summary.op( tag='training_pr_%s' % pidx, predictions=tf.cast( tf.argmax( train_score, axis=-1), tf.float32), labels=tf.cast(train_label, tf.bool), display_name='training_precision_recall_%s' % pidx) else: pr_summary.op( tag='training_pr', predictions=tf.cast( tf.argmax( train_scores, axis=-1), tf.float32), labels=tf.cast(train_labels, tf.bool), display_name='training_precision_recall') log.info('Added training summaries.') with tf.variable_scope('cnn', tf.AUTO_REUSE) as scope: # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=train_means_image, # Normalize with train data training=False, output_size=dataset_module.output_size, input_normalization=dataset_module.input_normalization) val_scores, _, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad_images: oh_dims = int(val_scores.get_shape()[-1]) target_scores = tf.one_hot(val_labels, oh_dims) * val_scores val_gradients = tf.gradients(target_scores, val_images)[0] log.info('Built validation model.') # Check the shapes of labels and scores val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, weights=config.loss_weights, dataset_module=dataset_module) # Add loss tensorboard tracking if isinstance(val_loss, list): for lidx, tl in enumerate(val_loss): tf.summary.scalar('validation_loss_%s' % lidx, tl) val_loss = tf.add_n(val_loss) else: tf.summary.scalar('validation_loss', val_loss) # Add a score for the validation set val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, # TODO pred=val_scores, labels=val_labels) # Add aux scores if requested val_aux = {} if hasattr(dataset_module, 'aux_scores'): for m in dataset_module.aux_scores: val_aux[m] = eval_metrics.metric_interpreter( metric=m, pred=val_scores, labels=val_labels) # [0] # TODO: Fix for multiloss # Prepare tensorboard summaries if config.tensorboard_images: if len(val_images.get_shape()) == 4: tf_fun.image_summaries( val_images, tag='Validation') if (np.asarray( val_labels.get_shape().as_list()) > 1).sum() > 2: tf_fun.image_summaries( val_labels, tag='Validation_targets') tf_fun.image_summaries( val_scores, tag='Validation_predictions') if isinstance(val_accuracy, list): for vidx, va in enumerate(val_accuracy): tf.summary.scalar('validation_accuracy_%s' % vidx, va) else: tf.summary.scalar('validation_accuracy', val_accuracy) if config.pr_curve: if isinstance(val_scores, list): for pidx, val_score in enumerate(val_scores): val_label = val_labels[:, pidx] pr_summary.op( tag='validation_pr_%s' % pidx, predictions=tf.cast( tf.argmax( val_score, axis=-1), tf.float32), labels=tf.cast(val_label, tf.bool), display_name='validation_precision_recall_%s' % pidx) else: pr_summary.op( tag='validation_pr', predictions=tf.cast( tf.argmax( val_scores, axis=-1), tf.float32), labels=tf.cast(val_labels, tf.bool), display_name='validation_precision_recall') log.info('Added validation summaries.') # Set up summaries and saver if not hasattr(config, 'max_to_keep'): config.max_to_keep = None saver = tf.train.Saver( var_list=tf.global_variables(), max_to_keep=config.max_to_keep) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading if placeholder_data: coord, threads = None, None else: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } if grad_images: train_dict['train_gradients'] = train_gradients val_dict['val_gradients'] = val_gradients if isinstance(train_accuracy, list): for tidx, (ta, va) in enumerate(zip(train_accuracy, val_accuracy)): train_dict['train_accuracy_%s' % tidx] = ta val_dict['val_accuracy_%s' % tidx] = va else: train_dict['train_accuracy_0'] = train_accuracy val_dict['val_accuracy_0'] = val_accuracy if load_and_evaluate_ckpt is not None: # Remove the train operation and add a ckpt pointer del train_dict['train_op'] if hasattr(dataset_module, 'aux_score'): # Attach auxillary scores to tensor dicts for m in dataset_module.aux_scores: train_dict['train_aux_%s' % m] = train_aux[m] val_dict['val_aux_%s' % m] = val_aux[m] # Start training loop if old_lr is not None: config.lr = old_lr np.save( os.path.join( dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') if load_and_evaluate_ckpt is not None: return evaluation.evaluation_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params, placeholder_data=placeholder_data) else: output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') if output_dict is not None: py_utils.save_npys( data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def get_data(self): """Get the names of files.""" res_height = self.video_frame_size[0] / self.resize_factor if self.process_data: py_utils.make_dir(self.temp_dir) videos_m4v = glob(os.path.join(self.video_dir, '*.m4v')) ca2_npy = glob(os.path.join(self.neural_dir_ca, '*.npy')) ca2_match = filter(re.compile('(.+)_MV_(.+)').match, ca2_npy) num_frame_sets = np.arange(self.total_frames / self.video_timesteps) frame_sets_idx = num_frame_sets.reshape(1, -1).repeat( self.video_timesteps, axis=1).reshape(-1) frame_idx = np.arange(self.total_frames) fr = [] res_data = [] video_idx = [] for idx, v in tqdm(enumerate(videos_m4v), total=len(videos_m4v)): vname = v.split(os.path.sep)[-1].split('.')[0] # Load the video, resize, and normalize to [0, 1] # vid = np.load(v.replace('.m4v', '.npy')) vid = mp.VideoFileClip(v) clip_resized = vid.resize(height=res_height) fr += [vid.fps] frames = np.asarray( [frame for frame in clip_resized.iter_frames()]) if frames[0].max() > 1: frames = frames.astype(np.float32) / 255. frames = frames[:self.total_frames] # Find matching ca2+ vids it_ca2 = filter( re.compile('(.+)_%s_(.+)' % vname).match, ca2_match) for tr, samp_path in enumerate(it_ca2): # Resample, detrend, zscore samp = resample_detrend_zscore(np.load(samp_path), new_fps=False) samp = samp.transpose(2, 0, 1) samp = samp[:len(frames)] esamp_file = '%s.npy' % samp_path.strip('full_frame.npy') # self.ephys_suffix) esamp_file = esamp_file.replace(self.ca_split, self.ephys_split) esamp = np.load(esamp_file)[:len(frames)] dffsearch = os.path.join( os.path.sep.join(samp_path.split(os.path.sep)[:4]), self.dff_split, self.dff_suffix, samp_path.split(os.path.sep)[-1]) dffglob = glob('%s_cell*' % dffsearch.strip('full_frame.npy')) dffsamp = np.asarray( [np.load(d)[:len(frames)] for d in dffglob]) dffsamp = resample_detrend_zscore(dffsamp, new_fps=False) dffsamp = dffsamp.transpose() dffsamp = dffsamp[:len(frames)] for fs in np.unique(frame_sets_idx): fi = frame_sets_idx == fs # Trim samp and vid and save to npz in temp folder fname = os.path.join(self.temp_dir, '%s_%s_fs_%s' % (vname, tr, fs)) video_idx += [idx] res_data += [fname] it_esamp = esamp[:self.model_timesteps] samp = samp[:self.model_timesteps] if self.smooth_ephys: it_esamp = smooth(it_esamp, window='flat', mode='same', window_len=5)[5:-5] try: np.savez( fname, ca=samp, # [fi], # Make sure this is correct! ephys=it_esamp, frame_idx=frame_idx[fi], dff=dffsamp[:self.model_timesteps], # [fi], video=frames[fi]) except Exception as e: import ipdb ipdb.set_trace() np.save(os.path.join(self.temp_dir, 'video_idx'), video_idx) else: res_data = glob(os.path.join(self.temp_dir, '*.npz')) res_data = [ rs for rs in res_data if 'retina' not in rs.split(os.path.sep)[-1] ] # video_idx = np.load( # os.path.join( # self.temp_dir, # 'video_idx.npy')) assert len(res_data) res_data = np.asarray(res_data) video_idx = np.asarray([ f.split(os.path.sep)[-1].split('g_')[0] for f in res_data ]).astype(int) # Calculate reliability unique_videos, video_counts = np.unique(video_idx, return_counts=True) repeated_videos = np.in1d(video_idx, unique_videos[np.where(video_counts > 1)[0]]) repeated_videos = video_idx[repeated_videos] dff_reliability, ca_reliability = [], [] e_reliability, variability = [], [] mask = np.load(self.cell_mask) for video in tqdm(np.unique(repeated_videos), desc='Calculating reliability', total=len(repeated_videos) // 2): it_idx = np.in1d(video_idx, video) sel_data = res_data[it_idx] data_a = np.load(sel_data[0]) data_b = np.load(sel_data[1]) dff_a = data_a['dff'] dff_b = data_b['dff'] ephys_a = data_a['ephys'] ephys_b = data_b['ephys'] ca_a = data_a['ca'].reshape(dff_a.shape[0], -1) ca_b = data_b['ca'].reshape(dff_a.shape[0], -1) if self.binarize_spikes: ephys_a = (ephys_a > 0).astype(np.int32) ephys_b = (ephys_b > 0).astype(np.int32) proc_ca_a, proc_ca_b = [], [] for cell in range(mask.shape[-1]): it_cells = mask[:, :, cell].ravel() cell_locations = np.where(it_cells) proc_ca_a += [ np.asarray([ ca_a[idx, cell_locations] for idx in range(ca_a.shape[0]) ]) ] proc_ca_b += [ np.asarray([ ca_b[idx, cell_locations] for idx in range(ca_b.shape[0]) ]) ] ca_rs = [] var = [] for pa, pb in zip(proc_ca_a, proc_ca_b): # Loop through cells pa = pa.squeeze() pb = pb.squeeze() rscores = [ np.corrcoef(pa[:, idx], pb[:, idx])[0, 1] for idx in range(pa.shape[-1]) ] arg_idx = np.argmax(rscores) rscores = self.aggregate(rscores) var += [[np.std(pa[:, arg_idx]), np.std(pb[:, arg_idx])]] ca_rs += [rscores] # ca_rs = self.aggregate(ca_rs) ca_reliability += [ca_rs] variability += [var] dff_reliability += [ np.mean([ np.corrcoef(dff_a[:, idx], dff_b[:, idx])[0, 1] for idx in range(dff_a.shape[-1]) ]) ] e_reliability += [ np.corrcoef(ephys_a.ravel(), ephys_b.ravel())[0, 1] ] ca_reliability = np.asarray(ca_reliability) dff_reliability = np.asarray(dff_reliability) e_reliability = np.asarray(e_reliability) variability = np.stack([np.asarray(v) for v in variability]).mean(0).mean(-1) # Bimodal difference at 0.8 mask_thresh_idx = variability > 0.8 mask = (mask * mask_thresh_idx[None, None, :]).transpose(2, 0, 1) np.savez(os.path.join(self.temp_dir, '%s_reliabilities' % self.output_name), ca_reliability=ca_reliability, dff_reliability=dff_reliability, proc_mask=mask, mask_thresh_idx=mask_thresh_idx, e_reliability=e_reliability, variability=variability) print 'Ca2+ video reliability: %s' % ca_reliability[:, mask_thresh_idx].mean( ) print 'dff video reliability: %s' % np.mean(dff_reliability) print 'ephys video reliability: %s' % np.nanmean(e_reliability) # Stimulus period is 5.15 seconds. Ca2+ is 111 frames at 15fps. # Means that there is 7.326 seconds of recording from Ca2+. # Video is variable framerate. # For each video resample ca2+ to 25fps then align both. # Trim this down to 76 for 5000 ms of ca2+ data # Create CV folds # num_files = len(np.unique(video_idx)) # cv_range = np.random.permutation(num_files) # cv_split = np.round(num_files * self.cv_split).astype(int) # train_idx = cv_range[:cv_split] # validation_idx = cv_range[cv_split:] # train_idx = np.in1d(video_idx, train_idx) # validation_idx = np.in1d(video_idx, validation_idx) validation_idx = [] validation_idx = np.asarray([ np.where(video_idx == reps)[0][0] for reps in np.unique(repeated_videos) ]) train_idx = ~np.in1d(video_idx, validation_idx) train_files = res_data[train_idx] validation_files = res_data[ validation_idx] # np.in1d(video_idx, validation_idx)] import ipdb ipdb.set_trace() if self.shuffle: rand_idx = np.random.permutation(len(train_files)) train_files = train_files[rand_idx] rand_idx = np.random.permutation(len(validation_files)) validation_files = validation_files[rand_idx] # Build CV dict cv_files = {} cv_files[self.folds['train']] = train_files cv_files[self.folds['val']] = validation_files return cv_files
def sigmoid_fun(x): """Apply sigmoid to maps before mAP.""" return 1 / (1 + np.exp(x)) df = pd.read_csv( os.path.join( 'data_to_process_for_jk', 'generalize_snemi_experiment_data', 'generalize_snemi_experiment_data_grouped.csv')) sel, subsel = 0, 3 out_dir = os.path.join('data_to_process_for_jk', 'generalization_membranes') py_utils.make_dir(out_dir) datasets = np.unique(df.val_dataset) models = np.unique(df.model) for d in tqdm( datasets, desc='Cycling through datasets', total=len(datasets)): f = plt.figure() plt.suptitle(d) for idx, m in enumerate(models): data = np.load( df[np.logical_and( df.model == m, df.val_dataset == d)].file_name.values[0]) data = data['val_dict'][sel] logs = data['logits'] if idx == 0:
def calculate_map(it_val_dict, exp_label, config, map_dir='maps', auto_adjust=False): """Calculate map and ARAND for segmentation performance.""" py_utils.make_dir(map_dir) def get_segments(x, threshold=0.5, comp=np.greater): """Watershed boundary map.""" distance = ndi.distance_transform_edt(comp(x, threshold).astype(float)) local_maxi = peak_local_max(distance, indices=False, footprint=np.ones((10, 10))) markers = ndi.label(local_maxi)[0] return watershed(-distance, markers).astype(np.int32) maps, arands = [], [] if it_val_dict is not None: for ol in tqdm(range(len(it_val_dict)), desc='Evaluating mAPs'): eval_dicts = it_val_dict[ol] eval_logits = eval_dicts['logits'] eval_labels = eval_dicts['labels'] for log, lab in zip(eval_logits, eval_labels): lab = lab.squeeze() log_shape = log.shape if len(log.shape) > 2: if log_shape[-1] > 1: log = -1 * log[..., [0, 1]].mean(-1) lab = (lab[..., [0, 1]].mean(-1) > 0.5).astype( np.int32) if auto_adjust and lab.mean() < 0.5: lab = 1. - lab sig_pred = 1. - sigmoid_fun(log.squeeze()) # First calculate map maps += [ average_precision_score(y_score=sig_pred.ravel(), y_true=lab.ravel()) ] # Then get ARAND on segments pred_segs = get_segments(sig_pred) lab_segs = get_segments(lab) lab_mask = (lab == 0).astype(np.int32) pred_segs *= lab_mask lab_segs *= lab_mask arands += [adapted_rand(pred_segs, lab_segs)] out_path = os.path.join(map_dir, exp_label) print('mAP is: %s' % np.mean(maps)) print('ARAND is: %s' % np.mean(arands)) print('Saved to: %s.npz' % out_path) np.savez(out_path, maps=maps, arands=arands, config=config, val_dict=it_val_dict) else: print('Received an empty validation dict.') return maps, arands
def extract_vgg_features(cm_type='contextual_vector_vd', layer_name='pool3', output_type='sparse_pool', project_name=None, model_type='vgg16', timesteps=5, dtype=tf.float32): """Main extraction and training script.""" assert project_name is not None, 'Need a project name.' # 1. Get file paths and load config config = Config() config.cm_type = cm_type project_path = config.projects[project_name] # 2. Assert the model is there and load neural data. print 'Loading preprocessed data...' data = np.load(os.path.join(project_path, '%s.npz' % project_name)) neural_data = data['data_matrix'] if config.round_neural_data: neural_data = np.round(neural_data) # TODO: across_session_data_matrix is subtracted version images = data['all_images'].astype(np.float32) # Remove zeroed columns from neural data channel_check = np.abs(neural_data).sum(0) > 0 neural_data = neural_data[:, channel_check] # TODO: create AUX dict with each channel's X/Y output_aux = {'loss': config.loss_type} # None rfs = rf_sizes.get_eRFs(model_type)[layer_name] # 3. Create a output directory if necessary and save a timestamped numpy. model_description = '%s_%s_%s_%s_%s_%s' % ( cm_type, layer_name, output_type, project_name, model_type, timesteps) dt_stamp = '%s_%s' % (model_description, str(datetime.now()).replace( ' ', '_').replace(':', '_').replace('-', '_')) project_dir = os.path.join(config.results, project_name) out_dir = os.path.join(project_dir, dt_stamp) checkpoint_dir = os.path.join(out_dir, 'checkpoints') dirs = [config.results, config.summaries, out_dir] [py_utils.make_dir(x) for x in dirs] print '-' * 60 print('Training model:' + out_dir) print '-' * 60 # 4. Prepare data on CPU neural_shape = list(neural_data.shape) num_neurons = neural_shape[-1] with tf.device('/cpu:0'): train_images = tf.placeholder(dtype=dtype, name='train_images', shape=[config.train_batch_size] + config.img_shape) train_neural = tf.placeholder(dtype=dtype, name='train_neural', shape=[config.train_batch_size] + [num_neurons]) val_images = tf.placeholder(dtype=dtype, name='val_images', shape=[config.val_batch_size] + config.img_shape) val_neural = tf.placeholder(dtype=dtype, name='val_neural', shape=[config.val_batch_size] + [num_neurons]) # 5. Prepare model on GPU with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: vgg = vgg16.Vgg16(vgg16_npy_path=config.vgg16_weight_path) train_mode = tf.get_variable(name='training', initializer=False) vgg.build( train_images, output_shape=1000, # hardcode train_mode=train_mode, final_layer=layer_name) # Select a layer activities = vgg[layer_name] # Feature reduce with a 1x1 conv if config.reduce_features is not None: vgg, activities, reduce_weights = ff.pool_ff_interpreter( self=vgg, it_neuron_op='1x1conv', act=activities, it_name='feature_reduce', out_channels=config.reduce_features, aux=None) else: reduce_weights = None # Add con-model if requested if cm_type is not None and cm_type != 'none': norms = normalizations.normalizations() activities, cm_weights, _ = norms[cm_type]( x=activities, r_in=rfs['r_in'], j_in=rfs['j_in'], timesteps=timesteps, lesions=config.lesions, train=True) else: cm_weights = None # Create output layer for N-recording channels activities = tf.nn.dropout(activities, 0.5) vgg, output_activities, output_weights = ff.pool_ff_interpreter( self=vgg, it_neuron_op=output_type, act=activities, it_name='output', out_channels=num_neurons, aux=output_aux) # Prepare the loss function loss, _ = loss_utils.loss_interpreter(logits=output_activities, labels=train_neural, loss_type=config.loss_type) # Add contextual model WD if config.reduce_features is not None and reduce_weights is not None: loss += loss_utils.add_wd(weights=reduce_weights, wd_dict=config.wd_types) # Add contextual model WD if config.cm_wd_types is not None and cm_weights is not None: loss += loss_utils.add_wd(weights=cm_weights, wd_dict=config.cm_wd_types) # Add WD to output layer if config.wd_types is not None: loss += loss_utils.add_wd(weights=output_weights, wd_dict=config.wd_types) # Finetune the learning rates train_op = loss_utils.optimizer_interpreter( loss=loss, lr=config.lr, optimizer=config.optimizer) # Calculate metrics train_accuracy = eval_metrics.metric_interpreter( metric=config.metric, pred=output_activities, labels=train_neural) # Add summaries for debugging tf.summary.image('train images', train_images) tf.summary.image('validation images', val_images) tf.summary.scalar("loss", loss) tf.summary.scalar("training accuracy", train_accuracy) # Setup validation op scope.reuse_variables() # Validation graph is the same as training except no batchnorm val_vgg = vgg16.Vgg16(vgg16_npy_path=config.vgg16_weight_path) val_vgg.build(val_images, output_shape=1000, final_layer=layer_name) # Select a layer val_activities = val_vgg[layer_name] # Add feature reduction if requested if config.reduce_features is not None: val_vgg, val_activities, _ = ff.pool_ff_interpreter( self=val_vgg, it_neuron_op=config.reduce_type, act=val_activities, it_name='feature_reduce', out_channels=config.reduce_features, aux=None) else: reduce_weights = None # Add con-model if requested if cm_type is not None and cm_type != 'none': val_activities, _, _ = norms[cm_type](x=val_activities, r_in=rfs['r_in'], j_in=rfs['j_in'], timesteps=timesteps, lesions=config.lesions, train=False) # Create output layer for N-recording channels val_vgg, val_output_activities, _ = ff.pool_ff_interpreter( self=val_vgg, it_neuron_op=output_type, act=val_activities, it_name='output', out_channels=num_neurons, aux=output_aux) # Prepare the loss function val_loss, _ = loss_utils.loss_interpreter( logits=val_output_activities, labels=val_neural, loss_type=config.loss_type, max_spikes=config.max_spikes) # Calculate metrics val_accuracy = eval_metrics.metric_interpreter( metric=config.metric, pred=val_output_activities, labels=val_neural) tf.summary.scalar('validation loss', val_loss) tf.summary.scalar('validation accuracy', val_accuracy) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run(tf.global_variables_initializer()) summary_dir = os.path.join(config.summaries, dt_stamp) summary_writer = tf.summary.FileWriter(summary_dir, sess.graph) # Start training loop train_vars = { 'images': train_images, 'neural_data': train_neural, 'loss': loss, 'score': train_accuracy, 'train_op': train_op } if cm_weights is not None: for k, v in cm_weights.iteritems(): train_vars[k] = v val_vars = { 'images': val_images, 'neural_data': val_neural, 'loss': val_loss, 'score': val_accuracy, } extra_params = { 'cm_type': cm_type, 'layer_name': layer_name, 'output_type': output_type, 'project_name': project_name, 'model_type': model_type, 'lesions': config.lesions, 'timesteps': timesteps } np.savez(os.path.join(out_dir, 'training_config_file'), config=config, extra_params=extra_params) train_cv_out, val_cv_out, weights = training.training_loop( config=config, neural_data=neural_data, images=images, target_size=config.img_shape[:2], sess=sess, train_vars=train_vars, val_vars=val_vars, summary_op=summary_op, summary_writer=summary_writer, checkpoint_dir=checkpoint_dir, summary_dir=summary_dir, saver=saver) np.savez(os.path.join(out_dir, 'data'), config=config, extra_params=extra_params, train_cv_out=train_cv_out, val_cv_out=val_cv_out, weight=weights)
def main( experiment_name, im_ext='.pdf', transform_loss=None, # 'log', colors='Paired', flip_axis=False, port_fwd=False, num_steps=np.inf, exclude=None, list_experiments=False, out_dir='analysis_data'): """Plot results of provided experiment name.""" config = Config() if list_experiments: db.list_experiments() return if port_fwd: config.db_ssh_forward = True py_utils.make_dir(out_dir) # Get experiment data if ',' in experiment_name: exps = experiment_name.split(',') perf = [] for exp in exps: perf += db.get_performance(experiment_name=exp) experiment_name = exps[0] else: perf = db.get_performance(experiment_name=experiment_name) if len(perf) == 0: raise RuntimeError('Could not find any results.') structure_names = [x['model'].split('/')[-1] for x in perf] datasets = [x['val_dataset'] for x in perf] steps = [float(x['step']) for x in perf] training_loss = [float(x['train_loss']) for x in perf] validation_loss = [float(x['val_loss']) for x in perf] training_score = [float(x['train_score']) for x in perf] validation_score = [float(x['val_score']) for x in perf] summary_dirs = [x['summary_path'] for x in perf] ckpts = [x['ckpt_path'] for x in perf] params = [x['num_params'] for x in perf] lrs = [x['lr'] for x in perf] # Pass data into a pandas DF df = pd.DataFrame(np.vstack( (structure_names, datasets, steps, params, training_loss, training_score, validation_loss, validation_score, summary_dirs, ckpts, lrs)).transpose(), columns=[ 'model names', 'datasets', 'training iteration', 'params', 'training loss', 'training accuracy', 'validation loss', 'validation accuracy', 'summary_dirs', 'checkpoints', 'lrs' ]) df['training loss'] = pd.to_numeric(df['training loss'], errors='coerce') df['validation accuracy'] = pd.to_numeric(df['validation accuracy'], errors='coerce') df['training accuracy'] = pd.to_numeric(df['training accuracy'], errors='coerce') df['training iteration'] = pd.to_numeric(df['training iteration'], errors='coerce') df['params'] = pd.to_numeric(df['params'], errors='coerce') df['lrs'] = pd.to_numeric(df['lrs'], errors='coerce') # Plot TTA dfs = [] print(len(df)) uni_structure_names = np.unique(structure_names) max_num_steps = num_steps # (20000 / 32) * num_epochs # min_num_steps = 1 for m in tqdm(uni_structure_names, total=len(uni_structure_names)): it_df = df[df['model names'] == m] it_df = it_df[it_df['training iteration'] < max_num_steps] # sorted_df = it_df.sort_values('training loss') # max_vals = sorted_df.groupby(['datasets']).first() sorted_df = [] different_models = np.unique(it_df['summary_dirs']) num_models = len(different_models) for model in different_models: # Grab each model then sort by training iteration sel_data = it_df[it_df['summary_dirs'] == model] sel_data = sel_data.sort_values('training iteration') # Smooth the sorted validation scores for tta sel_data['tta'] = ndimage.gaussian_filter1d( sel_data['validation accuracy'], 3) sel_data['num_runs'] = num_models sorted_df += [sel_data] sorted_df = pd.concat(sorted_df) dfs += [sorted_df] # Get max scores and TTAs dfs = pd.concat(dfs) scores = dfs.groupby(['lrs', 'datasets', 'model names'], as_index=False).max() # skipna=True) losses = dfs.groupby(['lrs', 'datasets', 'model names'], as_index=False).min() # skipna=True) ttas = dfs.groupby(['lrs', 'datasets', 'model names'], as_index=False).mean() # skipna=True) # Combine into a single DF print('Sort by val loss, then validate each (make a new dataloader)') scores['tta'] = ttas['validation accuracy'] scores['validation loss'] = losses['validation loss'] # Save datasets to csv filename = 'raw_data_%s.csv' % experiment_name dfs.to_csv(os.path.join(out_dir, filename)) filename = 'scores_%s.csv' % experiment_name scores.to_csv(os.path.join(out_dir, filename)) # Save an easy-to-parse csv for test datasets and fix for automated processing trim_ckpts, trim_models = [], [] for idx in range(len(scores)): ckpt = scores.iloc[idx]['checkpoints'] ckpt = '%s-%s' % (ckpt, ckpt.split('.')[0].split('_')[-1]) model = scores.iloc[idx]['model names'] trim_ckpts += [ckpt] trim_models += [model] # trimmed_ckpts = pd.DataFrame(trim_ckpts, columns=['checkpoints']) # trimmed_models = pd.DataFrame(trim_models, columns=['model']) trimmed_ckpts = pd.DataFrame(trim_ckpts) trimmed_models = pd.DataFrame(trim_models) trimmed_ckpts.to_csv( os.path.join(out_dir, 'checkpoints_%s.csv' % experiment_name)) trimmed_models.to_csv( os.path.join(out_dir, 'models_%s.csv' % experiment_name)) # Add indicator variable to group different model types during plotting scores['model_idx'] = 0 model_groups = ['fgru', 'resnet', 'unet', 'hgru'] for idx, m in enumerate(model_groups): scores['model_idx'][scores['model names'].str.contains( m, regex=False)] = idx keep_groups = np.where(~np.in1d(model_groups, 'hgru'))[0] scores = scores[scores['model_idx'].isin(keep_groups)] # Print scores to console print scores # Create max accuracy plots and aggregated dataset num_groups = len(keep_groups) # agg_df = [] f = plt.figure() sns.set(context='paper', font='Arial', font_scale=.5) sns.set_style("white") sns.despine() count = 1 for idx in keep_groups: plt.subplot(1, num_groups, count) sel_df = scores[scores['model_idx'] == idx] # sel_df = sel_df.groupby( # ['datasets', 'model names'], as_index=False).aggregate('max') # agg_df += [sel_df] sns.pointplot(data=sel_df, x='datasets', y='validation accuracy', hue='model names') plt.ylim([0.4, 1.1]) count += 1 plt.savefig(os.path.join(out_dir, 'max_%s.png' % experiment_name), dpi=300) filename = 'agg_data_%s.csv' % experiment_name # agg_df = pd.concat(agg_df) # agg_df.to_csv(os.path.join(out_dir, filename)) plt.close(f) # Create tta plots f = plt.figure() sns.set(context='paper', font='Arial', font_scale=.5) sns.set_style("white") sns.despine() count = 1 for idx in keep_groups: plt.subplot(1, num_groups, count) sel_df = scores[scores['model_idx'] == idx] # sel_df = sel_df.groupby( # ['datasets', 'model names'], as_index=False).aggregate('mean') sns.pointplot(data=sel_df, x='datasets', y='tta', hue='model names') plt.ylim([0.4, 1.1]) count += 1 plt.savefig(os.path.join(out_dir, 'tta_%s.png' % experiment_name), dpi=300) plt.close(f)
def main(experiment, model, train, val, checkpoint, use_db=True, test=False, reduction=0, random=True, add_config=None, gpu_device=['/gpu:0'], cpu_device='/cpu:0', num_gpus=False, transfer=False, placeholders=False, save_test_npz=True, num_batches=None, map_out='test_maps', out_dir=None): """Interpret and run a model.""" main_config = Config() dt_string = py_utils.get_dt_stamp() log = logger.get( os.path.join(main_config.log_dir, '%s_%s' % (experiment, dt_string))) if num_gpus: gpu_device = ['/gpu:%d' % i for i in range(num_gpus)] if test and save_test_npz and out_dir is None: raise RuntimeError('You must specify an out_dir.') if use_db: exp_params = db.get_parameters(log=log, experiment=experiment, random=random)[0] else: exp = py_utils.import_module(experiment, pre_path='experiments') exp_params = exp.experiment_params() exp_params['_id'] = -1 exp_params['experiment'] = experiment if model is not None: exp_params['model'] = model else: assert len(exp_params['model']) > 1, 'No model name supplied.' exp_params['model'] = exp_params['model'][0] if train is not None: exp_params['train_dataset'] = train if val is not None: exp_params['val_dataset'] = val # if reduction or out_dir is not None or transfer: # fine_tune = get_fine_tune_params( # out_dir=out_dir, reduction=reduction) # else: # pass results = model_tools.build_model(exp_params=exp_params, dt_string=dt_string, log=log, test=test, config=main_config, use_db=use_db, num_batches=num_batches, map_out=map_out, placeholders=placeholders, add_config=add_config, gpu_device=gpu_device, cpu_device=cpu_device, checkpoint=checkpoint) if test and save_test_npz: # Save results somewhere safe py_utils.make_dir(out_dir) results['checkpoint'] = checkpoint results['model'] = model results['experiment'] = experiment np.savez(os.path.join(out_dir, results['exp_label']), **results) log.info('Finished.')
def training_loop( config, coord, sess, summary_op, summary_writer, saver, restore_saver, threads, directories, train_dict, val_dict, exp_label, num_params, use_db, log, placeholders=False, checkpoint=None, save_weights=False, save_checkpoints=False, save_activities=False, save_gradients=False): """Run the model training loop.""" if checkpoint is not None: restore_saver.restore(sess, checkpoint) print 'Restored checkpoint %s' % checkpoint if not hasattr(config, 'early_stop'): config.early_stop = np.inf val_perf = np.asarray([np.inf]) step = 0 best_val_dict = None if save_weights: try: weight_dict = {v.name: v for v in tf.trainable_variables()} val_dict = dict( val_dict, **weight_dict) except Exception: raise RuntimeError('Failed to find weights to save.') else: weight_dict = None if hasattr(config, 'early_stop'): it_early_stop = config.early_stop else: it_early_stop = np.inf if hasattr(config, "adaptive_train"): adaptive_train = config.adaptive_train else: adaptive_train = False if placeholders: train_images = placeholders['train']['images'] val_images = placeholders['val']['images'] train_labels = placeholders['train']['labels'] val_labels = placeholders['val']['labels'] train_batches = len(train_images) / config.train_batch_size train_batch_idx = np.arange( train_batches).reshape(-1, 1).repeat( config.train_batch_size) train_images = train_images[:len(train_batch_idx)] train_labels = train_labels[:len(train_batch_idx)] val_batches = len(val_images) / config.val_batch_size val_batch_idx = np.arange( val_batches).reshape(-1, 1).repeat( config.val_batch_size) val_images = val_images[:len(val_batch_idx)] val_labels = val_labels[:len(val_batch_idx)] # Check that labels are appropriate shape tf_label_shape = train_dict['train_labels'].get_shape().as_list() np_label_shape = train_labels.shape if len(tf_label_shape) == 2 and len(np_label_shape) == 1: train_labels = train_labels[..., None] val_labels = val_labels[..., None] elif len(tf_label_shape) == len(np_label_shape): pass else: raise RuntimeError( 'Mismatch label shape np: %s vs. tf: %s' % ( np_label_shape, tf_label_shape)) # Start training train_losses = [] train_logits = [] for epoch in tqdm( range(config.epochs), desc='Epoch', total=config.epochs): for train_batch in range(train_batches): io_start_time = time.time() data_idx = train_batch_idx == train_batch it_train_images = train_images[data_idx] it_train_labels = train_labels[data_idx] if isinstance(it_train_images[0], basestring): it_train_images = np.asarray( [ data_to_tfrecords.load_image(im) for im in it_train_images]) feed_dict = { train_dict['train_images']: it_train_images, train_dict['train_labels']: it_train_labels } ( train_score, train_loss, it_train_dict, timer) = training_step( sess=sess, train_dict=train_dict, config=config, feed_dict=feed_dict) train_losses.append(train_loss) if step % config.validation_period == 0: val_score, val_lo, it_val_dict, duration = validation_step( sess=sess, val_dict=val_dict, config=config, log=log, val_images=val_images, val_labels=val_labels, val_batch_idx=val_batch_idx, val_batches=val_batches) # Save progress and important data try: val_check = np.where(val_lo < val_perf)[0] if not len(val_check): it_early_stop -= 1 print 'Deducted from early stop count.' else: it_early_stop = config.early_stop best_val_dict = it_val_dict print 'Reset early stop count.' if it_early_stop <= 0: print 'Early stop triggered. Ending early.' print 'Best validation loss: %s' % np.min(val_perf) return save_progress( config=config, val_check=val_check, weight_dict=weight_dict, it_val_dict=it_val_dict, exp_label=exp_label, step=step, directories=directories, sess=sess, saver=saver, val_score=val_score, val_loss=val_lo, train_score=train_score, train_loss=train_loss, timer=duration, num_params=num_params, log=log, summary_op=summary_op, summary_writer=summary_writer, save_activities=save_activities, save_gradients=save_gradients, save_checkpoints=save_checkpoints) except Exception as e: log.info('Failed to save checkpoint: %s' % e) # Hack to get the visulations... clean this up later if "BSDS500_test_orientation_viz" in config.experiment: # .model == "BSDS_inh_perturb" or config.model == "BSDS_exc_perturb": # from matplotlib import pyplot as plt;plt.plot(it_train_dict['train_logits'].squeeze(), "r", label="Perturb");plt.plot(it_train_dict['train_labels'].squeeze()[-6:], 'b', label="GT");plt.legend();plt.show() # from matplotlib import pyplot as plt;plt.imshow((it_train_dict['impatch'].squeeze() + np.asarray([123.68, 116.78, 103.94])[None, None]).astype(np.uint8));plt.show() # from matplotlib import pyplot as plt;dd = it_train_dict["grad0"];plt.imshow(np.abs(dd.squeeze()).mean(-1) / (np.abs(dd.squeeze()).std(-1) + 1e-4));plt.show() # from matplotlib import pyplot as plt;dd = it_train_dict['mask'];plt.imshow(dd.squeeze().mean(-1));plt.show() train_logits.append([it_train_dict["train_logits"].ravel()]) out_dir = "circuits_{}".format(config.out_dir) py_utils.make_dir(out_dir) out_target = os.path.join(out_dir, "{}_{}".format(config.model, config.train_dataset)) np.save("{}_optim".format(out_target), [sess.run(tf.trainable_variables())]) # , it_train_dict["conv"]]) np.save("{}_perf".format(out_target), train_losses) np.save("{}_curves".format(out_target), train_logits) np.save("{}_label".format(out_target), it_train_dict["train_labels"]) """ if config.model == "BSDS_inh_perturb": np.save("inh_perturbs/optim", sess.run(tf.trainable_variables()[0])) np.save("inh_perturbs/perf", train_losses) np.save("inh_perturbs/curves", train_logits) np.save("inh_perturbs/label", it_train_dict["train_labels"]) if config.model == "BSDS_exc_perturb": np.save("exc_perturbs/optim", sess.run(tf.trainable_variables()[0])) np.save("exc_perturbs/perf", train_losses) np.save("exc_perturbs/curves", train_logits) np.save("exc_perturbs/label", it_train_dict["train_labels"]) """ # Training status and validation accuracy val_status( log=log, dt=datetime.now(), step=step, train_loss=train_loss, rate=config.val_batch_size / duration, timer=float(duration), score_function=config.score_function, train_score=train_score, val_score=val_score, val_loss=val_lo, best_val_loss=np.min(val_perf), summary_dir=directories['summaries']) else: # Training status io_duration = time.time() - io_start_time train_status( log=log, dt=datetime.now(), step=step, train_loss=train_loss, rate=config.val_batch_size / duration, timer=float(duration), io_timer=float(io_duration), lr=it_train_dict['lr'], score_function=config.score_function, train_score=train_score) # End iteration val_perf = np.concatenate([val_perf, [val_lo]]) step += 1 # Adaptive ending if adaptive_train and train_loss <= adaptive_train: break if adaptive_train and train_loss <= adaptive_train: break else: try: while not coord.should_stop(): ( train_score, train_loss, it_train_dict, duration) = training_step( sess=sess, config=config, train_dict=train_dict) io_start_time = time.time() if step % config.validation_period == 0: val_score, val_lo, it_val_dict, duration = validation_step( sess=sess, val_dict=val_dict, config=config, log=log) # Save progress and important data try: val_check = np.where(val_lo < val_perf)[0] if not len(val_check): it_early_stop -= 1 print 'Deducted from early stop count.' else: it_early_stop = config.early_stop best_val_dict = it_val_dict print 'Reset early stop count.' if it_early_stop <= 0: print 'Early stop triggered. Ending early.' print 'Best validation loss: %s' % np.min(val_perf) break val_perf = save_progress( config=config, val_check=val_check, weight_dict=weight_dict, it_val_dict=it_val_dict, exp_label=exp_label, step=step, directories=directories, sess=sess, saver=saver, val_score=val_score, val_loss=val_lo, val_perf=val_perf, train_score=train_score, train_loss=train_loss, timer=duration, num_params=num_params, log=log, use_db=use_db, summary_op=summary_op, summary_writer=summary_writer, save_activities=save_activities, save_gradients=save_gradients, save_checkpoints=save_checkpoints) except Exception as e: log.info('Failed to save checkpoint: %s' % e) # Training status and validation accuracy val_status( log=log, dt=datetime.now(), step=step, train_loss=train_loss, rate=config.val_batch_size / duration, timer=float(duration), score_function=config.score_function, train_score=train_score, val_score=val_score, val_loss=val_lo, best_val_loss=np.min(val_perf), summary_dir=directories['summaries']) else: # Training status io_duration = time.time() - io_start_time train_status( log=log, dt=datetime.now(), step=step, train_loss=train_loss, rate=config.val_batch_size / duration, timer=float(duration), io_timer=float(io_duration), lr=it_train_dict['lr'], score_function=config.score_function, train_score=train_score) # End iteration step += 1 except tf.errors.OutOfRangeError: log.info( 'Done training for %d epochs, %d steps.' % ( config.epochs, step)) log.info('Saved to: %s' % directories['checkpoints']) finally: coord.request_stop() coord.join(threads) sess.close() print 'Best %s loss: %s' % (config.val_loss_function, val_perf[0]) if hasattr(config, 'get_map') and config.get_map: tf_fun.calculate_map(best_val_dict, exp_label, config) return
import os import re import argparse import numpy as np from matplotlib import pyplot as plt from utils import py_utils OUTDIR = 'npy_log' py_utils.make_dir(OUTDIR) def main(log, save_data=False, plot_data=False, data_check='Validation', val_key=r'(?<=(Validation\saccuracy\s=\s))\d\.\d+', train_key=r'(?<=(Training\saccuracy\s=\s))\d\.\d+'): """Parse a log file for the key string extending to the next whitespace.""" train_data, val_data = [], [] with open(log, 'rb') as f: for line in f: if data_check in line: train_data += [float(re.search(train_key, line).group())] val_data += [float(re.search(val_key, line).group())] val_data = np.array(val_data) if plot_data: f = plt.figure() plt.plot(val_data, label='Validation') plt.plot(train_data, label='Training') plt.xlabel('Iterations of training (x2000)')
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, num_models=3, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_dir='/media/data_cifs/AllenData/DataForTrain/all_stimulus_template', stimulus_type='tfrecord', top_n=0, preload_stim=False, target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join( allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()[ 'experiment_name']} if query_db: perf = query_hp_hist( exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT for gd in data_files: mt = gd.split( os.path.sep)[-1].split( template_exp + '_')[-1].split('_' + 'val')[0] it_data = np.load(gd).item() sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view( np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] bp = np.argmax(sperfs) fxs += [xy[0]] fys += [xy[1]] fperfs += [sperfs[bp]] fargs += [sargs[bp]] fmodel_types += [sel_mts[bp]] fexps += [sexps[bp]] xs = np.asarray(fxs) ys = np.asarray(fys) perfs = np.asarray(fperfs) arg_perf = np.asarray(fargs) exps = np.asarray(fexps) model_types = np.asarray(fmodel_types) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] # sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] sorted_perfs = [np.argsort(it_perfs)[::-1][top_n]] for idx in sorted_perfs: perf = sel_exp_query( experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % ( model_file, model_file.split(os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull stimuli stim_dir = os.path.join( main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] # stim_val_data = [x for x in stim_files if 'val.tfrecords' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Store sparse noise for reference sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): val_images = tf.placeholder( tf.float32, shape=[1] + [x for x in dataset_module.im_size]) # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append(os.path.join('models', 'structs', sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') grad_image = tf.gradients(model.output, val_images)[0] print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) saver.restore(sess, model_ckpt) # Set up exemplar threading if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name] elif target_model == 'dog': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name] else: raise NotImplementedError val_tensors = { 'images': val_images, # 'labels': val_labels, 'filts': fname, 'responses': model[target_layer], 'labels': model['output'], 'grads': grad_image } all_images, all_preds, all_grads, all_responses = [], [], [], [] num_steps = 10000 stimuli = os.path.join( stimulus_dir, 'locally_sparse_noise_8deg_template.pkl') stimuli = pickle.load(open(stimuli, 'rb'))[:num_steps] ih, iw = 304, 608 ns, sh, sw = stimuli.shape sh, sw = ih, iw sh = int(sh) sw = int(sw) tb = int((ih - sh) // 2) lr = int((iw - sw) // 2) gns = 32 cnst = 127.5 for step in range(num_steps): # step, stim in enumerate(stimuli): # step in range(num_steps): chosen_stim = np.random.permutation(ns)[0] if preload_stim: it_stim = stimuli[chosen_stim].astype(np.float32) it_stim = it_stim.astype(np.float32) noise_im = (misc.imresize(it_stim, [sh, sw], interp='nearest')) noise_im = cv2.copyMakeBorder( noise_im.squeeze(), tb, tb, lr, lr, cv2.BORDER_CONSTANT, value=cnst) else: stim_noise = scipy.sparse.csr_matrix(scipy.sparse.random(ih // gns, iw // gns, density=0.05)).todense() stim_noise_mask = stim_noise == 0 stim_noise[stim_noise > 0.5] = 255. stim_noise[stim_noise < 0.5] = 0. stim_noise[stim_noise_mask] = cnst noise_im = (misc.imresize(stim_noise.squeeze(), [sh, sw], interp='nearest')) if np.random.rand() < 0.5: noise_im = np.fliplr(noise_im) if np.random.rand() < 0.5: noise_im = np.flipud(noise_im) # noise_im = (misc.imresize(it_stim, [ih, iw], interp='nearest'))[None, :, :, None] noise_im = noise_im / 255. noise_im = noise_im[None, :, :, None] assert noise_im.max() <= 1 val_vals = sess.run(val_tensors.values(), feed_dict={val_images: noise_im}) val_dict = {k: v for k, v in zip(val_tensors.keys(), val_vals)} all_images += [noise_im] # val_dict['images']] # all_responses += [val_dict['responses']] all_preds += [val_dict['labels'].squeeze()] # all_grads += [val_dict['grads'].squeeze()] print 'Finished step %s' % step # Process and save data all_images = np.concatenate(all_images).squeeze() ev, vals = peakdet(all_preds, np.median(all_preds)) sp = np.zeros_like(all_preds) sp[ev[:, 0].astype(int)] = 1 plt.imshow(np.matmul(all_images.reshape(all_images.shape[0], -1).transpose(), all_preds).reshape(ih, iw));plt.show() plt.imshow(np.matmul(all_images.reshape(all_images.shape[0], -1).transpose(), sp).reshape(ih, iw));plt.show() filters = val_dict['filts'][0].squeeze().transpose(2, 0, 1) import ipdb;ipdb.set_trace() all_grads = np.asarray(all_grads) all_preds = np.asarray(all_preds).reshape(-1, 1) import ipdb;ipdb.set_trace() # res_f = all_responses.reshape(ne * h * w, k) # res_g = res_grads.reshape(ne, rh * rw) # i_cov = np.cov(res_i.transpose()) # f_cov = np.cov(res_f.transpose()) # g_cov = np.cov(res_g.transpose()) # sp = (all_preds > all_preds.mean()).astype(np.float32) # res_i = res_g # ev, vals = peakdet(all_preds, 0.5) # sp = np.zeros_like(all_preds) # sp[ev[:, 0].astype(int)] = 1 # slen = ne # nsp = np.sum(sp) # number of spikes # swid = rh * rw # Msz = np.dot(np.dot(slen, swid), ne) # Size of full stimulus matrix # rowlen = 1830 # np.dot(swid, ne) # Length of a single row of stimulus matrix # Compute raw mean and covariance # RawMu = np.mean(res_i, 0).T # RawCov = np.dot(res_i.T, res_i) / (slen-1.) - (RawMu*np.vstack(RawMu)*slen) / (slen-1.) # Compute spike-triggered mean and covariance # iisp = np.nonzero((sp > 0.)) # spvec = sp[iisp] # STA = np.divide(np.dot(spvec.T, res_i[iisp[0],:]).T, nsp) # STC = np.dot(res_i[iisp[0],:].T, np.multiply(res_i[iisp[0],:], ml.repmat(spvec, rowlen, 1).T))/(nsp-1.) - (STA*np.vstack(STA)*nsp)/(nsp-1.) # res_i_cov = np.matmul(res_i.transpose(), res_i) # inv_res_i = np.linalg.pinv(res_i_cov) # sta = inv_res_i * np.matmul(res_i.transpose(), all_preds) # sta = inv_res_i * np.matmul(res_i.transpose(), spike_preds) # sti = (1. / float(ne)) * (np.linalg.pinv(i_cov) * np.matmul(res_i, all_preds)) # sta = (1. / float(ne)) * (np.linalg.pinv(f_cov) * np.matmul(res_f, all_preds)) # sta = sta.reshape(h, w) # stg = (1. / float(ne)) * np.matmul(all_grads.reshape(h * w, ne), all_preds) # stg = stg.reshape(h, w) np.savez( os.path.join(data_dir, 'data'), images=all_images, pred=all_preds, filters=filters, STA=STA, fits=fits, grads=all_grads) if target_model != 'dog': save_mosaic( maps=filters, # [0].squeeze().transpose(2, 0, 1), output=os.path.join(data_dir, '%s_filters' % target_layer), rc=8, cc=4, title='%s filters' % ( target_layer)) else: import ipdb;ipdb.set_trace() print 'Complete.'
e = 'for_alekh' dirs = [ 'berson_001', 'berson_010', 'berson_100', 'snemi_001', 'snemi_010', 'snemi_100', 'seung_berson_001', 'seung_berson_010', 'seung_berson_100', 'seung_snemi_001', 'seung_snemi_010', 'seung_snemi_100', ] for d in tqdm(dirs, total=len(dirs)): z = np.load(glob(os.path.join(p, d, '*.npz'))[0])['test_dict'] out_path = os.path.join(p, e, d) label_path = os.path.join(out_path, 'groundTruth') pred_path = os.path.join(out_path, 'predictions') py_utils.make_dir(out_path) py_utils.make_dir(label_path) py_utils.make_dir(pred_path) for idx, i in enumerate(z): label = i['labels'] pred = tf_fun.sigmoid_fun(i['logits']) np.save(os.path.join(label_path, '%s.npy' % idx), label) np.save(os.path.join(pred_path, '%s.npy' % idx), pred)
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, num_models=3, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_type='tfrecord', top_n=1, grad='lrp', target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join( allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()[ 'experiment_name']} if query_db: perf = query_hp_hist( exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT for gd in data_files: mt = gd.split( os.path.sep)[-1].split( template_exp + '_')[-1].split('_' + 'val')[0] it_data = np.load(gd).item() sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view( np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] scores = [] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] # Only get top conv/sep spots sperfs = sperfs[sel_mts != 'dog'] sperfs = sperfs[sel_mts != 'DoG'] scores += [sperfs.mean() / sperfs.std()] best_fits = np.argmax(np.asarray(scores)) xs = np.asarray([uxys[best_fits][0]]) ys = np.asarray([uxys[best_fits][1]]) sel_idx = (xys == uxys[best_fits]).sum(axis=-1) == 2 perfs = np.asarray(perfs[sel_idx]) exps = np.asarray(exps[sel_idx]) model_types = np.asarray(model_types[sel_idx]) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] for idx in sorted_perfs: perf = sel_exp_query( experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % ( model_file, model_file.split(os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull stimuli stim_dir = os.path.join( main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] # stim_val_data = [x for x in stim_files if 'val.tfrecords' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Store sparse noise for reference sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): if stimulus_type == 'sparse_noise': pass elif stimulus_type == 'drifting_grating': pass elif stimulus_type == 'tfrecord': val_images, val_labels = data_loader.inputs( dataset=stim_val_data, batch_size=1, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=[None], # dataset_module.preprocess, num_epochs=1, tf_reader_settings=dataset_module.tf_reader, shuffle=False ) # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append(os.path.join('models', 'structs', sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') if grad == 'vanilla': grad_image = tf.gradients(model.output, val_images)[0] elif grad == 'lrp': eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map( {'Relu': 'GradLRP'}): grad_image = tf.gradients(model.output, val_images)[0] elif grad == 'cam': eval_graph = tf.Graph() with eval_graph.as_default(): with eval_graph.gradient_override_map( {'Relu': 'GuidedRelu'}): grad_image = tf.gradients(model.output, val_images)[0] else: raise NotImplementedError print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) ) saver.restore(sess, model_ckpt) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name] elif target_model == 'dog' or target_model == 'DoG': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name] else: raise NotImplementedError val_tensors = { 'images': val_images, 'labels': val_labels, 'filts': fname, 'responses': model.output, # model[target_layer], 'grads': grad_image } all_images, all_preds, all_grads, all_responses = [], [], [], [] step = 0 try: while not coord.should_stop(): val_vals = sess.run(val_tensors.values()) val_dict = {k: v for k, v in zip(val_tensors.keys(), val_vals)} all_images += [val_dict['images']] all_responses += [val_dict['responses']] all_preds += [val_dict['labels'].squeeze()] all_grads += [val_dict['grads'].squeeze()] print 'Finished step %s' % step step += 1 except: print 'Finished tfrecords' finally: coord.request_stop() coord.join(threads) sess.close() # Process and save data # if target_model != 'dog': # filters = val_dict['filts'][0].squeeze().transpose(2, 0, 1) all_images = np.concatenate(all_images).squeeze() all_grads = np.asarray(all_grads) all_preds = np.asarray(all_preds).reshape(-1, 1) all_responses = np.asarray(all_responses).squeeze() np.savez( os.path.join(data_dir, 'data'), images=all_images, pred=all_preds, # filters=filters, grads=all_grads) # if target_model != 'dog': # save_mosaic( # maps=filters, # [0].squeeze().transpose(2, 0, 1), # output=os.path.join(data_dir, '%s_filters' % target_layer), # rc=8, # cc=4, # title='%s filters' % ( # target_layer)) print 'Complete.'
def plot_fits( experiment='760_cells_2017_11_04_16_29_09', query_db=False, template_exp='ALLEN_selected_cells_1', process_pnodes=False, allen_dir='/home/drew/Documents/Allen_Brain_Observatory', output_dir='tests/ALLEN_files', stimulus_dir='/media/data_cifs/AllenData/DataForTrain/all_stimulus_template', stimulus_type='tfrecord', top_n=100, recalc=False, preload_stim=False, target_layer='conv1_1', # conv1_1, sep_conv1_1, dog1_1 target_model='conv2d'): # conv2d, sep_conv2d, dog """Plot fits across the RF. experiment: Name of Allen experiment you're plotting. query_db: Use data from DB versus data in Numpys. num_models: The number of architectures you're testing. template_exp: The name of the contextual_circuit model template used.""" sys.path.append(allen_dir) from allen_config import Allen_Brain_Observatory_Config if process_pnodes: from pnodes_declare_datasets_loop import query_hp_hist, sel_exp_query else: from declare_datasets_loop import query_hp_hist, sel_exp_query config = Config() main_config = Allen_Brain_Observatory_Config() db_config = credentials.postgresql_connection() files = glob( os.path.join(allen_dir, main_config.multi_exps, experiment, '*.npz')) assert len(files), 'Couldn\'t find files.' out_data, xs, ys = [], [], [] perfs, model_types, exps, arg_perf = [], [], [], [] count = 0 for f in files: data = np.load(f) d = { 'x': data['rf_data'].item()['on_center_x'], 'y': data['rf_data'].item()['on_center_y'], # x: files['dataset_method'].item()['x_min'], # y: files['dataset_method'].item()['y_min'], } exp_name = { 'experiment_name': data['dataset_method'].item()['experiment_name'] } if query_db: perf = query_hp_hist(exp_name['experiment_name'], db_config=db_config) if perf is None: print 'No fits for: %s' % exp_name['experiment_name'] else: raise NotImplementedError d['perf'] = perf d['max_val'] = np.max(perf) out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(d['perf'])] count += 1 else: data_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_losses.npy')) # Scores has preds, labels has GT score_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_scores.npy')) # Scores has preds, labels has GT lab_files = glob( os.path.join( main_config.ccbp_exp_evals, exp_name['experiment_name'], '*val_labels.npy')) # Scores has preds, labels has GT for gd, sd, ld in zip(data_files, score_files, lab_files): mt = gd.split(os.path.sep)[-1].split(template_exp + '_')[-1].split('_' + 'val')[0] if not recalc: it_data = np.load(gd).item() else: lds = np.load(ld).item() sds = np.load(sd).item() it_data = { k: np.corrcoef(lds[k], sds[k])[0, 1] for k in sds.keys() } sinds = np.asarray(it_data.keys())[np.argsort(it_data.keys())] sit_data = [it_data[idx] for idx in sinds] d['perf'] = sit_data d['max_val'] = np.max(sit_data) d['max_idx'] = np.argmax(sit_data) d['mt'] = mt out_data += [d] xs += [np.round(d['x'])] ys += [np.round(d['y'])] perfs += [np.max(sit_data)] arg_perf += [np.argmax(sit_data)] exps += [gd.split(os.path.sep)[-2]] model_types += [mt] count += 1 # Package as a df xs = np.round(np.asarray(xs)).astype(int) ys = np.round(np.asarray(ys)).astype(int) perfs = np.asarray(perfs) arg_perf = np.asarray(arg_perf) exps = np.asarray(exps) model_types = np.asarray(model_types) # Filter to only keep top-scoring values at each x/y (dirty trick) fxs, fys, fperfs, fmodel_types, fexps, fargs = [], [], [], [], [], [] xys = np.vstack((xs, ys)).transpose() cxy = np.ascontiguousarray( # Unique rows xys).view(np.dtype((np.void, xys.dtype.itemsize * xys.shape[1]))) _, idx = np.unique(cxy, return_index=True) uxys = xys[idx] for xy in uxys: sel_idx = (xys == xy).sum(axis=-1) == 2 sperfs = perfs[sel_idx] sexps = exps[sel_idx] sargs = arg_perf[sel_idx] sel_mts = model_types[sel_idx] bp = np.argmax(sperfs) fxs += [xy[0]] fys += [xy[1]] fperfs += [sperfs[bp]] fargs += [sargs[bp]] fmodel_types += [sel_mts[bp]] fexps += [sexps[bp]] xs = np.asarray(fxs) ys = np.asarray(fys) perfs = np.asarray(fperfs) arg_perf = np.asarray(fargs) exps = np.asarray(fexps) model_types = np.asarray(fmodel_types) umt, model_types_inds = np.unique(model_types, return_inverse=True) # Get weights for the top-n fitting models of each type it_perfs = perfs[model_types == target_model] it_exps = exps[model_types == target_model] # it_args = arg_perf[model_types == target_model] sorted_perfs = np.argsort(it_perfs)[::-1][:top_n] perf = sel_exp_query(experiment_name=it_exps[sorted_perfs[0]], model=target_model, db_config=db_config) dummy_sel_model = perf[-1] print 'Using %s' % dummy_sel_model model_file = dummy_sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Pull responses dataset_module = py_utils.import_module( model_dir=config.dataset_info, dataset=dummy_sel_model['experiment_name']) dataset_module = dataset_module.data_processing() with tf.device('/cpu:0'): val_images = tf.placeholder(tf.float32, shape=[1] + [x for x in dataset_module.im_size]) # Pull stimuli stim_dir = os.path.join(main_config.tf_record_output, dummy_sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] # Mean normalize log = logger.get(os.path.join(output_dir, 'sta_logs', target_model)) data_dir = os.path.join(output_dir, 'data', target_model) py_utils.make_dir(data_dir) sys.path.append( os.path.join('models', 'structs', dummy_sel_model['experiment_name'])) model_dict = __import__(target_model) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=stim_mean_data, training=True, # FIXME output_size=dataset_module.output_size) with tf.device('/gpu:0'): with tf.variable_scope('cnn') as scope: val_scores, model_summary = model.build( data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') grad_image = tf.gradients(model.output, val_images)[0] print(json.dumps(model_summary, indent=4)) # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) all_filters = [] all_rfs = [] max_scores = [] for idx in sorted_perfs: perf = sel_exp_query(experiment_name=it_exps[idx], model=target_model, db_config=db_config) # perf_steps = np.argsort([v['training_step'] for v in perf])[::-1] perf_steps = [v['validation_loss'] for v in perf] max_score = np.max(perf_steps) arg_perf_steps = np.argmax(perf_steps) sel_model = perf[arg_perf_steps] # perf_steps[it_args[idx]]] print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) model_meta = '%s.meta' % model_ckpt # Store sparse noise for reference stim_dir = os.path.join(main_config.tf_record_output, sel_model['experiment_name']) stim_files = glob(stim_dir + '*') stim_meta_file = [x for x in stim_files if 'meta' in x][0] stim_val_data = [x for x in stim_files if 'train.tfrecords' in x][0] stim_val_mean = [x for x in stim_files if 'train_means' in x][0] assert stim_meta_file is not None assert stim_val_data is not None assert stim_val_mean is not None stim_meta_data = np.load(stim_meta_file).item() rf_stim_meta_data = stim_meta_data['rf_data'] stim_mean_data = np.load( stim_val_mean).items()[0][1].item()['image']['mean'] rf_stim_meta_data = rf_stim_meta_data.values()[0][0] sparse_rf_on = { 'center_x': rf_stim_meta_data.get('on_center_x', None), 'center_y': rf_stim_meta_data.get('on_center_y', None), 'width_x': rf_stim_meta_data.get('on_width_x', None), 'width_y': rf_stim_meta_data.get('on_width_y', None), 'distance': rf_stim_meta_data.get('on_distance', None), 'area': rf_stim_meta_data.get('on_area', None), 'rotation': rf_stim_meta_data.get('on_rotation', None), } sparse_rf_off = { 'center_x': rf_stim_meta_data.get('off_center_x', None), 'center_y': rf_stim_meta_data.get('off_center_y', None), 'width_x': rf_stim_meta_data.get('off_width_x', None), 'width_y': rf_stim_meta_data.get('off_width_y', None), 'distance': rf_stim_meta_data.get('off_distance', None), 'area': rf_stim_meta_data.get('off_area', None), 'rotation': rf_stim_meta_data.get('off_rotation', None), } sparse_rf = {'on': sparse_rf_on, 'off': sparse_rf_off} # Set up exemplar threading if target_model == 'conv2d': fname = [ x for x in tf.global_variables() if 'conv1_1_filters:0' in x.name ] elif target_model == 'sep_conv2d': fname = [ x for x in tf.global_variables() if 'sep_conv1_1_filters:0' in x.name ] elif target_model == 'dog' or target_model == 'DoG': fname = [ x for x in tf.global_variables() if 'dog1_1_filters:0' in x.name ] else: raise NotImplementedError print 'Using %s' % sel_model model_file = sel_model['ckpt_file'].split('.')[0] model_ckpt = '%s.ckpt-%s' % (model_file, model_file.split( os.path.sep)[-1].split('_')[-1]) saver.restore(sess, model_ckpt) all_filters += [sess.run(fname)] all_rfs += [sparse_rf] max_scores += [max_score] np.savez('tests/ALLEN_files/filters/%s_%s_recalc_%s' % (experiment, target_model, recalc), rfs=all_rfs, perf=max_scores, filters=all_filters) print 'SAVED'
def main(experiment_name, list_experiments=False, gpu_device='/gpu:0'): """Create a tensorflow worker to run experiments in your DB.""" if list_experiments: exps = db.list_experiments() print '_' * 30 print 'Initialized experiments:' print '_' * 30 for l in exps: print l.values()[0] print '_' * 30 print 'You can add to the DB with: '\ 'python prepare_experiments.py --experiment=%s' % \ exps[0].values()[0] return if experiment_name is None: print 'No experiment specified. Pulling one out of the DB.' experiment_name = db.get_experiment_name() # Prepare to run the model config = Config() condition_label = '%s_%s' % (experiment_name, py_utils.get_dt_stamp()) experiment_label = '%s' % (experiment_name) log = logger.get(os.path.join(config.log_dir, condition_label)) experiment_dict = experiments.experiments()[experiment_name]() config = add_to_config(d=experiment_dict, config=config) # Globals config, exp_params = process_DB_exps( experiment_name=experiment_name, log=log, config=config) # Update config w/ DB params dataset_module = py_utils.import_module(model_dir=config.dataset_info, dataset=config.dataset) dataset_module = dataset_module.data_processing() # hardcoded class name train_data, train_means = get_data_pointers( dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[1], # TODO: SEARCH FOR INDEX. log=log) val_data, val_means = get_data_pointers(dataset=config.dataset, base_dir=config.tf_records, cv=dataset_module.folds.keys()[0], log=log) # Initialize output folders dir_list = { 'checkpoints': os.path.join(config.checkpoints, condition_label), 'summaries': os.path.join(config.summaries, condition_label), 'condition_evaluations': os.path.join(config.condition_evaluations, condition_label), 'experiment_evaluations': os.path.join( # DEPRECIATED config.experiment_evaluations, experiment_label), 'visualization': os.path.join(config.visualizations, condition_label), 'weights': os.path.join(config.condition_evaluations, condition_label, 'weights') } [py_utils.make_dir(v) for v in dir_list.values()] # Prepare data loaders on the cpu config.data_augmentations = py_utils.flatten_list( config.data_augmentations, log) with tf.device('/cpu:0'): train_images, train_labels = data_loader.inputs( dataset=train_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) val_images, val_labels = data_loader.inputs( dataset=val_data, batch_size=config.batch_size, model_input_image_size=dataset_module.model_input_image_size, tf_dict=dataset_module.tf_dict, data_augmentations=config.data_augmentations, num_epochs=config.epochs, tf_reader_settings=dataset_module.tf_reader, shuffle=config.shuffle) log.info('Created tfrecord dataloader tensors.') # Load model specification struct_name = config.model_struct.split(os.path.sep)[-1] try: model_dict = py_utils.import_module( dataset=struct_name, model_dir=os.path.join('models', 'structs', experiment_name).replace(os.path.sep, '.')) except IOError: print 'Could not find the model structure: %s' % experiment_name # Inject model_dict with hyperparameters if requested model_dict.layer_structure = hp_opt_utils.inject_model_with_hps( layer_structure=model_dict.layer_structure, exp_params=exp_params) # Prepare model on GPU with tf.device(gpu_device): with tf.variable_scope('cnn') as scope: # Training model if len(dataset_module.output_size) > 1: log.warning('Found > 1 dimension for your output size.' 'Converting to a scalar.') dataset_module.output_size = np.prod( dataset_module.output_size) if hasattr(model_dict, 'output_structure'): # Use specified output layer output_structure = model_dict.output_structure else: output_structure = None model = model_utils.model_class( mean=train_means, training=True, output_size=dataset_module.output_size) train_scores, model_summary = model.build( data=train_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built training model.') log.debug(json.dumps(model_summary, indent=4), verbose=0) print_model_architecture(model_summary) # Prepare the loss function train_loss, _ = loss_utils.loss_interpreter( logits=train_scores, labels=train_labels, loss_type=config.loss_function, dataset_module=dataset_module) # Add weight decay if requested if len(model.regularizations) > 0: train_loss = loss_utils.wd_loss( regularizations=model.regularizations, loss=train_loss, wd_penalty=config.regularization_strength) train_op = loss_utils.optimizer_interpreter( loss=train_loss, lr=config.lr, optimizer=config.optimizer, constraints=config.optimizer_constraints, model=model) log.info('Built training loss function.') train_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=train_scores, labels=train_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('train images', train_images) tf.summary.scalar('training loss', train_loss) tf.summary.scalar('training accuracy', train_accuracy) log.info('Added training summaries.') # Validation model scope.reuse_variables() val_model = model_utils.model_class( mean=val_means, training=True, output_size=dataset_module.output_size) val_scores, _ = val_model.build( # Ignore summary data=val_images, layer_structure=model_dict.layer_structure, output_structure=output_structure, log=log, tower_name='cnn') log.info('Built validation model.') val_loss, _ = loss_utils.loss_interpreter( logits=val_scores, labels=val_labels, loss_type=config.loss_function, dataset_module=dataset_module) val_accuracy = eval_metrics.metric_interpreter( metric=dataset_module.score_metric, pred=val_scores, labels=val_labels) # training accuracy if int(train_images.get_shape()[-1]) <= 3: tf.summary.image('val images', val_images) tf.summary.scalar('validation loss', val_loss) tf.summary.scalar('validation accuracy', val_accuracy) log.info('Added validation summaries.') # Set up summaries and saver saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # Initialize the graph sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) # Need to initialize both of these if supplying num_epochs to inputs sess.run( tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())) summary_writer = tf.summary.FileWriter(dir_list['summaries'], sess.graph) # Set up exemplar threading coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Create dictionaries of important training and validation information train_dict = { 'train_loss': train_loss, 'train_accuracy': train_accuracy, 'train_images': train_images, 'train_labels': train_labels, 'train_op': train_op, 'train_scores': train_scores } val_dict = { 'val_loss': val_loss, 'val_accuracy': val_accuracy, 'val_images': val_images, 'val_labels': val_labels, 'val_scores': val_scores, } # Start training loop np.save( os.path.join(dir_list['condition_evaluations'], 'training_config_file'), config) log.info('Starting training') output_dict = training.training_loop( config=config, db=db, coord=coord, sess=sess, summary_op=summary_op, summary_writer=summary_writer, saver=saver, threads=threads, summary_dir=dir_list['summaries'], checkpoint_dir=dir_list['checkpoints'], weight_dir=dir_list['weights'], train_dict=train_dict, val_dict=val_dict, train_model=model, val_model=val_model, exp_params=exp_params) log.info('Finished training.') model_name = config.model_struct.replace('/', '_') py_utils.save_npys(data=output_dict, model_name=model_name, output_string=dir_list['experiment_evaluations'])
def encode_dataset(dataset, train_shards=0, val_shards=0, force_val=False): config = Config() data_class = py_utils.import_module(module=dataset, pre_path=config.dataset_classes) data_proc = data_class.data_processing() data = data_proc.get_data() if len(data) == 2: files, labels = data nhot = None elif len(data) == 3: files, labels, nhot = data else: raise NotImplementedError targets = data_proc.targets im_size = data_proc.im_size if hasattr(data_proc, 'preprocess'): preproc_list = data_proc.preprocess else: preproc_list = [] if hasattr(data_proc, 'label_size'): label_size = data_proc.label_size else: label_size = None if hasattr(data_proc, 'label_size'): store_z = data_proc.store_z else: store_z = False if hasattr(data_proc, 'normalize_im'): normalize_im = data_proc.normalize_im else: normalize_im = False if not train_shards: ds_name = os.path.join(config.tf_records, data_proc.output_name) data_to_tfrecords(files=files, labels=labels, targets=targets, nhot=nhot, ds_name=ds_name, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im) else: assert val_shards > 0, 'Choose the number of val shards.' raise NotImplementedError('Needs support for nhot.') shard_dir = os.path.join(config.tf_records, data_proc.output_name) py_utils.make_dir(shard_dir) if not force_val: create_shards(it_shards=train_shards, shard_dir=shard_dir, key='train', files=files, labels=labels, targets=targets, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im) create_shards(it_shards=val_shards, shard_dir=shard_dir, key='val', files=files, labels=labels, targets=targets, im_size=im_size, label_size=label_size, preprocess=preproc_list, store_z=store_z, normalize_im=normalize_im)
def get_labels(self, files): """Process and save label images.""" labels = {} new_files = {} for k, images in files.iteritems(): # Replace extension and path with labels label_vec = [] file_vec = [] fold = images[0].split(os.path.sep)[-2] # New label dir proc_dir = os.path.join( images[0].split(fold)[0], fold, self.processed_labels) py_utils.make_dir(proc_dir) # New image dir proc_image_dir = os.path.join( self.config.data_root, self.name, self.images_dir, fold, self.processed_images) py_utils.make_dir(proc_image_dir) all_images = [] for im in images: it_label = im.split(os.path.sep)[-1] it_label_path = '%s%s' % (im.split('.')[0], self.lab_extension) it_label_path = it_label_path.replace( self.images_dir, self.labels_dir) # Process every label and duplicate images for each label_data = io.loadmat( it_label_path)['groundTruth'].reshape(-1) im_data = misc.imread(im) transpose_labels = False if not np.all(self.im_size == list(im_data.shape)): im_data = np.swapaxes(im_data, 0, 1) transpose_labels = True assert np.all( self.im_size == list(im_data.shape) ), 'Mismatched dimensions.' if self.fold_options[k] == 'duplicate': # Loop through all labels for idx, lab in enumerate(label_data): # Process labels ip_lab = lab.item()[1].astype(np.float32) if transpose_labels: ip_lab = np.swapaxes(ip_lab, 0, 1) it_im_name = '%s_%s' % (idx, it_label) it_lab_name = '%s.npy' % it_im_name.split('.')[0] out_lab = os.path.join(proc_dir, it_lab_name) np.save(out_lab, ip_lab) label_vec += [out_lab] # Process images proc_im = os.path.join(proc_image_dir, it_im_name) all_images.append(im_data) file_vec += [proc_im] elif self.fold_options[k] == 'mean': mean_labs = [] for idx, lab in enumerate(label_data): # Process labels ip_lab = lab.item()[1].astype(np.float32) if transpose_labels: ip_lab = np.swapaxes(ip_lab, 0, 1) mean_labs += [ip_lab] mean_lab = np.asarray(mean_labs).mean(0) out_lab = os.path.join( proc_dir, '%s.npy' % it_label.strip( self.im_extension)[0]) np.save(out_lab, mean_lab) label_vec += [out_lab] # Process images proc_im = os.path.join(proc_image_dir, it_label) all_images.append(im_data) file_vec += [proc_im] else: raise NotImplementedError # Zscore the images. # load all images from filevec into an asarray # calculate the mean # calculate the std # zscore # Overwrite the saved images... use floating point flag on imsave # Alternatively, save with scikit-image all_images = np.array(all_images).astype(np.float32) if k == 'train': mean_rgb_train = all_images.mean( 0, keepdims=True).mean( 1, keepdims=True).mean( 2, keepdims=True) std_rgb_train = all_images.std() all_images_z = (all_images - mean_rgb_train) / std_rgb_train for i, img in enumerate(all_images_z): np.save( '%s.npy' % file_vec[i].strip(self.im_extension), img.astype(np.float32)) # labels[k] = label_vec # new_files[k] = ['%s.npy' % i for i in file_vec] labels[k] = label_vec new_files[k] = ['%s.npy' % i for i in file_vec] return labels, new_files
# Config file_path = '/home/drew/Documents/hgru/results' out_path = 'movies' f = os.path.join(file_path, 'val_gradients.npz') im_key = 'val_images' grad_key = 'val_gradients' lab_key = 'val_labels' path_index = 3 nopath_index = 0 interp_frames = 100 # Process data data = np.load(f) images = data[im_key] grads = data[grad_key] labels = data[lab_key] py_utils.make_dir(out_path) # Render movies in a loop for idx in range(len(images)): render_movie( images=images, grads=grads, labels=labels, idx=idx, interp_frames=interp_frames, out_path=out_path)