def predict_volume_stack(models, data_generator, batch_size, grid_size, verbose=False): """ predict all the patches in a volume requires batch_size to be a divisor of the number of patches (prod(grid_size)) Note: we allow models to be a list or a single model. Normally, if you'd like to run a function over a list for some param, you can simply loop outside of the function. here, however, we are dealing with a generator, and want the output of that generator to be consistent for each model. Returns: if models isa list of more than one model: a tuple of model entried, each entry is a tuple of: all_true, all_pred, all_vol, <all_prior> if models is just one model: a tuple of all_true, all_pred, all_vol, <all_prior> """ if not isinstance(models, (list, tuple)): models = (models,) # compute the number of batches we need for one volume # we need the batch_size to be a divisor of nb_patches, # in order to loop through batches and form full volumes nb_patches = np.prod(grid_size) # assert np.mod(nb_patches, batch_size) == 0, \ # "batch_size %d should be a divisor of nb_patches %d" %(batch_size, nb_patches) nb_batches = ((nb_patches - 1) // batch_size) + 1 # go through the patches batch_gen = tqdm(range(nb_batches)) if verbose else range(nb_batches) for batch_idx in batch_gen: sample = next(data_generator) nb_vox = np.prod(sample[1].shape[1:-1]) do_prior = isinstance(sample[0], (list, tuple)) # pre-allocate all the data if batch_idx == 0: nb_labels = sample[1].shape[-1] all_vol = [np.zeros((nb_patches, nb_vox)) for f in models] all_true = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] all_pred = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] all_prior = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] # get in_vol, y_true, y_pred for idx, model in enumerate(models): # with timer.Timer('prediction', verbose): pred = model.predict(sample[0]) assert pred.shape[0] == batch_size, \ "batch size mismatch. sample has batch size %d, given batch size is %d" %(pred.shape[0], batch_size) input_batch = sample[0] if not do_prior else sample[0][0] # compute batch range batch_start = batch_idx * batch_size batch_end = np.minimum(batch_start + batch_size, nb_patches) batch_range = np.arange(batch_start, batch_end) batch_vox_idx = batch_end-batch_start # update stacks all_vol[idx][batch_range, :] = K.batch_flatten(input_batch)[0:batch_vox_idx, :] all_true[idx][batch_range, :] = K.batch_flatten(sample[1])[0:batch_vox_idx, :] all_pred[idx][batch_range, :] = K._batch_flatten(pred)[0:batch_vox_idx, :] if do_prior: all_prior[idx][batch_range, :] = K.batch_flatten(sample[0][1])[0:batch_vox_idx, :] # reshape probabilistic answers for idx, _ in enumerate(models): all_true[idx] = np.reshape(all_true[idx], [nb_patches, nb_vox, nb_labels]) all_pred[idx] = np.reshape(all_pred[idx], [nb_patches, nb_vox, nb_labels]) if do_prior: all_prior[idx] = np.reshape(all_prior[idx], [nb_patches, nb_vox, nb_labels]) # prepare output tuple ret = () for midx, _ in enumerate(models): if do_prior: ret += ((all_true[midx], all_pred[midx], all_vol[midx], all_prior[midx]), ) else: ret += ((all_true[midx], all_pred[midx], all_vol[midx]), ) if len(models) == 1: ret = ret[0] return ret
def predict_volume_stack(models, data_generator, batch_size, grid_size, verbose=False): """ predict all the patches in a volume requires batch_size to be a divisor of the number of patches (prod(grid_size)) Note: we allow models to be a list or a single model. Normally, if you'd like to run a function over a list for some param, you can simply loop outside of the function. here, however, we are dealing with a generator, and want the output of that generator to be consistent for each model. Returns: if models isa list of more than one model: a tuple of model entried, each entry is a tuple of: all_true, all_pred, all_vol, <all_prior> if models is just one model: a tuple of all_true, all_pred, all_vol, <all_prior> """ if not isinstance(models, (list, tuple)): models = (models,) # compute the number of batches we need for one volume # we need the batch_size to be a divisor of nb_patches, # in order to loop through batches and form full volumes nb_patches = np.prod(grid_size) # assert np.mod(nb_patches, batch_size) == 0, \ # "batch_size %d should be a divisor of nb_patches %d" %(batch_size, nb_patches) nb_batches = ((nb_patches - 1) // batch_size) + 1 # go through the patches batch_gen = tqdm(range(nb_batches)) if verbose else range(nb_batches) for batch_idx in batch_gen: sample = next(data_generator) nb_vox = np.prod(sample[1].shape[1:-1]) do_prior = isinstance(sample[0], (list, tuple)) # pre-allocate all the data if batch_idx == 0: nb_labels = sample[1].shape[-1] all_vol = [np.zeros((nb_patches, nb_vox)) for f in models] all_true = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] all_pred = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] all_prior = [np.zeros((nb_patches, nb_vox * nb_labels)) for f in models] # get in_vol, y_true, y_pred for idx, model in enumerate(models): # with timer.Timer('prediction', verbose): pred = model.predict(sample[0]) assert pred.shape[0] == batch_size, \ "batch size mismatch. sample has batch size %d, given batch size is %d" %(pred.shape[0], batch_size) input_batch = sample[0] if not do_prior else sample[0][0] # compute batch range batch_start = batch_idx * batch_size batch_end = np.minimum(batch_start + batch_size, nb_patches) batch_range = np.arange(batch_start, batch_end) batch_vox_idx = batch_end-batch_start # update stacks all_vol[idx][batch_range, :] = K.batch_flatten(input_batch)[0:batch_vox_idx, :] all_true[idx][batch_range, :] = K.batch_flatten(sample[1])[0:batch_vox_idx, :] all_pred[idx][batch_range, :] = K._batch_flatten(pred)[0:batch_vox_idx, :] if do_prior: all_prior[idx][batch_range, :] = K.batch_flatten(sample[0][1])[0:batch_vox_idx, :] # reshape probabilistic answers for idx, _ in enumerate(models): all_true[idx] = np.reshape(all_true[idx], [nb_patches, nb_vox, nb_labels]) all_pred[idx] = np.reshape(all_pred[idx], [nb_patches, nb_vox, nb_labels]) if do_prior: all_prior[idx] = np.reshape(all_prior[idx], [nb_patches, nb_vox, nb_labels]) # prepare output tuple ret = () for midx, _ in enumerate(models): if do_prior: ret += ((all_true[midx], all_pred[midx], all_vol[midx], all_prior[midx]), ) else: ret += ((all_true[midx], all_pred[midx], all_vol[midx]), ) if len(models) == 1: ret = ret[0] return ret