def get_data_pointers(dataset, base_dir, cv, log):
"""Get data file pointers."""
data_pointer = os.path.join(base_dir, '%s_%s.tfrecords' % (dataset, cv))
data_means = os.path.join(base_dir, '%s_%s_means.npy' % (dataset, cv))
log.info('Using %s tfrecords: %s' % (cv, data_pointer))
py_utils.check_path(data_pointer, log, '%s not found.' % data_pointer)
mean_loc = py_utils.check_path(
data_means, log, '%s not found for cv: %s.' % (data_means, cv))
data_means_image, data_means_label = None, None
if not mean_loc:
alt_data_pointer = data_means.replace('.npy', '.npz')
alt_data_pointer = py_utils.check_path(
alt_data_pointer, log, '%s not found.' % alt_data_pointer)
# TODO: Fix this API and make it more flexible. Kill npzs in Allen?
if not alt_data_pointer:
# No mean for this dataset
data_means = None
else:
log.info('Loading means from npz for cv: %s.' % cv)
data_means = np.load(alt_data_pointer)
data_means_vol = data_means[data_means.keys()[0]].item()
if 'image' in data_means_vol.keys():
data_means_image = data_means_vol['image']
if 'label' in data_means_vol.keys():
data_means_label = data_means_vol['label']
else:
data_means_image = np.load(data_means)
return data_pointer, data_means_image, data_means_label
def get_data_pointers(dataset, base_dir, cv, log):
data_pointer = os.path.join(base_dir, '%s_%s.tfrecords' % (dataset, cv))
data_means = os.path.join(base_dir, '%s_%s_means.npy' % (dataset, cv))
log.info('Using %s tfrecords: %s' % (cv, data_pointer))
py_utils.check_path(data_pointer, log, '%s not found.' % data_pointer)
py_utils.check_path(data_means, log, '%s not found.' % data_means)
data_means = np.load(data_means)
return data_pointer, data_means
def get_data_pointers(dataset, base_dir, cv, log):
"""Get data file pointers.
Parameters
----------
dataset : iterable object
base_dir : str
cv : dataset_module.folds.keys()
log : iterable object
Returns
-------
string that leads to directory of data
"""
data_pointer = os.path.join(base_dir, '%s_%s.tfrecords' % (dataset, cv))
data_means = os.path.join(base_dir, '%s_%s_means.npy' % (dataset, cv))
log.info('Using %s tfrecords: %s' % (cv, data_pointer))
py_utils.check_path(data_pointer, log, '%s not found.' % data_pointer)
mean_loc = py_utils.check_path(
data_means, log, '%s not found for cv: %s.' % (data_means, cv))
if not mean_loc:
alt_data_pointer = data_means.replace('.npy', '.npz')
alt_data_pointer = py_utils.check_path(
alt_data_pointer, log, '%s not found.' % alt_data_pointer)
# TODO: Fix this API and make it more flexible. Kill npzs in Allen?
if not alt_data_pointer:
# No mean for this dataset
data_means = None
else:
log.info('Loading means from npz for cv: %s.' % cv)
data_means = np.load(alt_data_pointer)
data_means = data_means[data_means.keys()[0]].item()['image']
else:
data_means = np.load(data_means)
return data_pointer, data_means