def get(data_path,
seg_path,
data_name='main',
seg_name='stack',
augment=False,
transform=False):
print('loading dataset...', data_path)
filename = data_path.split('/')[-1]
filename = filename.split('.')[0]
test_dataset = []
train_dataset = []
p_data = data_path
p_seg = seg_path
train_dataset.append({})
train_dataset[-1]['name'] = filename
train_dataset[-1]['nhood'] = pygt.malis.mknhood3d()
train_dataset[-1]['data'] = np.array(h5py.File(p_data)[data_name],
dtype=np.float32) / (2.**8)
train_dataset[-1]['components'] = np.array(h5py.File(p_seg)[seg_name])
train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph(
train_dataset[-1]['components'], train_dataset[-1]['nhood'])
if transform:
train_dataset[-1]['transform'] = {}
train_dataset[-1]['transform']['scale'] = (0.8, 1.2)
train_dataset[-1]['transform']['shift'] = (-0.2, 0.2)
if augment:
print 'augmenting...'
train_dataset = pygt.augment_data_simple(train_dataset,
trn_method='affinity')
for iset in range(len(train_dataset)):
train_dataset[iset]['data'] = train_dataset[iset]['data'][
None, :] # add a dummy dimension
train_dataset[iset]['components'] = train_dataset[iset][
'components'][None, :]
print(train_dataset[iset]['name'] + str(iset) + ' shape:' +
str(train_dataset[iset]['data'].shape))
return train_dataset, test_dataset
train_dataset = []
train_dataset.append({})
dname = 'tstvol-520-1-h5'
train_dataset[-1]['name'] = dname
train_dataset[-1]['nhood'] = pygt.malis.mknhood3d()
train_dataset[-1]['data'] = np.array(h5py.File(
join(path, dname, 'img_normalized.h5'), 'r')['main'],
dtype=float32)
train_dataset[-1]['components'] = np.array(
h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main'])
train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph(
train_dataset[-1]['components'], train_dataset[-1]['nhood'])
print('Training set contains ' + str(len(train_dataset)) + ' volumes')
print('Running the simple dataset augmenter...')
train_dataset = pygt.augment_data_simple(train_dataset)
print('Training set now contains ' + str(len(train_dataset)) + ' volumes')
for iset in range(len(train_dataset)):
train_dataset[iset]['data'] = train_dataset[iset]['data'][None, :]
train_dataset[iset]['components'] = train_dataset[iset]['components'][
None, :]
# Train set
test_dataset = []
test_dataset.append({})
dname = 'tstvol-520-2-h5'
test_dataset[-1]['name'] = dname
test_dataset[-1]['data'] = np.array(h5py.File(
join(path, dname, 'img_normalized.h5'), 'r')['main'],
dtype=float32)
# Load the datasets
path = "/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/"
# Train set
train_dataset = []
train_dataset.append({})
dname = "tstvol-520-1-h5"
train_dataset[-1]["name"] = dname
train_dataset[-1]["nhood"] = pygt.malis.mknhood3d()
train_dataset[-1]["data"] = np.array(h5py.File(join(path, dname, "img_normalized.h5"), "r")["main"], dtype=float32)
train_dataset[-1]["components"] = np.array(h5py.File(join(path, dname, "groundtruth_seg_thick.h5"), "r")["main"])
train_dataset[-1]["label"] = pygt.malis.seg_to_affgraph(train_dataset[-1]["components"], train_dataset[-1]["nhood"])
print("Training set contains " + str(len(train_dataset)) + " volumes")
print("Running the simple dataset augmenter...")
train_dataset = pygt.augment_data_simple(train_dataset)
print("Training set now contains " + str(len(train_dataset)) + " volumes")
for iset in range(len(train_dataset)):
train_dataset[iset]["data"] = train_dataset[iset]["data"][None, :]
train_dataset[iset]["components"] = train_dataset[iset]["components"][None, :]
# Train set
test_dataset = []
test_dataset.append({})
dname = "tstvol-520-2-h5"
test_dataset[-1]["name"] = dname
test_dataset[-1]["data"] = np.array(h5py.File(join(path, dname, "img_normalized.h5"), "r")["main"], dtype=float32)
# test_dataset[-1]['components'] = np.array(h5py.File(join(path,dname,'groundtruth_seg_thick.h5'),'r')['main'])
# test_dataset[-1]['nhood'] = pygt.malis.mknhood3d()
# test_dataset[-1]['label'] = pygt.malis.seg_to_affgraph(test_dataset[-1]['components'],test_dataset[-1]['nhood'])
train_dataset = []
train_dataset.append({})
dname = '500-distal'
train_dataset[-1]['name'] = dname
train_dataset[-1]['nhood'] = pygt.malis.mknhood3d()
train_dataset[-1]['data'] = np.array(h5py.File(
join(path, dname, 'im_uint8.h5'), 'r')['main'],
dtype=np.float32) / (2.**8)
train_dataset[-1]['components'] = np.array(
h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main'])
train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph(
train_dataset[-1]['components'], train_dataset[-1]['nhood'])
train_dataset[-1]['transform'] = {}
train_dataset[-1]['transform']['scale'] = (0.8, 1.2)
train_dataset[-1]['transform']['shift'] = (-0.2, 0.2)
train_dataset = pygt.augment_data_simple(train_dataset, trn_method='affinity')
for iset in range(len(train_dataset)):
print(train_dataset[iset].keys())
train_dataset[iset]['data'] = train_dataset[iset]['data'][
None, :] # add a dummy dimension
#train_dataset[iset]['label'] = train_dataset[iset]['label'][None,:]
train_dataset[iset]['components'] = train_dataset[iset]['components'][
None, :]
print(train_dataset[iset]['name'] + str(iset) + ' shape:' +
str(train_dataset[iset]['data'].shape))
## Testing datasets
test_dataset = []
for dname in ['ecs-tst-normalized']:
