def get(data_path, seg_path, data_name='main', seg_name='stack', augment=False, transform=False): print('loading dataset...', data_path) filename = data_path.split('/')[-1] filename = filename.split('.')[0] test_dataset = [] train_dataset = [] p_data = data_path p_seg = seg_path train_dataset.append({}) train_dataset[-1]['name'] = filename train_dataset[-1]['nhood'] = pygt.malis.mknhood3d() train_dataset[-1]['data'] = np.array(h5py.File(p_data)[data_name], dtype=np.float32) / (2.**8) train_dataset[-1]['components'] = np.array(h5py.File(p_seg)[seg_name]) train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph( train_dataset[-1]['components'], train_dataset[-1]['nhood']) if transform: train_dataset[-1]['transform'] = {} train_dataset[-1]['transform']['scale'] = (0.8, 1.2) train_dataset[-1]['transform']['shift'] = (-0.2, 0.2) if augment: print 'augmenting...' train_dataset = pygt.augment_data_simple(train_dataset, trn_method='affinity') for iset in range(len(train_dataset)): train_dataset[iset]['data'] = train_dataset[iset]['data'][ None, :] # add a dummy dimension train_dataset[iset]['components'] = train_dataset[iset][ 'components'][None, :] print(train_dataset[iset]['name'] + str(iset) + ' shape:' + str(train_dataset[iset]['data'].shape)) return train_dataset, test_dataset
train_dataset = [] train_dataset.append({}) dname = 'tstvol-520-1-h5' train_dataset[-1]['name'] = dname train_dataset[-1]['nhood'] = pygt.malis.mknhood3d() train_dataset[-1]['data'] = np.array(h5py.File( join(path, dname, 'img_normalized.h5'), 'r')['main'], dtype=float32) train_dataset[-1]['components'] = np.array( h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main']) train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph( train_dataset[-1]['components'], train_dataset[-1]['nhood']) print('Training set contains ' + str(len(train_dataset)) + ' volumes') print('Running the simple dataset augmenter...') train_dataset = pygt.augment_data_simple(train_dataset) print('Training set now contains ' + str(len(train_dataset)) + ' volumes') for iset in range(len(train_dataset)): train_dataset[iset]['data'] = train_dataset[iset]['data'][None, :] train_dataset[iset]['components'] = train_dataset[iset]['components'][ None, :] # Train set test_dataset = [] test_dataset.append({}) dname = 'tstvol-520-2-h5' test_dataset[-1]['name'] = dname test_dataset[-1]['data'] = np.array(h5py.File( join(path, dname, 'img_normalized.h5'), 'r')['main'], dtype=float32)
# Load the datasets path = "/groups/turaga/home/turagas/data/FlyEM/fibsem_medulla_7col/" # Train set train_dataset = [] train_dataset.append({}) dname = "tstvol-520-1-h5" train_dataset[-1]["name"] = dname train_dataset[-1]["nhood"] = pygt.malis.mknhood3d() train_dataset[-1]["data"] = np.array(h5py.File(join(path, dname, "img_normalized.h5"), "r")["main"], dtype=float32) train_dataset[-1]["components"] = np.array(h5py.File(join(path, dname, "groundtruth_seg_thick.h5"), "r")["main"]) train_dataset[-1]["label"] = pygt.malis.seg_to_affgraph(train_dataset[-1]["components"], train_dataset[-1]["nhood"]) print("Training set contains " + str(len(train_dataset)) + " volumes") print("Running the simple dataset augmenter...") train_dataset = pygt.augment_data_simple(train_dataset) print("Training set now contains " + str(len(train_dataset)) + " volumes") for iset in range(len(train_dataset)): train_dataset[iset]["data"] = train_dataset[iset]["data"][None, :] train_dataset[iset]["components"] = train_dataset[iset]["components"][None, :] # Train set test_dataset = [] test_dataset.append({}) dname = "tstvol-520-2-h5" test_dataset[-1]["name"] = dname test_dataset[-1]["data"] = np.array(h5py.File(join(path, dname, "img_normalized.h5"), "r")["main"], dtype=float32) # test_dataset[-1]['components'] = np.array(h5py.File(join(path,dname,'groundtruth_seg_thick.h5'),'r')['main']) # test_dataset[-1]['nhood'] = pygt.malis.mknhood3d() # test_dataset[-1]['label'] = pygt.malis.seg_to_affgraph(test_dataset[-1]['components'],test_dataset[-1]['nhood'])
train_dataset = [] train_dataset.append({}) dname = '500-distal' train_dataset[-1]['name'] = dname train_dataset[-1]['nhood'] = pygt.malis.mknhood3d() train_dataset[-1]['data'] = np.array(h5py.File( join(path, dname, 'im_uint8.h5'), 'r')['main'], dtype=np.float32) / (2.**8) train_dataset[-1]['components'] = np.array( h5py.File(join(path, dname, 'groundtruth_seg_thick.h5'), 'r')['main']) train_dataset[-1]['label'] = pygt.malis.seg_to_affgraph( train_dataset[-1]['components'], train_dataset[-1]['nhood']) train_dataset[-1]['transform'] = {} train_dataset[-1]['transform']['scale'] = (0.8, 1.2) train_dataset[-1]['transform']['shift'] = (-0.2, 0.2) train_dataset = pygt.augment_data_simple(train_dataset, trn_method='affinity') for iset in range(len(train_dataset)): print(train_dataset[iset].keys()) train_dataset[iset]['data'] = train_dataset[iset]['data'][ None, :] # add a dummy dimension #train_dataset[iset]['label'] = train_dataset[iset]['label'][None,:] train_dataset[iset]['components'] = train_dataset[iset]['components'][ None, :] print(train_dataset[iset]['name'] + str(iset) + ' shape:' + str(train_dataset[iset]['data'].shape)) ## Testing datasets test_dataset = [] for dname in ['ecs-tst-normalized']: