def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32') / 255.0
# Mask
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Segmentation
if 'aff' in class_keys or 'long' in class_keys:
fpath = os.path.join(dpath, info['dir'], info['seg'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Myelin
if 'mye' in class_keys:
fpath = os.path.join(dpath, info['dir'], info['mye'])
print(fpath)
mye = emio.imread(fpath).astype('uint8')
dset['mye'] = mye
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype(np.float32)
dset['img'] /= 255.0
# Mask
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype(np.uint8)
# Mitochondria
if 'mit' in info:
fpath = os.path.join(dpath, info['dir'], info['mit'])
print(fpath)
dset['mit'] = (emio.imread(fpath) > 0).astype(np.uint8)
else:
dset['mit'] = np.zeros_like(dset['msk'])
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32') / 255.0
# Mask
if tag == 'stitched_vol19-vol34':
# Train
fpath = os.path.join(dpath, info['dir'], 'msk_train.h5')
print(fpath)
dset['msk_train'] = emio.imread(fpath).astype('uint8')
# Validation
fpath = os.path.join(dpath, info['dir'], 'msk_val.h5')
print(fpath)
dset['msk_val'] = emio.imread(fpath).astype('uint8')
else:
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Segmentation
if 'aff' in class_keys or 'long' in class_keys:
fpath = os.path.join(dpath, info['dir'], info['seg'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32')
dset['img'] /= 255.0
# Segmentation
fpath = os.path.join(dpath, info['dir'], info['seg'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Mask
fpath = os.path.join(dpath, info['dir'], info['msk'] + '_train.h5')
print(fpath)
dset['msk_train'] = emio.imread(fpath).astype('uint8')
fpath = os.path.join(dpath, info['dir'], info['msk'] + '_val.h5')
print(fpath)
dset['msk_val'] = emio.imread(fpath).astype('uint8')
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, **kwargs):
dset = dict()
# Image
fpath = os.path.join(dpath, data_info['dir'], data_info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype(np.float32)
dset['img'] /= 255.0
# Mask
fpath = os.path.join(dpath, data_info['dir'], data_info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype(np.uint8)
# Segmentation
fpath = os.path.join(dpath, data_info['dir'], data_info['seg_d3_b0'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype(np.uint32)
# Background mask
idx = dset['seg'] == 1
dset['msk'][idx] = 0
# Membrane swirl
idx = dset['seg'] == 2
dset['seg'][idx] = 0
# Large lamellar structure
idx = dset['seg'] == 3
dset['msk'][idx] = 0
# Additoinal info
dset['loc'] = data_info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype(np.float32)
dset['img'] /= 255.0
# Mask
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype(np.uint8)
# Segmentation
fpath = os.path.join(dpath, info['dir'], info['seg_d3_b0'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype(np.uint32)
# Special case
if 'lamellae' in info:
idx = np.isin(dset['seg'], info['lamellae'])
dset['seg'][idx] = 0
# Glia
if 'glia' in class_keys:
if 'glia' in info:
fpath = os.path.join(dpath, info['dir'], info['glia'])
print(fpath)
dset['glia'] = emio.imread(fpath).astype(np.uint8)
else:
dset['glia'] = np.zeros_like(dset['msk'])
# Mask out
if 'rosetta' in info:
idx = np.isin(dset['seg'], info['rosetta'])
dset['msk'][idx] = 0
if 'esophagus' in info:
idx = np.isin(dset['seg'], info['esophagus'])
dset['msk'][idx] = 0
if 'glia_msk' in info:
idx = np.isin(dset['seg'], info['glia_msk'])
dset['msk'][idx] = 0
if 'dark_cell' in info:
idx = np.isin(dset['seg'], info['dark_cell'])
dset['msk'][idx] = 0
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], glia_mask=False, **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32')
dset['img'] /= 255.0
# Mask
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Segmentation
fpath = os.path.join(dpath, info['dir'], info['seg_d3_b0'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Boundary
if 'bdr' in class_keys:
fpath = os.path.join(dpath, info['dir'], info['bdr'])
print(fpath)
bdr = emio.imread(fpath).astype('uint32')
dset['bdr'] = bdr
# Glia
if 'glia' in class_keys:
fpath = os.path.join(dpath, info['dir'], info['glia'])
print(fpath)
dset['glia'] = emio.imread(fpath).astype('uint8')
# Glia mask
if glia_mask:
# Use original mask for glia detection.
assert 'msk' in dset
dset['gmsk'] = np.copy(dset['msk'])
# Mask out glia.
assert 'seg' in dset
gmsk = ~np.isin(dset['seg'], info['glia_ids'])
dset['msk'] &= gmsk
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, bdr=False, mye=False):
dset = dict()
# Image
fpath = os.path.join(dpath, info['dir'], info['img'])
print(fpath)
img = emio.imread(fpath).astype('float32') / 255.0
dset['img'] = img
# Segmentation
fpath = os.path.join(dpath, info['dir'], info['seg'])
print(fpath)
seg = emio.imread(fpath).astype('uint32')
dset['seg'] = seg
# Boundary (or affinity)
if bdr:
fpath = os.path.join(dpath, info['dir'], info['bdr'])
print(fpath)
seg = emio.imread(fpath).astype('uint32')
dset['aff'] = seg
dset['bdr'] = (seg == 0).astype('uint8')
# Myelin
if mye:
fpath = os.path.join(dpath, info['dir'], info['mye'])
print(fpath)
mye = emio.imread(fpath).astype('uint8')
dset['mye'] = mye
# Train mask
fpath = os.path.join(dpath, info['dir'], info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Additoinal info
dset['loc'] = info['loc']
return dset
def make_forward_scanner(opt, data_name=None):
# Cloud-volume
if opt.gs_input:
try:
from deepem.test import cv_utils
img = cv_utils.cutout(opt, opt.gs_input, dtype='uint8')
# Optional input histogram normalization
if opt.gs_input_norm:
assert len(opt.gs_input_norm) == 2
low, high = opt.gs_input_norm
img = normalize_per_slice(img, lowerfract=low, upperfract=high)
# [0, 255] -> [0.0, 1.0]
img = (img/255.).astype('float32')
# Optional input mask
if opt.gs_input_mask:
try:
msk = cv_utils.cutout(opt, opt.gs_input_mask, dtype='uint8')
img[msk > 0] = 0
except:
raise
except ImportError:
raise
else:
assert data_name is not None
print(data_name)
# Read an EM image.
if opt.dummy:
img = np.random.rand(*opt.dummy_inputsz[-3:]).astype('float32')
else:
fpath = os.path.join(opt.data_dir, data_name, opt.input_name)
img = emio.imread(fpath)
img = (img/255.).astype('float32')
# Border mirroring
if opt.mirror:
pad_width = [(x//2,x//2) for x in opt.mirror]
img = np.pad(img, pad_width, 'reflect')
# ForwardScanner
dataset = Dataset(spec=opt.in_spec)
dataset.add_data('input', img)
return ForwardScanner(dataset, opt.scan_spec, **opt.scan_params)
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
dname = tag[:-1] if tag[-1] == 'a' else tag
fpath = os.path.join(dpath, dname, info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32')
dset['img'] /= 255.0
# Mask
if dname == 'stitched_vol19-vol34':
fpath = os.path.join(dpath, dname, 'msk_train.h5')
print(fpath)
dset['msk_train'] = emio.imread(fpath).astype('uint8')
fpath = os.path.join(dpath, dname, 'msk_val.h5')
print(fpath)
dset['msk_val'] = emio.imread(fpath).astype('uint8')
else:
fpath = os.path.join(dpath, dname, info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Segmentation
if 'aff' in class_keys or 'long' in class_keys:
fpath = os.path.join(dpath, dname, info['seg'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Synapse (distillation)
if 'syn' in class_keys:
if 'syn' in info:
fpath = os.path.join(dpath, dname, info['syn'])
print(fpath)
syn = emio.imread(fpath).astype('float32')
else:
syn = np.zeros(dset['img'].shape, dtype='float32')
dset['syn'] = syn
# Myelin
if 'mye' in class_keys:
if 'mye' in info:
fpath = os.path.join(dpath, dname, info['mye'])
print(fpath)
mye = emio.imread(fpath).astype('uint8')
else:
mye = np.zeros(dset['img'].shape, dtype='uint8')
dset['mye'] = mye
# Blood vessel
if 'blv' in class_keys:
if 'blv' in info:
fpath = os.path.join(dpath, dname, info['blv'])
print(fpath)
blv = emio.imread(fpath).astype('uint8')
else:
blv = np.zeros(dset['img'].shape, dtype='uint8')
dset['blv'] = blv
# Additoinal info
dset['loc'] = info['loc']
return dset
def load_dataset(dpath, tag, info, class_keys=[], **kwargs):
assert len(class_keys) > 0
dset = dict()
# Image
dname = tag[:-1] if tag[-1] == 'a' else tag
fpath = os.path.join(dpath, dname, info['img'])
print(fpath)
dset['img'] = emio.imread(fpath).astype('float32')
dset['img'] /= 255.0
# Mask
if dname == 'stitched_vol19-vol34':
fpath = os.path.join(dpath, dname, 'msk_train.h5')
print(fpath)
dset['msk_train'] = emio.imread(fpath).astype('uint8')
fpath = os.path.join(dpath, dname, 'msk_val.h5')
print(fpath)
dset['msk_val'] = emio.imread(fpath).astype('uint8')
else:
fpath = os.path.join(dpath, dname, info['msk'])
print(fpath)
dset['msk'] = emio.imread(fpath).astype('uint8')
# Segmentation
if 'aff' in class_keys or 'long' in class_keys:
fpath = os.path.join(dpath, dname, info['seg'])
print(fpath)
dset['seg'] = emio.imread(fpath).astype('uint32')
# Synapse
if 'psd' in class_keys:
if 'psd' in info:
fpath = os.path.join(dpath, dname, info['psd'])
print(fpath)
psd = (emio.imread(fpath) > 0).astype('uint8')
else:
psd = np.zeros(dset['img'].shape, dtype='uint8')
dset['psd'] = psd
# Special volumes
special = [
'stitched_vol40-vol41', 'vol101', 'vol102', 'vol103', 'vol104'
]
if dname in special:
fpath = os.path.join(dpath, dname, 'psd_msk.h5')
print(fpath)
psd_msk = emio.imread(fpath).astype('uint8')
else:
psd_msk = dset['msk']
dset['psd_msk'] = psd_msk
# Myelin
if 'mye' in class_keys:
if 'mye' in info:
fpath = os.path.join(dpath, dname, info['mye'])
print(fpath)
mye = emio.imread(fpath).astype('uint8')
else:
mye = np.zeros(dset['img'].shape, dtype='uint8')
dset['mye'] = mye
# Blood vessel
if 'blv' in class_keys:
if 'blv' in info:
fpath = os.path.join(dpath, dname, info['blv'])
print(fpath)
blv = emio.imread(fpath).astype('uint8')
else:
blv = np.zeros(dset['img'].shape, dtype='uint8')
dset['blv'] = blv
# Additoinal info
dset['loc'] = info['loc']
return dset