def test_get_samples(self):
# TODO: use dataset that is always available instead of lodopab
if not LoDoPaBDataset.check_for_lodopab():
return
d = LoDoPaBDataset(impl='skimage')
angle_indices = range(0, d.shape[0][0], 2)
asd = AngleSubsetDataset(d, angle_indices)
obs_arr_asd, gt_arr_asd = asd.get_samples(range(3))
obs_arr, gt_arr = d.get_samples(range(3))
obs_arr_subset = np.asarray(obs_arr)[:, np.asarray(angle_indices), :]
self.assertEqual(obs_arr_asd.shape, obs_arr_subset.shape)
self.assertEqual(gt_arr_asd.shape, gt_arr.shape)
self.assertTrue(np.all(np.asarray(obs_arr_asd) == obs_arr_subset))
self.assertTrue(np.all(np.asarray(gt_arr_asd) == np.asarray(gt_arr)))
def test_generator(self):
if not LoDoPaBDataset.check_for_lodopab():
return
NUM_SAMPLES = 3
d = LoDoPaBDataset(impl='skimage')
for part in ['train', 'validation', 'test']:
samples = [d.get_sample(i, part) for i in range(NUM_SAMPLES)]
samples2 = [s for s in islice(d.generator(part), NUM_SAMPLES)]
for (s_obs, s_gt), (s2_obs, s2_gt) in zip(samples, samples2):
self.assertTrue(np.all(np.asarray(s_obs) == s2_obs))
self.assertTrue(np.all(np.asarray(s_gt) == s2_gt))
if d.rel_patient_ids is not None:
d2 = LoDoPaBDataset(sorted_by_patient=True, impl='skimage')
for part in ['train', 'validation', 'test']:
samples = [d2.get_sample(i, part) for i in range(NUM_SAMPLES)]
samples2 = [s for s in islice(d2.generator(part), NUM_SAMPLES)]
for (s_obs, s_gt), (s2_obs, s2_gt) in zip(samples, samples2):
self.assertTrue(np.all(np.asarray(s_obs) == s2_obs))
self.assertTrue(np.all(np.asarray(s_gt) == s2_gt))
def test_get_samples(self):
if not LoDoPaBDataset.check_for_lodopab():
return
KEY = range(420, 423)
d = LoDoPaBDataset(impl='skimage')
for part in ['train', 'validation', 'test']:
samples = [d.get_sample(i, part) for i in KEY]
samples2 = d.get_samples(KEY, part)
for (s_obs, s_gt), s2_obs, s2_gt in zip(samples, samples2[0],
samples2[1]):
self.assertTrue(np.all(np.asarray(s_obs) == s2_obs))
self.assertTrue(np.all(np.asarray(s_gt) == s2_gt))
if d.rel_patient_ids is not None:
d2 = LoDoPaBDataset(sorted_by_patient=True, impl='skimage')
for part in ['train', 'validation', 'test']:
samples = [d2.get_sample(i, part) for i in KEY]
samples2 = d2.get_samples(KEY, part)
for (s_obs, s_gt), s2_obs, s2_gt in zip(samples, samples2[0],
samples2[1]):
self.assertTrue(np.all(np.asarray(s_obs) == s2_obs))
self.assertTrue(np.all(np.asarray(s_gt) == s2_gt))
def test_patient_ids(self):
if not LoDoPaBDataset.check_for_lodopab():
return
d = LoDoPaBDataset(impl='skimage')
if d.rel_patient_ids is not None:
for part in ['train', 'validation', 'test']:
self.assertEqual(len(d.rel_patient_ids[part]), d.get_len(part))
self.assertTrue(
np.all(
np.unique(d.rel_patient_ids[part]) == range(
d.get_num_patients(part))))
self.assertTrue(
np.all(
np.diff(d.rel_patient_ids[part][
LoDoPaBDataset.get_idx_sorted_by_patient()[part]])
>= 0))
d2 = LoDoPaBDataset(sorted_by_patient=True, impl='skimage')
REL_PATIENT_ID = 42
ifp = d.get_indices_for_patient(REL_PATIENT_ID, part)
ifp2 = d2.get_indices_for_patient(REL_PATIENT_ID, part)
self.assertGreater(len(ifp), 0)
self.assertEqual(len(ifp), len(ifp2))
for i, i2 in zip(ifp[:3], ifp2[:3]):
self.assertEqual(d.get_sample(i, part),
d2.get_sample(i2, part))
def get_standard_dataset(name, **kwargs):
"""
Return a standard dataset by name.
The standard datasets are (currently):
``'ellipses'``
A typical synthetical CT dataset with ellipse phantoms.
`EllipsesDataset` is used as ground truth dataset, a ray
transform with parallel beam geometry using 30 angles is applied,
and white gaussian noise with a standard deviation of 2.5% (i.e.
``0.025 * mean(abs(observation))``) is added.
The ray transform that corresponds to the (noiseless) forward
operator is stored in the attribute `ray_trafo` of the dataset.
In order to avoid the inverse crime, the ground truth images of
shape (128, 128) are upscaled by bilinear interpolation to a
resolution of (400, 400) before the ray transform is applied (whose
discretization is different from the one of `ray_trafo`).
``'lodopab'``
The LoDoPaB-CT dataset, which is documented in the
preprint `<https://arxiv.org/abs/1910.01113>`_ hosted on
`<https://zenodo.org/record/3384092>`_.
It is a simulated low dose CT dataset based on real reconstructions
from the `LIDC-IDRI
<https://wiki.cancerimagingarchive.net/display/Public/LIDC-IDRI>`_
dataset.
The dataset contains 42895 pairs of images and projection data.
For simulation, a ray transform with parallel beam geometry using
1000 angles and 513 detector pixels is used. Poisson noise
corresponding to 4096 incident photons per pixel before attenuation
is applied to the projection data.
A ray transform that corresponds to the noiseless forward operator
is available via
:meth:`~dival.datasets.LoDoPaBDataset.get_ray_trafo` from this
dataset.
Parameters
----------
name : str
Name of the dataset.
kwargs : dict
Keyword arguments.
Supported parameters for the datasets are:
``'ellipses'``
impl : {``'skimage'``, ``'astra_cpu'``, ``'astra_cuda'``},\
optional
Implementation passed to :class:`odl.tomo.RayTransform`
fixed_seeds : dict or bool, optional
Seeds to use for random ellipse generation, passed to
:class:`.EllipsesDataset`.
``'lodopab'``
observation_model : {``'post-log'``, ``'pre-log'``}, optional
The observation model to use. Default is ``'post-log'``.
min_photon_count : float, optional
Replacement value for a simulated photon count of zero.
If ``observation_model == 'post-log'``, a value greater
than zero is required in order to avoid undefined values.
The default is 0.1, both for ``'post-log'`` and
``'pre-log'`` model.
Returns
-------
dataset : :class:`.Dataset`
The standard dataset.
It has an attribute `standard_dataset_name` that stores its name.
"""
name = name.lower()
if name == 'ellipses':
fixed_seeds = kwargs.pop('fixed_seeds', False)
ellipses_dataset = EllipsesDataset(image_size=128,
fixed_seeds=fixed_seeds)
NUM_ANGLES = 30
# image shape for simulation
IM_SHAPE = (400, 400) # images will be scaled up from (128, 128)
reco_space = ellipses_dataset.space
space = odl.uniform_discr(min_pt=reco_space.min_pt,
max_pt=reco_space.max_pt,
shape=IM_SHAPE,
dtype=np.float32)
reco_geometry = odl.tomo.parallel_beam_geometry(reco_space,
num_angles=NUM_ANGLES)
geometry = odl.tomo.parallel_beam_geometry(
space,
num_angles=NUM_ANGLES,
det_shape=reco_geometry.detector.shape)
impl = kwargs.pop('impl', 'astra_cuda')
ray_trafo = odl.tomo.RayTransform(space, geometry, impl=impl)
def get_reco_ray_trafo(impl=impl):
return odl.tomo.RayTransform(reco_space, reco_geometry, impl=impl)
reco_ray_trafo = get_reco_ray_trafo(impl=impl)
class _ResizeOperator(odl.Operator):
def __init__(self):
super().__init__(reco_space, space)
def _call(self, x, out):
out.assign(space.element(resize(x, IM_SHAPE, order=1)))
# forward operator
resize_op = _ResizeOperator()
forward_op = ray_trafo * resize_op
dataset = ellipses_dataset.create_pair_dataset(forward_op=forward_op,
noise_type='white',
noise_kwargs={
'relative_stddev':
True,
'stddev': 0.025
},
noise_seeds={
'train': 1,
'validation': 2,
'test': 3
})
dataset.get_ray_trafo = get_reco_ray_trafo
dataset.ray_trafo = reco_ray_trafo
elif name == 'lodopab':
dataset = LoDoPaBDataset(**kwargs)
else:
raise ValueError(
"unknown dataset '{}'. Known standard datasets are {}".format(
name, STANDARD_DATASET_NAMES))
dataset.standard_dataset_name = name
return dataset
help='noise model: mse, uncalib, gaussian, poisson, poissongaussian')
parser.add_argument('--reg',
type=float,
default=10,
help='regularization weight on prior std. dev.')
args = parser.parse_args()
# %% [markdown]
# ### Load Data
# %%
# update dival config file with correct path to data
dival.config.set_config('/lodopab_dataset/data_path', args.path)
dataset = LoDoPaBDataset(observation_model=args.obsmodel, impl='astra_cpu')
fbp = FBPReconstructor(dataset.get_ray_trafo(impl='astra_cpu'))
# compute padded (height, width) for sinograms to ensure they are multiples of 32
height, width = dataset.shape[0]
input_height = (height // 32 + 1) * 32 + 256
input_width = (width // 32 + 1) * 32 + 256
#input_height = (height // args.crop + 1) * args.crop
#input_width = (width // args.crop + 1) * args.crop
#pad_bottom = input_height - height
#pad_right = input_width - width
hdiff = input_height - height
wdiff = input_width - width
pad_top = hdiff // 2
pad_left = wdiff // 2
pad_bottom = hdiff - hdiff // 2
def get_standard_dataset(name, **kwargs):
"""
Return a standard dataset by name.
The standard datasets are (currently):
``'ellipses'``
A typical synthetical CT dataset with ellipse phantoms.
`EllipsesDataset` is used as ground truth dataset, a ray
transform with parallel beam geometry using 30 angles is applied,
and white gaussian noise with a standard deviation of 2.5% (i.e.
``0.025 * mean(abs(observation))``) is added.
In order to avoid the inverse crime, the ground truth images of
shape (128, 128) are upscaled by bilinear interpolation to a
resolution of (400, 400) before the ray transform is applied (whose
discretization is different from the one of :attr:`ray_trafo`).
Attributes of the returned dataset:
`ray_trafo` : :class:`odl.tomo.RayTransform`
Ray transform corresponding to the noiseless forward
operator.
``get_ray_trafo(**kwargs)`` : function
Function that returns a ray transform corresponding to the
noiseless forward operator. Keyword arguments (e.g. `impl`)
are forwarded to the :class:`RayTransform` constructor.
``'lodopab'``
The LoDoPaB-CT dataset, which is documented in the Data Descriptor
article `<https://www.nature.com/articles/s41597-021-00893-z>`_ and
hosted on `<https://zenodo.org/record/3384092>`_.
It is a simulated low dose CT dataset based on real reconstructions
from the `LIDC-IDRI
<https://wiki.cancerimagingarchive.net/display/Public/LIDC-IDRI>`_
dataset.
The dataset contains 42895 pairs of images and projection data.
For simulation, a ray transform with parallel beam geometry using
1000 angles and 513 detector pixels is used. Poisson noise
corresponding to 4096 incident photons per pixel before attenuation
is applied to the projection data.
Attributes of the returned dataset:
`ray_trafo` : :class:`odl.tomo.RayTransform`
Ray transform corresponding to the noiseless forward
operator.
Methods of the returned dataset:
``get_ray_trafo(**kwargs)``
Function that returns a ray transform corresponding to the
noiseless forward operator. Keyword arguments (e.g. `impl`)
are forwarded to the :class:`RayTransform` constructor.
Parameters
----------
name : str
Name of the dataset.
kwargs : dict
Keyword arguments.
Supported parameters for the datasets are:
``'ellipses'``
impl : {``'skimage'``, ``'astra_cpu'``, ``'astra_cuda'``},\
optional
Implementation passed to :class:`odl.tomo.RayTransform`
Default: ``'astra_cuda'``.
fixed_seeds : dict or bool, optional
Seeds to use for random ellipse generation, passed to
:meth:`.EllipsesDataset.__init__`.
Default: ``False``.
fixed_noise_seeds : dict or bool, optional
Seeds to use for noise generation, passed as `noise_seeds`
to :meth:`.GroundTruthDataset.create_pair_dataset`.
If ``True`` is passed (the default), the seeds
``{'train': 1, 'validation': 2, 'test': 3}`` are used.
``'lodopab'``
num_angles : int, optional
Number of angles to use from the full 1000 angles.
Must be a divisor of 1000.
observation_model : {``'post-log'``, ``'pre-log'``}, optional
The observation model to use. Default is ``'post-log'``.
min_photon_count : float, optional
Replacement value for a simulated photon count of zero.
If ``observation_model == 'post-log'``, a value greater
than zero is required in order to avoid undefined values.
The default is 0.1, both for ``'post-log'`` and
``'pre-log'`` model.
sorted_by_patient : bool, optional
Whether to sort the samples by patient id.
Useful to resplit the dataset.
Default: ``False``.
impl : {``'skimage'``, ``'astra_cpu'``, ``'astra_cuda'``},\
optional
Implementation passed to :class:`odl.tomo.RayTransform`
Default: ``'astra_cuda'``.
Returns
-------
dataset : :class:`.Dataset`
The standard dataset.
It has an attribute `standard_dataset_name` that stores its name.
"""
name = name.lower()
if name == 'ellipses':
fixed_seeds = kwargs.pop('fixed_seeds', False)
ellipses_dataset = EllipsesDataset(image_size=128,
fixed_seeds=fixed_seeds)
NUM_ANGLES = 30
# image shape for simulation
IM_SHAPE = (400, 400) # images will be scaled up from (128, 128)
reco_space = ellipses_dataset.space
space = odl.uniform_discr(min_pt=reco_space.min_pt,
max_pt=reco_space.max_pt,
shape=IM_SHAPE,
dtype=np.float32)
reco_geometry = odl.tomo.parallel_beam_geometry(reco_space,
num_angles=NUM_ANGLES)
geometry = odl.tomo.parallel_beam_geometry(
space,
num_angles=NUM_ANGLES,
det_shape=reco_geometry.detector.shape)
impl = kwargs.pop('impl', 'astra_cuda')
ray_trafo = odl.tomo.RayTransform(space, geometry, impl=impl)
reco_ray_trafo = odl.tomo.RayTransform(reco_space,
reco_geometry,
impl=impl)
# forward operator
resize_op = ResizeOperator(reco_space, space)
forward_op = ray_trafo * resize_op
noise_seeds = kwargs.pop('fixed_noise_seeds', True)
if isinstance(noise_seeds, bool):
noise_seeds = ({
'train': 1,
'validation': 2,
'test': 3
} if noise_seeds else None)
dataset = ellipses_dataset.create_pair_dataset(forward_op=forward_op,
noise_type='white',
noise_kwargs={
'relative_stddev':
True,
'stddev': 0.025
},
noise_seeds=noise_seeds)
dataset.get_ray_trafo = partial(odl.tomo.RayTransform, reco_space,
reco_geometry)
dataset.ray_trafo = reco_ray_trafo
elif name == 'lodopab':
num_angles = kwargs.pop('num_angles', None)
lodopab_kwargs = {}
for k in [
'observation_model', 'min_photon_count', 'sorted_by_patient',
'impl'
]:
if k in kwargs:
lodopab_kwargs[k] = kwargs.pop(k)
dataset = LoDoPaBDataset(**lodopab_kwargs)
if num_angles is not None:
dataset = get_angle_subset_dataset(dataset,
num_angles,
impl=kwargs.get(
'impl', 'astra_cuda'))
else:
raise ValueError(
"unknown dataset '{}'. Known standard datasets are {}".format(
name, STANDARD_DATASET_NAMES))
if kwargs:
warn('unused keyword arguments: {}'.format(', '.join(kwargs.keys())))
dataset.standard_dataset_name = name
return dataset
parser.add_argument('--crop', type=int, default=128, help='crop size')
parser.add_argument('--batch', type=int, default=4, help='batch size')
parser.add_argument('--epoch', type=int, default=300, help='num epochs')
parser.add_argument('--steps', type=int, default=50, help='steps per epoch')
parser.add_argument('--lr', type=float, default=0.0003, help='learning rate')
args = parser.parse_args()
if args.mode != "samplepoisson":
raise ValueError("Only support samplepoisson mode")
""" Load training and validation datasets """
# update dival config file with correct path to data
dival.config.set_config('/lodopab_dataset/data_path', args.path)
dataset = LoDoPaBDataset(observation_model=args.obsmodel, impl='skimage')
#sinogram, _ = dataset.get_sample(0, part='train', out=(True,False))
#height, width = sinogram.shape[0:2]
#y_crop = 32*(height//32)
#x_crop = 32*(width//32)
#print('crop: ',y_crop,x_crop)
#crop_size = [y_crop,x_crop]
crop_size = [args.crop, args.crop]
def load_train_sinograms(batch_size, crop_size):
indices = np.arange(dataset.train_len)
batch = np.zeros((batch_size, crop_size[0], crop_size[1], 1))
while True:
