def __init__(self, dataset, subset_iterator, dropout_p, dropout_p_left, dropout_p_right, data_specs=None,
return_tuple=False, convert=None):
FiniteDatasetIterator.__init__(self, dataset, subset_iterator,data_specs,return_tuple,convert)
self.dropout_p=dropout_p
self.dropout_p_left=dropout_p_left
self.dropout_p_right=dropout_p_right
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
topo=None,
targets=None,
rng=None):
# TODO: Refactor, deduplicate with set_iteration_scheme
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if topo is None:
topo = getattr(self, '_iter_topo', False)
if targets is None:
targets = getattr(self, '_iter_targets', False)
if rng is None and mode.stochastic:
rng = self.rng
return FiniteDatasetIterator(
self, mode(self.X.shape[0], batch_size, num_batches, rng), topo,
targets)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
topo=None,
targets=None,
rng=None,
data_specs=None,
return_tuple=False):
if topo is not None or targets is not None:
raise ValueError("You should use the new interface iterator")
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
if data_specs is None:
data_specs = self.data_specs
return FiniteDatasetIterator(self,
mode(self.get_num_examples(), batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple)
def iterator(self,
mode=None,
data_specs=None,
batch_size=None,
num_batches=None,
rng=None,
return_tuple=False,
**kwargs):
"""
if data_specs is set to None, the aliases (or sources) and spaces
provided when the dataset object has been created will be used.
"""
if data_specs is None:
data_specs = (self._get_sources, self._get_spaces)
[mode, batch_size, num_batches, rng,
data_specs] = self._init_iterator(mode, batch_size, num_batches, rng,
data_specs)
convert = None
return FiniteDatasetIterator(self,
mode(self.get_num_examples(), batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
rng=None,
data_specs=None,
return_tuple=False):
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
if data_specs is None:
data_specs = getattr(self, '_iter_data_specs', None)
return FiniteDatasetIterator(self,
mode(self.n_samples, batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple)
def iterator(self, mode=None, batch_size=None, num_batches=None,
rng=None, data_specs=None,
return_tuple=False):
"""
Copied from dense_design_matrix, in order to fix uneven problem.
"""
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space,)
sub_sources = (source,)
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
if src == 'features' and \
getattr(self, 'view_converter', None) is not None:
conv_fn = (lambda batch, self=self, space=sp:
self.view_converter.get_formatted_batch(batch,
space))
else:
conv_fn = None
convert.append(conv_fn)
# TODO: Refactor
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
# hack to make the online augmentations run
FiniteDatasetIterator.uneven = False
iterator = FiniteDatasetIterator(self,
mode(self.X.shape[0],
batch_size,
num_batches,
rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
return iterator
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
topo=None,
targets=None,
rng=None,
data_specs=None,
return_tuple=False):
"""
method inherited from Dataset
"""
self.mode = mode
self.batch_size = batch_size
self._targets = targets
self._return_tuple = return_tuple
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
# if
self.conv_fn = lambda x: x.todense()
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space, )
sub_sources = (source, )
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
if src == 'features' or 'targets':
conv_fn = self.conv_fn
else:
conv_fn = None
convert.append(conv_fn)
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
return FiniteDatasetIterator(self,
mode(self.X.shape[0], batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
def iterator(self):
"""
:description: returns an iterator object over this dataset
"""
data_specs = self._iter_data_specs
mode = self._iter_mode
batch_size = 1
return FiniteDatasetIterator(self,
mode(self.n_examples, batch_size, None,
None),
data_specs=data_specs)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
rng=None,
data_specs=None,
return_tuple=False):
"""
.. todo::
WRITEME
"""
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space, )
sub_sources = (source, )
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
convert.append(None)
# TODO: Refactor
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
return FiniteDatasetIterator(self,
mode(len(self.trials), batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
def iterator(self,
mode=None,
batch_size=1,
num_batches=None,
rng=None,
data_specs=None,
return_tuple=False):
if num_batches is None:
num_batches = len(self.X1) / (batch_size)
mode = resolve_iterator_class(mode)
i = FiniteDatasetIterator(
self,
mode(len(self.X1), batch_size, num_batches, rng),
data_specs=data_specs,
)
return i
def iterator(self, mode=None, batch_size=None, num_batches=None,
topo=None, return_tuple=False, targets=None, rng=None, data_specs=None):
mode = resolve_iterator_class(mode)
self.data_specs = data_specs
if self.do_dropoout:
return MultiViewDatasetIteratorDropout(
self,
mode(self.data[0].shape[0],
batch_size, num_batches, rng),self.p_dropout,0.5,0.5,
data_specs=data_specs
)
else:
return FiniteDatasetIterator(
self,
mode(self.data[0].shape[0],
batch_size, num_batches, rng),
data_specs=data_specs
)
def iterator(self, mode=None, batch_size=None, num_batches=None,
topo=None, targets=None, rng=None, data_specs=None,
return_tuple=False):
space, source = self.data_specs
subspaces = space.components
subsources = source
mode = resolve_iterator_class("shuffled_sequential")
rng = self.rng
# rng = None
assert rng is not None
subset_iterator = mode(self.y.shape[0],
batch_size,
num_batches,
rng=rng)
return FiniteDatasetIterator(
self,
subset_iterator=subset_iterator,
data_specs=data_specs,
return_tuple=return_tuple,
convert=self.convert)
def iterator(self, mode=None, batch_size=None, num_batches=None,
topo=None, targets=None, rng=None, data_specs=None,
return_tuple=False):
if topo is not None or targets is not None:
if data_specs is not None:
raise ValueError('In DenseDesignMatrix.iterator, both the '
'"data_specs" argument and deprecated '
'arguments "topo" or "targets" were '
'provided.',
(data_specs, topo, targets))
warnings.warn("Usage of `topo` and `target` arguments are being "
"deprecated, and will be removed around November "
"7th, 2013. `data_specs` should be used instead.",
stacklevel=2)
# build data_specs from topo and targets if needed
if topo is None:
topo = getattr(self, '_iter_topo', False)
if topo:
# self.iterator is called without a data_specs, and with
# "topo=True", so we use the default topological space
# stored in self.X_topo_space
assert self.X_topo_space is not None
X_space = self.X_topo_space
else:
X_space = self.X_space
if targets is None:
targets = getattr(self, '_iter_targets', False)
if targets:
assert self.y is not None
y_space = self.data_specs[0].components[1]
space = CompositeSpace((X_space, y_space))
source = ('features', 'targets')
else:
space = X_space
source = 'features'
data_specs = (space, source)
convert = None
else:
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space,)
sub_sources = (source,)
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
if src == 'features' and \
getattr(self, 'view_converter', None) is not None:
conv_fn = (lambda batch, self=self, space=sp:
self.view_converter.get_formatted_batch(batch,
space))
else:
conv_fn = None
convert.append(conv_fn)
# TODO: Refactor
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
return FiniteDatasetIterator(self,
mode(self.X.shape[0],
batch_size,
num_batches,
rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
topo=None,
targets=None,
rng=None,
data_specs=None,
return_tuple=False):
warnings.warn(
"Overloading this method is not necessary with the new "
"interface change and this will be removed around November "
"7th 2013",
stacklevel=2)
if topo is not None or targets is not None:
if data_specs is not None:
raise ValueError(
"In DenseDesignMatrix.iterator, both "
"the `data_specs` argument and deprecated arguments "
"`topo` or `targets` were provided.",
(data_specs, topo, targets))
warnings.warn(
"Usage of `topo` and `target` arguments are being "
"deprecated, and will be removed around November 7th, "
"2013. `data_specs` should be used instead.",
stacklevel=2)
# build data_specs from topo and targets if needed
if topo is None:
topo = getattr(self, '_iter_topo', False)
if topo:
# self.iterator is called without a data_specs, and with
# "topo=True", so we use the default topological space
# stored in self.X_topo_space
assert self.X_topo_space is not None
X_space = self.X_topo_space
else:
X_space = self.X_space
if targets is None:
targets = getattr(self, '_iter_targets', False)
if targets:
assert self.y is not None
y_space = self.data_specs[0][1]
space = (X_space, y_space)
source = ('features', 'targets')
else:
space = X_space
source = 'features'
data_specs = (space, source)
_deprecated_interface = True
else:
_deprecated_interface = False
# TODO: Refactor
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
if data_specs is None:
data_specs = self._iter_data_specs
if _deprecated_interface:
return FiniteDatasetIteratorPyTables(self,
mode(self.X.shape[0],
batch_size, num_batches,
rng),
data_specs=data_specs,
return_tuple=return_tuple)
else:
return FiniteDatasetIterator(self,
mode(self.X.shape[0], batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
topo=None,
targets=None,
rng=None,
data_specs=None,
return_tuple=False):
if topo is not None or targets is not None:
warnings.warn(
"Usage of `topo` and `target` arguments are "
"being deprecated, and will be removed "
"around November 7th, 2013. `data_specs` "
"should be used instead. Here these two "
"arguments are not used",
stacklevel=2)
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space, )
sub_sources = (source, )
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
if src == 'features' and \
getattr(self, 'view_converter', None) is not None:
conv_fn = (lambda batch, self=self, space=sp: self.
view_converter.get_formatted_batch(batch, space))
else:
conv_fn = None
convert.append(conv_fn)
# TODO: Refactor
if mode is None:
if hasattr(self, '_iter_subset_class'):
mode = self._iter_subset_class
else:
raise ValueError('iteration mode not provided and no default '
'mode set for %s' % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
return FiniteDatasetIterator(self,
mode(self.X.shape[0], batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
def iterator(self,
mode=None,
batch_size=None,
num_batches=None,
rng=None,
data_specs=None,
return_tuple=False):
if data_specs is None:
data_specs = self._iter_data_specs
# If there is a view_converter, we have to use it to convert
# the stored data for "features" into one that the iterator
# can return.
space, source = data_specs
if isinstance(space, CompositeSpace):
sub_spaces = space.components
sub_sources = source
else:
sub_spaces = (space, )
sub_sources = (source, )
convert = []
for sp, src in safe_zip(sub_spaces, sub_sources):
if (src == "features"
and getattr(self, "view_converter", None) is not None):
if self.distorter is None:
conv_fn = (
lambda batch, self=self, space=sp: self.view_converter.
get_formatted_batch(batch, space))
else:
conv_fn = (lambda batch, self=self, space=sp: self.
distorter._distort(
self.view_converter.get_formatted_batch(
batch, space)))
else:
conv_fn = None
convert.append(conv_fn)
# TODO: Refactor
if mode is None:
if hasattr(self, "_iter_subset_class"):
mode = self._iter_subset_class
else:
raise ValueError("iteration mode not provided and no default "
"mode set for %s" % str(self))
else:
mode = resolve_iterator_class(mode)
if batch_size is None:
batch_size = getattr(self, "_iter_batch_size", None)
if num_batches is None:
num_batches = getattr(self, "_iter_num_batches", None)
if rng is None and mode.stochastic:
rng = self.rng
return FiniteDatasetIterator(self,
mode(self.X.shape[0], batch_size,
num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
