def __init__(self, dataset, subset_iterator, dropout_p, dropout_p_left, dropout_p_right, data_specs=None, return_tuple=False, convert=None): FiniteDatasetIterator.__init__(self, dataset, subset_iterator,data_specs,return_tuple,convert) self.dropout_p=dropout_p self.dropout_p_left=dropout_p_left self.dropout_p_right=dropout_p_right
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None): # TODO: Refactor, deduplicate with set_iteration_scheme if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if topo is None: topo = getattr(self, '_iter_topo', False) if targets is None: targets = getattr(self, '_iter_targets', False) if rng is None and mode.stochastic: rng = self.rng return FiniteDatasetIterator( self, mode(self.X.shape[0], batch_size, num_batches, rng), topo, targets)
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): if topo is not None or targets is not None: raise ValueError("You should use the new interface iterator") if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng if data_specs is None: data_specs = self.data_specs return FiniteDatasetIterator(self, mode(self.get_num_examples(), batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple)
def iterator(self, mode=None, data_specs=None, batch_size=None, num_batches=None, rng=None, return_tuple=False, **kwargs): """ if data_specs is set to None, the aliases (or sources) and spaces provided when the dataset object has been created will be used. """ if data_specs is None: data_specs = (self._get_sources, self._get_spaces) [mode, batch_size, num_batches, rng, data_specs] = self._init_iterator(mode, batch_size, num_batches, rng, data_specs) convert = None return FiniteDatasetIterator(self, mode(self.get_num_examples(), batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)
def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None, data_specs=None, return_tuple=False): if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng if data_specs is None: data_specs = getattr(self, '_iter_data_specs', None) return FiniteDatasetIterator(self, mode(self.n_samples, batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple)
def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None, data_specs=None, return_tuple=False): """ Copied from dense_design_matrix, in order to fix uneven problem. """ if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space,) sub_sources = (source,) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): if src == 'features' and \ getattr(self, 'view_converter', None) is not None: conv_fn = (lambda batch, self=self, space=sp: self.view_converter.get_formatted_batch(batch, space)) else: conv_fn = None convert.append(conv_fn) # TODO: Refactor if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng # hack to make the online augmentations run FiniteDatasetIterator.uneven = False iterator = FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert) return iterator
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): """ method inherited from Dataset """ self.mode = mode self.batch_size = batch_size self._targets = targets self._return_tuple = return_tuple if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. # if self.conv_fn = lambda x: x.todense() space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space, ) sub_sources = (source, ) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): if src == 'features' or 'targets': conv_fn = self.conv_fn else: conv_fn = None convert.append(conv_fn) if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) return FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)
def iterator(self): """ :description: returns an iterator object over this dataset """ data_specs = self._iter_data_specs mode = self._iter_mode batch_size = 1 return FiniteDatasetIterator(self, mode(self.n_examples, batch_size, None, None), data_specs=data_specs)
def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None, data_specs=None, return_tuple=False): """ .. todo:: WRITEME """ if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space, ) sub_sources = (source, ) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): convert.append(None) # TODO: Refactor if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng return FiniteDatasetIterator(self, mode(len(self.trials), batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)
def iterator(self, mode=None, batch_size=1, num_batches=None, rng=None, data_specs=None, return_tuple=False): if num_batches is None: num_batches = len(self.X1) / (batch_size) mode = resolve_iterator_class(mode) i = FiniteDatasetIterator( self, mode(len(self.X1), batch_size, num_batches, rng), data_specs=data_specs, ) return i
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, return_tuple=False, targets=None, rng=None, data_specs=None): mode = resolve_iterator_class(mode) self.data_specs = data_specs if self.do_dropoout: return MultiViewDatasetIteratorDropout( self, mode(self.data[0].shape[0], batch_size, num_batches, rng),self.p_dropout,0.5,0.5, data_specs=data_specs ) else: return FiniteDatasetIterator( self, mode(self.data[0].shape[0], batch_size, num_batches, rng), data_specs=data_specs )
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): space, source = self.data_specs subspaces = space.components subsources = source mode = resolve_iterator_class("shuffled_sequential") rng = self.rng # rng = None assert rng is not None subset_iterator = mode(self.y.shape[0], batch_size, num_batches, rng=rng) return FiniteDatasetIterator( self, subset_iterator=subset_iterator, data_specs=data_specs, return_tuple=return_tuple, convert=self.convert)
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): if topo is not None or targets is not None: if data_specs is not None: raise ValueError('In DenseDesignMatrix.iterator, both the ' '"data_specs" argument and deprecated ' 'arguments "topo" or "targets" were ' 'provided.', (data_specs, topo, targets)) warnings.warn("Usage of `topo` and `target` arguments are being " "deprecated, and will be removed around November " "7th, 2013. `data_specs` should be used instead.", stacklevel=2) # build data_specs from topo and targets if needed if topo is None: topo = getattr(self, '_iter_topo', False) if topo: # self.iterator is called without a data_specs, and with # "topo=True", so we use the default topological space # stored in self.X_topo_space assert self.X_topo_space is not None X_space = self.X_topo_space else: X_space = self.X_space if targets is None: targets = getattr(self, '_iter_targets', False) if targets: assert self.y is not None y_space = self.data_specs[0].components[1] space = CompositeSpace((X_space, y_space)) source = ('features', 'targets') else: space = X_space source = 'features' data_specs = (space, source) convert = None else: if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space,) sub_sources = (source,) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): if src == 'features' and \ getattr(self, 'view_converter', None) is not None: conv_fn = (lambda batch, self=self, space=sp: self.view_converter.get_formatted_batch(batch, space)) else: conv_fn = None convert.append(conv_fn) # TODO: Refactor if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng return FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): warnings.warn( "Overloading this method is not necessary with the new " "interface change and this will be removed around November " "7th 2013", stacklevel=2) if topo is not None or targets is not None: if data_specs is not None: raise ValueError( "In DenseDesignMatrix.iterator, both " "the `data_specs` argument and deprecated arguments " "`topo` or `targets` were provided.", (data_specs, topo, targets)) warnings.warn( "Usage of `topo` and `target` arguments are being " "deprecated, and will be removed around November 7th, " "2013. `data_specs` should be used instead.", stacklevel=2) # build data_specs from topo and targets if needed if topo is None: topo = getattr(self, '_iter_topo', False) if topo: # self.iterator is called without a data_specs, and with # "topo=True", so we use the default topological space # stored in self.X_topo_space assert self.X_topo_space is not None X_space = self.X_topo_space else: X_space = self.X_space if targets is None: targets = getattr(self, '_iter_targets', False) if targets: assert self.y is not None y_space = self.data_specs[0][1] space = (X_space, y_space) source = ('features', 'targets') else: space = X_space source = 'features' data_specs = (space, source) _deprecated_interface = True else: _deprecated_interface = False # TODO: Refactor if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng if data_specs is None: data_specs = self._iter_data_specs if _deprecated_interface: return FiniteDatasetIteratorPyTables(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple) else: return FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple)
def iterator(self, mode=None, batch_size=None, num_batches=None, topo=None, targets=None, rng=None, data_specs=None, return_tuple=False): if topo is not None or targets is not None: warnings.warn( "Usage of `topo` and `target` arguments are " "being deprecated, and will be removed " "around November 7th, 2013. `data_specs` " "should be used instead. Here these two " "arguments are not used", stacklevel=2) if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space, ) sub_sources = (source, ) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): if src == 'features' and \ getattr(self, 'view_converter', None) is not None: conv_fn = (lambda batch, self=self, space=sp: self. view_converter.get_formatted_batch(batch, space)) else: conv_fn = None convert.append(conv_fn) # TODO: Refactor if mode is None: if hasattr(self, '_iter_subset_class'): mode = self._iter_subset_class else: raise ValueError('iteration mode not provided and no default ' 'mode set for %s' % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, '_iter_batch_size', None) if num_batches is None: num_batches = getattr(self, '_iter_num_batches', None) if rng is None and mode.stochastic: rng = self.rng return FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)
def iterator(self, mode=None, batch_size=None, num_batches=None, rng=None, data_specs=None, return_tuple=False): if data_specs is None: data_specs = self._iter_data_specs # If there is a view_converter, we have to use it to convert # the stored data for "features" into one that the iterator # can return. space, source = data_specs if isinstance(space, CompositeSpace): sub_spaces = space.components sub_sources = source else: sub_spaces = (space, ) sub_sources = (source, ) convert = [] for sp, src in safe_zip(sub_spaces, sub_sources): if (src == "features" and getattr(self, "view_converter", None) is not None): if self.distorter is None: conv_fn = ( lambda batch, self=self, space=sp: self.view_converter. get_formatted_batch(batch, space)) else: conv_fn = (lambda batch, self=self, space=sp: self. distorter._distort( self.view_converter.get_formatted_batch( batch, space))) else: conv_fn = None convert.append(conv_fn) # TODO: Refactor if mode is None: if hasattr(self, "_iter_subset_class"): mode = self._iter_subset_class else: raise ValueError("iteration mode not provided and no default " "mode set for %s" % str(self)) else: mode = resolve_iterator_class(mode) if batch_size is None: batch_size = getattr(self, "_iter_batch_size", None) if num_batches is None: num_batches = getattr(self, "_iter_num_batches", None) if rng is None and mode.stochastic: rng = self.rng return FiniteDatasetIterator(self, mode(self.X.shape[0], batch_size, num_batches, rng), data_specs=data_specs, return_tuple=return_tuple, convert=convert)