def link_local(self): req_param(self, ['nifm', 'ifmshape', 'fshape']) opt_param(self, ['ofmlocs', 'links']) opt_param(self, ['deltasbuf', 'outputbuf']) opt_param(self, ['nofm'], self.nifm) opt_param(self, ['pooling'], False) opt_param(self, ['stride'], 1) opt_param(self, ['pad'], 0) assert len(self.ifmshape) == len(self.fshape) ofmshape = [] for dim in range(len(self.ifmshape)): assert self.ifmshape[dim] >= self.fshape[dim] num = self.ifmshape[dim] - self.fshape[dim] + 2 * self.pad ofmshape.extend([num // self.stride + 1]) self.ofmshape = tuple(ofmshape) self.negpad = -self.pad self.ifmsize = np.prod(self.ifmshape) self.ofmsize = np.prod(self.ofmshape) self.fpsize = np.prod(self.fshape) self.fsize = self.nifm * self.fpsize self.nout = self.nofm * self.ofmsize logger.debug('name=%s, nifm=%d, ifmshape=%s, ofmshape=%s', self.name, self.nifm, self.ifmshape, self.ofmshape)
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ['name']) opt_param(self, [ 'pre_act_dtype', 'output_dtype', 'deltas_dtype', 'weight_dtype', 'updates_dtype' ], np.float32) opt_param(self, ['prev_layer']) opt_param(self, ['activation'], Linear()) opt_param(self, ['is_local', 'is_data', 'is_cost'], False) opt_param(self, ['skip_act', 'has_params'], False) opt_param(self, ['prev_names'], []) opt_param(self, ['backend_type'], 'np.float32') if self.backend_type == 'np.float16': logger.info("Setting layer dtype to float16") for some_type in [ 'pre_act_dtype', 'output_dtype', 'deltas_dtype', 'weight_dtype', 'updates_dtype' ]: setattr(self, some_type, np.float16)
def __init__(self, **kwargs): self.accumulate = True # Reusing deltas not supported for RNNs yet self.reuse_deltas = False super(RNN, self).__init__(**kwargs) req_param(self, ['unrolls']) self.rec_layer = self.layers[1]
def __init__(self, **kwargs): opt_param(self, ['preprocess_done'], False) opt_param(self, ['dotransforms', 'square_crop'], True) opt_param(self, ['mean_norm', 'unit_norm'], False) opt_param(self, ['shuffle_macro'], False) opt_param(self, ['tdims'], 0) opt_param(self, ['label_list'], ['l_id']) opt_param(self, ['num_channels'], 3) opt_param(self, ['num_workers'], 6) opt_param(self, ['backend_type'], 'np.float32') self.__dict__.update(kwargs) if self.backend_type in ['float16', 'np.float16', 'numpy.float16']: self.backend_type = np.float16 elif self.backend_type in ['float32', 'np.float32', 'numpy.float32']: self.backend_type = np.float32 else: raise ValueError('Datatype not understood') logger.warning("Imageset initialized with dtype %s", self.backend_type) req_param(self, ['cropped_image_size', 'output_image_size', 'imageset', 'save_dir', 'repo_path', 'macro_size']) opt_param(self, ['image_dir'], os.path.join(self.repo_path, self.imageset)) self.rgb = True if self.num_channels == 3 else False self.norm_factor = 128. if self.mean_norm else 256. self.img_dtype = np.int8
def __init__(self, **kwargs): opt_param(self, ['preprocess_done'], False) opt_param(self, ['dotransforms', 'square_crop'], False) opt_param(self, ['mean_norm', 'unit_norm'], False) opt_param(self, ['shuffle_macro'], False) opt_param(self, ['tdims'], 0) opt_param(self, ['label_list'], ['l_id']) opt_param(self, ['num_channels'], 3) opt_param(self, ['num_workers'], 6) opt_param(self, ['backend_type'], 'np.float32') self.__dict__.update(kwargs) if self.backend_type in ['float16', 'np.float16', 'numpy.float16']: self.backend_type = np.float16 elif self.backend_type in ['float32', 'np.float32', 'numpy.float32']: self.backend_type = np.float32 else: raise ValueError('Datatype not understood') logger.warning("Imageset initialized with dtype %s", self.backend_type) req_param(self, ['cropped_image_size', 'output_image_size', 'imageset', 'save_dir', 'repo_path', 'macro_size']) opt_param(self, ['image_dir'], os.path.join(self.repo_path, self.imageset)) self.rgb = True if self.num_channels == 3 else False self.norm_factor = 128. if self.mean_norm else 256. self.img_dtype = np.int8
def initialize(self, kwargs): super(DataLayer, self).initialize(kwargs) self.reset_counter() if self.is_local is True: req_param(self, ['nofm', 'ofmshape']) self.nout = self.nofm * np.prod(self.ofmshape) else: req_param(self, ['nout'])
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ['dataset', 'model']) opt_param(self, ['backend']) opt_param(self, ['live'], False) if self.backend is not None: self.initialize(self.backend)
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ['layers', 'batch_size']) opt_param(self, ['step_print'], -1) opt_param(self, ['accumulate'], False) opt_param(self, ['reuse_deltas'], True) opt_param(self, ['timing_plots'], False)
def initialize(self, kwargs): super(RecurrentCostLayer, self).initialize(kwargs) req_param(self, ['cost', 'ref_layer']) opt_param(self, ['ref_label'], 'targets') self.targets = None self.cost.olayer = self.prev_layer self.cost.initialize(kwargs) self.deltas = self.cost.get_deltabuf()
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ['layers', 'batch_size']) opt_param(self, ['step_print'], -1) opt_param(self, ['accumulate'], False) opt_param(self, ['reuse_deltas'], True) opt_param(self, ['timing_plots'], False) opt_param(self, ['serialize_schedule'])
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ["layers", "batch_size"]) opt_param(self, ["step_print"], -1) opt_param(self, ["accumulate"], False) opt_param(self, ["reuse_deltas"], True) opt_param(self, ["timing_plots"], False) opt_param(self, ["serialize_schedule"])
def initialize(self, kwargs): req_param(self, ['nout', 'nin', 'unrolls', 'activation']) super(RecurrentOutputLayer, self).initialize(kwargs) self.weight_shape = (self.nout, self.nin) self.bias_shape = (self.nout, 1) opt_param(self, ['delta_shape'], (self.nin, self.batch_size)) # moved self.allocate_output_bufs() self.allocate_param_bufs()
def initialize(self, kwargs): if self.initialized: return self.__dict__.update(kwargs) req_param(self, ['backend', 'batch_size']) self.output = None self.deltas = None self.initialized = True
def initialize(self, kwargs): self.__dict__.update(kwargs) req_param(self, ['backend', 'batch_size']) self.output = None self.deltas = None if self.is_local: self.nofm = self.end_idx self.end_idx = np.prod(self.ifmshape) * self.end_idx self.nout = self.end_idx self.allocate_output_bufs()
def initialize(self, kwargs): req_param(self, ['weight_init_rec']) self.weight_rec_shape = (self.nout, self.nout) super(RecurrentLSTMLayer, self).initialize(kwargs) self.weight_shape = (self.nout, self.nin) self.bias_shape = (self.nout, 1) opt_param(self, ['delta_shape'], (self.nout, self.batch_size)) self.allocate_output_bufs() self.allocate_param_bufs()
def initialize(self, kwargs): req_param(self, ["ksize", "alpha", "beta"]) self.alpha = self.alpha * 1.0 / self.ksize super(CrossMapResponseNormLayer, self).initialize(kwargs) self.nout = self.nin self.ofmshape, self.nofm = self.ifmshape, self.nifm self.allocate_output_bufs() self.tempbuf = None if isinstance(self.backend, CPU) and not self.prev_layer.is_data: self.tempbuf = self.backend.empty((1, self.ifmshape[-2], self.ifmshape[-1], self.batch_size))
def initialize(self, kwargs): super(CrossMapPoolingLayer, self).initialize(kwargs) req_param(self, ['nofm']) self.initialize_local() self.allocate_output_bufs() self.allocate_param_bufs() opt_param(self, ['updatebuf'], None) if isinstance(self.backend, CPU): self.updatebuf = self.backend.empty((1, 1))
def __init__(self, **kwargs): self.accumulate = True super(Balance, self).__init__(**kwargs) req_param(self, ['classlayers', 'stylelayers']) self.cost_layer = self.classlayers[-1] self.out_layer = self.layers[-2] self.class_layer = self.classlayers[-2] self.branch_layer = self.stylelayers[-2] self.pathways = [self.layers, self.classlayers, self.stylelayers] self.kwargs = kwargs
def initialize(self, kwargs): """ Initialize the Batch Normalization transform. This function will be called from WeightLayer.initialize with a reference to the layer. Arguments: _eps (numeric, optional): value used for numerical stability when normalizing by variance _iscale (numeric, optional): explicitly set an affine scale value to be used in inference instead of calculated scale from training _ishift (numeric, optional): explicitly set an affine shift value to be used in inference instead of calculated shift from training """ self.__dict__.update(kwargs) self.dtype = self.layer.weight_dtype self.bigtype = np.float32 if self.dtype is np.float16 else self.dtype opt_param(self, ['_iscale', '_ishift']) opt_param(self, ['_eps'], 1e-6) req_param(self, ['layer']) self.backend = self.layer.backend self.is_local = self.layer.is_local self.batch_size = self.layer.batch_size if self.is_local: self.in1d = (self.layer.nofm, 1) self.ofmsize = self.layer.ofmsize self.orig_shape = (self.layer.nofm * self.ofmsize, self.batch_size) self.in_shape = (self.layer.nofm, self.ofmsize * self.batch_size) else: self.in_shape = (self.layer.nout, self.batch_size) self.in1d = (self.layer.nout, 1) self.train_mode = True logger.info("BatchNormalization set to train mode") self.nbatches = 0 self._xhat = self.backend.zeros(self.in_shape, dtype=self.dtype) self._mean = self.backend.zeros(self.in1d, dtype=self.bigtype) self._vars = self.backend.zeros(self.in1d, dtype=self.bigtype) # Global mean and var to be used during inference self._gmean = self.backend.zeros(self.in1d, dtype=self.bigtype) self._gvars = self.backend.zeros(self.in1d, dtype=self.bigtype) # learned params and their update buffers self._beta = self.backend.zeros(self.in1d, dtype=self.bigtype) self._gamma = self.backend.ones(self.in1d, dtype=self.bigtype) self.layer.params.extend([self._beta, self._gamma]) self._beta_updates = self.backend.zeros(self.in1d, dtype=self.bigtype) self._gamma_updates = self.backend.zeros(self.in1d, dtype=self.bigtype) self.layer.updates.extend([self._beta_updates, self._gamma_updates])
def initialize(self, kwargs): req_param(self, ['ksize', 'alpha', 'beta']) self.alpha = self.alpha * 1.0 / self.ksize super(CrossMapResponseNormLayer, self).initialize(kwargs) self.nout = self.nin self.ofmshape, self.nofm = self.ifmshape, self.nifm self.allocate_output_bufs() self.tempbuf = None if isinstance(self.backend, CPU) and not self.prev_layer.is_data: self.tempbuf = self.backend.empty( (1, self.ifmshape[-2], self.ifmshape[-1], self.batch_size))
def initialize(self, kwargs): super(CostLayer, self).initialize(kwargs) req_param(self, ['cost']) opt_param(self, ['ref_label'], 'targets') opt_param(self, ['raw_label'], False) opt_param(self, ['category_label'], 'l_id') self.reference = None self.cost.olayer = self.prev_layer kwargs['raw_label'] = self.raw_label self.cost.initialize(kwargs) self.deltas = self.cost.get_deltabuf()
def initialize(self, kwargs): self.__dict__.update(kwargs) opt_param(self, ['backend'], self.olayer.backend) opt_param(self, ['batch_size'], self.olayer.batch_size) opt_param(self, ['olayer_data'], 'output') req_param(self.olayer, [self.olayer_data]) # if not hasattr(self.olayer, self.olayer_data): # raise ValueError("Layer %s does not have buffer %s" % # (self.olayer.name, self.olayer_data)) # else: self.set_outputbuf(getattr(self.olayer, self.olayer_data))
def initialize(self, kwargs): super(WeightLayer, self).initialize(kwargs) req_param(self, ['weight_init', 'lrule_init', 'nin', 'nout']) opt_param(self, ['accumulate'], False) opt_param(self, ['batch_norm'], False) self.weight_init.initialize(self.backend) self.params = [] self.updates = [] if self.batch_norm: self.bn = BatchNorm() kwargs['layer'] = self self.bn.initialize(kwargs)
def initialize(self, kwargs): super(WeightLayer, self).initialize(kwargs) req_param(self, ['nin', 'nout']) opt_param(self, ['weight_init'], default_weight_init()) opt_param(self, ['lrule_init'], default_lrule_init()) opt_param(self, ['accumulate'], False) opt_param(self, ['batch_norm'], False) opt_param(self, ['mempool']) # Used for parallel mode self.weight_init.initialize(self.backend) self.params = [] self.updates = [] if self.batch_norm: self.bn = BatchNorm() kwargs['layer'] = self self.bn.initialize(kwargs)
def load(self, backend=None, experiment=None): ''' Imageset only supports nervanagpu based backends ''' if not hasattr(self.backend, 'ng'): raise DeprecationWarning("Only nervanagpu-based backends " "supported. For using cudanet backend, " "revert to neon 0.8.2 ") bdir = os.path.expanduser(self.save_dir) cachefile = os.path.join(bdir, 'dataset_cache.pkl') if not os.path.exists(cachefile): logger.error("Batch dir cache not found in %s:", cachefile) response = raw_input("Press Y to create, otherwise exit: ") if response == 'Y': from neon.util.batch_writer import (BatchWriter, BatchWriterImagenet) if self.imageset.startswith('I1K'): self.bw = BatchWriterImagenet(**self.__dict__) else: self.bw = BatchWriter(**self.__dict__) self.bw.run() logger.error('Done writing batches - please rerun to train.') else: logger.error('Exiting...') sys.exit() cstats = deserialize(cachefile, verbose=False) if cstats['macro_size'] != self.macro_size: raise NotImplementedError("Cached macro size %d different from " "specified %d, delete save_dir %s " "and try again.", cstats['macro_size'], self.macro_size, self.save_dir) # Set the max indexes of batches for each from the cache file self.maxval = cstats['nval'] + cstats['val_start'] - 1 self.maxtrain = cstats['ntrain'] + cstats['train_start'] - 1 # Make sure only those properties not by yaml are updated cstats.update(self.__dict__) self.__dict__.update(cstats) # Should also put (in addition to nclass), number of train/val images req_param(self, ['ntrain', 'nval', 'train_start', 'val_start', 'train_mean', 'val_mean', 'labels_dict'])
def __init__(self, **kwargs): self.accumulate = True super(BalanceMP, self).__init__(**kwargs) req_param(self, ['costpaths']) # Append the prefix to the costpaths for ckey in self.costpaths.keys(): self.costpaths[ckey] = self.prefixlayers + self.costpaths[ckey] self.cost_layer = self.costpaths['subject'][-1] self.branch_layer = self.costpaths['z'][-2] self.out_layer = self.layers[-2] softmaxlabels = filter(lambda x: x != 'z', self.costpaths.keys()) self.softlayers = [self.costpaths[ck][-2] for ck in softmaxlabels] self.pathways = [self.layers, self.costpaths['z']] self.pathways += [self.costpaths[ck] for ck in softmaxlabels] self.path_skip_act = [False, False] + [True for ck in softmaxlabels] self.kwargs = kwargs
def __init__(self, **kwargs): self.initialized = False self.__dict__.update(kwargs) req_param(self, ['name']) opt_param(self, ['pre_act_dtype', 'output_dtype', 'deltas_dtype', 'weight_dtype', 'updates_dtype'], np.float32) opt_param(self, ['prev_layer']) opt_param(self, ['activation'], Linear()) opt_param(self, ['is_local', 'is_data', 'is_cost'], False) opt_param(self, ['is_random'], False) opt_param(self, ['skip_act', 'has_params'], False) opt_param(self, ['prev_names'], []) opt_param(self, ['backend_type'], 'np.float32') if self.backend_type == 'np.float16': logger.info("Setting layer dtype to float16") for some_type in ['pre_act_dtype', 'output_dtype', 'deltas_dtype', 'weight_dtype', 'updates_dtype']: setattr(self, some_type, np.float16)
def __init__(self, **kwargs): self.is_local = True super(PoolingLayer, self).__init__(**kwargs) req_param(self, ['op']) opt_param(self, ['maxout'], 1)
def __init__(self, **kwargs): super(SubConvLayer, self).__init__(**kwargs) req_param(self, ['endidx'])
def initialize(self, kwargs): """ Initialize the Batch Normalization transform. This function will be called from WeightLayer.initialize with a reference to the layer. Arguments: _eps (numeric, optional): value used for numerical stability when normalizing by variance _iscale (numeric, optional): explicitly set an affine scale value to be used in inference instead of calculated scale from training _ishift (numeric, optional): explicitly set an affine shift value to be used in inference instead of calculated shift from training """ self.__dict__.update(kwargs) self.dtype = self.layer.weight_dtype opt_param(self, ['_eps'], 1e-6) opt_param(self, ['_rho'], 0.99) req_param(self, ['layer']) self.backend = self.layer.backend self.is_local = self.layer.is_local self.batch_size = self.layer.batch_size if self.is_local: self.in1d = (self.layer.nofm, 1) self.ofmsize = self.layer.ofmsize self.in_shape = (self.layer.nofm, self.ofmsize * self.batch_size) make_zbuf = self.backend.allocate_fragment else: self.in_shape = (self.layer.nout, self.batch_size) self.in1d = (self.layer.nout, 1) make_zbuf = self.backend.empty self.train_mode = True logger.info("BatchNormalization set to train mode") self._xhat = make_zbuf(self.in_shape, dtype=self.dtype, persist_values=False) self._mean = self.backend.zeros(self.in1d, dtype=self.dtype, persist_values=False) self._vars = self.backend.zeros(self.in1d, dtype=self.dtype, persist_values=False) # learned params and their update buffers self._beta = self.backend.zeros(self.in1d, dtype=self.dtype, persist_values=False) self._gamma = self.backend.ones(self.in1d, dtype=self.dtype, persist_values=False) self.layer.params.extend([self._beta, self._gamma]) if self.backend.is_dist: self._beta.ptype = self._gamma.ptype = 'replica' self._mean.ptype = self._vars.ptype = 'replica' # Global mean and var to be used during inference self._gmean = self.backend.zeros_like(self._mean, dtype=self.dtype, persist_values=True) self._gvars = self.backend.zeros_like(self._vars, dtype=self.dtype, persist_values=True) self._beta_updates = self.backend.zeros_like(self._beta, dtype=self.dtype) self._gamma_updates = self.backend.zeros_like(self._gamma, dtype=self.dtype) self.layer.updates.extend([self._beta_updates, self._gamma_updates])
def initialize(self, kwargs): super(ActivationLayer, self).initialize(kwargs) req_param(self, ['activation']) self.nout = self.nin self.allocate_output_bufs()
def initialize(self, kwargs): super(CompositeLayer, self).initialize(kwargs) req_param(self, ['sublayers']) self.has_params = True for subl in self.sublayers: subl.initialize(kwargs)
def __init__(self, **kwargs): super(SliceLayer, self).__init__(**kwargs) req_param(self, ['end_idx'])