def initialize(self, shape): """Initializes the uninitialized variable. Uninitialized variable is a variable created with the data array set to None. This method creates and initializes the data array. The shape of the variable can be left unknown until this method is called. Args: shape (tuple of int): Shape of the data array. """ xp = numpy if self._initial_backend != 'cuda' else cuda.cupy with cuda.get_device_from_id(self._initial_device): data = initializers.generate_array(self.initializer, shape, xp) ginit = self._grad_initializer grad = None if ginit is None else initializers.generate_array( ginit, shape, xp) self.data = data self.grad = grad # Convert the array for iDeep. if self._initial_backend == 'intel64': self.to_intel64()
def initialize(self, shape): """Initializes the uninitialized variable. Uninitialized variable is a variable created with the data array set to None. This method creates and initializes the data array. The shape of the variable can be left unknown until this method is called. Args: shape (tuple of int): Shape of the data array. """ xp = numpy if self._initial_backend != 'cuda' else cuda.cupy with cuda.get_device_from_id(self._initial_device): data = initializers.generate_array(self.initializer, shape, xp) ginit = self._grad_initializer grad = None if ginit is None else initializers.generate_array( ginit, shape, xp) self.array = data self.grad = grad # Convert the array for iDeep. if self._initial_backend == 'intel64': self.to_intel64()
def __init__(self, sample_rate=16000, window_size_sec=0.02, window_stride_sec=0.01, n_fft=512, n_filters=64, preemph=0.97, dither=1.0e-05): super(NemoMelSpecExtractor, self).__init__() self.log_zero_guard_value = 2**-24 win_length = int(window_size_sec * sample_rate) self.hop_length = int(window_stride_sec * sample_rate) self.n_filters = n_filters from scipy import signal as scipy_signal from librosa import stft as librosa_stft window_arr = scipy_signal.hann(win_length, sym=True) self.stft = lambda x: librosa_stft(x, n_fft=n_fft, hop_length=self.hop_length, win_length=win_length, window=window_arr, center=True) self.dither = dither self.preemph = preemph self.pad_align = 16 from librosa.filters import mel as librosa_mel self.fb_arr = librosa_mel(sample_rate, n_fft, n_mels=n_filters, fmin=0, fmax=(sample_rate / 2)) with self.init_scope(): self.window = initializers.generate_array(initializer=Constant( 0, dtype="float32"), shape=window_arr.shape, xp=self.xp, dtype=np.float32) self.register_persistent("window") self.fb = initializers.generate_array( initializer=Constant(0, dtype="float32"), shape=np.expand_dims(self.fb_arr, axis=0).shape, xp=self.xp, dtype="float32") self.register_persistent("fb")
def _init_array(self, initializer, default_value, size): if initializer is None: initializer = default_value initializer = initializers._get_initializer(initializer) return initializers.generate_array( initializer, size, self.xp, dtype=self._highprec_dtype, device=self.device)
def check_shaped_initializer(self, backend_config): initializer = self.target(dtype=self.dtype, **self.target_kwargs) xp = backend_config.xp w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(backend.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype)
def __init__(self, in_channels, out_channels, ksize, stride, pad, groups): super(CondenseComplexConv, self).__init__() with self.init_scope(): self.bn = L.BatchNormalization(size=in_channels) self.activ = F.relu self.conv = L.Convolution2D( in_channels=in_channels, out_channels=out_channels, ksize=ksize, stride=stride, pad=pad, nobias=True, groups=groups) self.c_shuffle = ChannelShuffle( channels=out_channels, groups=groups) self.index = initializers.generate_array( initializer=initializers._get_initializer(0), shape=(in_channels,), xp=self.xp, dtype=np.int32) self.register_persistent('index')
def __init__(self, initializer=None, shape=None, name=None): if initializer is None: initializer = constant.NaN() elif numpy.isscalar(initializer): initializer = constant.Constant(initializer) if shape is None: if isinstance(initializer, (numpy.ndarray, cuda.ndarray)): # parameter initialized by the initial array super(Parameter, self).__init__(initializer, name=name) else: # uninitialized parameter super(Parameter, self).__init__(name=name) dtype = getattr(initializer, 'dtype', None) self._grad_initializer = constant.NaN(dtype) else: # parameter initialized with a given shape if isinstance(initializer, (numpy.ndarray, cuda.ndarray)): xp = cuda.get_array_module(initializer) initializer = constant.Constant(initializer) else: xp = numpy data = initializers.generate_array(initializer, shape, xp) grad = xp.full_like(data, numpy.nan) super(Parameter, self).__init__(data, name=name, grad=grad) self.update_rule = None self.initializer = initializer
def load_pretrained_embedding_layer(id2unigram, embed, external_model, finetuning=False): xp = cuda.get_array_module(embed.W) n_vocab = len(id2unigram) dim = external_model.wv.syn0[0].shape[0] initialW = initializers.normal.Normal(1.0) weight = [] count = 0 for i in range(n_vocab): key = id2unigram[i] if key in external_model.wv.vocab: vec = external_model.wv[key] count += 1 else: if finetuning: vec = initializers.generate_array(initialW, (dim, ), xp) else: vec = xp.zeros(dim, dtype='f') weight.append(vec) weight = xp.reshape(weight, (n_vocab, dim)) embed.W = chainer.Parameter(initializer=weight) if count >= 1: print('Use {} pretrained embedding vectors\n'.format(count), file=sys.stderr)
def __init__(self, initializer=None, shape=None, name=None): if initializer is None: initializer = constant.NaN() elif numpy.isscalar(initializer): initializer = constant.Constant(initializer) if shape is None: if isinstance(initializer, (numpy.ndarray, cuda.ndarray)): # parameter initialized by the initial array super(Parameter, self).__init__(initializer, name=name) else: # uninitialized parameter super(Parameter, self).__init__(name=name) self.initializer = initializer dtype = getattr(initializer, 'dtype', numpy.float32) self._grad_initializer = constant.NaN(dtype) else: # parameter initialized with a given shape if isinstance(initializer, (numpy.ndarray, cuda.ndarray)): xp = cuda.get_array_module(initializer) initializer = constant.Constant(initializer) else: xp = numpy data = initializers.generate_array(initializer, shape, xp) grad = xp.full_like(data, numpy.nan) super(Parameter, self).__init__(data, name=name, grad=grad) self.update_rule = None
def grow_embedding_layers_with_pretrained_model(n_vocab_org, n_vocab_grown, rand_embed, pretrained_embed, external_model, id2unigram_grown, train=False): diff = n_vocab_grown - n_vocab_org d_rand = rand_embed.W.shape[1] d_pretrained = pretrained_embed.W.shape[ 1] #external_model.wv.syn0[0].shape[0] count = 0 initialW = initializers.normal.Normal(1.0) w2_rand = [] w2_pretrained = [] for i in range(n_vocab_org, n_vocab_grown): if train: # resume training vec_rand = initializers.generate_array(initialW, (d_rand, ), np) else: # test vec_rand = rand_embed.W[ 0].data # use pretrained vector of unknown token w2_rand.append(vec_rand) key = id2unigram_grown[i] if key in external_model.wv.vocab: vec_pretrained = external_model.wv[key] count += 1 else: vec_pretrained = np.zeros(d_pretrained, dtype='f') w2_pretrained.append(vec_pretrained) w2_rand = np.reshape(w2_rand, (diff, d_rand)) if cuda.get_array_module(rand_embed.W) == cuda.cupy: w2_rand = chainer.Variable(w2_rand) w2_rand.to_gpu() w_rand = F.concat((rand_embed.W, w2_rand), axis=0) rand_embed.W = chainer.Parameter(initializer=w_rand.data, name='W') w2_pretrained = np.reshape(w2_pretrained, (diff, d_pretrained)) if cuda.get_array_module(rand_embed.W) == cuda.cupy: w2_pretrained = chainer.Variable(w2_pretrained) w2_pretrained.to_gpu() w_pretrained = F.concat((pretrained_embed.W, w2_pretrained), 0) pretrained_embed.W = chainer.Parameter(initializer=w_pretrained.data, name='W') print('Grow embedding matrix: {} -> {}'.format(n_vocab_org, rand_embed.W.shape[0]), file=sys.stderr) print('Grow pretrained embedding matrix: {} -> {}'.format( n_vocab_org, pretrained_embed.W.shape[0]), file=sys.stderr) if count >= 1: print('Add {} pretrained embedding vectors'.format(count), file=sys.stderr)
def check_shaped_initializer(self, xp): initializer = initializers.Identity(scale=self.scale, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(backend.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype) testing.assert_allclose(w, self.scale * numpy.identity(len(self.shape)), **self.check_options)
def check_shaped_initializer(self, xp): initializer = initializers.Constant(fill_value=self.fill_value, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(backend.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype) testing.assert_allclose(w, numpy.full(self.shape, self.fill_value), **self.check_options)
def check_shaped_initializer(self, xp): initializer = initializers.Identity( scale=self.scale, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(cuda.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype) testing.assert_allclose( w, self.scale * numpy.identity(len(self.shape)), **self.check_options)
def check_shaped_initializer(self, xp): initializer = initializers.Constant( fill_value=self.fill_value, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(cuda.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype) testing.assert_allclose( w, numpy.full(self.shape, self.fill_value), **self.check_options)
def initialize_state(self, shape): if self.gpu: xp = cuda.cupy else: xp = np self.s = initializers.generate_array(initializers.Normal(), (shape[0], self.n_out), xp, dtype=xp.float32) self.y = Variable(xp.zeros((shape[0], self.n_out), dtype=xp.float32))
def initialize(self, shape): """Initializes the uninitialized variable. Uninitialized variable is a variable created with the data array set to None. This method creates and initializes the data array. The shape of the variable can be left unknown until this method is called. Args: shape (tuple of int): Shape of the data array. """ xp = numpy if self._initial_device is None else cuda.cupy with cuda.get_device_from_id(self._initial_device): data = initializers.generate_array(self.initializer, shape, xp) ginit = self._grad_initializer grad = None if ginit is None else initializers.generate_array( ginit, shape, xp) self._data[0] = data self._node._grad = grad
def __init__(self, units, in_units, drop_rate=0.5): super(CondenseLinear, self).__init__() drop_in_units = int(in_units * drop_rate) with self.init_scope(): self.dense = L.Linear(in_size=drop_in_units, out_size=units) self.index = initializers.generate_array( initializer=initializers._get_initializer(0), shape=(drop_in_units, ), xp=self.xp, dtype=np.int32) self.register_persistent('index')
def add_param(self, name, shape, dtype=numpy.float32, initializer=None): """Registers a parameter to the link. The registered parameter is saved and loaded on serialization and deserialization, and involved in the optimization. The data and gradient of the variable are initialized by NaN arrays. If ``initializer`` is not ``None``, the data is initialized by ``initializer``. If the supplied ``name`` argument corresponds to an uninitialized parameter (that is, one that was added with the :meth:`add_uninitialized_param` method), ``name`` will be removed from the set of uninitialized parameters. The parameter is set to an attribute of the link with the given name. Args: name (str): Name of the parameter. This name is also used as the attribute name. Any uninitialized parameters with the same name will be removed. shape (int or tuple of ints): Shape of the parameter array. dtype: Data type of the parameter array. initializer(chainer.initializer.Initializer): If it is not ``None``, the data is initialized with the given initializer. Note that in this case ``dtype`` argument is ignored. """ d = self.__dict__ if name in d: raise AttributeError( 'cannot register a new parameter %s: attribute exists' % name) if initializer is None: data = self.xp.full(shape, numpy.nan, dtype=dtype) else: data = initializers.generate_array(initializer, shape, self.xp) u = self._uninitialized_params.get(name) if u is None: grad = self.xp.full_like(data, numpy.nan) else: if u._cleared: grad = None elif u._zeroed: grad = self.xp.zeros_like(data) else: grad = self.xp.full_like(data, numpy.nan) var = variable.Variable(data, volatile='auto', name=name) var.grad = grad self._params.append(name) d[name] = var if name in self._uninitialized_params: del self._uninitialized_params[name]
def initialize(self, shape): """Initializes the uninitialized variable. Uninitialized variable is a variable created with the data array set to None. This method creates and initializes the data array. The shape of the variable can be left unknown until this method is called. Args: shape (tuple of int): Shape of the data array. """ data = initializers.generate_array(self.initializer, shape, numpy) ginit = self._grad_initializer grad = None if ginit is None else initializers.generate_array( ginit, shape, numpy) if self._initial_device >= 0: data = cuda.to_gpu(data, device=self._initial_device) if grad is not None: grad = cuda.to_gpu(grad, device=self._initial_device) self._data[0] = data self._node._grad = grad
def grow_embedding_layer_without_pretrained_model(n_vocab_org, n_vocab_grown, rand_embed, train=False): xp = cuda.get_array_module(rand_embed.W) diff = n_vocab_grown - n_vocab_org d_rand = rand_embed.W.shape[1] initialW = initializers.normal.Normal(1.0) if train: w2_rand = initializers.generate_array(initialW, (diff, d_rand), xp) else: w2_rand = xp.zeros((diff, d_rand), dtype='f') w_rand = F.concat((rand_embed.W, w2_rand), axis=0) rand_embed.W = chainer.Parameter(initializer=w_rand.data, name='W') print('Grow embedding matrix: {} -> {}'.format(n_vocab_org, rand_embed.W.shape[0]), file=sys.stderr)
def grow_embedding_layer_with_pretrained_model(n_vocab_org, n_vocab_grown, rand_embed, external_model, id2unigram_grown, train=False, fasttext=False): diff = n_vocab_grown - n_vocab_org d_rand = rand_embed.W.shape[1] count = 0 initialW = initializers.normal.Normal(1.0) w2_rand = [] # [MEMO] the following error happened if fasttext=True: # cupy.cuda.cudnn.CuDNNError: CUDNN_STATUS_INTERNAL_ERROR: b'CUDNN_STATUS_INTERNAL_ERROR' wv_vocab = external_model.wv if fasttext else external_model.wv.vocab for i in range(n_vocab_org, n_vocab_grown): key = id2unigram_grown[i] if key in wv_vocab: vec_rand = external_model.wv[key] count += 1 elif train: vec_rand = initializers.generate_array(initialW, (d_rand, ), np) else: vec_rand = rand_embed.W[ 0].data # use pretrained vector of unknown token w2_rand.append(vec_rand) w2_rand = np.reshape(w2_rand, (diff, d_rand)) if cuda.get_array_module(rand_embed.W) == cuda.cupy: w2_rand = chainer.Variable(w2_rand) w2_rand.to_gpu() w_rand = F.concat((rand_embed.W, w2_rand), axis=0) rand_embed.W = chainer.Parameter(initializer=w_rand.data, name='W') print('Grow embedding matrix: {} -> {}'.format(n_vocab_org, rand_embed.W.shape[0]), file=sys.stderr) if count >= 1: print('Add {} pretrained embedding vectors'.format(count), file=sys.stderr)
def check_shaped_initializer(self, xp): initializer = self.target(scale=0.1, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(cuda.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype)
def _generate_array(self, xp, dtype=None, device=None): initializer = initializers.Zero(dtype) return initializers.generate_array(initializer, (), xp, device=device)
def _init_array(initializer, default_value, size, dtype): if initializer is None: initializer = default_value initializer = initializers._get_initializer(initializer) return initializers.generate_array(initializer, size, numpy, dtype=dtype)
def check_shaped_initializer(self, xp): initializer = initializers.Orthogonal(scale=2.0, dtype=self.dtype) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(backend.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype)
def check_shaped_initializer(self, xp): initializer = self.target(dtype=self.dtype, **self.target_kwargs) w = initializers.generate_array(initializer, self.shape, xp) self.assertIs(backend.get_array_module(w), xp) self.assertTupleEqual(w.shape, self.shape) self.assertEqual(w.dtype, self.dtype)
def _generate_array(self, dtype=None): initializer = initializers.Zero(dtype) return initializers.generate_array(initializer, (), numpy)