Beispiel #1
0
    def __call__(self, shape, dtype=None):

        if self.nb_filters is not None:
            kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
        else:
            kernel_shape = (int(self.input_dim), self.kernel_size[-1])

        fan_in, fan_out = initializers._compute_fans(
            tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
        )

        if self.criterion == 'glorot':
            s = 1. / (fan_in + fan_out)
        elif self.criterion == 'he':
            s = 1. / fan_in
        else:
            raise ValueError('Invalid criterion: ' + self.criterion)
        rng = RandomState(self.seed)
        modulus = rng.rayleigh(scale=s, size=kernel_shape)
        phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
        weight_real = modulus * np.cos(phase)
        weight_imag = modulus * np.sin(phase)
        weight = np.concatenate([weight_real, weight_imag], axis=-1)

        return weight
Beispiel #2
0
    def __call__(self, shape, dtype=None):

        if self.nb_filters is not None:
            kernel_shape = shape
            # kernel_shape = tuple(self.kernel_size) + (int(self.input_dim),
            #                      self.nb_filters)
        else:
            kernel_shape = (int(self.input_dim), self.kernel_size[-1])

        fan_in, fan_out = _compute_fans(
            # tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
            kernel_shape
        )

        # fix for ValueError: The initial value's shape (...) is not compatible with the explicitly supplied `shape` argument
        reim_shape = list(kernel_shape)
        reim_shape[-1] //= 2
        reim_shape = tuple(reim_shape)

        if self.criterion == 'glorot':
            s = 1. / (fan_in + fan_out)
        elif self.criterion == 'he':
            s = 1. / fan_in
        else:
            raise ValueError('Invalid criterion: ' + self.criterion)
        rng = RandomState(self.seed)
        modulus = rng.rayleigh(scale=s, size=reim_shape)
        phase = rng.uniform(low=-np.pi, high=np.pi, size=reim_shape)
        weight_real = modulus * np.cos(phase)
        weight_imag = modulus * np.sin(phase)
        weight = np.concatenate([weight_real, weight_imag], axis=-1)

        return weight
Beispiel #3
0
 def convolution(self):
     #initialize my conv kernel
     self.kernels_real = []
     self.kernels_imag = []
     for i, filter_size in enumerate(self.filter_sizes):
         with tf.name_scope('conv-pool-%s' % filter_size):
             filter_shape = [filter_size, filter_size, 1, self.num_filters]
             input_dim = 2
             fan_in = np.prod(filter_shape[:-1])
             fan_out = (filter_shape[-1] * np.prod(filter_shape[:2]))
             s = 1. / fan_in
             rng = RandomState(23455)
             modulus = rng.rayleigh(scale=s, size=filter_shape)
             phase = rng.uniform(low=-np.pi, high=np.pi, size=filter_shape)
             W_real = modulus * np.cos(phase)
             W_imag = modulus * np.sin(phase)
             W_real = tf.Variable(W_real, dtype='float32')
             W_imag = tf.Variable(W_imag, dtype='float32')
             self.kernels_real.append(W_real)
             self.kernels_imag.append(W_imag)
             # self.para.append(W_real)
             # self.para.append(W_imag)
     self.num_filters_total = self.num_filters * len(self.filter_sizes)
     self.qa_real = self.narrow_convolution(
         tf.expand_dims(self.M_qa_real, -1),
         self.kernels_real) - self.narrow_convolution(
             tf.expand_dims(self.M_qa_imag, -1), self.kernels_imag)
     print(self.qa_real)
     self.qa_imag = self.narrow_convolution(
         tf.expand_dims(self.M_qa_imag, -1),
         self.kernels_real) + self.narrow_convolution(
             tf.expand_dims(self.M_qa_real, -1), self.kernels_imag)
     print(self.qa_imag)
Beispiel #4
0
 def narrow_convolutionandpool_real_imag(self,embedding_real,embedding_imag):
     pooled_outputs_real=[]
     pooled_outputs_imag=[]
     for i,filter_size in enumerate(self.filter_sizes):
         filter_shape = [filter_size,self.embedding_size,1,self.num_filters]
         input_dim=2
         fan_in = np.prod(filter_shape[:-1])
         fan_out = (filter_shape[-1] * np.prod(filter_shape[:2]))
         s=1./fan_in
         rng=RandomState(23455)
         b = tf.Variable(tf.constant(0.1, shape=[self.num_filters]), name="b")
         modulus=rng.rayleigh(scale=s,size=filter_shape)
         phase=rng.uniform(low=-np.pi,high=np.pi,size=filter_shape)
         W_real=modulus*np.cos(phase)
         W_imag=modulus*np.sin(phase)
         W_real = tf.Variable(W_real,dtype = 'float32')
         W_imag = tf.Variable(W_imag,dtype = 'float32')
         conv_real = tf.nn.conv2d(embedding_real,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
         cov_imag=tf.nn.conv2d(embedding_imag,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
         cov_real_imag=tf.nn.conv2d(embedding_imag,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
         cov_imag_real=tf.nn.conv2d(embedding_real,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
         qa_real=conv_real-cov_imag
         qa_imag=cov_real_imag + cov_imag_real
         h_real = tf.nn.relu(tf.nn.bias_add(qa_real, b), name="relu")
         h_imag = tf.nn.relu(tf.nn.bias_add(qa_imag, b), name="relu")
         pooled_real = tf.nn.max_pool(
                 h_real,
                 ksize=[1, self.embedding_size - filter_size + 1, 1, 1],
                 strides=[1, 1, 1, 1],
                 padding='VALID',
                 name="pool")
         pooled_imag = tf.nn.max_pool(
                 h_imag,
                 ksize=[1, self.embedding_size - filter_size + 1, 1, 1],
                 strides=[1, 1, 1, 1],
                 padding='VALID',
                 name="pool")
         pooled_outputs_real.append(pooled_real)
         pooled_outputs_imag.append(pooled_imag)
     self.h_pool_real = tf.concat(pooled_outputs_real, 3)
     self.h_pool_imag = tf.concat(pooled_outputs_imag, 3)
     self.num_filters_total = self.num_filters * len(self.filter_sizes)
     self.h_pool_real=tf.reshape(self.h_pool_real, [-1, self.num_filters_total])
     self.h_pool_imag=tf.reshape(self.h_pool_imag, [-1, self.num_filters_total])
     h_drop_real = tf.nn.dropout(self.h_pool_real, self.dropout_keep_prob)
     h_drop_imag = tf.nn.dropout(self.h_pool_imag, self.dropout_keep_prob)
     h_drop=tf.concat([h_drop_real,h_drop_imag],1)
     return h_drop
Beispiel #5
0
    def __call__(self, shape, dtype=None):

        if self.nb_filters is not None:
            kernel_shape = tuple(self.kernel_size) + (int(
                self.input_dim), self.nb_filters)
        else:
            kernel_shape = (int(self.input_dim), self.kernel_size[-1])

# keras.initializers._compute_fans
        fan_in, fan_out = _compute_fans(
            tuple(self.kernel_size) + (self.input_dim, self.nb_filters))

        # Quaternion operations start here

        if self.criterion == 'glorot':
            s = 1. / np.sqrt(2 * (fan_in + fan_out))
        elif self.criterion == 'he':
            s = 1. / np.sqrt(2 * fan_in)
        else:
            raise ValueError('Invalid criterion: ' + self.criterion)

        #Generating randoms and purely imaginary quaternions :
        number_of_weights = np.prod(kernel_shape)
        v_i = np.random.uniform(0.0, 1.0, number_of_weights)
        v_j = np.random.uniform(0.0, 1.0, number_of_weights)
        v_k = np.random.uniform(0.0, 1.0, number_of_weights)
        #Make these purely imaginary quaternions unitary
        for i in range(0, number_of_weights):
            norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001
            v_i[i] /= norm
            v_j[i] /= norm
            v_k[i] /= norm
        v_i = v_i.reshape(kernel_shape)
        v_j = v_j.reshape(kernel_shape)
        v_k = v_k.reshape(kernel_shape)

        rng = RandomState(self.seed)
        modulus = rng.rayleigh(scale=s, size=kernel_shape)
        phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)

        weight_r = modulus * np.cos(phase)
        weight_i = modulus * v_i * np.sin(phase)
        weight_j = modulus * v_j * np.sin(phase)
        weight_k = modulus * v_k * np.sin(phase)
        weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
                                axis=-1)

        return weight
Beispiel #6
0
 def set_weight(self,num_unit,dim):
     input_dim = (self.total_embedding_dim - self.filter_sizes[0] + 1) * self.num_filters * dim
     unit=num_unit
     kernel_shape = [input_dim,unit]
     fan_in_f=np.prod(kernel_shape)
     s = np.sqrt(1. / fan_in_f)
     rng=RandomState(23455)
     modulus_f=rng.rayleigh(scale=s,size=kernel_shape)
     phase_f=rng.uniform(low=-np.pi,high=np.pi,size=kernel_shape)
     real_init=modulus_f*np.cos(phase_f)
     imag_init=modulus_f*np.sin(phase_f)
     real_kernel=tf.Variable(real_init,name='real_kernel')
     real_kernel=tf.to_float(real_kernel)
     imag_kernel=tf.Variable(imag_init,name='imag_kernel')
     imag_kernel=tf.to_float(imag_kernel)
     return real_kernel,imag_kernel
Beispiel #7
0
    def __call__(self, shape, dtype=None):
        fan_in = np.prod(shape[:-1]) if self.flattened is True else shape[-2]
        fan_out = shape[-1]

        if self.criterion == 'glorot':
            s = 2. / (fan_in + fan_out)
        elif self.criterion == 'he':
            s = 2. / fan_in
        else:
            raise ValueError('Invalid criterion: ' + self.criterion)

        rng = RandomState(self.seed)
        modulus = rng.rayleigh(scale = s, size = shape)
        phase = rng.uniform(low = -np.pi, high = np.pi, size = shape)
        weight_real = modulus * np.cos(phase)
        weight_image = modulus * np.sin(phase)
        # weight = np.concatenate([weight_real, weight_image], axis = -1)
        weight = np.stack([weight_real, weight_image])
        return weight
Beispiel #8
0
def _complex_he_init(shape):
    """
    Initialize kernels to be independent from each other as much as possible, as
    proposed in 'Deep Complex Networks' (https://arxiv.org/pdf/1705.09792.pdf).
    """
    fan_in, fan_out = _calculate_fan_in_and_fan_out(shape)
    s = 2. / fan_in

    rng = RandomState()
    modulus = rng.rayleigh(scale=s, size=shape)
    phase = rng.uniform(low=-np.pi, high=np.pi, size=shape)

    weight_real = modulus * np.cos(phase)
    weight_real = torch.from_numpy(weight_real).float()

    weight_im = modulus * np.sin(phase)
    weight_im = torch.from_numpy(weight_im).float()

    return weight_real, weight_im
Beispiel #9
0
    def __call__(self, shape, dtype=None):

        fan_in = self.shape[0]
        fan_out = self.shape[1]

        # Quaternion operations start here

        if self.criterion == 'glorot':
            s = 1. / np.sqrt(2 * (fan_in + fan_out))
        elif self.criterion == 'he':
            s = 1. / np.sqrt(2 * fan_in)
        else:
            raise ValueError('Invalid criterion: ' + self.criterion)

        #Generating randoms and purely imaginary quaternions :
        number_of_weights = np.prod(self.shape)
        v_i = np.random.uniform(0.0, 1.0, number_of_weights)
        v_j = np.random.uniform(0.0, 1.0, number_of_weights)
        v_k = np.random.uniform(0.0, 1.0, number_of_weights)
        #Make these purely imaginary quaternions unitary
        for i in range(0, number_of_weights):
            norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001
            v_i[i] /= norm
            v_j[i] /= norm
            v_k[i] /= norm
        v_i = v_i.reshape(self.shape)
        v_j = v_j.reshape(self.shape)
        v_k = v_k.reshape(self.shape)

        rng = RandomState(self.seed)
        modulus = rng.rayleigh(scale=s, size=self.shape)
        phase = rng.uniform(low=-np.pi, high=np.pi, size=self.shape)

        weight_r = modulus * np.cos(phase)
        weight_i = modulus * v_i * np.sin(phase)
        weight_j = modulus * v_j * np.sin(phase)
        weight_k = modulus * v_k * np.sin(phase)

        weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
                                axis=-1)

        return weight
Beispiel #10
0
def ComplexInit(tensor, init_criterion, seed):
    if isinstance(tensor, Variable):
        ComplexInit(tensor.data, init_criterion, seed)
        return tensor
    filter_type = None
    if len(tensor.size()) == 3:
        filter_type = 'Conv1d'
        num_rows = int(tensor.size()[0] / 2) * tensor.size()[1]
        num_cols = tensor.size()[2]
        kernel_size = (tensor.size()[2], )
    elif len(tensor.size()) == 4:
        filter_type = 'Conv2d'
        num_rows = int(tensor.size()[0] / 2) * tensor.size()[1]
        num_cols = tensor.size()[2] * tensor.size()[3]
        kernel_size = (tensor.size()[2], tensor.size()[3])
    else:
        sys.exit('The convolution type not support.')
    kernel_shape = (int(tensor.size()[0] / 2), tensor.size()[1]) + kernel_size
    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
    if init_criterion == 'glorot':
        desired_var = 1. / (fan_in + fan_out)
    elif init_criterion == 'he':
        desired_var = 1. / fan_in
    else:
        raise ValueError('invalid init critierion', init_criterion)

    rng = RandomState(seed)
    modulus = rng.rayleigh(scale=desired_var, size=kernel_shape)
    phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
    weight_real = modulus * np.cos(phase)
    weight_imag = modulus * np.sin(phase)

    weight = np.concatenate([weight_real, weight_imag], axis=0)
    temp_weight = torch.from_numpy(weight).float()
    tensor.copy_(temp_weight)
    del temp_weight
    return tensor