def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = initializers._compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters) ) if self.criterion == 'glorot': s = 1. / (fan_in + fan_out) elif self.criterion == 'he': s = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_real = modulus * np.cos(phase) weight_imag = modulus * np.sin(phase) weight = np.concatenate([weight_real, weight_imag], axis=-1) return weight
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = shape # kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), # self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) fan_in, fan_out = _compute_fans( # tuple(self.kernel_size) + (self.input_dim, self.nb_filters) kernel_shape ) # fix for ValueError: The initial value's shape (...) is not compatible with the explicitly supplied `shape` argument reim_shape = list(kernel_shape) reim_shape[-1] //= 2 reim_shape = tuple(reim_shape) if self.criterion == 'glorot': s = 1. / (fan_in + fan_out) elif self.criterion == 'he': s = 1. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=reim_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=reim_shape) weight_real = modulus * np.cos(phase) weight_imag = modulus * np.sin(phase) weight = np.concatenate([weight_real, weight_imag], axis=-1) return weight
def convolution(self): #initialize my conv kernel self.kernels_real = [] self.kernels_imag = [] for i, filter_size in enumerate(self.filter_sizes): with tf.name_scope('conv-pool-%s' % filter_size): filter_shape = [filter_size, filter_size, 1, self.num_filters] input_dim = 2 fan_in = np.prod(filter_shape[:-1]) fan_out = (filter_shape[-1] * np.prod(filter_shape[:2])) s = 1. / fan_in rng = RandomState(23455) modulus = rng.rayleigh(scale=s, size=filter_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=filter_shape) W_real = modulus * np.cos(phase) W_imag = modulus * np.sin(phase) W_real = tf.Variable(W_real, dtype='float32') W_imag = tf.Variable(W_imag, dtype='float32') self.kernels_real.append(W_real) self.kernels_imag.append(W_imag) # self.para.append(W_real) # self.para.append(W_imag) self.num_filters_total = self.num_filters * len(self.filter_sizes) self.qa_real = self.narrow_convolution( tf.expand_dims(self.M_qa_real, -1), self.kernels_real) - self.narrow_convolution( tf.expand_dims(self.M_qa_imag, -1), self.kernels_imag) print(self.qa_real) self.qa_imag = self.narrow_convolution( tf.expand_dims(self.M_qa_imag, -1), self.kernels_real) + self.narrow_convolution( tf.expand_dims(self.M_qa_real, -1), self.kernels_imag) print(self.qa_imag)
def narrow_convolutionandpool_real_imag(self,embedding_real,embedding_imag): pooled_outputs_real=[] pooled_outputs_imag=[] for i,filter_size in enumerate(self.filter_sizes): filter_shape = [filter_size,self.embedding_size,1,self.num_filters] input_dim=2 fan_in = np.prod(filter_shape[:-1]) fan_out = (filter_shape[-1] * np.prod(filter_shape[:2])) s=1./fan_in rng=RandomState(23455) b = tf.Variable(tf.constant(0.1, shape=[self.num_filters]), name="b") modulus=rng.rayleigh(scale=s,size=filter_shape) phase=rng.uniform(low=-np.pi,high=np.pi,size=filter_shape) W_real=modulus*np.cos(phase) W_imag=modulus*np.sin(phase) W_real = tf.Variable(W_real,dtype = 'float32') W_imag = tf.Variable(W_imag,dtype = 'float32') conv_real = tf.nn.conv2d(embedding_real,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1") cov_imag=tf.nn.conv2d(embedding_imag,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1") cov_real_imag=tf.nn.conv2d(embedding_imag,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1") cov_imag_real=tf.nn.conv2d(embedding_real,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1") qa_real=conv_real-cov_imag qa_imag=cov_real_imag + cov_imag_real h_real = tf.nn.relu(tf.nn.bias_add(qa_real, b), name="relu") h_imag = tf.nn.relu(tf.nn.bias_add(qa_imag, b), name="relu") pooled_real = tf.nn.max_pool( h_real, ksize=[1, self.embedding_size - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name="pool") pooled_imag = tf.nn.max_pool( h_imag, ksize=[1, self.embedding_size - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name="pool") pooled_outputs_real.append(pooled_real) pooled_outputs_imag.append(pooled_imag) self.h_pool_real = tf.concat(pooled_outputs_real, 3) self.h_pool_imag = tf.concat(pooled_outputs_imag, 3) self.num_filters_total = self.num_filters * len(self.filter_sizes) self.h_pool_real=tf.reshape(self.h_pool_real, [-1, self.num_filters_total]) self.h_pool_imag=tf.reshape(self.h_pool_imag, [-1, self.num_filters_total]) h_drop_real = tf.nn.dropout(self.h_pool_real, self.dropout_keep_prob) h_drop_imag = tf.nn.dropout(self.h_pool_imag, self.dropout_keep_prob) h_drop=tf.concat([h_drop_real,h_drop_imag],1) return h_drop
def __call__(self, shape, dtype=None): if self.nb_filters is not None: kernel_shape = tuple(self.kernel_size) + (int( self.input_dim), self.nb_filters) else: kernel_shape = (int(self.input_dim), self.kernel_size[-1]) # keras.initializers._compute_fans fan_in, fan_out = _compute_fans( tuple(self.kernel_size) + (self.input_dim, self.nb_filters)) # Quaternion operations start here if self.criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif self.criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + self.criterion) #Generating randoms and purely imaginary quaternions : number_of_weights = np.prod(kernel_shape) v_i = np.random.uniform(0.0, 1.0, number_of_weights) v_j = np.random.uniform(0.0, 1.0, number_of_weights) v_k = np.random.uniform(0.0, 1.0, number_of_weights) #Make these purely imaginary quaternions unitary for i in range(0, number_of_weights): norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001 v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(kernel_shape) v_j = v_j.reshape(kernel_shape) v_k = v_k.reshape(kernel_shape) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_r = modulus * np.cos(phase) weight_i = modulus * v_i * np.sin(phase) weight_j = modulus * v_j * np.sin(phase) weight_k = modulus * v_k * np.sin(phase) weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) return weight
def set_weight(self,num_unit,dim): input_dim = (self.total_embedding_dim - self.filter_sizes[0] + 1) * self.num_filters * dim unit=num_unit kernel_shape = [input_dim,unit] fan_in_f=np.prod(kernel_shape) s = np.sqrt(1. / fan_in_f) rng=RandomState(23455) modulus_f=rng.rayleigh(scale=s,size=kernel_shape) phase_f=rng.uniform(low=-np.pi,high=np.pi,size=kernel_shape) real_init=modulus_f*np.cos(phase_f) imag_init=modulus_f*np.sin(phase_f) real_kernel=tf.Variable(real_init,name='real_kernel') real_kernel=tf.to_float(real_kernel) imag_kernel=tf.Variable(imag_init,name='imag_kernel') imag_kernel=tf.to_float(imag_kernel) return real_kernel,imag_kernel
def __call__(self, shape, dtype=None): fan_in = np.prod(shape[:-1]) if self.flattened is True else shape[-2] fan_out = shape[-1] if self.criterion == 'glorot': s = 2. / (fan_in + fan_out) elif self.criterion == 'he': s = 2. / fan_in else: raise ValueError('Invalid criterion: ' + self.criterion) rng = RandomState(self.seed) modulus = rng.rayleigh(scale = s, size = shape) phase = rng.uniform(low = -np.pi, high = np.pi, size = shape) weight_real = modulus * np.cos(phase) weight_image = modulus * np.sin(phase) # weight = np.concatenate([weight_real, weight_image], axis = -1) weight = np.stack([weight_real, weight_image]) return weight
def _complex_he_init(shape): """ Initialize kernels to be independent from each other as much as possible, as proposed in 'Deep Complex Networks' (https://arxiv.org/pdf/1705.09792.pdf). """ fan_in, fan_out = _calculate_fan_in_and_fan_out(shape) s = 2. / fan_in rng = RandomState() modulus = rng.rayleigh(scale=s, size=shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=shape) weight_real = modulus * np.cos(phase) weight_real = torch.from_numpy(weight_real).float() weight_im = modulus * np.sin(phase) weight_im = torch.from_numpy(weight_im).float() return weight_real, weight_im
def __call__(self, shape, dtype=None): fan_in = self.shape[0] fan_out = self.shape[1] # Quaternion operations start here if self.criterion == 'glorot': s = 1. / np.sqrt(2 * (fan_in + fan_out)) elif self.criterion == 'he': s = 1. / np.sqrt(2 * fan_in) else: raise ValueError('Invalid criterion: ' + self.criterion) #Generating randoms and purely imaginary quaternions : number_of_weights = np.prod(self.shape) v_i = np.random.uniform(0.0, 1.0, number_of_weights) v_j = np.random.uniform(0.0, 1.0, number_of_weights) v_k = np.random.uniform(0.0, 1.0, number_of_weights) #Make these purely imaginary quaternions unitary for i in range(0, number_of_weights): norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001 v_i[i] /= norm v_j[i] /= norm v_k[i] /= norm v_i = v_i.reshape(self.shape) v_j = v_j.reshape(self.shape) v_k = v_k.reshape(self.shape) rng = RandomState(self.seed) modulus = rng.rayleigh(scale=s, size=self.shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=self.shape) weight_r = modulus * np.cos(phase) weight_i = modulus * v_i * np.sin(phase) weight_j = modulus * v_j * np.sin(phase) weight_k = modulus * v_k * np.sin(phase) weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) return weight
def ComplexInit(tensor, init_criterion, seed): if isinstance(tensor, Variable): ComplexInit(tensor.data, init_criterion, seed) return tensor filter_type = None if len(tensor.size()) == 3: filter_type = 'Conv1d' num_rows = int(tensor.size()[0] / 2) * tensor.size()[1] num_cols = tensor.size()[2] kernel_size = (tensor.size()[2], ) elif len(tensor.size()) == 4: filter_type = 'Conv2d' num_rows = int(tensor.size()[0] / 2) * tensor.size()[1] num_cols = tensor.size()[2] * tensor.size()[3] kernel_size = (tensor.size()[2], tensor.size()[3]) else: sys.exit('The convolution type not support.') kernel_shape = (int(tensor.size()[0] / 2), tensor.size()[1]) + kernel_size fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) if init_criterion == 'glorot': desired_var = 1. / (fan_in + fan_out) elif init_criterion == 'he': desired_var = 1. / fan_in else: raise ValueError('invalid init critierion', init_criterion) rng = RandomState(seed) modulus = rng.rayleigh(scale=desired_var, size=kernel_shape) phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) weight_real = modulus * np.cos(phase) weight_imag = modulus * np.sin(phase) weight = np.concatenate([weight_real, weight_imag], axis=0) temp_weight = torch.from_numpy(weight).float() tensor.copy_(temp_weight) del temp_weight return tensor