def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = initializers._compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
)
if self.criterion == 'glorot':
s = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_real = modulus * np.cos(phase)
weight_imag = modulus * np.sin(phase)
weight = np.concatenate([weight_real, weight_imag], axis=-1)
return weight
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = shape
# kernel_shape = tuple(self.kernel_size) + (int(self.input_dim),
# self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
fan_in, fan_out = _compute_fans(
# tuple(self.kernel_size) + (self.input_dim, self.nb_filters)
kernel_shape
)
# fix for ValueError: The initial value's shape (...) is not compatible with the explicitly supplied `shape` argument
reim_shape = list(kernel_shape)
reim_shape[-1] //= 2
reim_shape = tuple(reim_shape)
if self.criterion == 'glorot':
s = 1. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 1. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=reim_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=reim_shape)
weight_real = modulus * np.cos(phase)
weight_imag = modulus * np.sin(phase)
weight = np.concatenate([weight_real, weight_imag], axis=-1)
return weight
def convolution(self):
#initialize my conv kernel
self.kernels_real = []
self.kernels_imag = []
for i, filter_size in enumerate(self.filter_sizes):
with tf.name_scope('conv-pool-%s' % filter_size):
filter_shape = [filter_size, filter_size, 1, self.num_filters]
input_dim = 2
fan_in = np.prod(filter_shape[:-1])
fan_out = (filter_shape[-1] * np.prod(filter_shape[:2]))
s = 1. / fan_in
rng = RandomState(23455)
modulus = rng.rayleigh(scale=s, size=filter_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=filter_shape)
W_real = modulus * np.cos(phase)
W_imag = modulus * np.sin(phase)
W_real = tf.Variable(W_real, dtype='float32')
W_imag = tf.Variable(W_imag, dtype='float32')
self.kernels_real.append(W_real)
self.kernels_imag.append(W_imag)
# self.para.append(W_real)
# self.para.append(W_imag)
self.num_filters_total = self.num_filters * len(self.filter_sizes)
self.qa_real = self.narrow_convolution(
tf.expand_dims(self.M_qa_real, -1),
self.kernels_real) - self.narrow_convolution(
tf.expand_dims(self.M_qa_imag, -1), self.kernels_imag)
print(self.qa_real)
self.qa_imag = self.narrow_convolution(
tf.expand_dims(self.M_qa_imag, -1),
self.kernels_real) + self.narrow_convolution(
tf.expand_dims(self.M_qa_real, -1), self.kernels_imag)
print(self.qa_imag)
def narrow_convolutionandpool_real_imag(self,embedding_real,embedding_imag):
pooled_outputs_real=[]
pooled_outputs_imag=[]
for i,filter_size in enumerate(self.filter_sizes):
filter_shape = [filter_size,self.embedding_size,1,self.num_filters]
input_dim=2
fan_in = np.prod(filter_shape[:-1])
fan_out = (filter_shape[-1] * np.prod(filter_shape[:2]))
s=1./fan_in
rng=RandomState(23455)
b = tf.Variable(tf.constant(0.1, shape=[self.num_filters]), name="b")
modulus=rng.rayleigh(scale=s,size=filter_shape)
phase=rng.uniform(low=-np.pi,high=np.pi,size=filter_shape)
W_real=modulus*np.cos(phase)
W_imag=modulus*np.sin(phase)
W_real = tf.Variable(W_real,dtype = 'float32')
W_imag = tf.Variable(W_imag,dtype = 'float32')
conv_real = tf.nn.conv2d(embedding_real,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
cov_imag=tf.nn.conv2d(embedding_imag,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
cov_real_imag=tf.nn.conv2d(embedding_imag,W_real,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
cov_imag_real=tf.nn.conv2d(embedding_real,W_imag,strides=[1, 1, 1, 1],padding='VALID',name="conv-1")
qa_real=conv_real-cov_imag
qa_imag=cov_real_imag + cov_imag_real
h_real = tf.nn.relu(tf.nn.bias_add(qa_real, b), name="relu")
h_imag = tf.nn.relu(tf.nn.bias_add(qa_imag, b), name="relu")
pooled_real = tf.nn.max_pool(
h_real,
ksize=[1, self.embedding_size - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_imag = tf.nn.max_pool(
h_imag,
ksize=[1, self.embedding_size - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_outputs_real.append(pooled_real)
pooled_outputs_imag.append(pooled_imag)
self.h_pool_real = tf.concat(pooled_outputs_real, 3)
self.h_pool_imag = tf.concat(pooled_outputs_imag, 3)
self.num_filters_total = self.num_filters * len(self.filter_sizes)
self.h_pool_real=tf.reshape(self.h_pool_real, [-1, self.num_filters_total])
self.h_pool_imag=tf.reshape(self.h_pool_imag, [-1, self.num_filters_total])
h_drop_real = tf.nn.dropout(self.h_pool_real, self.dropout_keep_prob)
h_drop_imag = tf.nn.dropout(self.h_pool_imag, self.dropout_keep_prob)
h_drop=tf.concat([h_drop_real,h_drop_imag],1)
return h_drop
def __call__(self, shape, dtype=None):
if self.nb_filters is not None:
kernel_shape = tuple(self.kernel_size) + (int(
self.input_dim), self.nb_filters)
else:
kernel_shape = (int(self.input_dim), self.kernel_size[-1])
# keras.initializers._compute_fans
fan_in, fan_out = _compute_fans(
tuple(self.kernel_size) + (self.input_dim, self.nb_filters))
# Quaternion operations start here
if self.criterion == 'glorot':
s = 1. / np.sqrt(2 * (fan_in + fan_out))
elif self.criterion == 'he':
s = 1. / np.sqrt(2 * fan_in)
else:
raise ValueError('Invalid criterion: ' + self.criterion)
#Generating randoms and purely imaginary quaternions :
number_of_weights = np.prod(kernel_shape)
v_i = np.random.uniform(0.0, 1.0, number_of_weights)
v_j = np.random.uniform(0.0, 1.0, number_of_weights)
v_k = np.random.uniform(0.0, 1.0, number_of_weights)
#Make these purely imaginary quaternions unitary
for i in range(0, number_of_weights):
norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001
v_i[i] /= norm
v_j[i] /= norm
v_k[i] /= norm
v_i = v_i.reshape(kernel_shape)
v_j = v_j.reshape(kernel_shape)
v_k = v_k.reshape(kernel_shape)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_r = modulus * np.cos(phase)
weight_i = modulus * v_i * np.sin(phase)
weight_j = modulus * v_j * np.sin(phase)
weight_k = modulus * v_k * np.sin(phase)
weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
axis=-1)
return weight
def set_weight(self,num_unit,dim):
input_dim = (self.total_embedding_dim - self.filter_sizes[0] + 1) * self.num_filters * dim
unit=num_unit
kernel_shape = [input_dim,unit]
fan_in_f=np.prod(kernel_shape)
s = np.sqrt(1. / fan_in_f)
rng=RandomState(23455)
modulus_f=rng.rayleigh(scale=s,size=kernel_shape)
phase_f=rng.uniform(low=-np.pi,high=np.pi,size=kernel_shape)
real_init=modulus_f*np.cos(phase_f)
imag_init=modulus_f*np.sin(phase_f)
real_kernel=tf.Variable(real_init,name='real_kernel')
real_kernel=tf.to_float(real_kernel)
imag_kernel=tf.Variable(imag_init,name='imag_kernel')
imag_kernel=tf.to_float(imag_kernel)
return real_kernel,imag_kernel
def __call__(self, shape, dtype=None):
fan_in = np.prod(shape[:-1]) if self.flattened is True else shape[-2]
fan_out = shape[-1]
if self.criterion == 'glorot':
s = 2. / (fan_in + fan_out)
elif self.criterion == 'he':
s = 2. / fan_in
else:
raise ValueError('Invalid criterion: ' + self.criterion)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale = s, size = shape)
phase = rng.uniform(low = -np.pi, high = np.pi, size = shape)
weight_real = modulus * np.cos(phase)
weight_image = modulus * np.sin(phase)
# weight = np.concatenate([weight_real, weight_image], axis = -1)
weight = np.stack([weight_real, weight_image])
return weight
def _complex_he_init(shape):
"""
Initialize kernels to be independent from each other as much as possible, as
proposed in 'Deep Complex Networks' (https://arxiv.org/pdf/1705.09792.pdf).
"""
fan_in, fan_out = _calculate_fan_in_and_fan_out(shape)
s = 2. / fan_in
rng = RandomState()
modulus = rng.rayleigh(scale=s, size=shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=shape)
weight_real = modulus * np.cos(phase)
weight_real = torch.from_numpy(weight_real).float()
weight_im = modulus * np.sin(phase)
weight_im = torch.from_numpy(weight_im).float()
return weight_real, weight_im
def __call__(self, shape, dtype=None):
fan_in = self.shape[0]
fan_out = self.shape[1]
# Quaternion operations start here
if self.criterion == 'glorot':
s = 1. / np.sqrt(2 * (fan_in + fan_out))
elif self.criterion == 'he':
s = 1. / np.sqrt(2 * fan_in)
else:
raise ValueError('Invalid criterion: ' + self.criterion)
#Generating randoms and purely imaginary quaternions :
number_of_weights = np.prod(self.shape)
v_i = np.random.uniform(0.0, 1.0, number_of_weights)
v_j = np.random.uniform(0.0, 1.0, number_of_weights)
v_k = np.random.uniform(0.0, 1.0, number_of_weights)
#Make these purely imaginary quaternions unitary
for i in range(0, number_of_weights):
norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + 0.0001
v_i[i] /= norm
v_j[i] /= norm
v_k[i] /= norm
v_i = v_i.reshape(self.shape)
v_j = v_j.reshape(self.shape)
v_k = v_k.reshape(self.shape)
rng = RandomState(self.seed)
modulus = rng.rayleigh(scale=s, size=self.shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=self.shape)
weight_r = modulus * np.cos(phase)
weight_i = modulus * v_i * np.sin(phase)
weight_j = modulus * v_j * np.sin(phase)
weight_k = modulus * v_k * np.sin(phase)
weight = np.concatenate([weight_r, weight_i, weight_j, weight_k],
axis=-1)
return weight
def ComplexInit(tensor, init_criterion, seed):
if isinstance(tensor, Variable):
ComplexInit(tensor.data, init_criterion, seed)
return tensor
filter_type = None
if len(tensor.size()) == 3:
filter_type = 'Conv1d'
num_rows = int(tensor.size()[0] / 2) * tensor.size()[1]
num_cols = tensor.size()[2]
kernel_size = (tensor.size()[2], )
elif len(tensor.size()) == 4:
filter_type = 'Conv2d'
num_rows = int(tensor.size()[0] / 2) * tensor.size()[1]
num_cols = tensor.size()[2] * tensor.size()[3]
kernel_size = (tensor.size()[2], tensor.size()[3])
else:
sys.exit('The convolution type not support.')
kernel_shape = (int(tensor.size()[0] / 2), tensor.size()[1]) + kernel_size
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
if init_criterion == 'glorot':
desired_var = 1. / (fan_in + fan_out)
elif init_criterion == 'he':
desired_var = 1. / fan_in
else:
raise ValueError('invalid init critierion', init_criterion)
rng = RandomState(seed)
modulus = rng.rayleigh(scale=desired_var, size=kernel_shape)
phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape)
weight_real = modulus * np.cos(phase)
weight_imag = modulus * np.sin(phase)
weight = np.concatenate([weight_real, weight_imag], axis=0)
temp_weight = torch.from_numpy(weight).float()
tensor.copy_(temp_weight)
del temp_weight
return tensor
