def frisky(args):
song = K.hard_sigmoid(args)
#list = np.reshape(list, (32, self.timesteps, input_dim, 1))
# song -= 0.5
# song = K.sign(song)
#song = K.permute_dimensions(song, (1, 0, 2))
return song
def hard_sigmoid(x):
"""Hard sigmoid activation function.
A faster approximation of the sigmoid activation.
Piecewise linear approximation of the sigmoid function.
Ref: 'https://en.wikipedia.org/wiki/Hard_sigmoid'
For example:
>>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32)
>>> b = tf.keras.activations.hard_sigmoid(a)
>>> b.numpy()
array([0. , 0.3, 0.5, 0.7, 1. ], dtype=float32)
Args:
x: Input tensor.
Returns:
The hard sigmoid activation, defined as:
- `if x < -2.5: return 0`
- `if x > 2.5: return 1`
- `if -2.5 <= x <= 2.5: return 0.2 * x + 0.5`
"""
return backend.hard_sigmoid(x)
def step_do(self, step_in, states):
x_i =( K.dot(step_in, self.kernel_i) )
x_f =( K.dot(step_in, self.kernel_f) )
x_c =( K.dot(step_in, self.kernel_c) )
x_o = ( K.dot(step_in, self.kernel_o) )
h_tm1= states[0] # previous memory state
c_tm1 = states[1] # previous carry state
h2 = states[2]
c_tm2 = states[3]
h_next = h2
c_next = c_tm2
h_tm3 = states[4]
c_tm3 = states[5]
if self.inter_attention:
# step_in2 = K.relu(K.dot(step_in,self.transform)+self.biase3)
a = K.concatenate([h_tm1,step_in],axis=1); a1 = K.tanh(K.dot(a,self.attention_h));a1 = K.sqrt(K.sum(a1**2,axis=1))#a1 = K.squeeze((K.dot(a1,self.attention_h2)),1)#
a = K.concatenate([h2,step_in],axis=1); a2 = K.tanh(K.dot(a,self.attention_h));a2 = K.sqrt(K.sum(a2**2,axis=1))#a2 = K.squeeze((K.dot(a2,self.attention_h2)),1)#
a = K.concatenate([h_tm3,step_in],axis=1); a3 = K.tanh(K.dot(a,self.attention_h));a3 = K.sqrt(K.sum(a3**2,axis=1))#a3 = K.squeeze((K.dot(a3,self.attention_h2)),1)#
a = K.concatenate([K.expand_dims(a1,1),K.expand_dims(a2,1)],axis=1); a = K.concatenate([a,K.expand_dims(a3,1)],axis=1)
a = K.softmax(a); a1 = K.expand_dims(a[:,0],1); a2 = K.expand_dims(a[:,1],1); a3 = K.expand_dims(a[:,2],1);
h2 = K.relu(a1*h_tm1 + a2*h2 + a3*h_tm3)
a = K.concatenate([c_tm1,step_in],axis=1); a1 = K.tanh(K.dot(a,self.attention_c));a1 = K.sqrt(K.sum(a1**2,axis=1))#a1 = K.squeeze((K.dot(a1,self.attention_c2)),1)#
a = K.concatenate([c_tm2,step_in],axis=1); a2 = K.tanh(K.dot(a,self.attention_c));a2 = K.sqrt(K.sum(a2**2,axis=1))#a2 = K.squeeze((K.dot(a2,self.attention_c2)),1)#
a = K.concatenate([c_tm3,step_in],axis=1); a3 = K.tanh(K.dot(a,self.attention_c));a3 = K.sqrt(K.sum(a3**2,axis=1))#a3 = K.squeeze((K.dot(a3,self.attention_c2)),1)#
a = K.concatenate([K.expand_dims(a1,1),K.expand_dims(a2,1)],axis=1); a = K.concatenate([a,K.expand_dims(a3,1)],axis=1)
a = K.softmax(a); a1 = K.expand_dims(a[:,0],1); a2 = K.expand_dims(a[:,1],1); a3 = K.expand_dims(a[:,2],1);
c2 = K.relu(a1*c_tm1 + a2*c_tm2 + a3*c_tm3)
else:
h2 = (h_tm1 + h2 + h_tm3)/3
c2 = (c_tm1 + c_tm2 + c_tm3)/3
i = K.hard_sigmoid(x_i + K.dot(h2,self.recurrent_kernel_i))
f = K.hard_sigmoid(x_f + K.dot(h2,self.recurrent_kernel_f))
o = K.hard_sigmoid(x_o + K.dot(h2,self.recurrent_kernel_o))
m = K.hard_sigmoid(x_c + K.dot(h2,self.recurrent_kernel_c))
c = (f * c2 + i * m )
h = (o * K.tanh( c ))
return h, [h,c,h_tm1,c_tm1,h_next,c_next] #80.59,81.79
def hard_sigmoid(x):
"""
Hard Sigmoid activation function.
>>> round(hard_sigmoid(-1), 1)
0.3
"""
return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def hard_sigmoid(x):
"""
Hard Sigmoid activation function.
>>> round(hard_sigmoid(-1), 1)
0.3
"""
return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def lstm_step(self, inputs, h, c, param_group):
z = K.dot(inputs, self.kernel[param_group])
z += K.dot(h, self.recurrent_kernel[param_group])
z = K.bias_add(z, self.bias[param_group])
z0 = z[:, :self.units]
z1 = z[:, self.units:2 * self.units]
z2 = z[:, 2 * self.units:3 * self.units]
z3 = z[:, 3 * self.units:]
i = K.hard_sigmoid(z0)
f = K.hard_sigmoid(z1)
c = f * c + i * K.tanh(z2)
o = K.hard_sigmoid(z3)
h = o * K.tanh(c)
return h, c
def step_do(self, step_in, states):
x_i = K.dot(step_in, self.kernel_i)
x_f = K.dot(step_in, self.kernel_f)
x_c = K.dot(step_in, self.kernel_c)
x_o = K.dot(step_in, self.kernel_o)
h_tm1 = states[0] # previous memory state
c_tm1 = states[1] # previous carry state
i = K.hard_sigmoid(x_i + K.dot(h_tm1, self.recurrent_kernel_i))
f = K.hard_sigmoid(x_f + K.dot(h_tm1, self.recurrent_kernel_f))
o = K.hard_sigmoid(x_o + K.dot(h_tm1, self.recurrent_kernel_o))
m = x_c + K.dot(h_tm1, self.recurrent_kernel_c)
c = f * c_tm1 + i * m
h = o * K.tanh(c)
ch = K.concatenate([c, h])
return ch, [h, c]
def call(self, inputs, states): h_tm1 = states[0] # previous memory state c_tm1 = states[1] # previous carry state x_i = K.dot(inputs, self.kernel_i) x_f = K.dot(inputs, self.kernel_f) x_c = K.dot(inputs, self.kernel_c) x_o = K.dot(inputs, self.kernel_o) i = K.hard_sigmoid(x_i + K.dot(h_tm1, self.recurrent_kernel_i)) f = K.hard_sigmoid(x_f + K.dot(h_tm1, self.recurrent_kernel_f)) c = f * c_tm1 + i * K.tanh(x_c + K.dot(h_tm1, self.recurrent_kernel_c)) o = K.hard_sigmoid(x_o + K.dot(h_tm1, self.recurrent_kernel_o)) h = o * K.tanh(c) return h, [h, c]
def step_do2(self, step_in, states):
x_i =( K.dot(step_in, self.kernel_i2) )
x_f =( K.dot(step_in, self.kernel_f2) )
x_c =( K.dot(step_in, self.kernel_c2) )
x_o = ( K.dot(step_in, self.kernel_o2) )
h_tm1= states[0] # previous memory state
c_tm1 = states[1] # previous carry state
i = K.hard_sigmoid(x_i + K.dot(h_tm1,self.recurrent_kernel_i2))
f = K.hard_sigmoid(x_f + K.dot(h_tm1,self.recurrent_kernel_f2))
o = K.hard_sigmoid(x_o + K.dot(h_tm1,self.recurrent_kernel_o2))
m = K.tanh(x_c + K.dot(h_tm1,self.recurrent_kernel_c2))
c = (f * c_tm1 + i * m )
h = (o * K.tanh( c ))
return h, [h,c] #80.59,81.79
def regression_and_classification(y_true, y_pred):
#if y_true is 0, treated as a negative
# if y_true > 1, treated as a positive
# nothing in between
y_true_binarized = K.cast(K.greater_equal(y_true, 1.0), K.floatx())
hard_sigmoid_pred = K.hard_sigmoid(y_pred)
linearized_hard_sigmoid_pred = (0.2*y_pred) + 0.5
binary_crossentropy_loss = K.mean(K.binary_crossentropy(
output=hard_sigmoid_pred,
target=y_true_binarized), axis=-1)
mse_loss = K.mean(2.71*K.square(linearized_hard_sigmoid_pred - y_true)
*K.cast(K.greater(y_true, 1.0), K.floatx()),
axis=-1)
return binary_crossentropy_loss + mse_loss
def hard_sigmoid(x):
"""Hard sigmoid activation function.
A faster approximation of the sigmoid activation.
For example:
>>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32)
>>> b = tf.keras.activations.hard_sigmoid(a)
>>> b.numpy()
array([0. , 0.3, 0.5, 0.7, 1. ], dtype=float32)
Args:
x: Input tensor.
Returns:
The hard sigmoid activation, defined as:
- `if x < -2.5: return 0`
- `if x > 2.5: return 1`
- `if -2.5 <= x <= 2.5: return 0.2 * x + 0.5`
"""
return K.hard_sigmoid(x)
def hard_sigmoid(x):
return K.hard_sigmoid(x)
def hardtanh(x):
return 2 * K.hard_sigmoid(2.5 * x) - 1
def hard_sigmoid(x):
return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def to_ret(x):
correct = tf.sign(tf.where(tf.equal(x, 0.), tf.ones_like(x), x))
s = K.hard_sigmoid(tf.abs(x))
rand = tf.random_uniform(x.get_shape())
return tf.constant(val, tf.float32) * tf.where(tf.less(rand, s),
correct, -1 * correct)
def to_ret(x):
x_1 = tf.constant(val, tf.float32) * tf.sign(x)
s = K.hard_sigmoid(tf.abs(x))
rand = tf.random_uniform(x.get_shape())
return tf.where(tf.less(rand, s), x_1, tf.zeros_like(x_1))
def Vote(y_true, y_pred):
vote_value = hc.vote_value - 0.5
sig_pred = K.hard_sigmoid((y_pred - vote_value) * 5)
return categorical_crossentropy(y_true, sig_pred)
def call_input_gate(self):
Iiw = K.dot(self.inputs, self.Wki)
Irw = K.dot(self.h_tm1, self.Wri)
i = Iiw + Irw
self.i = K.hard_sigmoid(i)
def call(self, inputs):
s = K.zeros((K.shape(inputs)[0],self.units))
init_states = [s,s,s,s,s,s]
outputs = K.rnn(self.step_do, inputs, init_states)[1]
if self.intra_attention:
# self.attention1_1 = self.attention1[:self.units,:]
# self.attention1_2 = self.attention1[self.units:,:]
for i in range(inputs.shape[1]):
step_in = inputs[:,i,:]
h = outputs[:,i,:]
h_atten = K.concatenate([h,step_in],axis=1)
h_atten=K.hard_sigmoid(K.dot(h_atten,self.attention1) + 1*self.biase1)
h_atten=(K.dot(h_atten,self.attention2))
h_atten = K.elu(1*h_atten*h+0*self.biase2)
if i ==0:
output_atten = h_atten
else:
output_atten = K.concatenate([output_atten,h_atten])
outputs = Reshape((inputs.shape[1],self.units))(output_atten)
init_states2 = [s,s,s,s,s,s]
input2 = K.reverse(inputs,axes=1)
outputs2 = K.rnn(self.step_do, input2, init_states2)[1]
if self.intra_attention:
# self.attention1_1 = self.attention1[:self.units,:]
# self.attention1_2 = self.attention1[self.units:,:]
for i in range(inputs.shape[1]):
step_in = inputs[:,i,:]
h = outputs2[:,i,:]
h_atten = K.concatenate([h,step_in],axis=1)
h_atten=K.hard_sigmoid(K.dot(h_atten,self.attention1) + 1*self.biase1)
h_atten=(K.dot(h_atten,self.attention2))
h_atten = K.elu(1*h_atten*h+0*self.biase2)
if i ==0:
output_atten = h_atten
else:
output_atten = K.concatenate([output_atten,h_atten])
outputs2 = Reshape((inputs.shape[1],self.units))(output_atten)
outputs2 = K.reverse(outputs2,axes=1)
outputs = (K.concatenate([outputs,outputs2]))
'''
if self.intra_attention:
self.attention1_1 = self.attention1[:2*self.units,:]
self.attention1_2 = self.attention1[2*self.units:,:]
for i in range(inputs.shape[1]):
step_in = inputs[:,i,:]
h = outputs[:,i,:]
h_atten=K.relu(K.dot(h,self.attention1_1) + 0*self.biase1) ##################0
h_atten=(K.dot(h_atten,self.attention2))
h_b=K.relu(K.dot(step_in,self.attention1_2)+0*self.biase2) ##################1
h_b=(K.dot(h_b,self.attention2_2))
h_atten = K.tanh(h_atten*h + h_b)
if i ==0:
output_atten = h_atten
else:
output_atten = K.concatenate([output_atten,h_atten])
outputs = Reshape((inputs.shape[1],2*self.units))(output_atten)
'''
return outputs
def call_output_gate(self):
Oiw = K.dot(self.inputs, self.Wko)
Orw = K.dot(self.h_tm1, self.Wro)
o = Oiw + Orw
self.o = K.hard_sigmoid(o)
def call_forget_gate(self):
Fiw = K.dot(self.inputs, self.Wkf)
Frw = K.dot(self.h_tm1, self.Wrf)
f = Fiw + Frw
self.f = K.hard_sigmoid(f)
