def frisky(args): song = K.hard_sigmoid(args) #list = np.reshape(list, (32, self.timesteps, input_dim, 1)) # song -= 0.5 # song = K.sign(song) #song = K.permute_dimensions(song, (1, 0, 2)) return song
def hard_sigmoid(x): """Hard sigmoid activation function. A faster approximation of the sigmoid activation. Piecewise linear approximation of the sigmoid function. Ref: 'https://en.wikipedia.org/wiki/Hard_sigmoid' For example: >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) >>> b = tf.keras.activations.hard_sigmoid(a) >>> b.numpy() array([0. , 0.3, 0.5, 0.7, 1. ], dtype=float32) Args: x: Input tensor. Returns: The hard sigmoid activation, defined as: - `if x < -2.5: return 0` - `if x > 2.5: return 1` - `if -2.5 <= x <= 2.5: return 0.2 * x + 0.5` """ return backend.hard_sigmoid(x)
def step_do(self, step_in, states): x_i =( K.dot(step_in, self.kernel_i) ) x_f =( K.dot(step_in, self.kernel_f) ) x_c =( K.dot(step_in, self.kernel_c) ) x_o = ( K.dot(step_in, self.kernel_o) ) h_tm1= states[0] # previous memory state c_tm1 = states[1] # previous carry state h2 = states[2] c_tm2 = states[3] h_next = h2 c_next = c_tm2 h_tm3 = states[4] c_tm3 = states[5] if self.inter_attention: # step_in2 = K.relu(K.dot(step_in,self.transform)+self.biase3) a = K.concatenate([h_tm1,step_in],axis=1); a1 = K.tanh(K.dot(a,self.attention_h));a1 = K.sqrt(K.sum(a1**2,axis=1))#a1 = K.squeeze((K.dot(a1,self.attention_h2)),1)# a = K.concatenate([h2,step_in],axis=1); a2 = K.tanh(K.dot(a,self.attention_h));a2 = K.sqrt(K.sum(a2**2,axis=1))#a2 = K.squeeze((K.dot(a2,self.attention_h2)),1)# a = K.concatenate([h_tm3,step_in],axis=1); a3 = K.tanh(K.dot(a,self.attention_h));a3 = K.sqrt(K.sum(a3**2,axis=1))#a3 = K.squeeze((K.dot(a3,self.attention_h2)),1)# a = K.concatenate([K.expand_dims(a1,1),K.expand_dims(a2,1)],axis=1); a = K.concatenate([a,K.expand_dims(a3,1)],axis=1) a = K.softmax(a); a1 = K.expand_dims(a[:,0],1); a2 = K.expand_dims(a[:,1],1); a3 = K.expand_dims(a[:,2],1); h2 = K.relu(a1*h_tm1 + a2*h2 + a3*h_tm3) a = K.concatenate([c_tm1,step_in],axis=1); a1 = K.tanh(K.dot(a,self.attention_c));a1 = K.sqrt(K.sum(a1**2,axis=1))#a1 = K.squeeze((K.dot(a1,self.attention_c2)),1)# a = K.concatenate([c_tm2,step_in],axis=1); a2 = K.tanh(K.dot(a,self.attention_c));a2 = K.sqrt(K.sum(a2**2,axis=1))#a2 = K.squeeze((K.dot(a2,self.attention_c2)),1)# a = K.concatenate([c_tm3,step_in],axis=1); a3 = K.tanh(K.dot(a,self.attention_c));a3 = K.sqrt(K.sum(a3**2,axis=1))#a3 = K.squeeze((K.dot(a3,self.attention_c2)),1)# a = K.concatenate([K.expand_dims(a1,1),K.expand_dims(a2,1)],axis=1); a = K.concatenate([a,K.expand_dims(a3,1)],axis=1) a = K.softmax(a); a1 = K.expand_dims(a[:,0],1); a2 = K.expand_dims(a[:,1],1); a3 = K.expand_dims(a[:,2],1); c2 = K.relu(a1*c_tm1 + a2*c_tm2 + a3*c_tm3) else: h2 = (h_tm1 + h2 + h_tm3)/3 c2 = (c_tm1 + c_tm2 + c_tm3)/3 i = K.hard_sigmoid(x_i + K.dot(h2,self.recurrent_kernel_i)) f = K.hard_sigmoid(x_f + K.dot(h2,self.recurrent_kernel_f)) o = K.hard_sigmoid(x_o + K.dot(h2,self.recurrent_kernel_o)) m = K.hard_sigmoid(x_c + K.dot(h2,self.recurrent_kernel_c)) c = (f * c2 + i * m ) h = (o * K.tanh( c )) return h, [h,c,h_tm1,c_tm1,h_next,c_next] #80.59,81.79
def hard_sigmoid(x): """ Hard Sigmoid activation function. >>> round(hard_sigmoid(-1), 1) 0.3 """ return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def hard_sigmoid(x): """ Hard Sigmoid activation function. >>> round(hard_sigmoid(-1), 1) 0.3 """ return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def lstm_step(self, inputs, h, c, param_group): z = K.dot(inputs, self.kernel[param_group]) z += K.dot(h, self.recurrent_kernel[param_group]) z = K.bias_add(z, self.bias[param_group]) z0 = z[:, :self.units] z1 = z[:, self.units:2 * self.units] z2 = z[:, 2 * self.units:3 * self.units] z3 = z[:, 3 * self.units:] i = K.hard_sigmoid(z0) f = K.hard_sigmoid(z1) c = f * c + i * K.tanh(z2) o = K.hard_sigmoid(z3) h = o * K.tanh(c) return h, c
def step_do(self, step_in, states): x_i = K.dot(step_in, self.kernel_i) x_f = K.dot(step_in, self.kernel_f) x_c = K.dot(step_in, self.kernel_c) x_o = K.dot(step_in, self.kernel_o) h_tm1 = states[0] # previous memory state c_tm1 = states[1] # previous carry state i = K.hard_sigmoid(x_i + K.dot(h_tm1, self.recurrent_kernel_i)) f = K.hard_sigmoid(x_f + K.dot(h_tm1, self.recurrent_kernel_f)) o = K.hard_sigmoid(x_o + K.dot(h_tm1, self.recurrent_kernel_o)) m = x_c + K.dot(h_tm1, self.recurrent_kernel_c) c = f * c_tm1 + i * m h = o * K.tanh(c) ch = K.concatenate([c, h]) return ch, [h, c]
def call(self, inputs, states): h_tm1 = states[0] # previous memory state c_tm1 = states[1] # previous carry state x_i = K.dot(inputs, self.kernel_i) x_f = K.dot(inputs, self.kernel_f) x_c = K.dot(inputs, self.kernel_c) x_o = K.dot(inputs, self.kernel_o) i = K.hard_sigmoid(x_i + K.dot(h_tm1, self.recurrent_kernel_i)) f = K.hard_sigmoid(x_f + K.dot(h_tm1, self.recurrent_kernel_f)) c = f * c_tm1 + i * K.tanh(x_c + K.dot(h_tm1, self.recurrent_kernel_c)) o = K.hard_sigmoid(x_o + K.dot(h_tm1, self.recurrent_kernel_o)) h = o * K.tanh(c) return h, [h, c]
def step_do2(self, step_in, states): x_i =( K.dot(step_in, self.kernel_i2) ) x_f =( K.dot(step_in, self.kernel_f2) ) x_c =( K.dot(step_in, self.kernel_c2) ) x_o = ( K.dot(step_in, self.kernel_o2) ) h_tm1= states[0] # previous memory state c_tm1 = states[1] # previous carry state i = K.hard_sigmoid(x_i + K.dot(h_tm1,self.recurrent_kernel_i2)) f = K.hard_sigmoid(x_f + K.dot(h_tm1,self.recurrent_kernel_f2)) o = K.hard_sigmoid(x_o + K.dot(h_tm1,self.recurrent_kernel_o2)) m = K.tanh(x_c + K.dot(h_tm1,self.recurrent_kernel_c2)) c = (f * c_tm1 + i * m ) h = (o * K.tanh( c )) return h, [h,c] #80.59,81.79
def regression_and_classification(y_true, y_pred): #if y_true is 0, treated as a negative # if y_true > 1, treated as a positive # nothing in between y_true_binarized = K.cast(K.greater_equal(y_true, 1.0), K.floatx()) hard_sigmoid_pred = K.hard_sigmoid(y_pred) linearized_hard_sigmoid_pred = (0.2*y_pred) + 0.5 binary_crossentropy_loss = K.mean(K.binary_crossentropy( output=hard_sigmoid_pred, target=y_true_binarized), axis=-1) mse_loss = K.mean(2.71*K.square(linearized_hard_sigmoid_pred - y_true) *K.cast(K.greater(y_true, 1.0), K.floatx()), axis=-1) return binary_crossentropy_loss + mse_loss
def hard_sigmoid(x): """Hard sigmoid activation function. A faster approximation of the sigmoid activation. For example: >>> a = tf.constant([-3.0,-1.0, 0.0,1.0,3.0], dtype = tf.float32) >>> b = tf.keras.activations.hard_sigmoid(a) >>> b.numpy() array([0. , 0.3, 0.5, 0.7, 1. ], dtype=float32) Args: x: Input tensor. Returns: The hard sigmoid activation, defined as: - `if x < -2.5: return 0` - `if x > 2.5: return 1` - `if -2.5 <= x <= 2.5: return 0.2 * x + 0.5` """ return K.hard_sigmoid(x)
def hard_sigmoid(x): return K.hard_sigmoid(x)
def hardtanh(x): return 2 * K.hard_sigmoid(2.5 * x) - 1
def hard_sigmoid(x): return K.eval(K.hard_sigmoid(K.variable(x))).tolist()
def to_ret(x): correct = tf.sign(tf.where(tf.equal(x, 0.), tf.ones_like(x), x)) s = K.hard_sigmoid(tf.abs(x)) rand = tf.random_uniform(x.get_shape()) return tf.constant(val, tf.float32) * tf.where(tf.less(rand, s), correct, -1 * correct)
def to_ret(x): x_1 = tf.constant(val, tf.float32) * tf.sign(x) s = K.hard_sigmoid(tf.abs(x)) rand = tf.random_uniform(x.get_shape()) return tf.where(tf.less(rand, s), x_1, tf.zeros_like(x_1))
def Vote(y_true, y_pred): vote_value = hc.vote_value - 0.5 sig_pred = K.hard_sigmoid((y_pred - vote_value) * 5) return categorical_crossentropy(y_true, sig_pred)
def call_input_gate(self): Iiw = K.dot(self.inputs, self.Wki) Irw = K.dot(self.h_tm1, self.Wri) i = Iiw + Irw self.i = K.hard_sigmoid(i)
def call(self, inputs): s = K.zeros((K.shape(inputs)[0],self.units)) init_states = [s,s,s,s,s,s] outputs = K.rnn(self.step_do, inputs, init_states)[1] if self.intra_attention: # self.attention1_1 = self.attention1[:self.units,:] # self.attention1_2 = self.attention1[self.units:,:] for i in range(inputs.shape[1]): step_in = inputs[:,i,:] h = outputs[:,i,:] h_atten = K.concatenate([h,step_in],axis=1) h_atten=K.hard_sigmoid(K.dot(h_atten,self.attention1) + 1*self.biase1) h_atten=(K.dot(h_atten,self.attention2)) h_atten = K.elu(1*h_atten*h+0*self.biase2) if i ==0: output_atten = h_atten else: output_atten = K.concatenate([output_atten,h_atten]) outputs = Reshape((inputs.shape[1],self.units))(output_atten) init_states2 = [s,s,s,s,s,s] input2 = K.reverse(inputs,axes=1) outputs2 = K.rnn(self.step_do, input2, init_states2)[1] if self.intra_attention: # self.attention1_1 = self.attention1[:self.units,:] # self.attention1_2 = self.attention1[self.units:,:] for i in range(inputs.shape[1]): step_in = inputs[:,i,:] h = outputs2[:,i,:] h_atten = K.concatenate([h,step_in],axis=1) h_atten=K.hard_sigmoid(K.dot(h_atten,self.attention1) + 1*self.biase1) h_atten=(K.dot(h_atten,self.attention2)) h_atten = K.elu(1*h_atten*h+0*self.biase2) if i ==0: output_atten = h_atten else: output_atten = K.concatenate([output_atten,h_atten]) outputs2 = Reshape((inputs.shape[1],self.units))(output_atten) outputs2 = K.reverse(outputs2,axes=1) outputs = (K.concatenate([outputs,outputs2])) ''' if self.intra_attention: self.attention1_1 = self.attention1[:2*self.units,:] self.attention1_2 = self.attention1[2*self.units:,:] for i in range(inputs.shape[1]): step_in = inputs[:,i,:] h = outputs[:,i,:] h_atten=K.relu(K.dot(h,self.attention1_1) + 0*self.biase1) ##################0 h_atten=(K.dot(h_atten,self.attention2)) h_b=K.relu(K.dot(step_in,self.attention1_2)+0*self.biase2) ##################1 h_b=(K.dot(h_b,self.attention2_2)) h_atten = K.tanh(h_atten*h + h_b) if i ==0: output_atten = h_atten else: output_atten = K.concatenate([output_atten,h_atten]) outputs = Reshape((inputs.shape[1],2*self.units))(output_atten) ''' return outputs
def call_output_gate(self): Oiw = K.dot(self.inputs, self.Wko) Orw = K.dot(self.h_tm1, self.Wro) o = Oiw + Orw self.o = K.hard_sigmoid(o)
def call_forget_gate(self): Fiw = K.dot(self.inputs, self.Wkf) Frw = K.dot(self.h_tm1, self.Wrf) f = Fiw + Frw self.f = K.hard_sigmoid(f)