def unary_op(op_str, a): if op_str == "Abs": return ng.abs(a) elif op_str == "Acos": return ng.acos(a) elif op_str == "Asin": return ng.asin(a) elif op_str == "Atan": return ng.atan(a) elif op_str == "Ceiling": return ng.ceiling(a) elif op_str == "Cos": return ng.cos(a) elif op_str == "Cosh": return ng.cosh(a) elif op_str == "Floor": return ng.floor(a) elif op_str == "log": return ng.log(a) elif op_str == "exp": return ng.exp(a) elif op_str == "negative": return ng.negative(a) elif op_str == "Sign": return ng.sign(a) elif op_str == "Sin": return ng.sin(a) elif op_str == "Sinh": return ng.sinh(a) elif op_str == "Sqrt": return ng.sqrt(a) elif op_str == "Tan": return ng.tan(a) elif op_str == "Tanh": return ng.tanh(a)
def __call__(self, x): """ Returns the hyperbolic tangent. Arguments: x (Tensor or optree): Input value Returns: Tensor or optree: Output activation """ return ng.tanh(x)
def Tanh(self, cntk_op, inputs): """ Returns element-wise tanh of inputs[0]. Arguments: inputs: List of inputs to this node. Returns: A ngraph Op. """ return ng.tanh(inputs[0]).named(cntk_op.uid)
def Tanh(self, cntk_op, inputs): """ Returns element-wise tanh of inputs[0]. Arguments: cntk_op: CNTK operation to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ return ng.tanh(inputs[0]).named(cntk_op.uid)
def unary_op(op_str, a): if op_str == 'Abs': return ng.abs(a) elif op_str == 'Acos': return ng.acos(a) elif op_str == 'Asin': return ng.asin(a) elif op_str == 'Atan': return ng.atan(a) elif op_str == 'Ceiling': return ng.ceiling(a) elif op_str == 'Cos': return ng.cos(a) elif op_str == 'Cosh': return ng.cosh(a) elif op_str == 'Floor': return ng.floor(a) elif op_str == 'log': return ng.log(a) elif op_str == 'exp': return ng.exp(a) elif op_str == 'negative': return ng.negative(a) elif op_str == 'Reverse': return ng.reverse(a, np.array([1]), 'index') elif op_str == 'Sign': return ng.sign(a) elif op_str == 'Sin': return ng.sin(a) elif op_str == 'Sinh': return ng.sinh(a) elif op_str == 'Sqrt': return ng.sqrt(a) elif op_str == 'Tan': return ng.tan(a) elif op_str == 'Tanh': return ng.tanh(a)
def Tanh(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """Calculate the hyperbolic tangent of the input tensor elementwise.""" return ng.tanh(ng_inputs[0])
W1 = ng.variable(axes=[hidden_feature_axis, tmp_axis_2], initial_value=init) W2 = ng.variable(axes=[hidden_feature_axis, tmp_axis_2], initial_value=init) v = ng.variable(axes=[tmp_axis_2], initial_value=init) input_time_steps = output_time_steps = time_steps u_list = [] # a list of target probability distributions of every output time step for i in range(output_time_steps): # compute attention vector for output time step i u_i_list = [] # a list of attention scores u_i W2_di = ng.dot(W2, ng.slice_along_axis(dec_h_out, axis=rec_axis, idx=i)) for j in range(input_time_steps): # compute attention score for output time step i with input time step j W1_ej = ng.dot(W1, ng.slice_along_axis(enc_h_out, axis=rec_axis, idx=j)) score = ng.dot(v, ng.tanh(W1_ej + W2_di)) # u_i = v * tanh(W1 * e_j + W2 * d_i) u_i_list.append(score) output_prob = ng.softmax(ng.stack(u_i_list, axis=ax.Y, pos=0), ax.Y) u_list.append(output_prob) pointer_out = ng.stack(u_list, axis=rec_axis, pos=2) # specify loss function, calculate loss and update weights one_hot_target = ng.one_hot(inputs['tgt_txt'], axis=ax.Y) loss = ng.cross_entropy_multi(pointer_out, one_hot_target, usebits=True) mean_cost = ng.mean(loss, out_axes=[])
input_time_steps = output_time_steps = time_steps u_list = [ ] # a list of target probability distributions of every output time step for i in range(output_time_steps): # compute attention vector for output time step i u_i_list = [] # a list of attention scores u_i W2_di = ng.dot(W2, ng.slice_along_axis(dec_h_out, axis=rec_axis, idx=i)) for j in range(input_time_steps): # compute attention score for output time step i with input time step j W1_ej = ng.dot(W1, ng.slice_along_axis(enc_h_out, axis=rec_axis, idx=j)) score = ng.dot( v, ng.tanh(W1_ej + W2_di)) # u_i = v * tanh(W1 * e_j + W2 * d_i) u_i_list.append(score) output_prob = ng.softmax(ng.stack(u_i_list, axis=ax.Y, pos=0), ax.Y) u_list.append(output_prob) pointer_out = ng.stack(u_list, axis=rec_axis, pos=2) # specify loss function, calculate loss and update weights one_hot_target = ng.one_hot(inputs['tgt_txt'], axis=ax.Y) loss = ng.cross_entropy_multi(pointer_out, one_hot_target, usebits=True) mean_cost = ng.mean(loss, out_axes=[]) optimizer = RMSProp(decay_rate=0.96, learning_rate=args.lr,
def __call__(self, H_concat, states=None, output=None, reset_cells=True, input_data=None): """ Arguments: ---------- H_concat: Concatenated forward and reverse unrolled outputs of the `MatchLSTMCell_withAttention` cell states: previous LSTM state output: hidden state from previous timestep reset_cells: argument to reset a cell input_data: the ArrayIterator object for training data (contains information of length of each sentence) """ rec_axis_pr = H_concat.axes.recurrent_axis() const_one = ng.constant(const=1, axes=[self.dummy_axis]) b_k_lists = [] # rec_axis_hy=H_hy.axes.recurrent_axis() for i in range(0, 2): if output is None: h_k_old = ng.constant(axes=[self.F, self.N], const=0) else: h_k_old = ng.cast_axes(output, [self.F, self.N]) sum_1 = ng.dot( self.V_answer, ng.cast_axes(H_concat, [self.lstm_feature_new, rec_axis_pr, self.N])) sum_1 = ng.cast_axes( sum_1, [self.hidden_rows, self.hidden_cols_para, self.N]) int_sum2 = ng.dot(self.W_a, h_k_old) int_sum = int_sum2 # +self.b_a int_sum = ng.ExpandDims(int_sum, self.dummy_axis, 1) # Following notations from the paper # Compute Attention Vector F_i_int = sum_1 + ng.axes_with_order( ng.dot(int_sum, self.e_q), [self.hidden_rows, self.hidden_cols_para, self.N]) F_i = ng.tanh(F_i_int) # Attention Vector b_k_sum1 = ng.dot(self.v_lr, F_i) # This masking with -inf for length of para>max_para ensures that # when we do softmax over these values we get a 0 mask_loss_new = ng.log(ng.dot(const_one, input_data['para_len'])) mask_loss_new = ng.axes_with_order( ng.cast_axes(mask_loss_new, [self.N, self.hidden_cols_para]), [self.hidden_cols_para, self.N]) # Add mask to the required logits b_k = ng.softmax(b_k_sum1 + mask_loss_new) b_k_req = ng.softmax(b_k_sum1 + mask_loss_new) b_k_repeated = ng.cast_axes( ng.dot(self.e_q2, ng.ExpandDims(b_k, self.dummy_axis, 0)), [H_concat.axes[0], rec_axis_pr, self.N]) inputs_lstm = ng.sum(ng.multiply(H_concat, b_k_repeated), rec_axis_pr) # LSTM Cell calculations if self.out_axes is None: self.out_axes = self.feature_axes + inputs_lstm.axes.batch_axis( ) if states is None: states = self.initialize_states(inputs_lstm.axes.batch_axis(), reset_cells=reset_cells) assert self.out_axes == states['h'].axes for gate in self._gate_names: transform = self.gate_transform[gate] gate_input = self.i2h[gate](inputs_lstm) + self.h2h[gate]( states['h']) self.gate_output[gate] = ng.cast_role(transform(gate_input), self.out_axes) states['c'] = (states['c'] * self.gate_output['f'] + self.gate_output['i'] * self.gate_output['g']) states['h'] = self.gate_output['o'] * self.activation(states['c']) states['h'] = ng.cast_role(states['h'], self.out_axes) output = states['h'] # append required outputs b_k_lists.append(b_k_req) return b_k_lists
def __call__(self, H_pr, h_ip, states, output=None, reset_cells=True, input_data=None): """ Arguments: ---------- H_pr : Encoding for question h_ip: Sliced input of paragraph encoding for a particular time step states: State of the LSTM cell output: previous hidden state input_data: the ArrayIterator object for training data (contains information of length of each sentence) """ # get recurrent axis for question rec_axis_pr = H_pr.axes.recurrent_axis() const_one = ng.constant(const=1, axes=[self.dummy_axis]) # if first word in a paragraph is encountered, assign the previous LSTM # hidden state as zeros if output is None: h_r_old = ng.constant(axes=[self.F, self.N], const=0) else: h_r_old = ng.cast_axes(output, [self.F, self.N]) # Compute attention vector sum_1 = ng.dot(self.W_q, H_pr) sum_1 = ng.cast_axes(sum_1, [self.hidden_rows, self.hidden_cols_ques, self.N]) int_sum1 = ng.dot(self.W_p, h_ip) int_sum2 = ng.dot(self.W_r, h_r_old) int_sum = int_sum1 + int_sum2 + self.b_p int_sum = ng.ExpandDims(int_sum, self.dummy_axis, 1) # making for the attention vector req_mask = ng.axes_with_order( ng.cast_axes(ng.dot(self.e_q2, input_data['question_len']), [self.hidden_rows, self.N, self.hidden_cols_ques]), [self.hidden_rows, self.hidden_cols_ques, self.N]) req_mask_2 = ng.axes_with_order( ng.cast_axes(ng.dot(const_one, input_data['question_len']), [self.N, self.hidden_cols_ques]), [self.hidden_cols_ques, self.N]) G_i_int = sum_1 + ng.multiply( req_mask, ng.axes_with_order( ng.dot(int_sum, self.e_q), [self.hidden_rows, self.hidden_cols_ques, self.N])) G_i = ng.tanh(G_i_int) # Attention Vector at_sum1 = ng.dot(self.w_lr, G_i) at = ng.softmax(at_sum1 + ng.log(req_mask_2)) at_repeated = ng.cast_axes( ng.dot(self.e_q2, ng.ExpandDims(at, self.dummy_axis, 0)), [self.F, rec_axis_pr, self.N]) # Stack the 2 vectors as per the equation in the paper z1 = h_ip z2 = ng.sum(ng.multiply(H_pr, at_repeated), rec_axis_pr) # represents the inp to lstm_cell # ng.concat_along_axis([z1,z2],self.F) inputs_lstm = ng.dot(self.ZX, z1) + ng.dot(self.ZY, z2) # LSTM cell computations (from LSTM brach in ngraph) if self.out_axes is None: self.out_axes = self.feature_axes + inputs_lstm.axes.batch_axis() if states is None: states = self.initialize_states(inputs_lstm.axes.batch_axis(), reset_cells=reset_cells) assert self.out_axes == states['h'].axes for gate in self._gate_names: transform = self.gate_transform[gate] gate_input = self.i2h[gate](inputs_lstm) + self.h2h[gate]( states['h']) self.gate_output[gate] = ng.cast_role(transform(gate_input), self.out_axes) states['c'] = (states['c'] * self.gate_output['f'] + self.gate_output['i'] * self.gate_output['g']) states['h'] = self.gate_output['o'] * self.activation(states['c']) states['h'] = ng.cast_role(states['h'], self.out_axes) # return unrolled output and state of LSTM cell return ng.cast_axes(states['h'], axes=[self.F, self.N]), states
def tanh(x, name=None): return ng.tanh(x).named(name)
def Tanh(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op return ng.tanh(ng_inputs[0])