def MatMul(self, tf_node, inputs): """ Multiplies matrix `a` by matrix `b`. The inputs must be two-dimensional, the inner dimensions must match (possibly after transpose). Arguments: tf_node: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: a, b, transpose_a, transpose_b, a_is_sparse, b_is_sparse, name """ # get inputs left, right = inputs if tf_node.attr['transpose_a'].b: left = ng.Transpose(left) if tf_node.attr['transpose_b'].b: right = ng.Transpose(right) # check shape assert len(left.axes) == len(right.axes) == 2 assert left.axes[1].length == right.axes[0].length # cast axis left_casted = ng.cast_axes(left, [left.axes[0], right.axes[0] - 1]) # result op result_op = ng.dot(left_casted, right, name=tf_node.name) # return return result_op
def matmul(left, right, transpose_a=False, transpose_b=False, name=None): """ Only support 2d matmul for now. """ # Transpose if transpose_a: left = ng.Transpose(left) if transpose_b: right = ng.Transpose(right) # Check shape assert len(left.axes) == len(right.axes) == 2 assert left.axes[1].length == right.axes[0].length # step 1: cast left (pos_1, pos_0), right (pos_1, pos_0) => # left (temp , pos_1), right (pos_1, pos_0) # step 2: perform left dot right, result # (temp, pos_0) # step 3: cast back to (post_1, pos_0) left_temp_axes = ng.make_axes( [ng.make_axis(left.axes[0].length), right.axes[0]]) left = ng.cast_axes(left, axes=left_temp_axes) # Result op result_op = ng.dot(left, right).named(name) result_op = cast_to_pos_axes(result_op) # Return return result_op.named(name)
def Gemm(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op # Y = alpha * (A @ B) + beta * C input_a, input_b, input_c = ng_inputs alpha = onnx_node.get_attribute_value('alpha', 1) # Scalar multiplier for A @ B beta = onnx_node.get_attribute_value( 'beta', 1) # Scalar multiplier for input tensor C broadcast = onnx_node.get_attribute_value('broadcast', 1) # Should C be broadcast? trans_a = onnx_node.get_attribute_value('transA', False) # Should A be transposed? trans_b = onnx_node.get_attribute_value('transB', False) # Should B be transposed? if not broadcast: logger.warning( 'Gemm node (%s): import does not support broadcast value %s', onnx_node.name, broadcast) if trans_a: input_a = ng.Transpose(input_a) if trans_b: input_b = ng.Transpose(input_b) input_a, input_b = cast_axes_for_matmul(input_a, input_b) a_dot_b = ng.dot(input_a, input_b) a_dot_b = cast_to_pos_axes(a_dot_b) return alpha * a_dot_b + beta * input_c
def create_loss_and_learner(model, labels, learning_rate, momentum_coef=0.0, wdecay=0.0, nesterov=False, gradient_clip_norm=None, gradient_clip_value=None): """ Auxiliary function to create loss function (cross entropy and softmax) and trainer using stochastic gradient descent with momentum. Arguments: model - imported model labels - placeholder for one-hot labels array learning_rate - learning rate for trainer momentum_coef - coefficient of momentum (deafult 0.0) wdecay - amount of weight decay (default 0.0) nesterov - use nesterov accelerated gradient (dafault False) gradient_clip_norm - target gradient norm (default None) gradient_clip_value - value to element-wise clip gradients (default None) Returns: Loss function (mean for batch) """ if model.axes.lengths != labels.axes.lengths: labels = ng.Transpose(labels) assert model.axes.lengths == labels.axes.lengths model = ng.cast_axes(model, axes=labels.axes) loss = ng.cross_entropy_multi(ng.softmax(model), labels) optimizer = GradientDescentMomentum(learning_rate, momentum_coef, wdecay, gradient_clip_norm, gradient_clip_value, nesterov) return ng.sequential([optimizer(loss), ng.mean(loss, out_axes=())])
def cross_entropy_with_softmax(model, labels): """ Auxiliary function to add cross entropy and softmax (loss function) to imported model for training. Arguments: model - imported model labels - placeholder for one-hot labels array Returns: Loss function (mean for batch) """ if model.axes.lengths != labels.axes.lengths: model = ng.Transpose(model) assert model.axes.lengths == labels.axes.lengths model = ng.cast_axes(model, axes=labels.axes) loss = ng.cross_entropy_multi(ng.softmax(model), labels) return ng.mean(loss, out_axes=())
def Transpose(onnx_node, ng_inputs): # type: (NodeWrapper, List[TensorOp]) -> Op """Transpose the input tensor similar to numpy.transpose. By default, reverse the dimensions, but if `perm` attribute is specified permute the axes according to the values given. """ data = ng_inputs[0] permute_axes = onnx_node.get_attribute_value('perm') if permute_axes: input_template = ''.join( [ascii_letters[i] for i in range(len(data.axes))]) output_template = ''.join([ascii_letters[i] for i in permute_axes]) ng_op = reorder_axes(data, input_template, output_template) else: ng_op = ng.Transpose(data) return cast_to_pos_axes(ng_op)
def CrossEntropyWithSoftmax(self, cntk_op, inputs): """ Computes the softmax cross entropy between the inputs[0] and inputs[1]. Arguments: cntk_op: CNTK operation to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ cast_0, cast_1 = squeeze_axes(inputs) if cast_0.axes.lengths != cast_1.axes.lengths: cast_0 = ng.Transpose(cast_0) assert cast_0.axes.lengths == cast_1.axes.lengths cast_0 = ng.cast_axes(cast_0, axes=cast_1.axes) loss = ng.cross_entropy_multi(ng.softmax(cast_0), cast_1) return ng.mean(loss, out_axes=()).named(cntk_op.uid)