def extract(node): param = node.pb.elu_param attrs = collect_attributes(param) attrs['operation'] = 'elu' Activation.update_node_stat(node, attrs) return ELUFrontExtractor.enabled
def test_activation_elu_infer(self): graph = build_graph(self.nodes_attributes, [ ('node_1', 'activation_node'), ('activation_node', 'node_3') ], { 'node_1': { 'value': np.array([6, -4, -2, -1]) }, 'activation_node': { 'operation': 'elu', 'alpha': 1.0, }, 'node_3': { 'value': None } }) graph.graph['layout'] = 'NCHW' activation_node = Node(graph, 'activation_node') Activation.infer(activation_node) exp_shape = np.array([227, 227, 227, 227]) res_shape = graph.node['node_3']['shape'] res_value = graph.node['node_3']['value'] exp_value = np.array([6., -0.98168436, -0.86466472, -0.63212056]) for i, value in enumerate(exp_shape): self.assertEqual(res_shape[i], value) for i, value in enumerate(exp_value): self.assertAlmostEqual(res_value[i], value)
def add_activation_function_after_node(graph: Graph, node: Node, activation_function: str): """ The function adds node with activation function defined by string 'activation_function' which gets input from the node 'node'. :param graph: graph to operate on. :param node: node to add activation after. :param activation_function: string defining the activation function. These values are read from TensorFlow* object detection API pipeline configuration file :return: activation function node. """ if activation_function == 'SOFTMAX': # softmax to be applied to the confidence softmax_conf_op = Softmax(graph, dict(axis=-1, nchw_layout=True)) activation_node = softmax_conf_op.create_node([node], dict(name=node.name + '/softmax')) elif activation_function == 'SIGMOID': # sigmoid activation function to be applied to the confidence sigmoid_conf_op = Activation( graph, dict(operation='sigmoid', nchw_layout=True)) activation_node = sigmoid_conf_op.create_node([node], dict(name=node.name + '/sigmoid')) elif activation_function == 'IDENTITY': # in case of Identity do nothing and just use result from the input node activation_node = node else: raise Error('Unknown post-processing activation function "{}".'.format( activation_function)) return activation_node
def test_activation_infer(self): graph = build_graph(self.nodes_attributes, [ ('node_1', 'activation_node'), ('activation_node', 'node_3') ], { 'node_1': { 'value': np.array([0, 7, 3, -1]) }, 'activation_node': { 'operation': 'relu6' }, 'node_3': { 'value': None } }) graph.graph['layout'] = 'NCHW' activation_node = Node(graph, 'activation_node') Activation.infer(activation_node) exp_shape = np.array([227, 227, 227, 227]) res_shape = graph.node['node_3']['shape'] res_value = graph.node['node_3']['value'] exp_value = np.array([0, 6, 3, 0]) for i, value in enumerate(exp_shape): self.assertEqual(res_shape[i], value) for i, value in enumerate(exp_value): self.assertEqual(res_value[i], value)
def extract(node): attrs = get_mxnet_layer_attrs(node.symbol_dict) act_type = attrs.str('act_type', 'leaky') if act_type == 'prelu': prelu_attrs = { 'channel_shared': 1, 'filler_type': 'constant', 'filler_value': 0, 'min': 0, 'max': 1, 'mean': 0, 'std': 0, 'sparse': -1, 'variance_norm': "caffe.FillerParameter.FAN_IN" } PreluOp.update_node_stat(node, prelu_attrs) elif act_type == 'elu': Activation.update_node_stat(node, {'operation': act_type}) elif act_type == 'leaky': negative_slope = attrs.float('slope', 0.25) ReLU.update_node_stat(node, {'negative_slope': negative_slope}) else: raise Error( "Operation '{}' not supported. Please register it as custom op. " + refer_to_faq_msg(86), act_type) return LeakyReLUFrontExtractor.enabled
def extract(node): attrs = get_mxnet_layer_attrs(node.symbol_dict) act_type = attrs.str('act_type', 'relu') if act_type == 'sigmoid' or act_type == 'tanh': Activation.update_node_stat(node, {'operation': act_type}) elif act_type == 'relu': ReLU.update_node_stat(node) else: raise Error( "Operation '{}' not supported. Please register it as custom op. " + refer_to_faq_msg(86), act_type) return ActivationFrontExtractor.enabled
def extract(node): Activation.update_node_stat(node, {'operation': 'tanh'}) return Tanh.enabled
def extract(node): alpha = onnx_attr(node, 'alpha', 'f', default=1.0) Activation.update_node_stat(node, {'operation': 'elu', 'alpha': alpha}) return EluFrontExtractor.enabled
def extract(node): Activation.update_node_stat(node, {'operation': 'sigmoid'}) return Sigmoid.enabled
def replace_op(self, graph: Graph, node: Node): input_node = node.in_node() memory_pair_input = unique_id('id') memory_pair_output = unique_id('id') # Input -> FullyConnected fc_layer_after_input_attrs = { 'name': 'input_fullyconnected', 'num_output': node.gifo_x_weights_shape[0], 'bias_term': True } embed_input(fc_layer_after_input_attrs, 1, 'weights', node.gifo_x_weights) embed_input(fc_layer_after_input_attrs, 2, 'biases', node.gifo_biases) fc_layer_after_input = InnerProduct( graph, fc_layer_after_input_attrs).create_node([input_node]) prev_lstm_output = Memory( graph, { 'name': 'prev_memory_output', 'id': memory_pair_input, 'index': 1, 'size': 2, 'shape': np.array([node.gifo_r_weights_shape[1]], dtype=np.int64) }).create_node() # *Memory(output) -> FullyConnected fc_layer_from_prev_state_attrs = { 'name': 'prev_memory_output_fullyconnected', 'num_output': node.gifo_r_weights_shape[0], 'bias_term': False } embed_input(fc_layer_from_prev_state_attrs, 1, 'weights', node.gifo_r_weights) fc_layer_from_prev_state = InnerProduct( graph, fc_layer_from_prev_state_attrs).create_node([prev_lstm_output]) # Memory -> FullyConnected \ # *Eltwise(sum) # Input -> FullyConnected / join_input_prev_state_sum = Eltwise(graph, { 'name': 'join_input_eltwise', 'operation': 'sum' }).create_node([fc_layer_from_prev_state, fc_layer_after_input]) # *Eltwise(sum) -> Split # it is split into 4 nodes: Act, Eltw*3 # the following order is mandatory # ___Tanh # / # Split ---(2)Eltwise(sum) # |\ # | \__(3)Eltwise(sum) # |____(4)Eltwise(sum) split_joined_input = Split( graph, { 'name': 'join_input_split', 'axis': 1, 'num_split': 4, 'out_ports_count': 4, }).create_node([join_input_prev_state_sum]) prev_lstm_state = Memory( graph, { 'name': 'prev_memory_state', 'id': memory_pair_output, 'index': 1, 'size': 2, 'shape': np.array([node.input_gate_weights.shape[0]], dtype=np.int64) }).create_node() # *Memory(state) -> *ScaleShift(input) state_input_scaleshift_attrs = { 'name': 'input_scaleshift', 'bias_term': False } embed_input(state_input_scaleshift_attrs, 1, 'weights', node.input_gate_weights) state_input_scaleshift = ScaleShiftOp( graph, state_input_scaleshift_attrs).create_node([prev_lstm_state]) # *Memory(state) -> *ScaleShift(forget) state_forget_scaleshift_attrs = { 'name': 'forget_scaleshift', 'bias_term': False } embed_input(state_forget_scaleshift_attrs, 1, 'weights', node.forget_gate_weights) state_forget_scaleshift = ScaleShiftOp( graph, state_forget_scaleshift_attrs).create_node([prev_lstm_state]) # Split \ # (2)Eltwise(sum) # Memory(state) -> *ScaleShift(input) / join_prev_lstm_input_joined_input_sum = Eltwise( graph, { 'name': 'join_prev_lstm_input_joined_input_eltwise', 'operation': 'sum' }).create_node([(split_joined_input, 1), state_input_scaleshift]) # Split \ # (3)Eltwise(sum) # Memory(state) -> *ScaleShift(forget) / join_prev_lstm_input_joined_forget_sum = Eltwise( graph, { 'name': 'join_prev_lstm_input_joined_forget_sum', 'operation': 'sum' }).create_node([(split_joined_input, 2), state_forget_scaleshift]) # Split -> Tanh remember_tahn = Activation(graph, { 'name': 'remember_tahnv', 'operation': 'tanh' }).create_node([(split_joined_input, 0)]) # Split -> (2)Eltwise(sum) -> *Sigmoid remember_sigmoid = Activation(graph, { 'name': 'remember_sigmoid', 'operation': 'sigmoid' }).create_node([join_prev_lstm_input_joined_input_sum]) # Split -> (3)Eltwise(sum) -> **Sigmoid forget_sigmoid = Activation(graph, { 'name': 'forget_sigmoid', 'operation': 'sigmoid' }).create_node([join_prev_lstm_input_joined_forget_sum]) # *Memory(state) \ # (6)Eltwise(mul) # Split -> (3)Eltwise(sum) -> **Sigmoid / join_forget_prev_state_mul = Eltwise(graph, { 'name': 'join_forget_prev_state_mul', 'operation': 'mul' }).create_node([forget_sigmoid, prev_lstm_state]) # Split -> Tahn \ # (5)Eltwise(mul) # Split -> (2)Eltwise(sum) -> *Sigmoid / join_remember_candidates_mul = Eltwise(graph, { 'name': 'join_remember_candidates_mul', 'operation': 'mul' }).create_node([remember_tahn, remember_sigmoid]) # (5)Eltwise(mul) \ # (7)Eltwise(sum) # (6)Eltwise(mul) / join_forget_remember_sum = Eltwise(graph, { 'name': 'join_forget_remember_sum', 'operation': 'sum' }).create_node( [join_forget_prev_state_mul, join_remember_candidates_mul]) # (7)Eltwise(sum) -> Clamp join_forget_clamp = Clamp( graph, { 'name': 'join_forget_clamp', 'max': node.clip_value, 'min': -node.clip_value }).create_node([join_forget_remember_sum]) # # Clamp -> (2)Memory(state) Memory( graph, { 'name': 'next_lstm_state', 'id': memory_pair_output, 'index': 0, 'size': 2, 'shape': np.array([node.input_gate_weights.shape[0]], dtype=np.int64) }).create_node([join_forget_clamp]) # Clamp -> (2)Tahn state_filtered_tahn = Activation(graph, { 'name': 'state_filtered_tahn', 'operation': 'tanh' }).create_node([join_forget_clamp]) # Clamp -> (2)ScaleShift clamp_scaleshift_attrs = { 'name': 'clamp_scaleshift', 'bias_term': False } embed_input(clamp_scaleshift_attrs, 1, 'weights', node.output_gate_weights) clamp_scaleshift = ScaleShiftOp( graph, clamp_scaleshift_attrs).create_node([join_forget_clamp]) # Split \ # (4)Eltwise(sum) # Clamp -> (2)ScaleShift / join_next_lstm_input_joined_input_sum = Eltwise( graph, { 'name': 'join_next_lstm_input_joined_input_sum', 'operation': 'sum' }).create_node([(split_joined_input, 3), clamp_scaleshift]) # (4)Eltwise(sum) -> (3)Sigmoid output_sigmoid = Activation(graph, { 'name': 'output_sigmoid', 'operation': 'sigmoid' }).create_node([join_next_lstm_input_joined_input_sum]) # (4)Eltwise(sum) -> (3)Sigmoid \ # (5)Eltwise(mul) # Clamp -> (2)Tahn / joined_output_mul = Eltwise(graph, { 'name': 'joined_output_mul', 'operation': 'mul' }).create_node([state_filtered_tahn, output_sigmoid]) # (5)Eltwise(mul) -> (3)FullyConnected fc_output_attrs = { 'name': 'FullyConnected', 'num_output': node.projection_weights_shape[0], 'bias_term': False } embed_input(fc_output_attrs, 1, 'weights', node.projection_weights) fc_output = InnerProduct(graph, fc_output_attrs).create_node( [joined_output_mul]) # / (2)Memory(output) # (3)FullyConnected # \ Output (any next node) (edge created automatically after replacement) Memory( graph, { 'name': 'next_lstm_output', 'id': memory_pair_input, 'index': 0, 'size': 2, 'shape': np.array([node.gifo_r_weights_shape[1]], dtype=np.int64) }).create_node([fc_output]) return [fc_output.id]
def extract(node): Activation.update_node_stat(node, {'operation': 'relu6'}) return True
def replace_pattern(self, graph: Graph, match: [str, Node]): node = match['activation'] Activation.update_node_stat(node, dict(operation=node.type.lower()))
def extract(node): Activation.update_node_stat(node, {'operation': 'elu'}) return Elu.enabled
def extract(node): Activation.update_node_stat(node, {'operation': 'exp'}) return __class__.enabled