def _convert_lstm(converter: KerasConverter, k_op: "keras.layers.LSTM"): assert k_op.stateful is False, "[KerasConverter] Currently, LSTM.stateful is not supported" assert k_op.go_backwards is False, "[KerasConverter] Currently, LSTM.go_backwards is not supported" x = converter.get_variable(converter.get_input_tensor(k_op)[0]) w_input = converter.convert_to_constant_variable(k_op.kernel, OrderCN) w_hidden = converter.convert_to_constant_variable(k_op.recurrent_kernel, OrderCN) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) else: b = None y, c = LSTM(None, k_op.use_bias, k_op.return_sequences, use_initial_c=False, use_initial_h=False, activation=k_op.activation.__name__, recurrent_activation=k_op.recurrent_activation.__name__)( x, w_input, w_hidden, b) k_outputs = converter.get_output_tensor(k_op) converter.set_variable(k_outputs[0], y) if k_op.return_state: converter.set_variable(k_outputs[1], None) converter.set_variable(k_outputs[2], c)
def _convert_conv2d_transpose(converter: KerasConverter, k_op: "keras.layers.Conv2DTranspose"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C])) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) if dilation_rate != (1, 1): raise NotImplementedError( "[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported" ) padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]), parse_padding(k_op.padding, ksize[1], dilation_rate[1])) y, = Deconvolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d(converter: KerasConverter, k_op: "keras.layers.Conv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) if k_op.data_format == "channels_first": assert x.order == OrderNCHW elif k_op.data_format == "channels_last": assert x.order == OrderNHWC else: raise ValueError(f"[KerasConverter] Unknown data format is detected: {k_op.data_format}") w = converter.convert_to_constant_variable(k_op.kernel, OrderHWCN) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) if k_op.padding == "valid": padding = (0, 0) elif k_op.padding == "same": padding = (ksize[0] // 2, ksize[1] // 2) else: raise ValueError(f"[KerasConverter] Unknown padding: {k_op.padding}") y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding, dilation_rate=dilation_rate)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d(converter: KerasConverter, k_op: "keras.layers.Conv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.C, Axis.N])) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]), parse_padding(k_op.padding, ksize[1], dilation_rate[1])) y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding, dilation_rate=dilation_rate)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d(converter: KerasConverter, k_op: "keras.layers.Conv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.C, Axis.N])) x, padding = convolution_handler_preprocess( x, ksize=k_op.kernel_size, padding=k_op.padding, dilation_rate=k_op.dilation_rate, data_format=k_op.data_format) y, = Convolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=padding, dilation_rate=k_op.dilation_rate)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_lstm(converter: KerasConverter, k_op: "keras.layers.LSTM"): assert k_op.stateful is False, "[KerasConverter] Currently, LSTM.stateful is not supported" assert k_op.go_backwards is False, "[KerasConverter] Currently, LSTM.go_backwards is not supported" # Structure of LSTM layer was changed in v2.0.9 (https://github.com/fchollet/keras/pull/7943) if "2.0.9" <= keras.__version__: cell = k_op.cell # type: keras.layers.LSTMCell else: cell = k_op # type: keras.layers.LSTM x = converter.get_variable(converter.get_input_tensor(k_op)[0]) x.order.unify(OrderNTC) w_input = converter.convert_to_constant_variable(cell.kernel, OrderCN) w_hidden = converter.convert_to_constant_variable(cell.recurrent_kernel, OrderCN) if k_op.use_bias: b = converter.convert_to_constant_variable(cell.bias, OrderC) else: b = None y, c = LSTM(None, cell.use_bias, k_op.return_sequences, use_initial_c=False, use_initial_h=False, activation=cell.activation.__name__, recurrent_activation=cell.recurrent_activation.__name__)(x, w_input, w_hidden, b) k_outputs = converter.get_output_tensor(k_op) converter.set_variable(k_outputs[0], y) if k_op.return_state: converter.set_variable(k_outputs[1], None) converter.set_variable(k_outputs[2], c)
def _convert_conv2d_transpose(converter: KerasConverter, k_op: "keras.layers.Conv2DTranspose"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) if k_op.data_format == "channels_first": assert x.order == OrderNCHW elif k_op.data_format == "channels_last": assert x.order == OrderNHWC else: raise ValueError( f"[KerasConverter] Unknown data format is detected: {k_op.data_format}" ) w = converter.convert_to_constant_variable(k_op.kernel, OrderHWNC) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) if k_op.padding == "valid": padding = (0, 0) elif k_op.padding == "same": # @see https://github.com/tensorflow/tensorflow/blob/e5cf6f0c13b6053e4c58af6a951b204fde263172/tensorflow/python/ops/nn_ops.py#L507-L519 pad_extra_shape = [k - 1 for k in ksize] if any(p % 2 != 0 for p in pad_extra_shape): raise NotImplementedError( f"[KerasConverter] Currently WebDNN doesn't supports different size padding: " f" (pad_extra_shape)=f{pad_extra_shape}") padding = tuple(p // 2 for p in pad_extra_shape) w = converter.convert_to_constant_variable(k_op.kernel, OrderHWNC) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) if k_op.padding == "valid": padding = (0, 0) elif k_op.padding == "same": padding = (ksize[0] // 2, ksize[1] // 2) else: raise ValueError(f"[KerasConverter] Unknown padding: {k_op.padding}") y, = Deconvolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_dense(converter: KerasConverter, k_op: "keras.layers.Dense"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) w = converter.convert_to_constant_variable(k_op.kernel, OrderCN) y, = Linear(None)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_dense(converter: KerasConverter, k_op: "keras.layers.Dense"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) w = converter.convert_to_constant_variable(k_op.kernel, Order([None, None])) y, = Tensordot(None, axes=[x.order.axes[-1], w.order.axes[0]])(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, Order([None])) b.order.axes[0].unify(w.order.axes[1]) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d_transpose(converter: KerasConverter, k_op: "keras.layers.Conv2DTranspose"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C])) if tuple(k_op.dilation_rate) != (1, 1): raise NotImplementedError( "[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported" ) padding = (parse_padding(k_op.padding, k_op.kernel_size[0], k_op.dilation_rate[0]), parse_padding(k_op.padding, k_op.kernel_size[1], k_op.dilation_rate[1])) if any(p[0] != p[1] for p in padding): pad_col2im = tuple(p[0] if p[0] == p[1] else 0 for p in padding) pad_extra = tuple((0, 0) if p[0] == p[1] else p for p in padding) y, = Deconvolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=pad_col2im)(x, w) if k_op.data_format == "channels_first": y = y[:, :, pad_extra[0][0]:-pad_extra[0][1], pad_extra[1][0]:-pad_extra[1][1]] elif k_op.data_format == "channels_last": y = y[:, pad_extra[0][0]:-pad_extra[0][1], pad_extra[1][0]:-pad_extra[1][1], :] else: raise NotImplementedError( f"Unknown data format: {k_op.data_format}") else: y, = Deconvolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=tuple(p[0] for p in padding))(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_embedding(converter: KerasConverter, k_op: "keras.layers.Embedding"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) x = x.reinterpret_axes(OrderNT) w = converter.convert_to_constant_variable(k_op.embeddings, OrderCN) y, = Embedding(None)(x, w) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_embedding(converter: KerasConverter, k_op: keras.layers.Embedding): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) if x.order == OrderNC: x, = ReinterpretAxis(None, in_order=OrderNC, out_order=OrderNT)(x) w = converter.convert_to_constant_variable(k_op.embeddings, OrderCN) y, = Embedding(None)(x, w) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_separable_conv2d(converter: KerasConverter, k_op: "keras.layers.SeparableConv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) axis_c_in = Axis.C axis_c_out = Axis() axis_depth_multiplier = Axis() w_depthwise = converter.convert_to_constant_variable( k_op.depthwise_kernel, Order([Axis.KH, Axis.KW, axis_c_in, axis_depth_multiplier])) w_pointwise = converter.convert_to_constant_variable( k_op.pointwise_kernel, Order([Axis.KH, Axis.KW, axis_c_in, axis_c_out])) w_pointwise = w_pointwise.reshape( shape=[ x.shape_dict[axis_c_in], k_op.depth_multiplier, w_pointwise.shape_dict[axis_c_out] ], order=Order([axis_c_in, axis_depth_multiplier, axis_c_out])) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]), parse_padding(k_op.padding, ksize[1], dilation_rate[1])) if any(p[0] != p[1] for p in padding): raise NotImplementedError( "[KerasConverter] \"Different size padding\" is not supported yet") padding = tuple(p[0] for p in padding) h, = Im2Col(None, ksize=ksize, stride=stride, padding=padding, dilation_rate=dilation_rate)(x) # TODO: Support depth-wise convolution natively # Currently, depth-wise convolution is not supported natively, and emulated by composition of small convolution operations. ys = [] for i in range(h.shape_dict[axis_c_in]): # 1. Depthwise convolution # # Ideal | Current implementation # ----------------------------------+---------------------------------------------------- # h.axes=[N, H, W, KH, KW, C_in] | g_sub.axes=[N, H, W, KH, KW] # w.axes=[KH, KW, C_in, DM] | w_sub.axes=[KH, KW, DM] # g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM] h_sub, = Slice( None, indices=AxisKeyDict( h.order.axes, [i if a == axis_c_in else slice(None) for a in h.order.axes]))(h) w_depthwise_sub = w_depthwise[:, :, i, :] g_sub, = Tensordot(None, axes=((Axis.KH, Axis.KW), (Axis.KH, Axis.KW)))(h_sub, w_depthwise_sub) # 2. Pointwise (projection) convolution # # Ideal | Current implementation # ----------------------------------+---------------------------------------------------- # g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM] # w.axes=[DM, Cin, C_out] | w_sub.axes=[DM, C_out] # y.axes=[N, H, W, C_out] | y_sub.axes=[N, H, W, C_out] w_pointwise_sub = w_pointwise[i, :, :] y_sub, = Tensordot(None, axes=((axis_depth_multiplier, ), (axis_depth_multiplier, )))(g_sub, w_pointwise_sub) ys.append(y_sub) # Sum up all sub convolution results to one while len(ys) > 1: ys.append(ys.pop(0) + ys.pop(0)) y = ys[0] # reinterpret axis "C_out" as C axes = list(y.order.axes) i = axes.index(axis_c_out) axes.pop(i) axes.insert(i, Axis.C) y = y.reinterpret_axes(Order(axes)) if k_op.data_format == "channels_last": y = y.transpose(OrderNHWC) elif k_op.data_format == "channels_first": y = y.transpose(OrderNCHW) else: raise NotImplementedError( f"[KerasConverter] Unknown data format: {k_op.data_format}") if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)