def avg_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): x = converter.get_variable(tf_op.inputs[0]) data_format = tf_op.get_attr("data_format") check_data_format(x, data_format) ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...] assert ksize[x.order.axes_dict[Axis.N]] == 1 assert ksize[x.order.axes_dict[Axis.C]] == 1 ksize = (ksize[x.order.axes_dict[Axis.H]], ksize[x.order.axes_dict[Axis.W]]) stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...] assert stride[x.order.axes_dict[Axis.N]] == 1 assert stride[x.order.axes_dict[Axis.C]] == 1 stride = (stride[x.order.axes_dict[Axis.H]], stride[x.order.axes_dict[Axis.W]]) padding = ( parse_padding(tf_op.get_attr("padding"), ksize[0], 1), parse_padding(tf_op.get_attr("padding"), ksize[1], 1), ) x, padding = convert_odd_padding_to_concat(x, padding=padding) if any(p > 0 for p in padding): console.warning( "[KerasConverter] keras.layers.AveragePooling computes average by dividing number of valid elements in window " "(without padding element), but WebDNN divides it by the number of elements including padding element, so different " "result will be generated on the edge.") y, = AveragePooling2D(None, ksize=ksize, stride=stride, padding=padding, cover_all=False)(x) converter.set_variable(tf_op.outputs[0], y)
def max_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): x = converter.get_variable(tf_op.inputs[0]) data_format = tf_op.get_attr("data_format") check_data_format(x, data_format) ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...] assert ksize[x.order.axes_dict[Axis.N]] == 1 assert ksize[x.order.axes_dict[Axis.C]] == 1 ksize = (ksize[x.order.axes_dict[Axis.H]], ksize[x.order.axes_dict[Axis.W]]) stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...] assert stride[x.order.axes_dict[Axis.N]] == 1 assert stride[x.order.axes_dict[Axis.C]] == 1 stride = (stride[x.order.axes_dict[Axis.H]], stride[x.order.axes_dict[Axis.W]]) padding = ( parse_padding(tf_op.get_attr("padding"), ksize[0], 1), parse_padding(tf_op.get_attr("padding"), ksize[1], 1), ) x, padding = convert_odd_padding_to_concat(x, padding=padding, value=-1.0e10) y, = MaxPooling2D(None, ksize=ksize, stride=stride, padding=padding, cover_all=False)(x) converter.set_variable(tf_op.outputs[0], y)
def _convert_conv2d_transpose(converter: KerasConverter, k_op: "keras.layers.Conv2DTranspose"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C])) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) if dilation_rate != (1, 1): raise NotImplementedError( "[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported" ) padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]), parse_padding(k_op.padding, ksize[1], dilation_rate[1])) if any(p[0] != p[1] for p in padding): raise NotImplementedError( "[KerasConverter] \"Different size padding\" is not supported yet") padding = tuple(p[0] for p in padding) y, = Deconvolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def max_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): x = converter.get_variable(tf_op.inputs[0]) data_format = tf_op.get_attr("data_format") check_data_format(x, data_format) ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...] assert ksize[x.order.axes_dict[Axis.N]] == 1 assert ksize[x.order.axes_dict[Axis.C]] == 1 ksize = (ksize[x.order.axes_dict[Axis.H]], ksize[x.order.axes_dict[Axis.W]]) stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...] assert stride[x.order.axes_dict[Axis.N]] == 1 assert stride[x.order.axes_dict[Axis.C]] == 1 stride = (stride[x.order.axes_dict[Axis.H]], stride[x.order.axes_dict[Axis.W]]) x, padding = convolution_handler_preprocess( x, ksize=ksize, padding=tf_op.get_attr("padding"), dilation_rate=(1, 1), data_format=data_format) y, = MaxPooling2D(None, ksize=ksize, stride=stride, padding=padding, cover_all=False)(x) converter.set_variable(tf_op.outputs[0], y)
def _convert_conv2d(converter: KerasConverter, k_op: "keras.layers.Conv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.C, Axis.N])) x, padding = convolution_handler_preprocess( x, ksize=k_op.kernel_size, padding=k_op.padding, dilation_rate=k_op.dilation_rate, data_format=k_op.data_format) y, = Convolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=padding, dilation_rate=k_op.dilation_rate)(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): x = converter.get_variable(tf_op.inputs[0]) data_format = tf_op.get_attr("data_format") check_data_format(x, data_format) w = converter.get_variable(tf_op.inputs[1]) # HWCN w.order.unify(Order([Axis.KH, Axis.KW, Axis.C, Axis.N])) ksize = (w.shape_dict[Axis.KH], w.shape_dict[Axis.KW]) stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...] assert stride[x.order.axes_dict[Axis.N]] == 1 assert stride[x.order.axes_dict[Axis.C]] == 1 stride = (stride[x.order.axes_dict[Axis.H]], stride[x.order.axes_dict[Axis.W]]) x, padding = convolution_handler_preprocess( x, ksize=ksize, padding=tf_op.get_attr("padding"), dilation_rate=(1, 1), data_format=data_format) y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w) converter.set_variable(tf_op.outputs[0], y)
def _convert_up_sampling2d(converter: KerasConverter, k_op: "keras.layers.UpSampling2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) #concat # TODO raise NotImplementedError( '[KerasConverter] keras.layers.UpSampling2D is not supported')
def convert_layer_global_average_pooling2d(converter: KerasConverter, k_op: "keras.layers.GlobalAveragePooling2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) y, = AveragePooling2D(None, ksize=(x.shape_dict[Axis.H], x.shape_dict[Axis.W]), stride=(1, 1), padding=(0, 0))(x) # flatten without changing memory layout z = y.reshape([y.shape[0], mul(y.shape[1:])], OrderNC) converter.set_variable(converter.get_output_tensor(k_op)[0], z)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.MaxPooling2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) padding = ( parse_padding(k_op.padding, k_op.pool_size[0], 1), parse_padding(k_op.padding, k_op.pool_size[1], 1) ) x, padding = convert_odd_padding_to_concat(x, padding=padding, value=-1.0e10) y, = MaxPooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False)(x) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d_transpose(converter: KerasConverter, k_op: "keras.layers.Conv2DTranspose"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) w = converter.convert_to_constant_variable( k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C])) if tuple(k_op.dilation_rate) != (1, 1): raise NotImplementedError( "[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported" ) padding = (parse_padding(k_op.padding, k_op.kernel_size[0], k_op.dilation_rate[0]), parse_padding(k_op.padding, k_op.kernel_size[1], k_op.dilation_rate[1])) if any(p[0] != p[1] for p in padding): pad_col2im = tuple(p[0] if p[0] == p[1] else 0 for p in padding) pad_extra = tuple((0, 0) if p[0] == p[1] else p for p in padding) y, = Deconvolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=pad_col2im)(x, w) if k_op.data_format == "channels_first": y = y[:, :, pad_extra[0][0]:-pad_extra[0][1], pad_extra[1][0]:-pad_extra[1][1]] elif k_op.data_format == "channels_last": y = y[:, pad_extra[0][0]:-pad_extra[0][1], pad_extra[1][0]:-pad_extra[1][1], :] else: raise NotImplementedError( f"Unknown data format: {k_op.data_format}") else: y, = Deconvolution2D(None, ksize=k_op.kernel_size, stride=k_op.strides, padding=tuple(p[0] for p in padding))(x, w) if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.AveragePooling2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) padding = ( parse_padding(k_op.padding, k_op.pool_size[0], 1), parse_padding(k_op.padding, k_op.pool_size[1], 1) ) x, padding = convert_odd_padding_to_concat(x, padding=padding) divide_without_padding = any(p > 0 for p in padding) # handling tensorflow style padding https://github.com/mil-tokyo/webdnn/issues/694 y, = AveragePooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False, divide_without_padding=divide_without_padding)(x) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): x = converter.get_variable(tf_op.inputs[0]) data_format = tf_op.get_attr("data_format") check_data_format(x, data_format) w = converter.get_variable(tf_op.inputs[1]) # HWCN w.order.unify(Order([Axis.KH, Axis.KW, Axis.C, Axis.N])) ksize = (w.shape_dict[Axis.KH], w.shape_dict[Axis.KW]) stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...] assert stride[x.order.axes_dict[Axis.N]] == 1 assert stride[x.order.axes_dict[Axis.C]] == 1 stride = (stride[x.order.axes_dict[Axis.H]], stride[x.order.axes_dict[Axis.W]]) input_size = np.array([x.shape_dict[Axis.H], x.shape_dict[Axis.W]]) padding = np.array([ parse_padding(tf_op.get_attr("padding"), ksize[0], 1), parse_padding(tf_op.get_attr("padding"), ksize[1], 1) ]) apron_size = (input_size + padding.sum(axis=1) - ksize) % stride # cancel padding by apron if possible for i in (0, 1): if padding[i, 0] > apron_size[i]: padding[i, 0] -= apron_size[i] apron_size[i] = 0 else: apron_size[i] -= padding[i, 0] padding[i, 0] = 0 if padding[i, 1] > apron_size[i]: padding[i, 1] -= apron_size[i] apron_size[i] = 0 else: apron_size[i] -= padding[i, 1] padding[i, 1] = 0 padding = padding.tolist() x, padding = convert_odd_padding_to_concat(x, padding=padding) y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w) converter.set_variable(tf_op.outputs[0], y)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.AveragePooling2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) padding = ( parse_padding(k_op.padding, k_op.pool_size[0], 1), parse_padding(k_op.padding, k_op.pool_size[1], 1) ) x, padding = convert_odd_padding_to_concat(x, padding=padding) if any(p > 0 for p in padding): console.warning( "[KerasConverter] keras.layers.AveragePooling computes average by dividing number of valid elements in window " "(without padding element), but WebDNN divides it by the number of elements including padding element, so different " "result will be generated on the edge.") y, = AveragePooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False)(x) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_zero_padding2d(converter: KerasConverter, k_op: "keras.layers.ZeroPadding2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) padding = k_op.padding top = padding[0][0] if top != padding[0][1]: # FIXME: This condition should be checked in each backend raise NotImplementedError( "[KerasConverter] In current implementation, Padding size of top and bottom must be same.") left = padding[1][0] if left != padding[1][1]: # FIXME: This condition should be checked in each backend raise NotImplementedError( "[KerasConverter] In current implementation, Padding size of left and right must be same.") y, = ZeroPadding2D(None, (top, left))(x) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_separable_conv2d(converter: KerasConverter, k_op: "keras.layers.SeparableConv2D"): x = converter.get_variable(converter.get_input_tensor(k_op)[0]) check_data_format(x, k_op.data_format) axis_c_in = Axis.C axis_c_out = Axis() axis_depth_multiplier = Axis() w_depthwise = converter.convert_to_constant_variable( k_op.depthwise_kernel, Order([Axis.KH, Axis.KW, axis_c_in, axis_depth_multiplier])) w_pointwise = converter.convert_to_constant_variable( k_op.pointwise_kernel, Order([Axis.KH, Axis.KW, axis_c_in, axis_c_out])) w_pointwise = w_pointwise.reshape( shape=[ x.shape_dict[axis_c_in], k_op.depth_multiplier, w_pointwise.shape_dict[axis_c_out] ], order=Order([axis_c_in, axis_depth_multiplier, axis_c_out])) ksize = tuple(k_op.kernel_size) stride = tuple(k_op.strides) dilation_rate = tuple(k_op.dilation_rate) padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]), parse_padding(k_op.padding, ksize[1], dilation_rate[1])) if any(p[0] != p[1] for p in padding): raise NotImplementedError( "[KerasConverter] \"Different size padding\" is not supported yet") padding = tuple(p[0] for p in padding) h, = Im2Col(None, ksize=ksize, stride=stride, padding=padding, dilation_rate=dilation_rate)(x) # TODO: Support depth-wise convolution natively # Currently, depth-wise convolution is not supported natively, and emulated by composition of small convolution operations. ys = [] for i in range(h.shape_dict[axis_c_in]): # 1. Depthwise convolution # # Ideal | Current implementation # ----------------------------------+---------------------------------------------------- # h.axes=[N, H, W, KH, KW, C_in] | g_sub.axes=[N, H, W, KH, KW] # w.axes=[KH, KW, C_in, DM] | w_sub.axes=[KH, KW, DM] # g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM] h_sub, = Slice( None, indices=AxisKeyDict( h.order.axes, [i if a == axis_c_in else slice(None) for a in h.order.axes]))(h) w_depthwise_sub = w_depthwise[:, :, i, :] g_sub, = Tensordot(None, axes=((Axis.KH, Axis.KW), (Axis.KH, Axis.KW)))(h_sub, w_depthwise_sub) # 2. Pointwise (projection) convolution # # Ideal | Current implementation # ----------------------------------+---------------------------------------------------- # g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM] # w.axes=[DM, Cin, C_out] | w_sub.axes=[DM, C_out] # y.axes=[N, H, W, C_out] | y_sub.axes=[N, H, W, C_out] w_pointwise_sub = w_pointwise[i, :, :] y_sub, = Tensordot(None, axes=((axis_depth_multiplier, ), (axis_depth_multiplier, )))(g_sub, w_pointwise_sub) ys.append(y_sub) # Sum up all sub convolution results to one while len(ys) > 1: ys.append(ys.pop(0) + ys.pop(0)) y = ys[0] # reinterpret axis "C_out" as C axes = list(y.order.axes) i = axes.index(axis_c_out) axes.pop(i) axes.insert(i, Axis.C) y = y.reinterpret_axes(Order(axes)) if k_op.data_format == "channels_last": y = y.transpose(OrderNHWC) elif k_op.data_format == "channels_first": y = y.transpose(OrderNCHW) else: raise NotImplementedError( f"[KerasConverter] Unknown data format: {k_op.data_format}") if k_op.use_bias: b = converter.convert_to_constant_variable(k_op.bias, OrderC) y = y + b y = do_activation(k_op.activation, y) converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_backprop_input_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"): input_sizes = converter.get_variable(tf_op.inputs[0]) if not isinstance(input_sizes, ConstantVariable): raise NotImplementedError( "[TensorFlowConverter] Conv2DBackpropInput with dynamic shape of output (input of convolution) variable is not supported." ) input_sizes = tuple(input_sizes.data.astype(np.int32).tolist()) w = converter.get_variable(tf_op.inputs[1]) # HWNC w.order.unify(Order([Axis.KH, Axis.KW, Axis.N, Axis.C])) gy = converter.get_variable(tf_op.inputs[2]) # NHWC data_format = tf_op.get_attr("data_format") check_data_format(gy, data_format) input_size = np.array([ input_sizes[gy.order.axes_dict[Axis.H]], input_sizes[gy.order.axes_dict[Axis.W]] ]) ksize = np.array([w.shape_dict[Axis.KH], w.shape_dict[Axis.KW]]) stride = np.array(tf_op.get_attr("strides")) assert stride[gy.order.axes_dict[Axis.N]] == 1 assert stride[gy.order.axes_dict[Axis.C]] == 1 stride = stride[[gy.order.axes_dict[Axis.H], gy.order.axes_dict[Axis.W]]] padding = np.array([ parse_padding(tf_op.get_attr("padding"), ksize[0], 1), parse_padding(tf_op.get_attr("padding"), ksize[1], 1) ]) x, = Deconvolution2D(None, ksize=ksize.tolist(), stride=stride.tolist(), padding=0)(gy, w) # Actual padding size is depend on 2 factors # 1. padding mode # 2. extra apron size (= (input size of convolution) - (size of the tensor expanded by deconvolution)) expanded_size = np.array([x.shape_dict[Axis.H], x.shape_dict[Axis.W]]) apron_size = input_size - (expanded_size - padding.sum(axis=1)) # cancel padding by apron if possible for i in (0, 1): if padding[i, 0] > apron_size[i]: padding[i, 0] -= apron_size[i] apron_size[i] = 0 else: apron_size[i] -= padding[i, 0] padding[i, 0] = 0 if padding[i, 1] > apron_size[i]: padding[i, 1] -= apron_size[i] apron_size[i] = 0 else: apron_size[i] -= padding[i, 1] padding[i, 1] = 0 # append extra apron for i, axis in enumerate((Axis.H, Axis.W)): if apron_size[i] == 0: continue data = np.zeros([ apron_size[i] if a == axis else x.shape_dict[a] for a in x.order.axes ]) x, = Concat(None, axis=axis)(x, ConstantVariable(data, x.order)) # crop without padding padding = padding.tolist() # type: List[List[int]] slice_h = slice(None) if padding[0] == [0, 0] else slice( padding[0][0], -padding[0][1]) slice_w = slice(None) if padding[1] == [0, 0] else slice( padding[1][0], -padding[1][1]) if data_format == b"NCHW": x = x[:, :, slice_h, slice_w] elif data_format == b"NHWC": x = x[:, slice_h, slice_w, :] else: raise NotImplementedError(f"Unknown data format: {data_format}") converter.set_variable(tf_op.outputs[0], x)