def avg_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"):
x = converter.get_variable(tf_op.inputs[0])
data_format = tf_op.get_attr("data_format")
check_data_format(x, data_format)
ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...]
assert ksize[x.order.axes_dict[Axis.N]] == 1
assert ksize[x.order.axes_dict[Axis.C]] == 1
ksize = (ksize[x.order.axes_dict[Axis.H]],
ksize[x.order.axes_dict[Axis.W]])
stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...]
assert stride[x.order.axes_dict[Axis.N]] == 1
assert stride[x.order.axes_dict[Axis.C]] == 1
stride = (stride[x.order.axes_dict[Axis.H]],
stride[x.order.axes_dict[Axis.W]])
padding = (
parse_padding(tf_op.get_attr("padding"), ksize[0], 1),
parse_padding(tf_op.get_attr("padding"), ksize[1], 1),
)
x, padding = convert_odd_padding_to_concat(x, padding=padding)
if any(p > 0 for p in padding):
console.warning(
"[KerasConverter] keras.layers.AveragePooling computes average by dividing number of valid elements in window "
"(without padding element), but WebDNN divides it by the number of elements including padding element, so different "
"result will be generated on the edge.")
y, = AveragePooling2D(None,
ksize=ksize,
stride=stride,
padding=padding,
cover_all=False)(x)
converter.set_variable(tf_op.outputs[0], y)
def max_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"):
x = converter.get_variable(tf_op.inputs[0])
data_format = tf_op.get_attr("data_format")
check_data_format(x, data_format)
ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...]
assert ksize[x.order.axes_dict[Axis.N]] == 1
assert ksize[x.order.axes_dict[Axis.C]] == 1
ksize = (ksize[x.order.axes_dict[Axis.H]],
ksize[x.order.axes_dict[Axis.W]])
stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...]
assert stride[x.order.axes_dict[Axis.N]] == 1
assert stride[x.order.axes_dict[Axis.C]] == 1
stride = (stride[x.order.axes_dict[Axis.H]],
stride[x.order.axes_dict[Axis.W]])
padding = (
parse_padding(tf_op.get_attr("padding"), ksize[0], 1),
parse_padding(tf_op.get_attr("padding"), ksize[1], 1),
)
x, padding = convert_odd_padding_to_concat(x,
padding=padding,
value=-1.0e10)
y, = MaxPooling2D(None,
ksize=ksize,
stride=stride,
padding=padding,
cover_all=False)(x)
converter.set_variable(tf_op.outputs[0], y)
def _convert_conv2d_transpose(converter: KerasConverter,
k_op: "keras.layers.Conv2DTranspose"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
w = converter.convert_to_constant_variable(
k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C]))
ksize = tuple(k_op.kernel_size)
stride = tuple(k_op.strides)
dilation_rate = tuple(k_op.dilation_rate)
if dilation_rate != (1, 1):
raise NotImplementedError(
"[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported"
)
padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]),
parse_padding(k_op.padding, ksize[1], dilation_rate[1]))
if any(p[0] != p[1] for p in padding):
raise NotImplementedError(
"[KerasConverter] \"Different size padding\" is not supported yet")
padding = tuple(p[0] for p in padding)
y, = Deconvolution2D(None, ksize=ksize, stride=stride, padding=padding)(x,
w)
if k_op.use_bias:
b = converter.convert_to_constant_variable(k_op.bias, OrderC)
y = y + b
y = do_activation(k_op.activation, y)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def max_pool_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"):
x = converter.get_variable(tf_op.inputs[0])
data_format = tf_op.get_attr("data_format")
check_data_format(x, data_format)
ksize = tuple(tf_op.get_attr("ksize")) # type: Tuple[int,...]
assert ksize[x.order.axes_dict[Axis.N]] == 1
assert ksize[x.order.axes_dict[Axis.C]] == 1
ksize = (ksize[x.order.axes_dict[Axis.H]],
ksize[x.order.axes_dict[Axis.W]])
stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...]
assert stride[x.order.axes_dict[Axis.N]] == 1
assert stride[x.order.axes_dict[Axis.C]] == 1
stride = (stride[x.order.axes_dict[Axis.H]],
stride[x.order.axes_dict[Axis.W]])
x, padding = convolution_handler_preprocess(
x,
ksize=ksize,
padding=tf_op.get_attr("padding"),
dilation_rate=(1, 1),
data_format=data_format)
y, = MaxPooling2D(None,
ksize=ksize,
stride=stride,
padding=padding,
cover_all=False)(x)
converter.set_variable(tf_op.outputs[0], y)
def _convert_conv2d(converter: KerasConverter, k_op: "keras.layers.Conv2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
w = converter.convert_to_constant_variable(
k_op.kernel, Order([Axis.KH, Axis.KW, Axis.C, Axis.N]))
x, padding = convolution_handler_preprocess(
x,
ksize=k_op.kernel_size,
padding=k_op.padding,
dilation_rate=k_op.dilation_rate,
data_format=k_op.data_format)
y, = Convolution2D(None,
ksize=k_op.kernel_size,
stride=k_op.strides,
padding=padding,
dilation_rate=k_op.dilation_rate)(x, w)
if k_op.use_bias:
b = converter.convert_to_constant_variable(k_op.bias, OrderC)
y = y + b
y = do_activation(k_op.activation, y)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"):
x = converter.get_variable(tf_op.inputs[0])
data_format = tf_op.get_attr("data_format")
check_data_format(x, data_format)
w = converter.get_variable(tf_op.inputs[1]) # HWCN
w.order.unify(Order([Axis.KH, Axis.KW, Axis.C, Axis.N]))
ksize = (w.shape_dict[Axis.KH], w.shape_dict[Axis.KW])
stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...]
assert stride[x.order.axes_dict[Axis.N]] == 1
assert stride[x.order.axes_dict[Axis.C]] == 1
stride = (stride[x.order.axes_dict[Axis.H]],
stride[x.order.axes_dict[Axis.W]])
x, padding = convolution_handler_preprocess(
x,
ksize=ksize,
padding=tf_op.get_attr("padding"),
dilation_rate=(1, 1),
data_format=data_format)
y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w)
converter.set_variable(tf_op.outputs[0], y)
def _convert_up_sampling2d(converter: KerasConverter,
k_op: "keras.layers.UpSampling2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
#concat
# TODO
raise NotImplementedError(
'[KerasConverter] keras.layers.UpSampling2D is not supported')
def convert_layer_global_average_pooling2d(converter: KerasConverter, k_op: "keras.layers.GlobalAveragePooling2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
y, = AveragePooling2D(None, ksize=(x.shape_dict[Axis.H], x.shape_dict[Axis.W]), stride=(1, 1), padding=(0, 0))(x)
# flatten without changing memory layout
z = y.reshape([y.shape[0], mul(y.shape[1:])], OrderNC)
converter.set_variable(converter.get_output_tensor(k_op)[0], z)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.MaxPooling2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
padding = (
parse_padding(k_op.padding, k_op.pool_size[0], 1),
parse_padding(k_op.padding, k_op.pool_size[1], 1)
)
x, padding = convert_odd_padding_to_concat(x, padding=padding, value=-1.0e10)
y, = MaxPooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False)(x)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_conv2d_transpose(converter: KerasConverter,
k_op: "keras.layers.Conv2DTranspose"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
w = converter.convert_to_constant_variable(
k_op.kernel, Order([Axis.KH, Axis.KW, Axis.N, Axis.C]))
if tuple(k_op.dilation_rate) != (1, 1):
raise NotImplementedError(
"[KerasConverter] keras.layers.Convolution2DTranspose with large dilation_rate is not supported"
)
padding = (parse_padding(k_op.padding, k_op.kernel_size[0],
k_op.dilation_rate[0]),
parse_padding(k_op.padding, k_op.kernel_size[1],
k_op.dilation_rate[1]))
if any(p[0] != p[1] for p in padding):
pad_col2im = tuple(p[0] if p[0] == p[1] else 0 for p in padding)
pad_extra = tuple((0, 0) if p[0] == p[1] else p for p in padding)
y, = Deconvolution2D(None,
ksize=k_op.kernel_size,
stride=k_op.strides,
padding=pad_col2im)(x, w)
if k_op.data_format == "channels_first":
y = y[:, :, pad_extra[0][0]:-pad_extra[0][1],
pad_extra[1][0]:-pad_extra[1][1]]
elif k_op.data_format == "channels_last":
y = y[:, pad_extra[0][0]:-pad_extra[0][1],
pad_extra[1][0]:-pad_extra[1][1], :]
else:
raise NotImplementedError(
f"Unknown data format: {k_op.data_format}")
else:
y, = Deconvolution2D(None,
ksize=k_op.kernel_size,
stride=k_op.strides,
padding=tuple(p[0] for p in padding))(x, w)
if k_op.use_bias:
b = converter.convert_to_constant_variable(k_op.bias, OrderC)
y = y + b
y = do_activation(k_op.activation, y)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.AveragePooling2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
padding = (
parse_padding(k_op.padding, k_op.pool_size[0], 1),
parse_padding(k_op.padding, k_op.pool_size[1], 1)
)
x, padding = convert_odd_padding_to_concat(x, padding=padding)
divide_without_padding = any(p > 0 for p in padding)
# handling tensorflow style padding https://github.com/mil-tokyo/webdnn/issues/694
y, = AveragePooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False,
divide_without_padding=divide_without_padding)(x)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_handler(converter: TensorFlowConverter, tf_op: "tf.Operation"):
x = converter.get_variable(tf_op.inputs[0])
data_format = tf_op.get_attr("data_format")
check_data_format(x, data_format)
w = converter.get_variable(tf_op.inputs[1]) # HWCN
w.order.unify(Order([Axis.KH, Axis.KW, Axis.C, Axis.N]))
ksize = (w.shape_dict[Axis.KH], w.shape_dict[Axis.KW])
stride = tuple(tf_op.get_attr("strides")) # type: Tuple[int,...]
assert stride[x.order.axes_dict[Axis.N]] == 1
assert stride[x.order.axes_dict[Axis.C]] == 1
stride = (stride[x.order.axes_dict[Axis.H]],
stride[x.order.axes_dict[Axis.W]])
input_size = np.array([x.shape_dict[Axis.H], x.shape_dict[Axis.W]])
padding = np.array([
parse_padding(tf_op.get_attr("padding"), ksize[0], 1),
parse_padding(tf_op.get_attr("padding"), ksize[1], 1)
])
apron_size = (input_size + padding.sum(axis=1) - ksize) % stride
# cancel padding by apron if possible
for i in (0, 1):
if padding[i, 0] > apron_size[i]:
padding[i, 0] -= apron_size[i]
apron_size[i] = 0
else:
apron_size[i] -= padding[i, 0]
padding[i, 0] = 0
if padding[i, 1] > apron_size[i]:
padding[i, 1] -= apron_size[i]
apron_size[i] = 0
else:
apron_size[i] -= padding[i, 1]
padding[i, 1] = 0
padding = padding.tolist()
x, padding = convert_odd_padding_to_concat(x, padding=padding)
y, = Convolution2D(None, ksize=ksize, stride=stride, padding=padding)(x, w)
converter.set_variable(tf_op.outputs[0], y)
def _convert_max_pooling2d(converter: KerasConverter, k_op: "keras.layers.AveragePooling2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
padding = (
parse_padding(k_op.padding, k_op.pool_size[0], 1),
parse_padding(k_op.padding, k_op.pool_size[1], 1)
)
x, padding = convert_odd_padding_to_concat(x, padding=padding)
if any(p > 0 for p in padding):
console.warning(
"[KerasConverter] keras.layers.AveragePooling computes average by dividing number of valid elements in window "
"(without padding element), but WebDNN divides it by the number of elements including padding element, so different "
"result will be generated on the edge.")
y, = AveragePooling2D(None, ksize=k_op.pool_size, stride=k_op.strides, padding=padding, cover_all=False)(x)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_zero_padding2d(converter: KerasConverter, k_op: "keras.layers.ZeroPadding2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
padding = k_op.padding
top = padding[0][0]
if top != padding[0][1]:
# FIXME: This condition should be checked in each backend
raise NotImplementedError(
"[KerasConverter] In current implementation, Padding size of top and bottom must be same.")
left = padding[1][0]
if left != padding[1][1]:
# FIXME: This condition should be checked in each backend
raise NotImplementedError(
"[KerasConverter] In current implementation, Padding size of left and right must be same.")
y, = ZeroPadding2D(None, (top, left))(x)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def _convert_separable_conv2d(converter: KerasConverter,
k_op: "keras.layers.SeparableConv2D"):
x = converter.get_variable(converter.get_input_tensor(k_op)[0])
check_data_format(x, k_op.data_format)
axis_c_in = Axis.C
axis_c_out = Axis()
axis_depth_multiplier = Axis()
w_depthwise = converter.convert_to_constant_variable(
k_op.depthwise_kernel,
Order([Axis.KH, Axis.KW, axis_c_in, axis_depth_multiplier]))
w_pointwise = converter.convert_to_constant_variable(
k_op.pointwise_kernel, Order([Axis.KH, Axis.KW, axis_c_in,
axis_c_out]))
w_pointwise = w_pointwise.reshape(
shape=[
x.shape_dict[axis_c_in], k_op.depth_multiplier,
w_pointwise.shape_dict[axis_c_out]
],
order=Order([axis_c_in, axis_depth_multiplier, axis_c_out]))
ksize = tuple(k_op.kernel_size)
stride = tuple(k_op.strides)
dilation_rate = tuple(k_op.dilation_rate)
padding = (parse_padding(k_op.padding, ksize[0], dilation_rate[0]),
parse_padding(k_op.padding, ksize[1], dilation_rate[1]))
if any(p[0] != p[1] for p in padding):
raise NotImplementedError(
"[KerasConverter] \"Different size padding\" is not supported yet")
padding = tuple(p[0] for p in padding)
h, = Im2Col(None,
ksize=ksize,
stride=stride,
padding=padding,
dilation_rate=dilation_rate)(x)
# TODO: Support depth-wise convolution natively
# Currently, depth-wise convolution is not supported natively, and emulated by composition of small convolution operations.
ys = []
for i in range(h.shape_dict[axis_c_in]):
# 1. Depthwise convolution
#
# Ideal | Current implementation
# ----------------------------------+----------------------------------------------------
# h.axes=[N, H, W, KH, KW, C_in] | g_sub.axes=[N, H, W, KH, KW]
# w.axes=[KH, KW, C_in, DM] | w_sub.axes=[KH, KW, DM]
# g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM]
h_sub, = Slice(
None,
indices=AxisKeyDict(
h.order.axes,
[i if a == axis_c_in else slice(None)
for a in h.order.axes]))(h)
w_depthwise_sub = w_depthwise[:, :, i, :]
g_sub, = Tensordot(None, axes=((Axis.KH, Axis.KW),
(Axis.KH, Axis.KW)))(h_sub,
w_depthwise_sub)
# 2. Pointwise (projection) convolution
#
# Ideal | Current implementation
# ----------------------------------+----------------------------------------------------
# g.axes=[N, H, W, C_in, DM] | g_sub.axes=[N, H, W, DM]
# w.axes=[DM, Cin, C_out] | w_sub.axes=[DM, C_out]
# y.axes=[N, H, W, C_out] | y_sub.axes=[N, H, W, C_out]
w_pointwise_sub = w_pointwise[i, :, :]
y_sub, = Tensordot(None,
axes=((axis_depth_multiplier, ),
(axis_depth_multiplier, )))(g_sub,
w_pointwise_sub)
ys.append(y_sub)
# Sum up all sub convolution results to one
while len(ys) > 1:
ys.append(ys.pop(0) + ys.pop(0))
y = ys[0]
# reinterpret axis "C_out" as C
axes = list(y.order.axes)
i = axes.index(axis_c_out)
axes.pop(i)
axes.insert(i, Axis.C)
y = y.reinterpret_axes(Order(axes))
if k_op.data_format == "channels_last":
y = y.transpose(OrderNHWC)
elif k_op.data_format == "channels_first":
y = y.transpose(OrderNCHW)
else:
raise NotImplementedError(
f"[KerasConverter] Unknown data format: {k_op.data_format}")
if k_op.use_bias:
b = converter.convert_to_constant_variable(k_op.bias, OrderC)
y = y + b
y = do_activation(k_op.activation, y)
converter.set_variable(converter.get_output_tensor(k_op)[0], y)
def conv2_d_backprop_input_handler(converter: TensorFlowConverter,
tf_op: "tf.Operation"):
input_sizes = converter.get_variable(tf_op.inputs[0])
if not isinstance(input_sizes, ConstantVariable):
raise NotImplementedError(
"[TensorFlowConverter] Conv2DBackpropInput with dynamic shape of output (input of convolution) variable is not supported."
)
input_sizes = tuple(input_sizes.data.astype(np.int32).tolist())
w = converter.get_variable(tf_op.inputs[1]) # HWNC
w.order.unify(Order([Axis.KH, Axis.KW, Axis.N, Axis.C]))
gy = converter.get_variable(tf_op.inputs[2]) # NHWC
data_format = tf_op.get_attr("data_format")
check_data_format(gy, data_format)
input_size = np.array([
input_sizes[gy.order.axes_dict[Axis.H]],
input_sizes[gy.order.axes_dict[Axis.W]]
])
ksize = np.array([w.shape_dict[Axis.KH], w.shape_dict[Axis.KW]])
stride = np.array(tf_op.get_attr("strides"))
assert stride[gy.order.axes_dict[Axis.N]] == 1
assert stride[gy.order.axes_dict[Axis.C]] == 1
stride = stride[[gy.order.axes_dict[Axis.H], gy.order.axes_dict[Axis.W]]]
padding = np.array([
parse_padding(tf_op.get_attr("padding"), ksize[0], 1),
parse_padding(tf_op.get_attr("padding"), ksize[1], 1)
])
x, = Deconvolution2D(None,
ksize=ksize.tolist(),
stride=stride.tolist(),
padding=0)(gy, w)
# Actual padding size is depend on 2 factors
# 1. padding mode
# 2. extra apron size (= (input size of convolution) - (size of the tensor expanded by deconvolution))
expanded_size = np.array([x.shape_dict[Axis.H], x.shape_dict[Axis.W]])
apron_size = input_size - (expanded_size - padding.sum(axis=1))
# cancel padding by apron if possible
for i in (0, 1):
if padding[i, 0] > apron_size[i]:
padding[i, 0] -= apron_size[i]
apron_size[i] = 0
else:
apron_size[i] -= padding[i, 0]
padding[i, 0] = 0
if padding[i, 1] > apron_size[i]:
padding[i, 1] -= apron_size[i]
apron_size[i] = 0
else:
apron_size[i] -= padding[i, 1]
padding[i, 1] = 0
# append extra apron
for i, axis in enumerate((Axis.H, Axis.W)):
if apron_size[i] == 0:
continue
data = np.zeros([
apron_size[i] if a == axis else x.shape_dict[a]
for a in x.order.axes
])
x, = Concat(None, axis=axis)(x, ConstantVariable(data, x.order))
# crop without padding
padding = padding.tolist() # type: List[List[int]]
slice_h = slice(None) if padding[0] == [0, 0] else slice(
padding[0][0], -padding[0][1])
slice_w = slice(None) if padding[1] == [0, 0] else slice(
padding[1][0], -padding[1][1])
if data_format == b"NCHW":
x = x[:, :, slice_h, slice_w]
elif data_format == b"NHWC":
x = x[:, slice_h, slice_w, :]
else:
raise NotImplementedError(f"Unknown data format: {data_format}")
converter.set_variable(tf_op.outputs[0], x)
