def __init__(self, name: Optional[str], n: float, k: float, alpha: float, beta: float):
super().__init__(name)
self.parameters["n"] = n
self.parameters["k"] = k
self.parameters["alpha"] = alpha
self.parameters["beta"] = beta
self.attributes.add(Tensorwise(Axis.N))
self.attributes.add(Tensorwise(Axis.H))
self.attributes.add(Tensorwise(Axis.W))
def __init__(self, name: Optional[str], ksize: IntOrTuple,
stride: IntOrTuple, padding: IntOrTuple,
dilation_rate: IntOrTuple):
super().__init__(name)
self.parameters["ksize"] = to_tuple(ksize)
self.parameters["stride"] = to_tuple(stride)
self.parameters["padding"] = to_tuple(padding)
self.parameters["dilation_rate"] = to_tuple(dilation_rate)
self.attributes.add(Tensorwise(Axis.N))
self.attributes.add(Tensorwise(Axis.C))
def __init__(self, name: Optional[str], ksize: IntOrTuple,
stride: IntOrTuple, padding: IntOrTuple,
dilation_rate: IntOrTuple, sections: List[int], axis: Axis):
super().__init__(name)
self.parameters["ksize"] = to_tuple(ksize)
self.parameters["stride"] = to_tuple(stride)
self.parameters["padding"] = to_tuple(padding)
self.parameters["dilation_rate"] = to_tuple(dilation_rate)
self.parameters["sections"] = list(sections)
self.parameters["axis"] = axis
if axis != Axis.N:
self.attributes.add(Tensorwise(self, Axis.N))
if axis != Axis.C:
self.attributes.add(Tensorwise(self, Axis.C))
def _split_axis(v: Variable, axis: Axis, graph):
"""
split variable by specified axis
"""
s1 = v.shape_dict[axis] // 2
s2 = v.shape_dict[axis] - s1
if isinstance(v, ConstantVariable):
v_datum = np.split(v.data, [s1], v.order.axes_dict[axis])
v1 = ConstantVariable(v_datum[0], v.order)
v2 = ConstantVariable(v_datum[1], v.order)
else:
v1 = Variable([s1 if a == axis else v.shape_dict[a] for a in v.order.axes], v.order)
v2 = Variable([s2 if a == axis else v.shape_dict[a] for a in v.order.axes], v.order)
ops = list(v.input_to)
if v.output_from is not None:
ops += [v.output_from]
for op in ops:
if all(isinstance(v, ConstantVariable) for v in op.inputs.values()):
op.fold_constance(graph)
elif isinstance(op, Tensordot):
# NOTE:
# "_split_tensordot" must be called before "_split_tensorwise".
#
# Let consider follow case:
#
# A.order = [Axis.X, Axis.Y]
# B.order = [Axis.Y, Axis.Z]
# C, = Tensordot(None, [Axis.Y, Axis.Z])(A, B) # -> C.order = [Axis.X, Axis.Y]
#
# In this case, tensordot operator has "Tensorwise[X]" and "Tensorwise[Y]" attributes, because "Tensordot" operation is
# tensorwise operation for each output axis. However, "Axis.Y" is also contained in reduced axes in "A". Therefore,
# "_split_tensorwise" incorrectly split "A".
#
_split_tensordot(graph, op, v, [v1, v2], axis)
elif Tensorwise.check_splittable(op, axis):
_split_tensorwise(graph, op, v, [v1, v2], axis)
elif isinstance(op, SplitAxis):
_split_splitaxis(graph, op, v, [v1, v2], axis)
elif isinstance(op, Concat):
_split_concat(graph, op, v, [v1, v2], axis)
elif isinstance(op, Im2Col):
_split_im2col(graph, op, v, [v1, v2], axis)
elif isinstance(op, PartialIm2Col):
_split_partial_im2col(graph, op, v, [v1, v2], axis)
elif isinstance(op, Reshape):
_split_reshape(graph, op, v, [v1, v2], axis)
else:
raise NotImplementedError(f"Variable is too large to handle in WebGL backend: {v}")
def __init__(self,
name: Optional[str],
ksize: IntOrTuple,
stride: IntOrTuple,
padding: IntOrTuple,
dilation_rate: Optional[IntOrTuple] = 1):
super().__init__(name)
self.parameters["ksize"] = assert_sequence_type(to_tuple(ksize),
(int, Placeholder),
message=f"""
[Convolution2D] Parameter "ksize" must be integer or tuple of integer""")
self.parameters["stride"] = assert_sequence_type(to_tuple(stride),
(int, Placeholder),
message=f"""
[Convolution2D] Parameter "stride" must be integer or tuple of integer""")
self.parameters["padding"] = assert_sequence_type(to_tuple(padding),
(int, Placeholder),
message=f"""
[Convolution2D] Parameter "padding" must be integer or tuple of integer""")
self.parameters["dilation_rate"] = assert_sequence_type(
to_tuple(dilation_rate), (int, Placeholder),
message=f"""
[Convolution2D] Parameter "dilation_rate" must be integer or tuple of integer"""
)
self.attributes.add(Tensorwise(self, Axis.N))
def exec(self):
x = self.inputs["x"]
x_shape_dict = x.shape_dict
N = x_shape_dict[Axis.N]
H2 = (x_shape_dict[Axis.H] + 2 * self.parameters["padding"][0] + self.parameters["stride"][0] - self.parameters["ksize"][0] - 1) // \
self.parameters["stride"][0] + 1
W2 = (x_shape_dict[Axis.W] + 2 * self.parameters["padding"][1] + self.parameters["stride"][1] - self.parameters["ksize"][1] - 1) // \
self.parameters["stride"][1] + 1
C2 = x_shape_dict[Axis.C]
if ((x_shape_dict[Axis.H] + 2 * self.parameters["padding"][0] - self.parameters["ksize"][0]) % self.parameters["stride"][0] != 0) or \
((x_shape_dict[Axis.W] + 2 * self.parameters["padding"][1] - self.parameters["ksize"][1]) % self.parameters["stride"][1] != 0):
# https://github.com/fchollet/keras/issues/5090#issuecomment-279495401
console.warning(
"[Pooling2D] Performing pooling with parameters which causes edge is ignored. " +
"Which edge (left / right) is ignored is different on frameworks," +
" so slightly different result will be generated.")
y = Variable([N, H2, W2, C2], OrderNHWC)
y.change_order(x.order) # output same order as input to preserve following reshape semantics
self.append_output("y", y)
for axis in x.order.axes:
if axis == Axis.H or axis == Axis.W:
continue
self.attributes.add(Tensorwise(self, axis))
return y,
def exec(self):
x = self.inputs["x"]
axis = self.parameters["axis"]
sections = [0] + self.parameters["sections"] + [x.shape_dict[axis]]
outputs = []
for i, i_from in enumerate(sections[:-1]):
i_to = sections[i + 1]
assert i_from < i_to, f"[SplitAxis] sections must be sorted ascending order: sections={sections}, sections[{i}]={i_from}, " \
f"sections[{i+1}]={i_to}"
out_shape = list(x.shape)
out_shape[x.order.axes_dict[axis]] = i_to - i_from
y = Variable(out_shape, x.order)
outputs.append(y)
self.append_output(f"y{i}", y)
for a in x.order.axes:
if a == axis:
continue
self.attributes.add(Tensorwise(self, a))
return outputs
def exec(self):
x = self.inputs["x"]
x_shape_dict = x.shape_dict
N = x_shape_dict[Axis.N]
H2 = (x_shape_dict[Axis.H] + 2 * self.PH - self.KH +
(self.SH - 1 if self.cover_all else 0)) // self.SH + 1
W2 = (x_shape_dict[Axis.W] + 2 * self.PW - self.KW +
(self.SW - 1 if self.cover_all else 0)) // self.SW + 1
C2 = x_shape_dict[Axis.C]
# odd_padding_height = (x_shape_dict[Axis.H] + 2 * self.PH - self.KH) % self.SH != 0
# odd_padding_width = (x_shape_dict[Axis.W] + 2 * self.PW - self.KW) % self.SW != 0
# if odd_padding_height or odd_padding_width:
# # https://github.com/fchollet/keras/issues/5090#issuecomment-279495401
# console.warning(
# "[Pooling2D] Performing pooling with parameters which causes edge is ignored. " +
# "Which edge (left / right) is ignored is different on frameworks," +
# " so slightly different result will be generated.")
y = Variable([N, H2, W2, C2], OrderNHWC)
y.change_order(
x.order
) # output same order as input to preserve following reshape semantics
self.append_output("y", y)
for axis in x.order.axes:
if axis == Axis.H or axis == Axis.W:
continue
self.attributes.add(Tensorwise(self, axis))
return y,
def __call__(self, x: Variable):
axis = self.parameters["axis"]
sections = [0] + self.parameters["sections"] + [x.shape_dict[axis]]
ys = [] # type: List[Tuple[str, Variable]]
for i, i_from in enumerate(sections[:-1]):
i_to = sections[i + 1]
assert i_from < i_to, f"[SplitAxis] sections must be sorted ascending order: sections={sections}, sections[{i}]={i_from}, " \
f"sections[{i+1}]={i_to}"
out_shape = list(x.shape)
out_shape[x.order.axes_dict[axis]] = i_to - i_from
y = Variable(out_shape, x.order)
ys.append((f"y{i}", y))
for a in x.order.axes:
if a == axis:
continue
self.attributes.add(Tensorwise(a))
self.append_input(f"x", x)
for key, y in ys:
self.append_output(key, y)
return tuple(y for _, y in ys)
def exec(self):
xs = [self.inputs[f"x{i}"] for i in range(len(self.inputs))]
axis = self.axis
axis_index = xs[0].order.axes_dict[axis]
axes = xs[0].order.axes
y_shape = list(xs[0].shape) # type: List[Placeholder]
y_shape[axis_index] = 0
y_order = xs[0].order
for a in y_order.axes:
if a == axis:
continue
self.attributes.add(Tensorwise(self, a))
for i, x in enumerate(xs):
assert x.order.check_same_axes(xs[0].order), f"""
[Concat] Input variable of Concat operator must have same axes
(x0.order.axes) = {xs[0].order.axes}
(x{i}.order.axes) = {xs[i].order.axes}"""
for other_axis in [other_axis for other_axis in axes if other_axis != axis]:
if Placeholder.check_resolved(xs[0].shape_dict[other_axis]) and Placeholder.check_resolved(x.shape_dict[other_axis]):
assert xs[0].shape_dict[other_axis] == x.shape_dict[other_axis], f"""
[Concat] Input variable of Concat operator must be same shape except the specified axis:
(x0.shape_dict[{other_axis}]) = {xs[0].shape_dict[other_axis]}
(x{i}.shape_dict[{other_axis}]) = {xs[i].shape_dict[other_axis]}"""
y_shape[axis_index] += x.shape_dict[axis]
y = Variable(y_shape, y_order)
self.append_output("y", y)
return y,
def __call__(self, x: Variable):
for axis in x.order.axes:
if axis == Axis.H or axis == Axis.W:
continue
self.attributes.add(Tensorwise(self, axis))
self.append_input("x", x)
return self.exec()
def exec(self):
x = self.inputs["x"]
y = Variable(
x.shape,
Order([
self.out_order.axes[self.in_order.axes_dict[a]]
for a in x.order.axes
]))
self.append_output("y", y)
for axis in x.order.axes:
self.attributes.add(Tensorwise(self, axis))
return y,
def exec(self):
reduced_axis = self.axis
x = self.inputs["x"]
y_axes = [axis for axis in x.order.axes if axis != reduced_axis]
y_shape = [x.shape_dict[axis] for axis in y_axes]
y_order = Order(y_axes)
# Add tensorwise attributes
for axis in y_order.axes:
self.attributes.add(Tensorwise(self, axis))
y = variable.Variable(y_shape, y_order)
self.append_output("y", y)
return y,
def exec(self):
A = self.inputs["A"]
B = self.inputs["B"]
c_shape_dict = AxisKeyDict()
for axis in A.order.axes:
if axis not in self.axes[0]:
c_shape_dict[axis] = A.shape_dict[axis]
for axis in B.order.axes:
if axis not in self.axes[1]:
c_shape_dict[axis] = B.shape_dict[axis]
C = Variable(list(c_shape_dict.values()), Order(list(c_shape_dict.keys())))
self.append_output("C", C)
for axis in C.order.axes:
self.attributes.add(Tensorwise(self, axis=axis))
return C,
def __call__(self, x: "variable.Variable"):
reduced_axis = self.axis
y_axes = list(x.order.axes)
y_shape = [
1 if axis == reduced_axis else x.shape_dict[axis]
for axis in y_axes
]
y_order = Order(y_axes)
y = variable.Variable(y_shape, y_order)
for axis in x.order.axes:
if axis != reduced_axis:
self.attributes.add(Tensorwise(axis))
self.append_input("x", x)
self.append_output("y", y)
return y,
def __call__(self, x: Variable):
assert self.in_order.check_same_axes(x.order), f"""
[ReinterpretAxis] Order mismatch:
(op.in_order) = {self.in_order}
(x.order) = {x.order}"""
y = Variable(
x.shape,
Order([
self.out_order.axes[self.in_order.axes_dict[a]]
for a in x.order.axes
]))
for axis in x.order.axes:
self.attributes.add(Tensorwise(axis))
self.append_input("x", x)
self.append_output("y", y)
return y,
def exec(self):
x = self.inputs["x"]
x_shape_dict = x.shape_dict
N = x_shape_dict[Axis.N]
H2 = self.outsize[0]
W2 = self.outsize[1]
C2 = x_shape_dict[Axis.C]
y = Variable([N, H2, W2, C2], OrderNHWC)
y.change_order(x.order) # output same order as input to preserve following reshape semantics
self.append_output("y", y)
for axis in x.order.axes:
if axis == Axis.H or axis == Axis.W:
continue
self.attributes.add(Tensorwise(self, axis))
return y,
def __call__(self, *xs: "variable.Variable"):
y_axes = []
y_shape_dict = AxisKeyDict()
# Check variable in descent order of the number of dimensions.
# Without this procedure, in case that x0.order=C and x1.order=NC, the output order is CN. Expected result is NC.
xs_order = [(i, x) for i, x in enumerate(xs)]
xs_order.sort(key=lambda d: d[1].ndim, reverse=True)
for i, x in xs_order:
for axis in x.order.axes:
if axis in y_axes:
if y_shape_dict[axis] == 1:
# broadcast
y_shape_dict[axis] = x.shape_dict[axis]
else:
y_axes.append(axis)
y_shape_dict[axis] = x.shape_dict[axis]
if Placeholder.check_resolved(x.shape_dict[axis]):
if Placeholder.check_resolved(y_shape_dict[axis]):
assert y_shape_dict[axis] == x.shape_dict[
axis] or x.shape_dict[axis] == 1, f"""
[Elementwise] All input variables of elementwise operator should be same shape:
(y.shape) = {y_shape_dict[a] for a in y_axes}
(x{i}.shape) = {x.shape}
(y.shape[{axis}]) = {y_shape_dict[axis]}
(x{i}.shape[{axis}]) = {x.shape_dict[axis]}"""
else:
y_shape_dict[axis] = x.shape_dict[axis]
# Add tensorwise attributes
for axis in y_axes:
self.attributes.add(Tensorwise(axis))
y = variable.Variable([y_shape_dict[axis] for axis in y_axes],
Order(y_axes))
for i, x in enumerate(xs):
self.append_input(f"x{i}", x)
self.append_output("y", y)
return y,
def __call__(self, x: Variable):
x_shape_dict = x.shape_dict
N = x_shape_dict[Axis.N]
H2 = (x_shape_dict[Axis.H] + 2 * self.PH - self.KH +
(self.SH - 1 if self.cover_all else 0)) // self.SH + 1
W2 = (x_shape_dict[Axis.W] + 2 * self.PW - self.KW +
(self.SW - 1 if self.cover_all else 0)) // self.SW + 1
C2 = x_shape_dict[Axis.C]
y = Variable([N, H2, W2, C2], OrderNHWC)
y.change_order(
x.order
) # output same order as input to preserve following reshape semantics
for axis in x.order.axes:
if axis == Axis.H or axis == Axis.W:
continue
self.attributes.add(Tensorwise(axis))
self.append_input("x", x)
self.append_output("y", y)
return y,
def __call__(self, x: Variable):
# assert index is valid
for axis, index in self.indices.items():
if axis in x.order.axes:
if isinstance(index, slice):
index = normalize_slice(index, x.shape_dict[axis])
if not (-x.shape_dict[axis] <= index.start <= x.shape_dict[axis]) or \
not (-x.shape_dict[axis] <= index.stop <= x.shape_dict[axis]):
raise ValueError(f"""
[Slice] Index {index} in {axis} is out of range:
(x.order) = {x.order}
(x.shape) = {x.shape}
(indices) = {self.indices}
(indices[{axis.name}]) = {index}
""")
if ((abs(index.stop - index.start) - 1) //
abs(index.step)) + 1 < 0:
raise ValueError(f"""
[Slice] Slice operator doesn't support 0-size output:
(x.order) = {x.order}
(x.shape) = {x.shape}
(indices) = {self.indices}
(indices[{axis.name}]) = {index}
""")
elif isinstance(index, int):
if not -x.shape_dict[axis] <= index < x.shape_dict[axis]:
raise ValueError(f"""
[Slice] Index {index} in {axis} is out of range:
(x.order) = {x.order}
(x.shape) = {x.shape}
(indices) = {self.indices}
(indices[{axis.name}]) = {index}
(valid range) = [{-x.shape_dict[axis]}, {x.shape_dict[axis]})
""")
elif index is None:
raise ValueError(f"""
[Slice] Axis {axis} is already exist:
(x.order) = {x.order}
(x.shape) = {x.shape}
(indices) = {self.indices}
(indices[{axis.name}]) = {index}
""")
else:
if index is not None:
raise ValueError(f"""
[Slice] Axis {axis} is not exist in input variable. In this case, index must be "None" (=insert new axis):
(x.order) = {x.order}
(x.shape) = {x.shape}
(indices) = {self.indices}
(indices[{axis.name}]) = {index}
""")
if all(isinstance(index, int) for index in self.indices.values()):
raise NotImplementedError(f"""
[Slice] Accessing to one element is not supported:
(indices) = {self.indices}
""")
self.append_input("x", x)
# add attribute
for axis in x.order.axes:
if axis in self.indices:
index = self.indices[axis]
if isinstance(
index, slice
) and index.start is None and index.stop is None and index.step is None:
# This axis is not sliced.
self.attributes.add(Tensorwise(self, axis))
else:
# This axis is not sliced.
self.attributes.add(Tensorwise(self, axis))
return self.exec()
def __call__(self, A: Variable, B: Variable):
for axis in self.axes[0]:
assert axis in A.order.axes, f"""
[Tensordot] Input variable "A" must have axes "{axis}":
(op) = {self}
(op.axes[0]) = {self.axes[0]}
(A) = {A}"""
for axis in A.order.axes:
if axis not in self.axes[0]:
assert axis in self.axes[1] or axis not in B.order.axes, f"""
[Tensordot] Axes of "A" which are not reduced must not be contained in "B":
(op) = {self}
(A.order.axes) = {A.order.axes}
(B.order.axes) = {B.order.axes}
(op.axes) = {self.axes}"""
for axis in self.axes[1]:
assert axis in B.order.axes, f"""
[Tensordot] Input variable "B" must have axes "{axis}":
(op) = {self}
(op.axes[1]) = {self.axes[1]}
(B) = {B}"""
for axis in B.order.axes:
if axis not in self.axes[1]:
assert axis in self.axes[0] or axis not in A.order.axes, f"""
[Tensordot] Axes of "B" which are not reduced must not be contained in "A":
(op) = {self}
(A.order.axes) = {A.order.axes}
(B.order.axes) = {B.order.axes}
(op.axes) = {self.axes}"""
reduction_size_a = mul(A.shape_dict[a] for a in self.axes[0])
reduction_size_b = mul(B.shape_dict[a] for a in self.axes[1])
assert reduction_size_a == reduction_size_b, f"""
[Tensordot] Reduction size of "A" and "B" must be same:
(A) = {A}
(B) = {B}
(axes) = {self.axes}
(reduction size of A) = {reduction_size_a}
(reduction size of B) = {reduction_size_b}
"""
c_shape_dict = AxisKeyDict()
for axis in A.order.axes:
if axis not in self.axes[0]:
c_shape_dict[axis] = A.shape_dict[axis]
for axis in B.order.axes:
if axis not in self.axes[1]:
c_shape_dict[axis] = B.shape_dict[axis]
C = Variable(list(c_shape_dict.values()),
Order(list(c_shape_dict.keys())))
for axis in C.order.axes:
self.attributes.add(Tensorwise(axis))
self.append_input("A", A)
self.append_input("B", B)
self.append_output("C", C)
return C,
def __init__(self, name: Optional[str], r: int):
super().__init__(name)
self.parameters["r"] = int(r)
self.attributes.add(Tensorwise(self, Axis.N))
def __init__(self, name: Optional[str], padding: IntOrTuple):
super().__init__(name)
self.parameters["padding"] = to_tuple(padding)
self.attributes.add(Tensorwise(Axis.C))
self.attributes.add(Tensorwise(Axis.N))
def _split_axis(v: Variable, axis: Axis, graph):
"""
split variable by specified axis
"""
s1 = v.shape_dict[axis] // 2
s2 = v.shape_dict[axis] - s1
if isinstance(v, ConstantVariable):
v_datum = np.split(v.data, [s1], v.order.axes_dict[axis])
v1 = ConstantVariable(v_datum[0], v.order)
v2 = ConstantVariable(v_datum[1], v.order)
else:
v1 = Variable([s1 if a == axis else v.shape_dict[a] for a in v.order.axes], v.order)
v2 = Variable([s2 if a == axis else v.shape_dict[a] for a in v.order.axes], v.order)
ops = list(v.input_to)
if v.output_from is not None:
ops += [v.output_from]
for op in ops:
if all(isinstance(v, ConstantVariable) for v in op.inputs.values()):
op.fold_constance()
elif isinstance(op, SplitAxis):
_split_splitaxis(graph, op, v, [v1, v2], axis)
elif isinstance(op, Concat):
_split_concat(graph, op, v, [v1, v2], axis)
elif isinstance(op, Im2Col):
_split_im2col(graph, op, v, [v1, v2], axis)
elif isinstance(op, PartialIm2Col):
_split_partial_im2col(graph, op, v, [v1, v2], axis)
elif isinstance(op, Reshape):
_split_reshape(graph, op, v, [v1, v2], axis)
elif isinstance(op, Sgemm):
_split_sgemm(graph, op, v, [v1, v2], axis)
elif Tensorwise.check_splittable(op, axis):
_split_tensorwise(graph, op, v, [v1, v2], axis)
else:
console.debug("-------------------------------------------------")
console.debug(f"{v}")
console.debug(f" original order: {v.order}")
console.debug(f" original shape: {v.shape}")
console.debug(f"")
console.debug(f" split axis: {axis}")
console.debug(f"")
console.debug(f" related operators:")
for related_op in ops:
console.debug(f" {related_op}")
console.debug(f"")
with open("cg-failed.dot", "w") as f:
f.write(traverse.dump_dot(graph))
raise NotImplementedError(f"Variable is too large to handle in WebGL backend: {v}")
def __init__(self, name: Optional[str]):
super().__init__(name)
self.attributes.add(Tensorwise(self, Axis.N))
self.attributes.add(Tensorwise(self, Axis.T))
