def draw_conv_weights(self):
for i, (name, weight) in enumerate(zip(self.layer_names, self._weights)):
if len(weight.shape) != 4:
return
for j, _w in enumerate(weight):
for k, _ww in enumerate(_w):
VisUtil.show_img(_ww, "{} {} filter {} channel {}".format(name, i+1, j+1, k+1))
def draw_conv_weights(self):
with self._sess.as_default():
for i, (name, weight) in enumerate(zip(self.layer_names, self._tf_weights)):
weight = weight.eval()
if len(weight.shape) != 4:
continue
for j, _w in enumerate(weight.transpose(2, 3, 0, 1)):
VisUtil.show_batch_img(_w, "{} {} filter {}".format(name, i + 1, j + 1))
def draw_conv_series(self, x, shape=None):
for xx in x:
VisUtil.show_img(VisUtil.trans_img(xx, shape), "Original")
activations = self._get_activations(np.array([xx]), predict=True)
for i, (layer, ac) in enumerate(zip(self._layers, activations)):
if len(ac.shape) == 4:
for n in ac:
_n, height, width = n.shape
a = int(ceil(sqrt(_n)))
g = np.ones((a * height + a, a * width + a), n.dtype)
g *= np.min(n)
_i = 0
for y in range(a):
for x in range(a):
if _i < _n:
g[y * height + y:(y + 1) * height + y, x * width + x:(x + 1) * width + x] = n[
_i, :, :]
_i += 1
# normalize to [0,1]
max_g = g.max()
min_g = g.min()
g = (g - min_g) / (max_g - min_g)
VisUtil.show_img(g, "Layer {} ({})".format(i + 1, layer.name))
else:
ac = ac[0]
length = sqrt(np.prod(ac.shape))
if length < 10:
continue
(height, width) = xx.shape[1:] if shape is None else shape[1:]
sqrt_shape = sqrt(height * width)
oh, ow = int(length * height / sqrt_shape), int(length * width / sqrt_shape)
VisUtil.show_img(ac[:oh*ow].reshape(oh, ow), "Layer {} ({})".format(i + 1, layer.name))
def draw_conv_series(self, x, shape=None):
x = np.asarray(x)
for xx in x:
VisUtil.show_img(VisUtil.trans_img(xx, shape), "Original")
for i, (layer, ac) in enumerate(zip(
self._layers, self._get_acts(np.array([xx.transpose(1, 2, 0)], dtype=np.float32)))):
if len(ac.shape) == 4:
VisUtil.show_batch_img(ac[0].transpose(2, 0, 1), "Layer {} ({})".format(i + 1, layer.name))
else:
ac = ac[0]
length = sqrt(np.prod(ac.shape))
if length < 10:
continue
(height, width) = xx.shape[1:] if shape is None else shape[1:]
sqrt_shape = sqrt(height * width)
oh, ow = int(length * height / sqrt_shape), int(length * width / sqrt_shape)
VisUtil.show_img(ac[:oh * ow].reshape(oh, ow), "Layer {} ({})".format(i + 1, layer.name))
def _draw_2d_network(self, radius=6, width=1200, height=800, padding=0.2,
plot_scale=2, plot_precision=0.03,
sub_layer_height_scale=0, **kwargs):
if not kwargs["show"] and not kwargs["mp4"]:
return
layers = len(self._layers) + 1
units = [layer.shape[0] for layer in self._layers] + [self._layers[-1].shape[1]]
whether_sub_layers = np.array([False] + [isinstance(layer, SubLayer) for layer in self._layers])
n_sub_layers = np.sum(whether_sub_layers) # type: int
plot_num = int(1 / plot_precision)
if plot_num % 2 == 1:
plot_num += 1
half_plot_num = int(plot_num * 0.5)
xf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale, plot_num)
yf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale, plot_num) * -1
input_x, input_y = np.meshgrid(xf, yf)
input_xs = np.c_[input_x.ravel(), input_y.ravel()]
activations = [activation.T.reshape(units[i + 1], plot_num, plot_num)
for i, activation in enumerate(self._get_activations(input_xs, predict=True))]
graphs = []
for j, activation in enumerate(activations):
graph_group = []
if j == len(activations) - 1:
classes = np.argmax(activation, axis=0)
else:
classes = None
for k, ac in enumerate(activation):
data = np.zeros((plot_num, plot_num, 3), np.uint8)
if j != len(activations) - 1:
mask = ac >= np.average(ac)
else:
mask = classes == k
data[mask], data[~mask] = [0, 165, 255], [255, 165, 0]
graph_group.append(data)
graphs.append(graph_group)
img = np.full([height, width, 3], 255, dtype=np.uint8)
axis0_padding = int(height / (layers - 1 + 2 * padding)) * padding + plot_num
axis0_step = (height - 2 * axis0_padding) / layers
sub_layer_decrease = int((1 - sub_layer_height_scale) * axis0_step)
axis0 = np.linspace(
axis0_padding,
height + n_sub_layers * sub_layer_decrease - axis0_padding,
layers, dtype=np.int)
axis0 -= sub_layer_decrease * np.cumsum(whether_sub_layers)
axis1_padding = plot_num
axis1 = [np.linspace(axis1_padding, width - axis1_padding, unit + 2, dtype=np.int)
for unit in units]
axis1 = [axis[1:-1] for axis in axis1]
colors, thicknesses, masks = [], [], []
for weight in self._weights:
line_info = VisUtil.get_line_info(weight.copy())
colors.append(line_info[0])
thicknesses.append(line_info[1])
masks.append(line_info[2])
for i, (y, xs) in enumerate(zip(axis0, axis1)):
for j, x in enumerate(xs):
if i == 0:
cv2.circle(img, (x, y), radius, (20, 215, 20), int(radius / 2))
else:
graph = graphs[i - 1][j]
img[y - half_plot_num:y + half_plot_num, x - half_plot_num:x + half_plot_num] = graph
if i > 0:
cv2.putText(img, self._layers[i - 1].name, (12, y - 36), cv2.LINE_AA, 0.6, (0, 0, 0), 1)
for i, y in enumerate(axis0):
if i == len(axis0) - 1:
break
for j, x in enumerate(axis1[i]):
new_y = axis0[i + 1]
whether_sub_layer = isinstance(self._layers[i], SubLayer)
for k, new_x in enumerate(axis1[i + 1]):
if whether_sub_layer and j != k:
continue
if masks[i][j][k]:
cv2.line(img, (x, y + half_plot_num), (new_x, new_y - half_plot_num),
colors[i][j][k], thicknesses[i][j][k])
return img
def _draw_img_network(self, img_shape, width=1200, height=800, padding=0.2,
sub_layer_height_scale=0, delay=1,
weight_average=None):
img_width, img_height = img_shape
half_width = int(img_width * 0.5) if img_width % 2 == 0 else int(img_width * 0.5) + 1
half_height = int(img_height * 0.5) if img_height % 2 == 0 else int(img_height * 0.5) + 1
layers = len(self._layers)
units = [layer.shape[1] for layer in self._layers]
whether_sub_layers = np.array([isinstance(layer, SubLayer) for layer in self._layers])
n_sub_layers = int(np.sum(whether_sub_layers))
_activations = [self._weights[0].copy().T]
for weight in self._weights[1:]:
_activations.append(weight.T.dot(_activations[-1]))
_graphs = []
for j, activation in enumerate(_activations):
_graph_group = []
for ac in activation:
ac = ac.reshape(img_width, img_height)
ac -= np.average(ac)
data = np.zeros((img_width, img_height, 3), np.uint8)
mask = ac >= 0.25
data[mask], data[~mask] = [0, 130, 255], [255, 130, 0]
_graph_group.append(data)
_graphs.append(_graph_group)
img = np.zeros((height, width, 3), np.uint8)
axis0_padding = int(height / (layers - 1 + 2 * padding)) * padding + img_height
axis0_step = (height - 2 * axis0_padding) / layers
sub_layer_decrease = int((1 - sub_layer_height_scale) * axis0_step)
axis0 = np.linspace(
axis0_padding,
height + n_sub_layers * sub_layer_decrease - axis0_padding,
layers, dtype=np.int)
axis0 -= sub_layer_decrease * np.cumsum(whether_sub_layers)
axis1_padding = img_width
axis1 = [np.linspace(axis1_padding, width - axis1_padding, unit + 2, dtype=np.int)
for unit in units]
axis1 = [axis[1:-1] for axis in axis1]
colors, thicknesses = [], []
color_weights = [weight.copy() for weight in self._weights]
color_min = [np.min(weight) for weight in color_weights]
color_max = [np.max(weight) for weight in color_weights]
color_average = [np.average(weight) for weight in color_weights] if weight_average is None else weight_average
for weight, weight_min, weight_max, weight_average in zip(
color_weights, color_min, color_max, color_average
):
line_info = VisUtil.get_line_info(weight, weight_min, weight_max, weight_average)
colors.append(line_info[0])
thicknesses.append(line_info[1])
for i, (y, xs) in enumerate(zip(axis0, axis1)):
for j, x in enumerate(xs):
graph = _graphs[i][j]
img[y - half_height:y + half_height, x - half_width:x + half_width] = graph
cv2.putText(img, self._layers[i].name, (12, y - 36), cv2.LINE_AA, 0.6, (255, 255, 255), 2)
for i, y in enumerate(axis0):
if i == len(axis0) - 1:
break
for j, x in enumerate(axis1[i]):
new_y = axis0[i + 1]
whether_sub_layer = isinstance(self._layers[i + 1], SubLayer)
for k, new_x in enumerate(axis1[i + 1]):
if whether_sub_layer and j != k:
continue
cv2.line(img, (x, y + half_height), (new_x, new_y - half_height),
colors[i + 1][j][k], thicknesses[i + 1][j][k])
cv2.imshow("Neural Network", img)
cv2.waitKey(delay)
return img
def _draw_detailed_network(self, radius=6, width=1200, height=800, padding=0.2,
plot_scale=2, plot_precision=0.03,
sub_layer_height_scale=0, delay=1,
weight_average=None):
layers = len(self._layers) + 1
units = [layer.shape[0] for layer in self._layers] + [self._layers[-1].shape[1]]
whether_sub_layers = np.array([False] + [isinstance(layer, SubLayer) for layer in self._layers])
n_sub_layers = int(np.sum(whether_sub_layers))
plot_num = int(1 / plot_precision)
if plot_num % 2 == 1:
plot_num += 1
half_plot_num = int(plot_num * 0.5)
xf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale, plot_num)
yf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale, plot_num) * -1
input_x, input_y = np.meshgrid(xf, yf)
input_xs = np.c_[input_x.ravel(), input_y.ravel()]
_activations = [activation.T.reshape(units[i + 1], plot_num, plot_num)
for i, activation in enumerate(self._get_activations(input_xs, predict=True))]
_graphs = []
for j, activation in enumerate(_activations):
_graph_group = []
for ac in activation:
data = np.zeros((plot_num, plot_num, 3), np.uint8)
mask = ac >= np.average(ac)
data[mask], data[~mask] = [0, 125, 255], [255, 125, 0]
_graph_group.append(data)
_graphs.append(_graph_group)
img = np.ones((height, width, 3), np.uint8) * 255
axis0_padding = int(height / (layers - 1 + 2 * padding)) * padding + plot_num
axis0_step = (height - 2 * axis0_padding) / layers
sub_layer_decrease = int((1 - sub_layer_height_scale) * axis0_step)
axis0 = np.linspace(
axis0_padding,
height + n_sub_layers * sub_layer_decrease - axis0_padding,
layers, dtype=np.int)
axis0 -= sub_layer_decrease * np.cumsum(whether_sub_layers)
axis1_padding = plot_num
axis1 = [np.linspace(axis1_padding, width - axis1_padding, unit + 2, dtype=np.int)
for unit in units]
axis1 = [axis[1:-1] for axis in axis1]
colors, thicknesses = [], []
color_weights = [weight.copy() for weight in self._weights]
color_min = [np.min(weight) for weight in color_weights]
color_max = [np.max(weight) for weight in color_weights]
color_average = [np.average(weight) for weight in color_weights] if weight_average is None else weight_average
for weight, weight_min, weight_max, weight_average in zip(
color_weights, color_min, color_max, color_average
):
line_info = VisUtil.get_line_info(weight, weight_min, weight_max, weight_average)
colors.append(line_info[0])
thicknesses.append(line_info[1])
for i, (y, xs) in enumerate(zip(axis0, axis1)):
for j, x in enumerate(xs):
if i == 0:
cv2.circle(img, (x, y), radius, (20, 215, 20), int(radius / 2))
else:
graph = _graphs[i - 1][j]
img[y - half_plot_num:y + half_plot_num, x - half_plot_num:x + half_plot_num] = graph
if i > 0:
cv2.putText(img, self._layers[i - 1].name, (12, y - 36), cv2.LINE_AA, 0.6, (0, 0, 0), 1)
for i, y in enumerate(axis0):
if i == len(axis0) - 1:
break
for j, x in enumerate(axis1[i]):
new_y = axis0[i + 1]
whether_sub_layer = isinstance(self._layers[i], SubLayer)
for k, new_x in enumerate(axis1[i + 1]):
if whether_sub_layer and j != k:
continue
cv2.line(img, (x, y + half_plot_num), (new_x, new_y - half_plot_num),
colors[i][j][k], thicknesses[i][j][k])
cv2.imshow("Neural Network", img)
cv2.waitKey(delay)
return img
def _draw_network(self, radius=6, width=1200, height=800, padding=0.2, sub_layer_height_scale=0, delay=1,
weight_average=None, activations=None):
layers = len(self._layers) + 1
units = [layer.shape[0] for layer in self._layers] + [self._layers[-1].shape[1]]
whether_sub_layers = np.array([False] + [isinstance(layer, SubLayer) for layer in self._layers])
n_sub_layers = int(np.sum(whether_sub_layers))
img = np.zeros((height, width, 3), np.uint8)
axis0_padding = int(height / (layers - 1 + 2 * padding)) * padding
axis0_step = (height - 2 * axis0_padding) / layers
sub_layer_decrease = int((1 - sub_layer_height_scale) * axis0_step)
axis0 = np.linspace(
axis0_padding,
height + n_sub_layers * sub_layer_decrease - axis0_padding,
layers, dtype=np.int)
axis0 -= sub_layer_decrease * np.cumsum(whether_sub_layers)
axis1_divide = [int(width / (unit + 1)) for unit in units]
axis1 = [np.linspace(divide, width - divide, units[i], dtype=np.int)
for i, divide in enumerate(axis1_divide)]
colors, thicknesses = [], []
color_weights = [weight.copy() for weight in self._weights]
color_min = [np.min(weight) for weight in color_weights]
color_max = [np.max(weight) for weight in color_weights]
color_average = [np.average(weight) for weight in color_weights] if weight_average is None else weight_average
for weight, weight_min, weight_max, weight_average in zip(
color_weights, color_min, color_max, color_average
):
line_info = VisUtil.get_line_info(weight, weight_min, weight_max, weight_average)
colors.append(line_info[0])
thicknesses.append(line_info[1])
activations = [np.average(np.abs(activation), axis=0) for activation in activations]
activations = [activation / np.max(activation) for activation in activations]
for i, (y, xs) in enumerate(zip(axis0, axis1)):
for j, x in enumerate(xs):
if i == 0:
cv2.circle(img, (x, y), radius, (20, 215, 20), int(radius / 2))
else:
activation = activations[i - 1][j]
try:
cv2.circle(img, (x, y), radius, (
int(255 * activation), int(255 * activation), int(255 * activation)), int(radius / 2))
except ValueError:
cv2.circle(img, (x, y), radius, (0, 0, 255), int(radius / 2))
if i > 0:
cv2.putText(img, self._layers[i - 1].name, (12, y - 36), cv2.LINE_AA, 0.6, (255, 255, 255), 2)
for i, y in enumerate(axis0):
if i == len(axis0) - 1:
break
for j, x in enumerate(axis1[i]):
new_y = axis0[i + 1]
whether_sub_layer = isinstance(self._layers[i], SubLayer)
for k, new_x in enumerate(axis1[i + 1]):
if whether_sub_layer and j != k:
continue
cv2.line(img, (x, y), (new_x, new_y), colors[i][j][k], thicknesses[i][j][k])
cv2.imshow("Neural Network", img)
cv2.waitKey(delay)
return img
def _handle_mp4(self, ims, animation_properties, name=None):
if name is None:
name = str(self)
if animation_properties[2] and ims:
VisUtil.make_mp4(ims, name)
def _draw_2d_network(self,
radius=6,
width=1200,
height=800,
padding=0.2,
plot_scale=2,
plot_precision=0.03,
sub_layer_height_scale=0,
**kwargs):
if not kwargs["show"] and not kwargs["mp4"]:
return
layers = len(self._layers) + 1
units = [layer.shape[0]
for layer in self._layers] + [self._layers[-1].shape[1]]
whether_sub_layers = np.array(
[False] + [isinstance(layer, SubLayer) for layer in self._layers])
n_sub_layers = np.sum(whether_sub_layers) # type: int
plot_num = int(1 / plot_precision)
if plot_num % 2 == 1:
plot_num += 1
half_plot_num = int(plot_num * 0.5)
xf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale,
plot_num)
yf = np.linspace(self._x_min * plot_scale, self._x_max * plot_scale,
plot_num) * -1
input_x, input_y = np.meshgrid(xf, yf)
input_xs = np.c_[input_x.ravel(), input_y.ravel()]
activations = [
activation.T.reshape(units[i + 1], plot_num, plot_num)
for i, activation in enumerate(
self._get_activations(input_xs, predict=True))
]
graphs = []
for j, activation in enumerate(activations):
graph_group = []
if j == len(activations) - 1:
classes = np.argmax(activation, axis=0)
else:
classes = None
for k, ac in enumerate(activation):
data = np.zeros((plot_num, plot_num, 3), np.uint8)
if j != len(activations) - 1:
mask = ac >= np.average(ac)
else:
mask = classes == k
data[mask], data[~mask] = [0, 165, 255], [255, 165, 0]
graph_group.append(data)
graphs.append(graph_group)
img = np.full([height, width, 3], 255, dtype=np.uint8)
axis0_padding = int(height /
(layers - 1 + 2 * padding)) * padding + plot_num
axis0_step = (height - 2 * axis0_padding) / layers
sub_layer_decrease = int((1 - sub_layer_height_scale) * axis0_step)
axis0 = np.linspace(axis0_padding,
height + n_sub_layers * sub_layer_decrease -
axis0_padding,
layers,
dtype=np.int)
axis0 -= sub_layer_decrease * np.cumsum(whether_sub_layers)
axis1_padding = plot_num
axis1 = [
np.linspace(axis1_padding,
width - axis1_padding,
unit + 2,
dtype=np.int) for unit in units
]
axis1 = [axis[1:-1] for axis in axis1]
colors, thicknesses, masks = [], [], []
for weight in self._weights:
line_info = VisUtil.get_line_info(weight.copy())
colors.append(line_info[0])
thicknesses.append(line_info[1])
masks.append(line_info[2])
for i, (y, xs) in enumerate(zip(axis0, axis1)):
for j, x in enumerate(xs):
if i == 0:
cv2.circle(img, (x, y), radius, (20, 215, 20),
int(radius / 2))
else:
graph = graphs[i - 1][j]
img[y - half_plot_num:y + half_plot_num,
x - half_plot_num:x + half_plot_num] = graph
if i > 0:
cv2.putText(img, self._layers[i - 1].name, (12, y - 36),
cv2.LINE_AA, 0.6, (0, 0, 0), 1)
for i, y in enumerate(axis0):
if i == len(axis0) - 1:
break
for j, x in enumerate(axis1[i]):
new_y = axis0[i + 1]
whether_sub_layer = isinstance(self._layers[i], SubLayer)
for k, new_x in enumerate(axis1[i + 1]):
if whether_sub_layer and j != k:
continue
if masks[i][j][k]:
cv2.line(img, (x, y + half_plot_num),
(new_x, new_y - half_plot_num),
colors[i][j][k], thicknesses[i][j][k])
return img
def _handle_mp4(self, ims, animation_properties, name=None):
if name is None:
name = str(self)
if animation_properties[2] and ims:
VisUtil.make_mp4(ims, name)