def get_mlp_layer_label_mismatch_penalties(non_assignment_penalty, class_or_reg,
list_of_activations=None):
""" Gets the label mismatch penalty for an MLP. """
rectifiers = neural_network.MLP_RECTIFIERS
sigmoids = neural_network.MLP_SIGMOIDS
non_linear_activations = rectifiers + sigmoids
mlp_layer_labels = neural_network.get_mlp_layer_labels(class_or_reg,
list_of_activations)
num_labels = len(mlp_layer_labels)
label_penalties = np.zeros((num_labels, num_labels))
for i in range(num_labels):
for j in range(i, num_labels):
labi = mlp_layer_labels[i]
labj = mlp_layer_labels[j]
if labi == labj:
cost = 0.0
elif (labi in rectifiers and labj in rectifiers) or \
(labi in sigmoids and labj in sigmoids):
cost = 0.1
elif labi in non_linear_activations and labj in non_linear_activations:
cost = 0.25
else:
cost = np.inf
label_penalties[i, j] = cost * non_assignment_penalty
label_penalties[j, i] = cost * non_assignment_penalty
return mlp_layer_labels, label_penalties
def generate_mlp_architectures(class_or_reg='reg'):
""" pylint: disable=bad-whitespace. """
# pylint: disable=bad-whitespace
all_layer_label_classes = get_mlp_layer_labels(class_or_reg)
last_layer_label = 'linear' if class_or_reg == 'reg' else 'softmax'
# Network 1
layer_labels = [
'ip', 'op', 'tanh', 'logistic', 'softplus', 'relu', 'elu',
last_layer_label
]
num_units_each_layer = [None, None, 32, 64, 16, 8, 8, None]
A = get_dok_mat_with_set_coords(8, [(0, 4), (2, 3), (3, 7), (4, 5), (4, 6),
(5, 2), (6, 2), (7, 1)])
mlp_1 = MultiLayerPerceptron(class_or_reg, layer_labels, A,
num_units_each_layer, all_layer_label_classes)
# Network 2
layer_labels = [
'ip', 'softplus', 'elu', 'tanh', 'logistic', last_layer_label, 'op'
]
num_units_each_layer = [None, 16, 16, 32, 64, None, None]
A = get_dok_mat_with_set_coords(7, [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5),
(5, 6)])
mlp_2 = MultiLayerPerceptron(class_or_reg, layer_labels, A,
num_units_each_layer, all_layer_label_classes)
# Network 3
layer_labels = [
'ip', 'tanh', 'logistic', 'logistic', 'tanh', 'elu', 'relu',
last_layer_label, 'op'
]
num_units_each_layer = [None, 8, 8, 8, 8, 16, 16, None, None]
A = get_dok_mat_with_set_coords(9, [(0, 1), (0, 2), (1, 3), (2, 4), (3, 5),
(3, 6), (4, 5), (4, 6), (5, 7), (6, 7),
(7, 8)])
mlp_3 = MultiLayerPerceptron(class_or_reg, layer_labels, A,
num_units_each_layer, all_layer_label_classes)
# Network 4
layer_labels = [
'ip', 'logistic', 'relu', 'softplus', last_layer_label, 'op', 'relu',
'tanh'
]
num_units_each_layer = [None, 8, 8, 16, None, None, 16, 8]
A = get_dok_mat_with_set_coords(8, [(0, 1), (0, 7), (1, 2), (2, 3), (2, 6),
(7, 6), (7, 3), (3, 4), (6, 4),
(4, 5)])
mlp_4 = MultiLayerPerceptron(class_or_reg, layer_labels, A,
num_units_each_layer, all_layer_label_classes)
return [
mlp_1,
mlp_2,
mlp_3,
mlp_4,
get_blocked_mlp(class_or_reg, 4, 3),
get_blocked_mlp(class_or_reg, 8, 2),
get_multidepth_mlp_eg1(class_or_reg),
get_multidepth_mlp_eg2(class_or_reg),
]
def _get_multidepth_mlp_eg12_common(class_or_reg):
""" A network with 2 linear layers mostly for debugging common operations. """
mlp_layer_labels = get_mlp_layer_labels(class_or_reg)
obtl_label = 'linear' if class_or_reg == 'reg' else 'softmax'
layer_labels = [
'ip', obtl_label, 'op', 'tanh', 'relu', 'leaky-relu', 'logistic',
'logistic', 'elu', obtl_label
]
num_units_in_each_layer = [
None, None, None, 64, 64, 128, 256, 64, 512, None
]
edges = [(0, 1), (0, 3), (0, 4), (3, 5), (4, 7), (5, 6), (7, 8), (3, 8), (7, 6), \
(6, 9), (8, 9), (1, 2), (9, 2)]
return layer_labels, num_units_in_each_layer, edges, mlp_layer_labels
def test_get_labels(self):
""" Tests labels generated for a CNN and MLP. """
self.report('Testing the labels for CNN and MLP.')
cnn_true_labels_7 = sorted([
'ip', 'op', 'fc', 'max-pool', 'avg-pool', 'softmax', 'res3',
'res5', 'res7', 'conv3', 'conv5', 'conv7'
])
cnn_true_labels_3 = sorted([
'ip', 'op', 'fc', 'max-pool', 'avg-pool', 'softmax', 'res3',
'conv3'
])
mlp_true_labels = sorted(['ip', 'op', 'relu', 'linear'])
cnn_ret_labels_7 = sorted(neural_network.get_cnn_layer_labels(7))
cnn_ret_labels_3 = sorted(neural_network.get_cnn_layer_labels(3))
mlp_ret_labels = sorted(
neural_network.get_mlp_layer_labels('reg', ['relu']))
assert cnn_true_labels_7 == cnn_ret_labels_7
assert cnn_true_labels_3 == cnn_ret_labels_3
assert mlp_true_labels == mlp_ret_labels
def get_blocked_mlp(class_or_reg,
num_blocks,
num_layers_per_block=None,
num_units_in_each_layer=None):
""" Creates a blocked MLP. """
# Create rectifiers and sigmoids
rectifiers = ['relu', 'elu', 'crelu', 'leaky-relu', 'softplus']
sigmoids = ['logistic', 'tanh']
obtl_label = 'linear' if class_or_reg == 'reg' else 'softmax'
np.random.shuffle(rectifiers)
np.random.shuffle(sigmoids)
rect_count = 0
sig_count = 0
# Preprocess args
# Create the network
layer_labels = ['ip']
num_units_in_each_layer = num_units_in_each_layer if num_units_in_each_layer \
is not None else _dflt_unit_sizes(num_blocks)
# Construct blocks
for block_idx in range(num_blocks):
if block_idx % 2 == 0:
layer_type = rectifiers[rect_count % len(rectifiers)]
rect_count += 1
else:
layer_type = sigmoids[sig_count % len(sigmoids)]
sig_count += 1
layer_labels.extend([layer_type] * num_layers_per_block)
num_units_in_each_layer.extend([num_units_in_each_layer[block_idx]] *
num_layers_per_block)
# A linear layer at the end of the block
layer_labels.extend([obtl_label, 'op'])
num_units_in_each_layer.extend([None, None])
conn_mat = get_feedforward_adj_mat(len(layer_labels))
all_layer_labels = get_mlp_layer_labels(class_or_reg)
return MultiLayerPerceptron(class_or_reg, layer_labels, conn_mat,
num_units_in_each_layer, all_layer_labels)
