def generate(filters):
return cell(
lambda channels: mo.siso_sequential(
[conv2d(D([channels]), D([1])),
batch_normalization(),
relu()]),
lambda num_inputs, node_id, channels: intermediate_node_fn(
num_inputs, node_id, channels, cell_ops), concat,
h_connections, 5, filters)
def pool_op(filters, filter_size, stride, pool_type):
if pool_type == 'avg':
pool = avg_pool2d(D([filter_size]), D([stride]))
elif pool_type == 'max':
pool = max_pool2d(D([filter_size]), D([stride]))
else:
pool = min_pool2d(D([filter_size]), D([stride]))
return mo.siso_sequential([pool, check_filters(filters)])
def dnn_net(num_classes):
h_nonlin_name = D(['relu', 'tanh', 'elu'])
h_opt_drop = D([0, 1])
return mo.siso_sequential([
flatten(),
mo.siso_repeat(
lambda: dnn_cell(D([64, 128, 256, 512, 1024]), h_nonlin_name,
h_opt_drop, D([0.25, 0.5, 0.75])), D([1, 2])),
dense(D([num_classes]))
])
def dnn_cell(h_num_hidden, h_nonlin_name, h_swap, h_opt_drop, h_opt_bn,
h_drop_rate):
return mo.siso_sequential([
dense(h_num_hidden),
nonlinearity(h_nonlin_name),
mo.siso_permutation([
lambda: mo.siso_optional(lambda: dropout(h_drop_rate), h_opt_drop),
lambda: mo.siso_optional(batch_normalization, h_opt_bn),
], h_swap)
])
def conv_op(filters, filter_size, stride, dilation_rate, spatial_separable):
if spatial_separable:
return mo.siso_sequential([
conv2d(D([filters]), D([[1, filter_size]]), D([[1, stride]])),
batch_normalization(),
relu(),
conv2d(D([filters]), D([[filter_size, 1]]), D([[stride, 1]])),
])
else:
return conv2d(D([filters]), D([filter_size]), D([stride]),
D([dilation_rate]))
def dnn_cell(h_num_hidden, h_nonlin_name, h_swap, h_opt_drop, h_opt_bn,
h_drop_keep_prob):
return mo.siso_sequential([
affine_simplified(h_num_hidden),
nonlinearity(h_nonlin_name),
mo.siso_permutation([
lambda: mo.siso_optional(lambda: dropout(h_drop_keep_prob),
h_opt_drop),
lambda: mo.siso_optional(batch_normalization, h_opt_bn),
], h_swap)
])
def intermediate_node_fn(num_inputs, node_id, filters, cell_ops):
return mo.siso_sequential([
add(num_inputs),
mo.siso_or(
{
'conv1': lambda: conv2d(D([filters]), D([1])),
'conv3': lambda: conv2d(D([filters]), D([3])),
'max3': lambda: max_pool2d(D([3]))
}, cell_ops[node_id]),
batch_normalization(),
relu()
])
def dnn_net(num_classes):
h_nonlin_name = D(['relu', 'relu6', 'crelu', 'elu', 'softplus'],
name='Mutatable')
h_swap = D([0, 1], name='Mutatable_sub')
h_opt_drop = D([0, 1], name='Mutatable_sub')
h_opt_bn = D([0, 1], name='Mutatable_sub')
return mo.siso_sequential([
mo.siso_repeat(
lambda: dnn_cell(D([64, 128, 256, 512, 1024]),
h_nonlin_name, h_swap, h_opt_drop, h_opt_bn,
D([0.25, 0.5, 0.75])), D([1, 2])),
affine_simplified(D([num_classes]))
])
def generate_search_space(stacks, num_cells_per_stack, num_nodes_per_cell,
num_init_filters):
search_space = [stem()]
cell_fn = create_cell_generator(num_nodes_per_cell)
num_filters = num_init_filters
for i in range(stacks):
if i > 0:
search_space.append(max_pool2d(D([2]), D([2])))
num_filters *= 2
for j in range(num_cells_per_stack):
search_space.append(cell_fn(num_filters))
search_space += [global_pool2d(), fc_layer(D([10]))]
return mo.siso_sequential(search_space)
def model(num_classes):
reduce_fn = lambda: conv2d(D(range(48, 129, 16)), D([3, 5, 7]), D([2]))
conv_fn = lambda: module(
D(range(48, 129, 16)), D([3, 5]), D([0, 1]), D([0, 1]), D([0.5, 0.1]),
D([2**i for i in xrange(6)]))
return mo.siso_sequential([
reduce_fn(),
conv_fn(),
reduce_fn(),
conv_fn(),
dense(D([num_classes]))
])
def aux_logits():
return mo.siso_sequential([
relu(),
avg_pool2d(D([5]), D([3]), D(['VALID'])),
conv2d(D([128]), D([1])),
batch_normalization(),
relu(),
global_convolution(D([768])),
batch_normalization(),
relu(),
flatten(),
fc_layer(D([10]))
])
def wrap_batch_norm_relu(io_pair,
add_relu=True,
add_bn=True,
weight_sharer=None,
name=None):
assert add_relu or add_bn
elements = [True, add_bn, add_relu]
module_fns = [
lambda: io_pair,
lambda: keras_batch_normalization(name=name,
weight_sharer=weight_sharer), relu
]
return mo.siso_sequential(
[module_fn() for i, module_fn in enumerate(module_fns) if elements[i]])
def generate_search_space(num_nodes_per_cell, num_normal_cells,
num_reduction_cells, init_filters, stem_multiplier):
global global_vars, hp_sharer
global_vars = {}
hp_sharer = hp.HyperparameterSharer()
hp_sharer.register('drop_path_keep_prob',
lambda: D([.7], name='drop_path_keep_prob'))
stem_in, stem_out = stem(int(init_filters * stem_multiplier))
progress_in, progress_out = mo.identity()
global_vars['progress'] = progress_out['Out']
normal_cell_fn = create_cell_generator(num_nodes_per_cell, False)
reduction_cell_fn = create_cell_generator(num_nodes_per_cell, True)
total_cells = num_normal_cells + num_reduction_cells
hasReduction = [False] * num_normal_cells
for i in range(num_reduction_cells):
hasReduction[int(
float(i + 1) / (num_reduction_cells + 1) *
num_normal_cells)] = True
inputs = [stem_out, stem_out]
filters = init_filters
aux_loss_idx = int(
float(num_reduction_cells) /
(num_reduction_cells + 1) * num_normal_cells) - 1
outs = {}
cells_created = 0.0
for i in range(num_normal_cells):
if hasReduction[i]:
filters *= 2
connect_new_cell(
reduction_cell_fn(filters, (cells_created + 1) / total_cells),
inputs)
cells_created += 1.0
connect_new_cell(
normal_cell_fn(filters, (cells_created + 1) / total_cells), inputs)
cells_created += 1.0
if i == aux_loss_idx:
aux_in, aux_out = aux_logits()
aux_in['In'].connect(inputs[-1]['Out'])
outs['Out0'] = aux_out['Out']
_, final_out = mo.siso_sequential([(None, inputs[-1]),
relu(),
global_pool2d(),
dropout(D([1.0])),
fc_layer(D([10]))])
outs['Out1'] = final_out['Out']
return {'In0': stem_in['In'], 'In1': progress_in['In']}, outs
def generate_stage(stage_num, num_nodes, filters, filter_size):
h_connections = [
Bool(name='%d_in_%d_%d' % (stage_num, in_id, out_id))
for (in_id,
out_id) in itertools.combinations(range(1, num_nodes + 1), 2)
]
return genetic_stage(
lambda: mo.siso_sequential([
conv2d(D([filters]), D([filter_size])),
batch_normalization(),
relu()
]), lambda num_inputs: intermediate_node_fn(num_inputs, filters), lambda
num_inputs: intermediate_node_fn(num_inputs, filters), h_connections,
num_nodes)
def enas_conv(out_filters, filter_size, separable, weight_sharer, name):
io_pair = (conv2D_depth_separable(filter_size, name, weight_sharer)
if separable else conv2D(filter_size, name, weight_sharer))
return mo.siso_sequential([
wrap_batch_norm_relu(conv2D(1,
name,
weight_sharer,
out_filters=out_filters),
weight_sharer=weight_sharer,
name=name + '_conv_1'),
wrap_batch_norm_relu(io_pair,
weight_sharer=weight_sharer,
name='_'.join(
[name, str(filter_size),
str(separable)]))
])
def generate_search_space(nodes_per_stage, filters_per_stage,
filter_size_per_stage):
search_space = []
for i in range(len(nodes_per_stage)):
search_space.append(
generate_stage(i, nodes_per_stage[i], filters_per_stage[i],
filter_size_per_stage[i]))
search_space.append(max_pool2d(D([3]), D([2]), D(['SAME'])))
search_space += [
flatten(),
fc_layer(D([1024])),
dropout(D([.5])),
fc_layer(D([10]))
]
return mo.siso_sequential(search_space)
def get_enas_search_space(num_classes, num_layers, out_filters, weight_sharer):
h_N = D([num_layers], name='num_layers')
return mo.siso_sequential([
enas_space(
h_N,
out_filters,
#mo.empty,
lambda: wrap_batch_norm_relu(conv2D(
3, 'stem', weight_sharer, out_filters=out_filters),
add_relu=False,
weight_sharer=weight_sharer,
name='stem'),
enas_repeat_fn,
['in'],
['out'],
weight_sharer),
global_pool(),
dropout(keep_prob=.9),
fc_layer(num_classes, 'softmax', weight_sharer),
])
def full_conv_op(filters, filter_size, stride, dilation_rate,
spatial_separable):
# Add bottleneck layer according to
# https://github.com/tensorflow/tpu/blob/master/models/official/amoeba_net/network_utils.py
if filter_size == 3 and spatial_separable:
reduced_filter_size = int(3 * filters / 8)
else:
reduced_filter_size = int(filters / 4)
if reduced_filter_size < 1:
return wrap_relu_batch_norm(
conv_op(filters, filter_size, stride, dilation_rate,
spatial_separable))
else:
return mo.siso_sequential([
wrap_relu_batch_norm(conv2d(D([reduced_filter_size]), D([1]))),
wrap_relu_batch_norm(
conv_op(reduced_filter_size, filter_size, stride,
dilation_rate, spatial_separable)),
wrap_relu_batch_norm(conv2d(D([filters]), D([1])))
])
def combine_with_concat(num_inputs, channels):
return mo.siso_sequential([concat(num_inputs), conv2d_cell(channels, 1)])
def separable_conv2d_cell(filters, kernel_size, stride=1):
return mo.siso_sequential([
separable_conv2d(filters, kernel_size, stride, 'relu'),
batch_normalization()
])
def depthwise_conv2d_cell(kernel_size, stride=1):
return mo.siso_sequential(
[depthwise_conv2d(kernel_size, stride, 'relu'),
batch_normalization()])
def conv2d_cell(filters, kernel_size, stride=1):
return mo.siso_sequential([
batch_normalization(),
conv2d(filters, kernel_size, stride, 'relu'),
])
def conv2d_cell(filters, kernel_size):
return mo.siso_sequential(
[conv2d_with_relu(filters, kernel_size),
batch_normalization()])
def dnn_cell(h_num_hidden, h_nonlin_name, h_opt_drop, h_drop_keep_prob):
return mo.siso_sequential([
dense(h_num_hidden),
nonlinearity(h_nonlin_name),
mo.siso_optional(lambda: dropout(h_drop_keep_prob), h_opt_drop)
])
def stem():
return mo.siso_sequential([
conv2d(D([128]), D([3])),
batch_normalization(),
relu(),
])
def wrap_relu_batch_norm(op):
return mo.siso_sequential([relu(), op, batch_normalization()])
def stem(filters):
return mo.siso_sequential(
[conv2d(D([filters]), D([3])),
batch_normalization()])
