def generate(filters): return cell( lambda channels: mo.siso_sequential( [conv2d(D([channels]), D([1])), batch_normalization(), relu()]), lambda num_inputs, node_id, channels: intermediate_node_fn( num_inputs, node_id, channels, cell_ops), concat, h_connections, 5, filters)
def pool_op(filters, filter_size, stride, pool_type): if pool_type == 'avg': pool = avg_pool2d(D([filter_size]), D([stride])) elif pool_type == 'max': pool = max_pool2d(D([filter_size]), D([stride])) else: pool = min_pool2d(D([filter_size]), D([stride])) return mo.siso_sequential([pool, check_filters(filters)])
def dnn_net(num_classes): h_nonlin_name = D(['relu', 'tanh', 'elu']) h_opt_drop = D([0, 1]) return mo.siso_sequential([ flatten(), mo.siso_repeat( lambda: dnn_cell(D([64, 128, 256, 512, 1024]), h_nonlin_name, h_opt_drop, D([0.25, 0.5, 0.75])), D([1, 2])), dense(D([num_classes])) ])
def dnn_cell(h_num_hidden, h_nonlin_name, h_swap, h_opt_drop, h_opt_bn, h_drop_rate): return mo.siso_sequential([ dense(h_num_hidden), nonlinearity(h_nonlin_name), mo.siso_permutation([ lambda: mo.siso_optional(lambda: dropout(h_drop_rate), h_opt_drop), lambda: mo.siso_optional(batch_normalization, h_opt_bn), ], h_swap) ])
def conv_op(filters, filter_size, stride, dilation_rate, spatial_separable): if spatial_separable: return mo.siso_sequential([ conv2d(D([filters]), D([[1, filter_size]]), D([[1, stride]])), batch_normalization(), relu(), conv2d(D([filters]), D([[filter_size, 1]]), D([[stride, 1]])), ]) else: return conv2d(D([filters]), D([filter_size]), D([stride]), D([dilation_rate]))
def dnn_cell(h_num_hidden, h_nonlin_name, h_swap, h_opt_drop, h_opt_bn, h_drop_keep_prob): return mo.siso_sequential([ affine_simplified(h_num_hidden), nonlinearity(h_nonlin_name), mo.siso_permutation([ lambda: mo.siso_optional(lambda: dropout(h_drop_keep_prob), h_opt_drop), lambda: mo.siso_optional(batch_normalization, h_opt_bn), ], h_swap) ])
def intermediate_node_fn(num_inputs, node_id, filters, cell_ops): return mo.siso_sequential([ add(num_inputs), mo.siso_or( { 'conv1': lambda: conv2d(D([filters]), D([1])), 'conv3': lambda: conv2d(D([filters]), D([3])), 'max3': lambda: max_pool2d(D([3])) }, cell_ops[node_id]), batch_normalization(), relu() ])
def dnn_net(num_classes): h_nonlin_name = D(['relu', 'relu6', 'crelu', 'elu', 'softplus'], name='Mutatable') h_swap = D([0, 1], name='Mutatable_sub') h_opt_drop = D([0, 1], name='Mutatable_sub') h_opt_bn = D([0, 1], name='Mutatable_sub') return mo.siso_sequential([ mo.siso_repeat( lambda: dnn_cell(D([64, 128, 256, 512, 1024]), h_nonlin_name, h_swap, h_opt_drop, h_opt_bn, D([0.25, 0.5, 0.75])), D([1, 2])), affine_simplified(D([num_classes])) ])
def generate_search_space(stacks, num_cells_per_stack, num_nodes_per_cell, num_init_filters): search_space = [stem()] cell_fn = create_cell_generator(num_nodes_per_cell) num_filters = num_init_filters for i in range(stacks): if i > 0: search_space.append(max_pool2d(D([2]), D([2]))) num_filters *= 2 for j in range(num_cells_per_stack): search_space.append(cell_fn(num_filters)) search_space += [global_pool2d(), fc_layer(D([10]))] return mo.siso_sequential(search_space)
def model(num_classes): reduce_fn = lambda: conv2d(D(range(48, 129, 16)), D([3, 5, 7]), D([2])) conv_fn = lambda: module( D(range(48, 129, 16)), D([3, 5]), D([0, 1]), D([0, 1]), D([0.5, 0.1]), D([2**i for i in xrange(6)])) return mo.siso_sequential([ reduce_fn(), conv_fn(), reduce_fn(), conv_fn(), dense(D([num_classes])) ])
def aux_logits(): return mo.siso_sequential([ relu(), avg_pool2d(D([5]), D([3]), D(['VALID'])), conv2d(D([128]), D([1])), batch_normalization(), relu(), global_convolution(D([768])), batch_normalization(), relu(), flatten(), fc_layer(D([10])) ])
def wrap_batch_norm_relu(io_pair, add_relu=True, add_bn=True, weight_sharer=None, name=None): assert add_relu or add_bn elements = [True, add_bn, add_relu] module_fns = [ lambda: io_pair, lambda: keras_batch_normalization(name=name, weight_sharer=weight_sharer), relu ] return mo.siso_sequential( [module_fn() for i, module_fn in enumerate(module_fns) if elements[i]])
def generate_search_space(num_nodes_per_cell, num_normal_cells, num_reduction_cells, init_filters, stem_multiplier): global global_vars, hp_sharer global_vars = {} hp_sharer = hp.HyperparameterSharer() hp_sharer.register('drop_path_keep_prob', lambda: D([.7], name='drop_path_keep_prob')) stem_in, stem_out = stem(int(init_filters * stem_multiplier)) progress_in, progress_out = mo.identity() global_vars['progress'] = progress_out['Out'] normal_cell_fn = create_cell_generator(num_nodes_per_cell, False) reduction_cell_fn = create_cell_generator(num_nodes_per_cell, True) total_cells = num_normal_cells + num_reduction_cells hasReduction = [False] * num_normal_cells for i in range(num_reduction_cells): hasReduction[int( float(i + 1) / (num_reduction_cells + 1) * num_normal_cells)] = True inputs = [stem_out, stem_out] filters = init_filters aux_loss_idx = int( float(num_reduction_cells) / (num_reduction_cells + 1) * num_normal_cells) - 1 outs = {} cells_created = 0.0 for i in range(num_normal_cells): if hasReduction[i]: filters *= 2 connect_new_cell( reduction_cell_fn(filters, (cells_created + 1) / total_cells), inputs) cells_created += 1.0 connect_new_cell( normal_cell_fn(filters, (cells_created + 1) / total_cells), inputs) cells_created += 1.0 if i == aux_loss_idx: aux_in, aux_out = aux_logits() aux_in['In'].connect(inputs[-1]['Out']) outs['Out0'] = aux_out['Out'] _, final_out = mo.siso_sequential([(None, inputs[-1]), relu(), global_pool2d(), dropout(D([1.0])), fc_layer(D([10]))]) outs['Out1'] = final_out['Out'] return {'In0': stem_in['In'], 'In1': progress_in['In']}, outs
def generate_stage(stage_num, num_nodes, filters, filter_size): h_connections = [ Bool(name='%d_in_%d_%d' % (stage_num, in_id, out_id)) for (in_id, out_id) in itertools.combinations(range(1, num_nodes + 1), 2) ] return genetic_stage( lambda: mo.siso_sequential([ conv2d(D([filters]), D([filter_size])), batch_normalization(), relu() ]), lambda num_inputs: intermediate_node_fn(num_inputs, filters), lambda num_inputs: intermediate_node_fn(num_inputs, filters), h_connections, num_nodes)
def enas_conv(out_filters, filter_size, separable, weight_sharer, name): io_pair = (conv2D_depth_separable(filter_size, name, weight_sharer) if separable else conv2D(filter_size, name, weight_sharer)) return mo.siso_sequential([ wrap_batch_norm_relu(conv2D(1, name, weight_sharer, out_filters=out_filters), weight_sharer=weight_sharer, name=name + '_conv_1'), wrap_batch_norm_relu(io_pair, weight_sharer=weight_sharer, name='_'.join( [name, str(filter_size), str(separable)])) ])
def generate_search_space(nodes_per_stage, filters_per_stage, filter_size_per_stage): search_space = [] for i in range(len(nodes_per_stage)): search_space.append( generate_stage(i, nodes_per_stage[i], filters_per_stage[i], filter_size_per_stage[i])) search_space.append(max_pool2d(D([3]), D([2]), D(['SAME']))) search_space += [ flatten(), fc_layer(D([1024])), dropout(D([.5])), fc_layer(D([10])) ] return mo.siso_sequential(search_space)
def get_enas_search_space(num_classes, num_layers, out_filters, weight_sharer): h_N = D([num_layers], name='num_layers') return mo.siso_sequential([ enas_space( h_N, out_filters, #mo.empty, lambda: wrap_batch_norm_relu(conv2D( 3, 'stem', weight_sharer, out_filters=out_filters), add_relu=False, weight_sharer=weight_sharer, name='stem'), enas_repeat_fn, ['in'], ['out'], weight_sharer), global_pool(), dropout(keep_prob=.9), fc_layer(num_classes, 'softmax', weight_sharer), ])
def full_conv_op(filters, filter_size, stride, dilation_rate, spatial_separable): # Add bottleneck layer according to # https://github.com/tensorflow/tpu/blob/master/models/official/amoeba_net/network_utils.py if filter_size == 3 and spatial_separable: reduced_filter_size = int(3 * filters / 8) else: reduced_filter_size = int(filters / 4) if reduced_filter_size < 1: return wrap_relu_batch_norm( conv_op(filters, filter_size, stride, dilation_rate, spatial_separable)) else: return mo.siso_sequential([ wrap_relu_batch_norm(conv2d(D([reduced_filter_size]), D([1]))), wrap_relu_batch_norm( conv_op(reduced_filter_size, filter_size, stride, dilation_rate, spatial_separable)), wrap_relu_batch_norm(conv2d(D([filters]), D([1]))) ])
def combine_with_concat(num_inputs, channels): return mo.siso_sequential([concat(num_inputs), conv2d_cell(channels, 1)])
def separable_conv2d_cell(filters, kernel_size, stride=1): return mo.siso_sequential([ separable_conv2d(filters, kernel_size, stride, 'relu'), batch_normalization() ])
def depthwise_conv2d_cell(kernel_size, stride=1): return mo.siso_sequential( [depthwise_conv2d(kernel_size, stride, 'relu'), batch_normalization()])
def conv2d_cell(filters, kernel_size, stride=1): return mo.siso_sequential([ batch_normalization(), conv2d(filters, kernel_size, stride, 'relu'), ])
def conv2d_cell(filters, kernel_size): return mo.siso_sequential( [conv2d_with_relu(filters, kernel_size), batch_normalization()])
def dnn_cell(h_num_hidden, h_nonlin_name, h_opt_drop, h_drop_keep_prob): return mo.siso_sequential([ dense(h_num_hidden), nonlinearity(h_nonlin_name), mo.siso_optional(lambda: dropout(h_drop_keep_prob), h_opt_drop) ])
def stem(): return mo.siso_sequential([ conv2d(D([128]), D([3])), batch_normalization(), relu(), ])
def wrap_relu_batch_norm(op): return mo.siso_sequential([relu(), op, batch_normalization()])
def stem(filters): return mo.siso_sequential( [conv2d(D([filters]), D([3])), batch_normalization()])