def create_block():
# first node of block
n1 = Node('N1')
for inpt in input_nodes:
n1.add_op(Connect(cell.graph, inpt, n1))
# second node of block
mlp_op_list = list()
mlp_op_list.append(Identity())
mlp_op_list.append(Dense(5, tf.nn.relu))
mlp_op_list.append(Dense(5, tf.nn.tanh))
mlp_op_list.append(Dense(10, tf.nn.relu))
mlp_op_list.append(Dense(10, tf.nn.tanh))
mlp_op_list.append(Dense(20, tf.nn.relu))
mlp_op_list.append(Dense(20, tf.nn.tanh))
n2 = Node('N2')
for op in mlp_op_list:
n2.add_op(op)
# third node of block
n3 = Node('N3')
for op in dropout_ops:
n3.add_op(op)
# 5 Blocks
block = Block()
block.add_node(n1)
block.add_node(n2)
block.add_node(n3)
block.add_edge(n1, n2)
block.add_edge(n2, n3)
return block
def add_dense_to_(node):
node.add_op(Identity()) # we do not want to create a layer in this case
activations = [None, tf.nn.relu, tf.nn.tanh, tf.nn.sigmoid]
for units in range(16, 97, 16):
for activation in activations:
node.add_op(Dense(units=units, activation=activation))
def set_ops(self, indexes):
"""
Set the operations for each node of each cell of the structure.
Args:
indexes (list): element of list can be float in [0, 1] or int.
output_node (ConstantNode): the output node of the Structure.
"""
cursor = 0
for c in self.struct:
num_nodes = c.num_nodes
c.set_ops(indexes[cursor:cursor + num_nodes])
cursor += num_nodes
self.graph.add_nodes_from(c.graph.nodes())
self.graph.add_edges_from(c.graph.edges())
output_nodes = get_output_nodes(self.graph)
if len(output_nodes) == 1:
node = ConstantNode(op=Identity(), name='Structure_Output')
self.graph.add_node(node)
self.graph.add_edge(output_nodes[0], node)
else:
node = ConstantNode(name='Structure_Output')
node.set_op(self.output_op(self.graph, node, output_nodes))
self.output_node = node
def add_dropout_op_(node):
node.add_op(Identity())
node.add_op(Dropout(rate=0.5))
node.add_op(Dropout(rate=0.4))
node.add_op(Dropout(rate=0.3))
node.add_op(Dropout(rate=0.2))
node.add_op(Dropout(rate=0.1))
node.add_op(Dropout(rate=0.05))
def create_conv_node(name):
n = Node(name)
n.add_op(Identity())
n.add_op(Conv1D(filter_size=3, num_filters=16))
n.add_op(Conv1D(filter_size=5, num_filters=16))
n.add_op(MaxPooling1D(pool_size=3, padding='same'))
n.add_op(MaxPooling1D(pool_size=5, padding='same'))
return n
def add_dense_op_(node):
node.add_op(Identity())
node.add_op(Dense(units=10))
node.add_op(Dense(units=50))
node.add_op(Dense(units=100))
node.add_op(Dense(units=200))
node.add_op(Dense(units=250))
node.add_op(Dense(units=500))
node.add_op(Dense(units=750))
node.add_op(Dense(units=1000))
def create_conv_node(name):
n = VariableNode(name)
n.add_op(Identity())
n.add_op(Conv1D(filter_size=3, num_filters=16))
n.add_op(MaxPooling1D(pool_size=3, padding='same'))
n.add_op(Dense(10, tf.nn.relu))
n.add_op(Conv1D(filter_size=5, num_filters=16))
n.add_op(MaxPooling1D(pool_size=5, padding='same'))
n.add_op(Dense(100, tf.nn.relu))
n.add_op(Conv1D(filter_size=10, num_filters=16))
n.add_op(MaxPooling1D(pool_size=10, padding='same'))
n.add_op(Dense(1000, tf.nn.relu))
return n
def create_dense_cell_type2(input_nodes):
"""MLP type 2
Args:
input_nodes (list(Node)): possible inputs of the current cell.
Returns:
Cell: a Cell instance.
"""
cell = Cell(input_nodes)
# first node of block
n1 = Node('N_0')
for inpt in input_nodes:
n1.add_op(Connect(cell.graph, inpt, n1))
# second node of block
mlp_op_list = list()
mlp_op_list.append(Identity())
mlp_op_list.append(Dense(5, tf.nn.relu))
mlp_op_list.append(Dense(10, tf.nn.relu))
mlp_op_list.append(Dense(20, tf.nn.relu))
mlp_op_list.append(Dense(40, tf.nn.relu))
mlp_op_list.append(Dense(80, tf.nn.relu))
mlp_op_list.append(Dense(160, tf.nn.relu))
mlp_op_list.append(Dense(320, tf.nn.relu))
n2 = Node('N_1')
for op in mlp_op_list:
n2.add_op(op)
# third
n3 = Node('N_2')
drop_ops = []
drop_ops.extend(dropout_ops)
for op in drop_ops:
n3.add_op(op)
# 1 Blocks
block1 = Block()
block1.add_node(n1)
block1.add_node(n2)
block1.add_node(n3)
block1.add_edge(n1, n2)
block1.add_edge(n2, n3)
cell.add_block(block1)
cell.set_outputs()
return cell
def create_mlp_node(node):
node.add_op(Identity())
node.add_op(Dense(100, tf.nn.relu))
node.add_op(Dense(100, tf.nn.tanh))
node.add_op(Dense(100, tf.nn.sigmoid))
node.add_op(Dropout(0.05))
node.add_op(Dense(500, tf.nn.relu))
node.add_op(Dense(500, tf.nn.tanh))
node.add_op(Dense(500, tf.nn.sigmoid))
node.add_op(Dropout(0.1))
node.add_op(Dense(1000, tf.nn.relu))
node.add_op(Dense(1000, tf.nn.tanh))
node.add_op(Dense(1000, tf.nn.sigmoid))
node.add_op(Dropout(0.2))
def add_mlp_op_(node):
node.add_op(Identity())
node.add_op(Dense(100, tf.nn.relu))
node.add_op(Dense(100, tf.nn.tanh))
node.add_op(Dense(100, tf.nn.sigmoid))
node.add_op(Dropout(0.3))
node.add_op(Dense(500, tf.nn.relu))
node.add_op(Dense(500, tf.nn.tanh))
node.add_op(Dense(500, tf.nn.sigmoid))
node.add_op(Dropout(0.4))
node.add_op(Dense(1000, tf.nn.relu))
node.add_op(Dense(1000, tf.nn.tanh))
node.add_op(Dense(1000, tf.nn.sigmoid))
node.add_op(Dropout(0.5))
def create_search_space(
input_shape=(100, ), output_shape=[(1),
(100, )], num_layers=5, **kwargs):
struct = KSearchSpace(input_shape, output_shape)
inp = struct.input_nodes[0]
# auto-encoder
units = [128, 64, 32, 16, 8, 16, 32, 64, 128]
# units = [32, 16, 32]
prev_node = inp
d = 1
for i in range(len(units)):
vnode = VariableNode()
vnode.add_op(Identity())
if d == 1 and units[i] < units[i + 1]:
d = -1
# print(min(1, units[i]), ' - ', max(1, units[i])+1)
for u in range(min(2, units[i]), max(2, units[i]) + 1, 2):
vnode.add_op(Dense(u, tf.nn.relu))
latente_space = vnode
else:
# print(min(units[i], units[i+d]), ' - ', max(units[i], units[i+d])+1)
for u in range(min(units[i], units[i + d]),
max(units[i], units[i + d]) + 1, 2):
vnode.add_op(Dense(u, tf.nn.relu))
struct.connect(prev_node, vnode)
prev_node = vnode
out2 = ConstantNode(op=Dense(100, name="output_1"))
struct.connect(prev_node, out2)
# regressor
prev_node = latente_space
# prev_node = inp
for _ in range(num_layers):
vnode = VariableNode()
for i in range(16, 129, 16):
vnode.add_op(Dense(i, tf.nn.relu))
struct.connect(prev_node, vnode)
prev_node = vnode
out1 = ConstantNode(op=Dense(1, name="output_0"))
struct.connect(prev_node, out1)
return struct
def add_mlp_ops_to(vnode):
# REG_L1 = 1.
# REG_L2 = 1.
vnode.add_op(Identity())
vnode.add_op(Dense(100, tf.nn.relu))
vnode.add_op(Dense(100, tf.nn.tanh))
vnode.add_op(Dense(100, tf.nn.sigmoid))
vnode.add_op(Dropout(0.05))
vnode.add_op(Dense(500, tf.nn.relu))
vnode.add_op(Dense(500, tf.nn.tanh))
vnode.add_op(Dense(500, tf.nn.sigmoid))
vnode.add_op(Dropout(0.1))
vnode.add_op(Dense(1000, tf.nn.relu))
vnode.add_op(Dense(1000, tf.nn.tanh))
vnode.add_op(Dense(1000, tf.nn.sigmoid))
vnode.add_op(Dropout(0.2))
def create_mlp_node(name):
n = VariableNode(name)
n.add_op(Identity())
n.add_op(Dense(100, tf.nn.relu))
n.add_op(Dense(100, tf.nn.tanh))
n.add_op(Dense(100, tf.nn.sigmoid))
n.add_op(Dropout(0.05))
n.add_op(Dense(500, tf.nn.relu))
n.add_op(Dense(500, tf.nn.tanh))
n.add_op(Dense(500, tf.nn.sigmoid))
n.add_op(Dropout(0.1))
n.add_op(Dense(1000, tf.nn.relu))
n.add_op(Dense(1000, tf.nn.tanh))
n.add_op(Dense(1000, tf.nn.sigmoid))
n.add_op(Dropout(0.2))
return n
def set_output_node(self, graph, output_nodes):
"""Set the output node of the search_space.
Args:
graph (nx.DiGraph): graph of the search_space.
output_nodes (Node): nodes of the current search_space without successors.
Returns:
Node: output node of the search_space.
"""
if len(output_nodes) == 1:
node = ConstantNode(op=Identity(), name='Structure_Output')
graph.add_node(node)
graph.add_edge(output_nodes[0], node)
else:
node = ConstantNode(name='Structure_Output')
op = Concatenate(self, output_nodes)
node.set_op(op=op)
return node
def add_lstm_(node):
node.add_op(Identity()) # we do not want to create a layer in this case
#activations = [None, tf.nn.relu, tf.nn.tanh, tf.nn.sigmoid]
for units in range(16, 97, 16):
node.add_op(tf.keras.layers.LSTM(units=units, return_sequences=False))
def add_conv_op_(node):
node.add_op(Identity())
node.add_op(Conv1D(filter_size=3, num_filters=8))
node.add_op(Conv1D(filter_size=4, num_filters=8))
node.add_op(Conv1D(filter_size=5, num_filters=8))
node.add_op(Conv1D(filter_size=6, num_filters=8))
def add_activation_op_(node):
node.add_op(Identity())
node.add_op(Activation(activation='relu'))
node.add_op(Activation(activation='tanh'))
node.add_op(Activation(activation='sigmoid'))
def add_pooling_op_(node):
node.add_op(Identity())
node.add_op(MaxPooling1D(pool_size=3, padding='same'))
node.add_op(MaxPooling1D(pool_size=4, padding='same'))
node.add_op(MaxPooling1D(pool_size=5, padding='same'))
node.add_op(MaxPooling1D(pool_size=6, padding='same'))