def create_search_space(
input_shape=(116, ), output_shape=(116, ), *args, **kwargs):
arch = AutoKArchitecture(input_shape, output_shape, regression=True)
source = prev_input = arch.input_nodes[0]
# look over skip connections within a range of 3 nodes
anchor_points = collections.deque([source], maxlen=3)
num_layers = 10
for _ in range(num_layers):
vnode = VariableNode()
add_dense_to_(vnode)
arch.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation='relu'))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
# ! for next iter
prev_input = cmerge
anchor_points.append(prev_input)
return arch
def __init__(self, input_shape, output_shape, *args, **kwargs):
super().__init__()
if type(input_shape) is tuple:
# we have only one input tensor here
op = Tensor(keras.layers.Input(input_shape, name="input_0"))
self.input_nodes = [ConstantNode(op=op, name='Input_0')]
elif type(input_shape) is list and all(map(lambda x: type(x) is tuple, input_shape)):
# we have a list of input tensors here
self.input_nodes = list()
for i in range(len(input_shape)):
op = Tensor(keras.layers.Input(
input_shape[i], name=f"input_{i}"))
inode = ConstantNode(op=op, name=f'Input_{i}')
self.input_nodes.append(inode)
else:
raise InputShapeOfWrongType(input_shape)
for node in self.input_nodes:
self.graph.add_node(node)
self.output_shape = output_shape
self.output_node = None
self._model = None
def create_cell_1(input_nodes):
"""Create a cell with convolution.
Args:
input_nodes (list(Node)): a list of input_nodes for this cell.
Returns:
Cell: the corresponding cell.
"""
cell = Cell(input_nodes)
def create_conv_block(input_nodes):
# first node of block
n1 = VariableNode('N1')
for inpt in input_nodes:
n1.add_op(Connect(cell.graph, inpt, n1))
def create_conv_node(name):
n = VariableNode(name)
n.add_op(Identity())
n.add_op(Conv1D(filter_size=3, num_filters=16))
n.add_op(MaxPooling1D(pool_size=3, padding='same'))
n.add_op(Dense(10, tf.nn.relu))
n.add_op(Conv1D(filter_size=5, num_filters=16))
n.add_op(MaxPooling1D(pool_size=5, padding='same'))
n.add_op(Dense(100, tf.nn.relu))
n.add_op(Conv1D(filter_size=10, num_filters=16))
n.add_op(MaxPooling1D(pool_size=10, padding='same'))
n.add_op(Dense(1000, tf.nn.relu))
return n
# second node of block
n2 = create_conv_node('N2')
n3 = create_conv_node('N3')
block = Block()
block.add_node(n1)
block.add_node(n2)
block.add_node(n3)
block.add_edge(n1, n2)
block.add_edge(n2, n3)
return block
block1 = create_conv_block(input_nodes)
block2 = create_conv_block(input_nodes)
block3 = create_conv_block(input_nodes)
cell.add_block(block1)
cell.add_block(block2)
cell.add_block(block3)
addNode = ConstantNode(name='Cell_out')
addNode.set_op(AddByPadding(cell.graph, addNode, cell.get_blocks_output()))
cell.set_outputs(node=addNode)
return cell
def create_search_space_old(
input_shape=(2, ), output_shape=(5, ), *args, **kwargs):
ss = AutoKSearchSpace(input_shape, output_shape, regression=True)
prev = ss.input_nodes[0]
for _ in range(3):
cn = ConstantNode(Dense(10, "relu"))
ss.connect(prev, cn)
prev = cn
cn = ConstantNode(Dense(5))
ss.connect(prev, cn)
return ss
def set_outputs(self, node=None):
"""Set output node of the current cell.
node (Node, optional): Defaults to None will create a Concatenation node for the last axis.
"""
if node is None:
stacked_nodes = self.get_blocks_output()
output_node = ConstantNode(name=f'Cell_{self.num}_Output')
output_node.set_op(
Concatenate(self.graph, output_node, stacked_nodes))
else:
output_node = node
self.output = output_node
def create_structure(input_shape=[(2, ), (2, ), (2, )],
output_shape=(1, ),
*args,
**kwargs):
network = KerasStructure(input_shape, output_shape)
input_nodes = network.input_nodes
cell1 = create_cell_1(input_nodes)
network.add_cell(cell1)
cell2 = create_cell_2([cell1.output])
network.add_cell(cell2)
cell3 = Cell(input_nodes + [cell1.output] + [cell2.output])
cnode = VariableNode(name='SkipCo')
nullNode = ConstantNode(op=Tensor([]), name='None')
cnode.add_op(Connect(cell3.graph, nullNode, cnode)) # SAME
cnode.add_op(Connect(cell3.graph, input_nodes[0], cnode))
cnode.add_op(Connect(cell3.graph, input_nodes[1], cnode))
cnode.add_op(Connect(cell3.graph, input_nodes[2], cnode))
cnode.add_op(Connect(cell3.graph, cell1.output, cnode))
cnode.add_op(Connect(cell3.graph, cell2.output, cnode)) # SAME
cnode.add_op(Connect(cell3.graph, input_nodes, cnode))
cnode.add_op(Connect(cell3.graph, [input_nodes[0], input_nodes[1]], cnode))
cnode.add_op(Connect(cell3.graph, [input_nodes[1], input_nodes[2]], cnode))
cnode.add_op(Connect(cell3.graph, [input_nodes[0], input_nodes[2]], cnode))
block = Block()
block.add_node(cnode)
cell3.add_block(block)
network.add_cell(cell3)
return network
def create_search_space(
input_shape=(100, ), output_shape=[(1),
(100, )], num_layers=5, **kwargs):
struct = KSearchSpace(input_shape, output_shape)
inp = struct.input_nodes[0]
# auto-encoder
units = [128, 64, 32, 16, 8, 16, 32, 64, 128]
# units = [32, 16, 32]
prev_node = inp
d = 1
for i in range(len(units)):
vnode = VariableNode()
vnode.add_op(Identity())
if d == 1 and units[i] < units[i + 1]:
d = -1
# print(min(1, units[i]), ' - ', max(1, units[i])+1)
for u in range(min(2, units[i]), max(2, units[i]) + 1, 2):
vnode.add_op(Dense(u, tf.nn.relu))
latente_space = vnode
else:
# print(min(units[i], units[i+d]), ' - ', max(units[i], units[i+d])+1)
for u in range(min(units[i], units[i + d]),
max(units[i], units[i + d]) + 1, 2):
vnode.add_op(Dense(u, tf.nn.relu))
struct.connect(prev_node, vnode)
prev_node = vnode
out2 = ConstantNode(op=Dense(100, name="output_1"))
struct.connect(prev_node, out2)
# regressor
prev_node = latente_space
# prev_node = inp
for _ in range(num_layers):
vnode = VariableNode()
for i in range(16, 129, 16):
vnode.add_op(Dense(i, tf.nn.relu))
struct.connect(prev_node, vnode)
prev_node = vnode
out1 = ConstantNode(op=Dense(1, name="output_0"))
struct.connect(prev_node, out1)
return struct
def create_search_space(input_shape=(
8,
5,
),
output_shape=(
8,
5,
),
num_layers=10,
*args,
**kwargs):
arch = KSearchSpace(input_shape, output_shape, regression=True)
source = prev_input = arch.input_nodes[0]
# look over skip connections within a range of the 2 previous nodes
anchor_points = collections.deque([source], maxlen=2)
for _ in range(num_layers):
vnode = VariableNode()
add_lstm_seq_(vnode)
arch.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation='relu'))
# cmerge.set_op(Concatenate(arch, [cell_output]))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
# ! for next iter
prev_input = cmerge
anchor_points.append(prev_input)
# prev_input = cell_output
cnode = ConstantNode()
add_lstm_oplayer_(cnode, 5)
arch.connect(prev_input, cnode)
return arch
def test_create_multiple_inputs_with_one_vnode(self):
from deephyper.search.nas.model.space import KSearchSpace
from deephyper.search.nas.model.space.node import VariableNode, ConstantNode
from deephyper.search.nas.model.space.op.op1d import Dense
from deephyper.search.nas.model.space.op.merge import Concatenate
struct = KSearchSpace([(5, ), (5, )], (1, ))
merge = ConstantNode()
merge.set_op(Concatenate(struct, struct.input_nodes))
vnode1 = VariableNode()
struct.connect(merge, vnode1)
vnode1.add_op(Dense(1))
struct.set_ops([0])
struct.create_model()
def create_mlp_block(cell, input_node):
# first node of block
n1 = ConstantNode(op=Dense(1000, tf.nn.relu), name='N1')
cell.graph.add_edge(input_node, n1) # fixed input of current block
# second node of block
n2 = ConstantNode(op=Dense(1000, tf.nn.relu), name='N2')
n3 = ConstantNode(op=Dense(1000, tf.nn.relu), name='N3')
block = Block()
block.add_node(n1)
block.add_node(n2)
block.add_node(n3)
block.add_edge(n1, n2)
block.add_edge(n2, n3)
return block
def create_structure(input_shape=(2, ), output_shape=(1, ), *args, **kwargs):
struct = AutoOutputStructure(input_shape, output_shape, regression=False)
n1 = VariableNode('N')
add_conv_op_(n1)
struct.connect(struct.input_nodes[0], n1)
n2 = VariableNode('N')
add_activation_op_(n2)
struct.connect(n1, n2)
n3 = VariableNode('N')
add_pooling_op_(n3)
struct.connect(n2, n3)
n4 = VariableNode('N')
add_conv_op_(n4)
struct.connect(n3, n4)
n5 = VariableNode('N')
add_activation_op_(n5)
struct.connect(n4, n5)
n6 = VariableNode('N')
add_pooling_op_(n6)
struct.connect(n5, n6)
n7 = ConstantNode(op=Flatten(), name='N')
struct.connect(n6, n7)
n8 = VariableNode('N')
add_dense_op_(n8)
struct.connect(n7, n8)
n9 = VariableNode('N')
add_activation_op_(n9)
struct.connect(n8, n9)
n10 = VariableNode('N')
add_dropout_op_(n10)
struct.connect(n9, n10)
n11 = VariableNode('N')
add_dense_op_(n11)
struct.connect(n10, n11)
n12 = VariableNode('N')
add_activation_op_(n12)
struct.connect(n11, n12)
n13 = VariableNode('N')
add_dropout_op_(n13)
struct.connect(n12, n13)
return struct
def create_search_space(
input_shape=[(10,), (10,)], output_shape=(10,), num_layers=10, *args, **kwargs
):
ss = KSearchSpace(input_shape, output_shape)
input1 = ss.input_nodes[0]
input2 = ss.input_nodes[1]
output_nodes = []
for
source = prev_input = arch.input_nodes[0]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
for _ in range(num_layers):
vnode = VariableNode()
add_dense_to_(vnode)
arch.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
# ! for next iter
prev_input = cmerge
anchor_points.append(prev_input)
return arch
def set_ops(self, indexes):
"""
Set the operations for each node of each cell of the structure.
Args:
indexes (list): element of list can be float in [0, 1] or int.
output_node (ConstantNode): the output node of the Structure.
"""
cursor = 0
for c in self.struct:
num_nodes = c.num_nodes
c.set_ops(indexes[cursor:cursor + num_nodes])
cursor += num_nodes
self.graph.add_nodes_from(c.graph.nodes())
self.graph.add_edges_from(c.graph.edges())
output_nodes = get_output_nodes(self.graph)
if len(output_nodes) == 1:
node = ConstantNode(op=Identity(), name='Structure_Output')
self.graph.add_node(node)
self.graph.add_edge(output_nodes[0], node)
else:
node = ConstantNode(name='Structure_Output')
node.set_op(self.output_op(self.graph, node, output_nodes))
self.output_node = node
def create_structure(input_shape=[(2,), (2,), (2,)], output_shape=(1,),
num_cell=8, *args, **kwargs):
# , output_op=AddByPadding)
network = KerasStructure(input_shape, output_shape)
input_nodes = network.input_nodes
# CELL 1
cell1 = create_cell_1(input_nodes)
network.add_cell(cell1)
# CELL Middle
inputs_skipco = [input_nodes, input_nodes[0],
input_nodes[1], input_nodes[2], cell1.output]
pred_cell = cell1
n = num_cell
for i in range(n):
cell_i = Cell(input_nodes + [cell1.output])
block1 = create_mlp_block(cell_i, pred_cell.output)
cell_i.add_block(block1)
cnode = VariableNode(name='SkipCo')
nullNode = ConstantNode(op=Tensor([]), name='None')
cnode.add_op(Connect(cell_i.graph, nullNode, cnode)) # SAME
for inpt in inputs_skipco:
cnode.add_op(Connect(cell_i.graph, inpt, cnode))
block2 = Block()
block2.add_node(cnode)
cell_i.add_block(block2)
# set_cell_output_add(cell2)
cell_i.set_outputs()
network.add_cell(cell_i)
# prep. for next iter
inputs_skipco.append(cell_i.output)
pred_cell = cell_i
# CELL LAST
cell_last = Cell([pred_cell.output])
block1 = create_mlp_block(cell_last, pred_cell.output)
cell_last.add_block(block1)
# set_cell_output_add(cell3)
cell_last.set_outputs()
network.add_cell(cell_last)
return network
def __init__(self,
input_shape,
output_shape,
output_op=None,
*args,
**kwargs):
self.graph = nx.DiGraph()
if type(input_shape) is tuple:
# we have only one input tensor here
op = Tensor(keras.layers.Input(input_shape, name="input_0"))
self.input_nodes = [ConstantNode(op=op, name='Input_0')]
elif type(input_shape) is list and all(
map(lambda x: type(x) is tuple, input_shape)):
# we have a list of input tensors here
self.input_nodes = list()
for i in range(len(input_shape)):
op = Tensor(
keras.layers.Input(input_shape[i], name=f"input_{i}"))
inode = ConstantNode(op=op, name=f'Input_{i}')
self.input_nodes.append(inode)
else:
raise RuntimeError(
f"input_shape must be either of type 'tuple' or 'list(tuple)' but is of type '{type(input_shape)}'!"
)
self.__output_shape = output_shape
self.output_node = None
self.output_op = Concatenate if output_op is None else output_op
self.struct = []
self.map_sh2int = {}
self._model = None
def create_structure(input_shape=[(2, ), (2, ), (2, )],
output_shape=(1, ),
*args,
**kwargs):
# , output_op=AddByPadding)
network = KerasStructure(input_shape, output_shape)
input_nodes = network.input_nodes
# CELL 1
cell1 = create_cell_1(input_nodes)
network.add_cell(cell1)
# CELL 2
cell2 = Cell(input_nodes + [cell1.output])
block1 = create_mlp_block(cell2, cell1.output)
cell2.add_block(block1)
cnode = VariableNode(name='SkipCo')
nullNode = ConstantNode(op=Tensor([]), name='None')
cnode.add_op(Connect(cell2.graph, nullNode, cnode)) # SAME
cnode.add_op(Connect(cell2.graph, input_nodes[0], cnode))
cnode.add_op(Connect(cell2.graph, input_nodes[1], cnode))
cnode.add_op(Connect(cell2.graph, input_nodes[2], cnode))
cnode.add_op(Connect(cell2.graph, cell1.output, cnode))
cnode.add_op(Connect(cell2.graph, input_nodes, cnode))
cnode.add_op(Connect(cell2.graph, [input_nodes[0], input_nodes[1]], cnode))
cnode.add_op(Connect(cell2.graph, [input_nodes[1], input_nodes[2]], cnode))
cnode.add_op(Connect(cell2.graph, [input_nodes[0], input_nodes[2]], cnode))
block2 = Block()
block2.add_node(cnode)
cell2.add_block(block2)
# set_cell_output_add(cell2)
cell2.set_outputs()
network.add_cell(cell2)
# CELL 3
cell3 = Cell([cell2.output])
block1 = create_mlp_block(cell3, cell2.output)
cell3.add_block(block1)
# set_cell_output_add(cell3)
cell3.set_outputs()
network.add_cell(cell3)
return network
def create_structure(input_shape=[(1, ), (942, ), (5270, ), (2048, )], output_shape=(1,), num_cells=2, *args, **kwargs):
struct = AutoOutputStructure(input_shape, output_shape, regression=True)
input_nodes = struct.input_nodes
output_submodels = [input_nodes[0]]
for i in range(1, 4):
vnode1 = VariableNode('N1')
add_mlp_op_(vnode1)
struct.connect(input_nodes[i], vnode1)
vnode2 = VariableNode('N2')
add_mlp_op_(vnode2)
struct.connect(vnode1, vnode2)
vnode3 = VariableNode('N3')
add_mlp_op_(vnode3)
struct.connect(vnode2, vnode3)
output_submodels.append(vnode3)
merge1 = ConstantNode(name='Merge')
merge1.set_op(Concatenate(struct, merge1, output_submodels))
vnode4 = VariableNode('N4')
add_mlp_op_(vnode4)
struct.connect(merge1, vnode4)
prev = vnode4
for i in range(num_cells):
vnode = VariableNode(f'N{i+1}')
add_mlp_op_(vnode)
struct.connect(prev, vnode)
merge = ConstantNode(name='Merge')
merge.set_op(AddByPadding(struct, merge, [vnode, prev]))
prev = merge
return struct
def create_search_space(
input_shape=(2, ), output_shape=(3, ), *args, **kwargs):
ss = KSearchSpace(input_shape, output_shape)
x = ss.input_nodes[0]
out_xor = ConstantNode(op=Dense(1), name="XOR")
ss.connect(x, out_xor)
out_and = ConstantNode(op=Dense(1), name="AND")
ss.connect(x, out_and)
out_or = ConstantNode(op=Dense(1), name="OR")
ss.connect(x, out_or)
out = ConstantNode(name="OUT")
out.set_op(Concatenate(ss, stacked_nodes=[out_xor, out_and, out_or]))
return ss
def create_search_space(input_shape=(2,), output_shape=(3,), *args, **kwargs):
ss = KSearchSpace(input_shape, output_shape)
x = ss.input_nodes[0]
# z = ConstantNode(op=Dense(4, activation="relu"), name="Z")
# ss.connect(x, z)
# x = z
out_xor = ConstantNode(op=Dense(1, activation="sigmoid"), name="out_XOR")
out_and = ConstantNode(op=Dense(1, activation="sigmoid"), name="out_AND")
out_or = ConstantNode(op=Dense(1, activation="sigmoid"), name="out_OR")
in_xor = VariableNode(name="in_XOR")
in_xor.add_op(Concatenate(ss, [x])) # 0
in_xor.add_op(Concatenate(ss, [x, out_and])) # 1
in_xor.add_op(Concatenate(ss, [x, out_or])) # 2
in_xor.add_op(Concatenate(ss, [x, out_and, out_or])) # 3
ss.connect(in_xor, out_xor)
in_and = VariableNode(name="in_AND")
in_and.add_op(Concatenate(ss, [x]))
in_and.add_op(Concatenate(ss, [x, out_xor]))
in_and.add_op(Concatenate(ss, [x, out_or]))
in_and.add_op(Concatenate(ss, [x, out_xor, out_or]))
ss.connect(in_and, out_and)
in_or = VariableNode(name="in_OR")
in_or.add_op(Concatenate(ss, [x]))
in_or.add_op(Concatenate(ss, [x, out_xor]))
in_or.add_op(Concatenate(ss, [x, out_and]))
in_or.add_op(Concatenate(ss, [x, out_xor, out_and]))
ss.connect(in_or, out_or)
out = ConstantNode(name="OUT")
out.set_op(Concatenate(ss, stacked_nodes=[out_xor, out_and, out_or]))
return ss
def set_output_node(self, graph, output_nodes):
"""Set the output node of the search_space.
Args:
graph (nx.DiGraph): graph of the search_space.
output_nodes (Node): nodes of the current search_space without successors.
Returns:
Node: output node of the search_space.
"""
if len(output_nodes) == 1:
node = ConstantNode(op=Identity(), name='Structure_Output')
graph.add_node(node)
graph.add_edge(output_nodes[0], node)
else:
node = ConstantNode(name='Structure_Output')
op = Concatenate(self, output_nodes)
node.set_op(op=op)
return node
def create_cell_1(input_nodes):
"""Create a cell with convolution.
Args:
input_nodes (list(Node)): a list of input_nodes for this cell.
Returns:
Cell: the corresponding cell.
"""
cell = Cell(input_nodes)
def create_block(input_node):
def add_mlp_ops_to(vnode):
# REG_L1 = 1.
# REG_L2 = 1.
vnode.add_op(Identity())
vnode.add_op(Dense(100, tf.nn.relu))
vnode.add_op(Dense(100, tf.nn.tanh))
vnode.add_op(Dense(100, tf.nn.sigmoid))
vnode.add_op(Dropout(0.05))
vnode.add_op(Dense(500, tf.nn.relu))
vnode.add_op(Dense(500, tf.nn.tanh))
vnode.add_op(Dense(500, tf.nn.sigmoid))
vnode.add_op(Dropout(0.1))
vnode.add_op(Dense(1000, tf.nn.relu))
vnode.add_op(Dense(1000, tf.nn.tanh))
vnode.add_op(Dense(1000, tf.nn.sigmoid))
vnode.add_op(Dropout(0.2))
# first node of block
n1 = VariableNode('N1')
add_mlp_ops_to(n1)
cell.graph.add_edge(input_node, n1) # fixed input of current block
# second node of block
n2 = VariableNode('N2')
add_mlp_ops_to(n2)
# third node of the block
n3 = VariableNode('N3')
add_mlp_ops_to(n3)
block = Block()
block.add_node(n1)
block.add_node(n2)
block.add_node(n3)
block.add_edge(n1, n2)
block.add_edge(n2, n3)
return block, (n1, n2, n3)
block1, _ = create_block(input_nodes[0])
block2, (vn1, vn2, vn3) = create_block(input_nodes[1])
# first node of block
m_vn1 = MirrorNode(node=vn1)
cell.graph.add_edge(input_nodes[2], m_vn1) # fixed input of current block
# second node of block
m_vn2 = MirrorNode(node=vn2)
# third node of the block
m_vn3 = MirrorNode(node=vn3)
block3 = Block()
block3.add_node(m_vn1)
block3.add_node(m_vn2)
block3.add_node(m_vn3)
block3.add_edge(m_vn1, m_vn2)
block3.add_edge(m_vn2, m_vn3)
cell.add_block(block1)
cell.add_block(block2)
cell.add_block(block3)
addNode = ConstantNode(name='Merging')
addNode.set_op(AddByPadding(cell.graph, addNode, cell.get_blocks_output()))
cell.set_outputs(node=addNode)
return cell
def create_search_space(
input_shape=(2, ), output_shape=(5, ), *args, **kwargs):
ss = KSearchSpace(input_shape, output_shape)
x = ss.input_nodes[0]
nunits = 10
hid_2 = ConstantNode(op=Dense(nunits, "relu"), name="hid_2")
out_2 = ConstantNode(op=Dense(1), name="out_2")
ss.connect(hid_2, out_2)
hid_3 = ConstantNode(op=Dense(nunits, "relu"), name="hid_3")
out_3 = ConstantNode(op=Dense(1), name="out_3")
ss.connect(hid_3, out_3)
hid_4 = ConstantNode(op=Dense(nunits, "relu"), name="hid_4")
out_4 = ConstantNode(op=Dense(1), name="out_4")
ss.connect(hid_4, out_4)
hid_5 = ConstantNode(op=Dense(nunits, "relu"), name="hid_5")
out_5 = ConstantNode(op=Dense(1), name="out_5")
ss.connect(hid_5, out_5)
hid_6 = ConstantNode(op=Dense(nunits, "relu"), name="hid_6")
out_6 = ConstantNode(op=Dense(1), name="out_6")
ss.connect(hid_6, out_6)
# L1 DEPENDENT ON DATA OVERALL
in_2 = VariableNode(name="in_2")
in_2.add_op(Concatenate(ss, [x]))
in_2.add_op(Concatenate(ss, [x, out_3]))
in_2.add_op(Concatenate(ss, [x, out_4]))
in_2.add_op(Concatenate(ss, [x, out_5]))
in_2.add_op(Concatenate(ss, [x, out_6]))
ss.connect(in_2, hid_2)
# L2 DEPENDANT ON DATA, L1 AND L3
in_3 = VariableNode(name="in_3")
in_3.add_op(Concatenate(ss, [x, out_2]))
in_3.add_op(Concatenate(ss, [x, out_4]))
ss.connect(in_3, hid_3)
# L3 DEPENDANT ON DATA, L1 L2 AND WIDTH
in_4 = VariableNode(name="in_4")
in_4.add_op(Concatenate(ss, [x]))
in_4.add_op(Concatenate(ss, [x, out_2]))
in_4.add_op(Concatenate(ss, [x, out_3]))
in_4.add_op(Concatenate(ss, [x, out_5]))
in_4.add_op(Concatenate(ss, [x, out_6]))
in_4.add_op(Concatenate(ss, [x, out_2, out_3]))
in_4.add_op(Concatenate(ss, [x, out_2, out_5]))
in_4.add_op(Concatenate(ss, [x, out_2, out_6]))
in_4.add_op(Concatenate(ss, [x, out_3, out_5]))
in_4.add_op(Concatenate(ss, [x, out_3, out_6]))
in_4.add_op(Concatenate(ss, [x, out_5, out_6]))
in_4.add_op(Concatenate(ss, [x, out_3, out_5, out_6]))
in_4.add_op(Concatenate(ss, [x, out_2, out_5, out_6]))
in_4.add_op(Concatenate(ss, [x, out_2, out_3, out_6]))
in_4.add_op(Concatenate(ss, [x, out_2, out_3, out_5]))
in_4.add_op(Concatenate(ss, [x, out_2, out_3, out_5, out_6]))
ss.connect(in_4, hid_4)
# HEIGHT DEPENDANT ON ALL MEASURES COMBINED AND DATA
in_5 = VariableNode(name="in_5")
in_5.add_op(Concatenate(ss, [x]))
in_5.add_op(Concatenate(ss, [x, out_2]))
in_5.add_op(Concatenate(ss, [x, out_3]))
in_5.add_op(Concatenate(ss, [x, out_4]))
in_5.add_op(Concatenate(ss, [x, out_6]))
in_5.add_op(Concatenate(ss, [x, out_2, out_3]))
in_5.add_op(Concatenate(ss, [x, out_2, out_4]))
in_5.add_op(Concatenate(ss, [x, out_2, out_6]))
in_5.add_op(Concatenate(ss, [x, out_3, out_4]))
in_5.add_op(Concatenate(ss, [x, out_3, out_6]))
in_5.add_op(Concatenate(ss, [x, out_4, out_6]))
in_5.add_op(Concatenate(ss, [x, out_3, out_4, out_6]))
in_5.add_op(Concatenate(ss, [x, out_2, out_4, out_6]))
in_5.add_op(Concatenate(ss, [x, out_2, out_3, out_6]))
in_5.add_op(Concatenate(ss, [x, out_2, out_3, out_4]))
in_5.add_op(Concatenate(ss, [x, out_2, out_3, out_4, out_6]))
ss.connect(in_5, hid_5)
# WIDTH DEPENDANT ON DATA AND CROSS SECTION
in_6 = VariableNode(name="in_6")
in_6.add_op(Concatenate(ss, [x]))
in_6.add_op(Concatenate(ss, [x, out_2]))
in_6.add_op(Concatenate(ss, [x, out_3]))
in_6.add_op(Concatenate(ss, [x, out_4]))
in_6.add_op(Concatenate(ss, [x, out_5]))
ss.connect(in_6, hid_6)
out = ConstantNode(name="OUT")
out.set_op(
Concatenate(ss, stacked_nodes=[out_2, out_3, out_4, out_5, out_6]))
return ss
def create_search_space(input_shape=None,
output_shape=None,
num_mpnn_cells=3,
num_dense_layers=2,
**kwargs):
"""Create a search space containing multiple Keras architectures
Args:
input_shape (list): the input shapes, e.g. [(3, 4), (5, 2)].
output_shape (tuple): the output shape, e.g. (12, ).
num_mpnn_cells (int): the number of MPNN cells.
num_dense_layers (int): the number of Dense layers.
Returns:
A search space containing multiple Keras architectures
"""
data = kwargs['data']
if data == 'qm7':
input_shape = [(8 + 1, 75), (8 + 1 + 10 + 1, 2), (8 + 1 + 10 + 1, 14),
(8 + 1, ), (8 + 1 + 10 + 1, )]
output_shape = (1, )
elif data == 'qm8':
input_shape = [(9 + 1, 75), (9 + 1 + 14 + 1, 2), (9 + 1 + 14 + 1, 14),
(9 + 1, ), (9 + 1 + 14 + 1, )]
output_shape = (16, )
elif data == 'qm9':
input_shape = [(9 + 1, 75), (9 + 1 + 16 + 1, 2), (9 + 1 + 16 + 1, 14),
(9 + 1, ), (9 + 1 + 16 + 1, )]
output_shape = (12, )
elif data == 'freesolv':
input_shape = [(24 + 1, 75), (24 + 1 + 25 + 1, 2),
(24 + 1 + 25 + 1, 14), (24 + 1, ), (24 + 1 + 25 + 1, )]
output_shape = (1, )
elif data == 'esol':
input_shape = [(55 + 1, 75), (55 + 1 + 68 + 1, 2),
(55 + 1 + 68 + 1, 14), (55 + 1, ), (55 + 1 + 68 + 1, )]
output_shape = (1, )
elif data == 'lipo':
input_shape = [(115 + 1, 75), (115 + 1 + 236 + 1, 2),
(115 + 1 + 236 + 1, 14), (115 + 1, ),
(115 + 1 + 236 + 1, )]
output_shape = (1, )
arch = KSearchSpace(input_shape, output_shape, regression=True)
source = prev_input = arch.input_nodes[0]
prev_input1 = arch.input_nodes[1]
prev_input2 = arch.input_nodes[2]
prev_input3 = arch.input_nodes[3]
prev_input4 = arch.input_nodes[4]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
count_gcn_layers = 0
count_dense_layers = 0
for _ in range(num_mpnn_cells):
graph_attn_cell = VariableNode()
mpnn_cell(graph_attn_cell) #
arch.connect(prev_input, graph_attn_cell)
arch.connect(prev_input1, graph_attn_cell)
arch.connect(prev_input2, graph_attn_cell)
arch.connect(prev_input3, graph_attn_cell)
arch.connect(prev_input4, graph_attn_cell)
cell_output = graph_attn_cell
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
prev_input = cmerge
anchor_points.append(prev_input)
count_gcn_layers += 1
global_pooling_node = VariableNode()
gather_cell(global_pooling_node)
arch.connect(prev_input, global_pooling_node)
prev_input = global_pooling_node
flatten_node = ConstantNode()
flatten_node.set_op(Flatten())
arch.connect(prev_input, flatten_node)
prev_input = flatten_node
for _ in range(num_dense_layers):
dense_node = ConstantNode()
dense_node.set_op(Dense(32, activation='relu'))
arch.connect(prev_input, dense_node)
prev_input = dense_node
count_dense_layers += 1
output_node = ConstantNode()
output_node.set_op(Dense(output_shape[0], activation='linear'))
arch.connect(prev_input, output_node)
return arch
def create_structure(input_shape=(2, ), output_shape=(1, ), *args, **kwargs):
struct = AutoOutputStructure(input_shape, output_shape, regression=False)
n1 = ConstantNode(op=Conv1D(filter_size=20, num_filters=128), name='N')
struct.connect(struct.input_nodes[0], n1)
n2 = ConstantNode(op=Activation(activation='relu'), name='N')
struct.connect(n1, n2)
n3 = ConstantNode(op=MaxPooling1D(pool_size=1, padding='same'), name='N')
struct.connect(n2, n3)
n4 = ConstantNode(op=Conv1D(filter_size=10, num_filters=128), name='N')
struct.connect(n3, n4)
n5 = ConstantNode(op=Activation(activation='relu'), name='N')
struct.connect(n4, n5)
n6 = ConstantNode(op=MaxPooling1D(pool_size=10, padding='same'), name='N')
struct.connect(n5, n6)
n7 = ConstantNode(op=Flatten(), name='N')
struct.connect(n6, n7)
n8 = ConstantNode(op=Dense(units=200), name='N')
struct.connect(n7, n8)
n9 = ConstantNode(op=Activation(activation='relu'), name='N')
struct.connect(n8, n9)
n10 = ConstantNode(op=Dropout(rate=0.1), name='N')
struct.connect(n9, n10)
n11 = ConstantNode(op=Dense(units=20), name='N')
struct.connect(n10, n11)
n12 = ConstantNode(op=Activation(activation='relu'), name='N')
struct.connect(n11, n12)
n13 = ConstantNode(op=Dropout(rate=0.1), name='N')
struct.connect(n12, n13)
return struct
def create_search_space(input_shape = None,
output_shape = None,
num_mpnn_layers=3,
num_dense_layers=2,
**kwargs):
"""
A function to create keras search sapce
Args:
input_shape: list of tuples
output_shape: a tuple
num_mpnn_layers: int, number of graph massage passing neural network layers
num_dense_layers: int, number of dense layers
**kwargs:
data: str, the dataset name
Returns:
arch: keras architecture
"""
data = kwargs['data']
if data == 'qm7':
input_shape = [(9+1, 75), (9+1+16+1, 2), (9+1+16+1, 14), (9+1, ), (9+1+16+1, )]
output_shape = (1, )
arch = KSearchSpace(input_shape, output_shape, regression=True)
source = prev_input = arch.input_nodes[0]
prev_input1 = arch.input_nodes[1]
prev_input2 = arch.input_nodes[2]
prev_input3 = arch.input_nodes[3]
prev_input4 = arch.input_nodes[4]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
count_mpnn_layers = 0
count_dense_layers = 0
for _ in range(num_mpnn_layers):
graph_attn_cell = VariableNode()
add_mpnn_to_(graph_attn_cell) #
arch.connect(prev_input, graph_attn_cell)
arch.connect(prev_input1, graph_attn_cell)
arch.connect(prev_input2, graph_attn_cell)
arch.connect(prev_input3, graph_attn_cell)
arch.connect(prev_input4, graph_attn_cell)
cell_output = graph_attn_cell
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
prev_input = cmerge
anchor_points.append(prev_input)
count_mpnn_layers += 1
global_pooling_node = VariableNode()
add_global_pooling_to_(global_pooling_node)
arch.connect(prev_input, global_pooling_node) # result from graph conv (?, 23, ?) --> Global pooling (?, 23)
prev_input = global_pooling_node
flatten_node = ConstantNode()
flatten_node.set_op(Flatten())
arch.connect(prev_input, flatten_node) # result from graph conv (?, 23) --> Flatten
prev_input = flatten_node
for _ in range(num_dense_layers):
dense_node = ConstantNode()
dense_node.set_op(Dense(32, activation='relu'))
arch.connect(prev_input, dense_node)
prev_input = dense_node
count_dense_layers += 1
output_node = ConstantNode()
output_node.set_op(Dense(output_shape[0], activation='linear'))
arch.connect(prev_input, output_node)
return arch
def create_structure(input_shape=[(1, ), (942, ), (5270, ), (2048, )],
output_shape=(1, ),
num_cells=2,
*args,
**kwargs):
struct = AutoOutputStructure(input_shape, output_shape, regression=True)
input_nodes = struct.input_nodes
output_submodels = [input_nodes[0]]
for i in range(1, 4):
cnode1 = ConstantNode(name='N', op=Dense(1000, tf.nn.relu))
struct.connect(input_nodes[i], cnode1)
cnode2 = ConstantNode(name='N', op=Dense(1000, tf.nn.relu))
struct.connect(cnode1, cnode2)
cnode3 = ConstantNode(name='N', op=Dense(1000, tf.nn.relu))
struct.connect(cnode2, cnode3)
output_submodels.append(cnode3)
merge1 = ConstantNode(name='Merge')
merge1.set_op(Concatenate(struct, merge1, output_submodels))
cnode4 = ConstantNode(name='N', op=Dense(1000, tf.nn.relu))
struct.connect(merge1, cnode4)
prev = cnode4
for i in range(num_cells):
cnode = ConstantNode(name='N', op=Dense(1000, tf.nn.relu))
struct.connect(prev, cnode)
merge = ConstantNode(name='Merge')
merge.set_op(AddByPadding(struct, merge, [cnode, prev]))
prev = merge
return struct