def create_search_space(
input_shape=(10,), output_shape=(7,), num_layers=10, *args, **kwargs
):
arch = AutoKSearchSpace(input_shape, output_shape, regression=True)
source = prev_input = arch.input_nodes[0]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
for _ in range(num_layers):
vnode = VariableNode()
add_dense_to_(vnode)
arch.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(arch, [cell_output], activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Tensor([]))
skipco.add_op(Connect(arch, anchor))
arch.connect(skipco, cmerge)
# ! for next iter
prev_input = cmerge
anchor_points.append(prev_input)
return arch
def test_create_more_nodes(self):
from deephyper.nas.space import AutoKSearchSpace
from deephyper.nas.space.node import VariableNode
from deephyper.nas.space.op.op1d import Dense
struct = AutoKSearchSpace((5, ), (1, ), regression=True)
vnode1 = VariableNode()
struct.connect(struct.input_nodes[0], vnode1)
vnode1.add_op(Dense(10))
vnode2 = VariableNode()
vnode2.add_op(Dense(10))
struct.connect(vnode1, vnode2)
struct.set_ops([0, 0])
falias = "test_auto_keras_search_spaceure"
struct.draw_graphviz(f"{falias}.dot")
model = struct.create_model()
from tensorflow.keras.utils import plot_model
plot_model(model, to_file=f"{falias}.png", show_shapes=True)
def build(
self,
input_shape,
output_shape,
regression=True,
num_units=(1, 11),
num_layers=10,
**kwargs
):
"""
Args:
input_shape (tuple, optional): True shape of inputs (no batch size dimension). Defaults to (2,).
output_shape (tuple, optional): True shape of outputs (no batch size dimension).. Defaults to (1,).
num_layers (int, optional): Maximum number of layers to have. Defaults to 10.
num_units (tuple, optional): Range of number of units such as range(start, end, step_size). Defaults to (1, 11).
regression (bool, optional): A boolean defining if the model is a regressor or a classifier. Defaults to True.
Returns:
AutoKSearchSpace: A search space object based on tf.keras implementations.
"""
ss = AutoKSearchSpace(input_shape, output_shape, regression=regression)
prev_node = ss.input_nodes[0]
for _ in range(num_layers):
vnode = VariableNode()
vnode.add_op(Identity())
for i in range(*num_units):
vnode.add_op(Dense(i, tf.nn.relu))
ss.connect(prev_node, vnode)
prev_node = vnode
return ss
def create_search_space(
input_shape=(32, 32, 3),
output_shape=(10, ),
num_filters=8,
num_blocks=4,
normal_cells=2,
reduction_cells=1,
repetitions=3,
*args,
**kwargs,
):
ss = AutoKSearchSpace(input_shape, output_shape, regression=False)
source = prev_input = ss.input_nodes[0]
# look over skip connections within a range of the 3 previous nodes
hidden_states = collections.deque([source, source], maxlen=2)
for ri in range(repetitions):
for nci in range(normal_cells):
# generate a normal cell
cout = generate_cell(
ss,
hidden_states,
num_blocks,
strides=1,
mime=ri + nci > 0,
num_filters=num_filters,
)
hidden_states.append(cout)
if ri < repetitions - 1: # we don't want the last cell to be a reduction cell
for rci in range(reduction_cells):
# generate a reduction cell
cout = generate_cell(
ss,
hidden_states,
num_blocks,
strides=2,
mime=ri + rci > 0,
num_filters=num_filters,
)
hidden_states.append(cout)
# out_node = ConstantNode(op=Dense(100, activation=tf.nn.relu))
out_dense = VariableNode()
out_dense.add_op(Identity())
for units in [10, 20, 50, 100, 200, 500, 1000]:
out_dense.add_op(Dense(units, activation=tf.nn.relu))
ss.connect(cout, out_dense)
out_dropout = VariableNode()
out_dropout.add_op(Identity())
for drop_rate in [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.8]:
out_dropout.add_op(Dropout(rate=drop_rate))
ss.connect(out_dense, out_dropout)
return ss
def test_create_search_space(input_shape=(2, ), output_shape=(1, ), **kwargs):
struct = AutoKSearchSpace(input_shape, output_shape, regression=True)
vnode1 = VariableNode()
for _ in range(1, 11):
vnode1.add_op(Operation(layer=tf.keras.layers.Dense(10)))
struct.connect(struct.input_nodes[0], vnode1)
struct.set_ops([0])
struct.create_model()
def build(
self,
input_shape,
output_shape,
regression=True,
num_layers=10,
dropout=0.0,
**kwargs,
):
ss = AutoKSearchSpace(input_shape, output_shape, regression=regression)
source = prev_input = ss.input_nodes[0]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
for _ in range(num_layers):
vnode = VariableNode()
self.add_dense_to_(vnode)
ss.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(AddByProjecting(ss, [cell_output],
activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Zero())
skipco.add_op(Connect(ss, anchor))
ss.connect(skipco, cmerge)
prev_input = cmerge
# ! for next iter
anchor_points.append(prev_input)
if dropout >= 0.0:
dropout_node = ConstantNode(op=Dropout(rate=dropout))
ss.connect(prev_input, dropout_node)
return ss
def build(self, input_shape, output_shape, regression=True, **kwargs):
ss = AutoKSearchSpace(input_shape, output_shape, regression=regression)
if type(input_shape) is list:
vnodes = []
for i in range(len(input_shape)):
vn = self.gen_vnode()
vnodes.append(vn)
ss.connect(ss.input_nodes[i], vn)
cn = ConstantNode()
cn.set_op(Concatenate(ss, vnodes))
vn = self.gen_vnode()
ss.connect(cn, vn)
else:
vnode1 = self.gen_vnode()
ss.connect(ss.input_nodes[0], vnode1)
return ss
class MultiInputsDenseSkipCoFactory(SpaceFactory):
def build(
self,
input_shape,
output_shape,
regression=True,
num_layers=10,
**kwargs,
):
self.ss = AutoKSearchSpace(input_shape,
output_shape,
regression=regression)
sub_graphs_outputs = []
for input_ in self.ss.input_nodes:
output_sub_graph = self.build_sub_graph(input_)
sub_graphs_outputs.append(output_sub_graph)
cmerge = ConstantNode()
cmerge.set_op(Concatenate(self.ss, sub_graphs_outputs))
output_sub_graph = self.build_sub_graph(cmerge)
return self.ss
def build_sub_graph(self, input_, num_layers=3):
source = prev_input = input_
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
for _ in range(num_layers):
vnode = VariableNode()
self.add_dense_to_(vnode)
self.ss.connect(prev_input, vnode)
# * Cell output
cell_output = vnode
cmerge = ConstantNode()
cmerge.set_op(
AddByProjecting(self.ss, [cell_output], activation="relu"))
for anchor in anchor_points:
skipco = VariableNode()
skipco.add_op(Zero())
skipco.add_op(Connect(self.ss, anchor))
self.ss.connect(skipco, cmerge)
prev_input = cmerge
# ! for next iter
anchor_points.append(prev_input)
return prev_input
def add_dense_to_(self, node):
node.add_op(
Identity()) # we do not want to create a layer in this case
activations = [
None, tf.nn.swish, tf.nn.relu, tf.nn.tanh, tf.nn.sigmoid
]
for units in range(50, 2000, 25):
for activation in activations:
node.add_op(Dense(units=units, activation=activation))
class MultiInputsDenseSkipCoFactory(SpaceFactory):
def build(
self,
input_shape,
output_shape,
regression=True,
num_layers=10,
**kwargs,
):
self.ss = AutoKSearchSpace(input_shape, output_shape, regression=regression)
self.shapes_to_vnodes = {}
sub_graphs_outputs = []
for input_ in self.ss.input_nodes:
output_sub_graph = self.build_sub_graph(input_)
sub_graphs_outputs.append(output_sub_graph)
cmerge = ConstantNode()
cmerge.set_op(Concatenate(self.ss, sub_graphs_outputs))
output_sub_graph = self.build_sub_graph(cmerge)
return self.ss
def build_sub_graph(self, input_, num_layers=3):
source = prev_input = input_
mirror = False
is_input = False
if type(source) is ConstantNode:
if type(source._op) is Tensor:
if "input_" in source._op.tensor.name:
is_input = True
input_name = source._op.tensor.name
input_shape = tuple(source._op.tensor.shape[1:])
if self.shapes_to_vnodes.get(input_shape) is None:
self.shapes_to_vnodes[input_shape] = []
else:
mirror = True
memory = self.shapes_to_vnodes[input_shape][::-1]
# look over skip connections within a range of the 3 previous nodes
anchor_points = collections.deque([source], maxlen=3)
for layer_i in range(num_layers):
if not(mirror):
vnode = VariableNode()
self.add_dense_to_(vnode)
if is_input:
self.shapes_to_vnodes[input_shape].append(vnode)
else:
vnode = MirrorNode(memory.pop())
self.ss.connect(prev_input, vnode)
# * Cell output
prev_node = vnode
if layer_i == num_layers-1:
return prev_node
cmerge = ConstantNode()
cmerge.set_op(Concatenate(self.ss, [prev_node]))
for anchor in anchor_points:
if not(mirror):
skipco = VariableNode()
if is_input:
self.shapes_to_vnodes[input_shape].append(skipco)
else:
skipco = MimeNode(memory.pop())
skipco.add_op(Zero())
skipco.add_op(Connect(self.ss, anchor))
self.ss.connect(skipco, cmerge)
prev_input = cmerge
# ! for next iter
anchor_points.append(prev_input)
return prev_input
def add_dense_to_(self, node):
node.add_op(Identity()) # we do not want to create a layer in this case
activations = [None, tf.nn.swish, tf.nn.relu, tf.nn.tanh, tf.nn.sigmoid]
for units in range(50, 2000, 25):
for activation in activations:
node.add_op(Dense(units=units, activation=activation))