def add_arch_dense_drop(base_arch):
# type: (List[Union[str, int, Tuple[Tuple[int, int], int]]]) -> List[Union[str, int, Tuple[Tuple[int, int], int]]]
drop_idx = []
idx = 0
for l in base_arch:
if helpers_other.arch_type(l) == 'drop':
drop_idx += [idx]
elif not drop_idx and helpers_other.arch_type(l) == 'dense':
drop_idx += [idx]
idx += 1
idx = random.choice(drop_idx)
dense_params = random.choice(const.mutations.fget()['dense_size'])
drop_params = 'drop%.2f' % random.choice(const.mutations.fget()['dropout'])
new_arch = base_arch[:]
if idx >= len(new_arch):
new_arch = new_arch[:idx] + [drop_params] + new_arch[idx:]
else:
new_arch = new_arch[:idx + 1] + [drop_params] + new_arch[idx + 1:]
if idx >= len(base_arch):
new_arch = new_arch[:idx] + [dense_params] + new_arch[idx:]
else:
new_arch = new_arch[:idx + 1] + [dense_params] + new_arch[idx + 1:]
return new_arch
def remove_layer(base_net):
# type: (Network) -> Network
"""
Creates a copy of given Network, but with removed layer, at a random index.
:param base_net: Network, which copy (with removed layer) will be returned.
:return: Copy of given network, with one less layer.
"""
if len(base_net.arch) <= const.min_depth != 0:
return add_layer(base_net)
layer_idx = random.randint(1, len(base_net.arch) - 2) # so that, Conv is always first, and Dense is always last.
layer_name = base_net.arch[layer_idx]
new_arch = base_net.arch[:layer_idx] + base_net.arch[layer_idx + 1:]
layer_idx += 1 # difference between net.arch and actual architecture. - First activation layer.
if helpers_other.arch_type(layer_name) in ['dense', 'drop'] and len(const.input_shape.fget()) > 2:
layer_idx += 1 # difference between net.arch and actual architecture. - Flatten layer.
return Network(
architecture=new_arch,
copy_model=helpers_other._remove_layer(base_net.model, layer_idx),
opt=base_net.opt,
activation=base_net.act,
callbacks=base_net.callbacks
)
def _add_layer(base_net, layer_name, layer_idx):
# type: (Network, Union[int, str, Tuple[Tuple[int], int]], int) -> Network
new_arch = base_net.arch[:layer_idx] + [layer_name] + base_net.arch[layer_idx:]
layer_idx += 1 # difference between net.arch and actual architecture. - First activation layer.
if helpers_other.arch_type(layer_name) in ['dense', 'drop']:
layer_idx += 1 # difference between net.arch and actual architecture. - Flatten layer.
if const.debug:
print('')
print('_add_layer')
print('Old arch: {}'.format(base_net.arch))
print('New arch: {}'.format(new_arch))
print('')
return Network(
architecture=new_arch,
copy_model=helpers_other._insert_layer(
base_net.model,
helpers_other.arch_to_layer(layer_name, base_net.act),
layer_idx),
opt=base_net.opt,
activation=base_net.act,
callbacks=base_net.callbacks
)
def remove_dense_drop(base_net):
# type: (Network) -> Network
"""
Removes a sequence of Dense layer, followed by Dropout layer/layers, in a given Network.\n
If no such sequence is found, then it adds one (Dropout + Dense), instead of removing it.
:param base_net: A Network, which copy, with mutations, will be returned.
:para: Shape of a singular x input to the Network.
:return: A Network, based on base_net, but with a sequence of Dense layer and a Dropout layers removed.
"""
drop_idx = []
for i_l, l in enumerate(base_net.arch):
if helpers_other.arch_type(l) == 'drop':
drop_idx += [i_l]
if not drop_idx:
return add_dense_drop(base_net)
if len(drop_idx) > 1:
curr_idx = random.randint(1, len(drop_idx))
else:
curr_idx = 1
if const.debug:
print('')
print('remove_dense_drop')
print('drop_idxs: {}'.format(drop_idx))
print('')
drop_arch_idx = drop_idx[curr_idx - 1]
return __remove_dense_drop(base_net, drop_arch_idx)
def __remove_dense_drop(base_net, drop_idx):
# type: (Network, int) -> Network
idx_start = drop_idx - 1
while helpers_other.arch_type(base_net.arch[idx_start]) == 'drop':
idx_start -= 1
idx_end = drop_idx
new_arch = base_net.arch[:idx_start] + base_net.arch[idx_end + 1:]
new_net = Network(
architecture=new_arch,
opt=base_net.opt,
activation=base_net.act,
callbacks=base_net.callbacks
)
if const.debug:
print('')
print('__remove_dense_drop')
print('Base arch: {}'.format(new_arch))
print('Idx start: {}'.format(idx_start))
print('Idx end: {}'.format(idx_end))
print('Index of drop layer in arch: {}'.format(drop_idx))
print('Drop layer: {}'.format(base_net.arch[drop_idx]))
print('Layer before: {}'.format(base_net.arch[drop_idx - 1]))
print('')
dim_offset = 2 if len(const.input_shape.fget()) > 2 else 1
for i_l, l in enumerate(new_net.model.layers[:idx_start + dim_offset - 1]):
if const.deep_debug:
print('')
print('__remove_dense_drop inside for loop till idx_start')
print('\tIdx: {}'.format(i_l))
print('\tOld layer type: {}'.format(type(base_net.model.get_layer(index=i_l))))
print('\tNew layer type: {}'.format(type(l)))
print('')
l.set_weights(base_net.model.get_layer(index=i_l).get_weights())
print('Rest new: {}'.format(new_net.model.layers[idx_start + dim_offset + 2:]))
print('{}'.format(idx_start + dim_offset + 2))
print('Rest old: {}'.format(base_net.model.layers[idx_end + dim_offset + 3:]))
print('{}'.format(idx_end + dim_offset + 3))
for i_l, l in enumerate(new_net.model.layers[idx_start + dim_offset + 2:], start=idx_end + dim_offset + 3):
if const.deep_debug:
print('')
print('__remove_dense_drop inside for loop from idx_end till end')
print('\tIdx: {}'.format(i_l))
print('\tOld layer type: {}'.format(type(base_net.model.get_layer(index=i_l))))
print('\tNew layer type: {}'.format(type(l)))
print('')
l.set_weights(base_net.model.get_layer(index=i_l).get_weights())
return new_net
def add_layer(base_net):
# type: (Network) -> Network
"""
Creates a copy of given Network, but with added layer, randomly derived from given parameters.
:param base_net: Network, which copy (with added layer) will be returned.
:return: Copy of given network, with additional layer inserted in a random position.
"""
layer_idx = random.randint(0, len(base_net.arch))
possible_layers = {}
if len(base_net.arch) + 1 > const.max_depth != 0:
remove_layer(base_net)
if layer_idx == 0:
possible_layers['conv'] = random.choice(const.mutations.fget()['kernel_size']), \
random.choice(const.mutations.fget()['conv_filters'])
elif layer_idx == len(base_net.arch):
possible_layers['dense'] = random.choice(const.mutations.fget()['dense_size'])
else:
prev_layer = base_net.arch[layer_idx - 1]
prev_type = helpers_other.arch_type(prev_layer)
next_layer = base_net.arch[layer_idx]
if prev_type in ['conv', 'max']:
possible_layers['conv'] = (random.choice(const.mutations.fget()['kernel_size']),
random.choice(const.mutations.fget()['conv_filters']))
if not prev_type == 'max' and helpers_other.can_add_max_number(base_net.arch):
possible_layers['max'] = 'max'
check_if_flat = lambda x: helpers_other.arch_type(x) in ['dense', 'drop']
if check_if_flat(next_layer):
possible_layers['dense'] = random.choice(const.mutations.fget()['dense_size'])
if check_if_flat(prev_layer) and not prev_type == 'drop':
possible_layers['drop'] = 'drop' + str(random.choice(const.mutations.fget()['dropout']))
layer_name = random.choice(possible_layers.values())
return _add_layer(base_net, layer_name, layer_idx)
def add_conv_max(base_net, conv_num=None):
# type: (Network, int) -> Network
"""
Adds a sequence of Convolutional layers, followed by MaxPool layer to a copy of a given Network.
:param base_net: Network, which copy (with added sequence) will be returned.
:param conv_num: Number of convolutional layers in a sequence.
Default value is a random choice from possibilities specified in mutations under 'n_conv' key, if there is only
one kernel size possible, or default number of convolution layers otherwise.
:return: Copy of given network, with additional sequence inserted in a position of maxpool layer,
or at the beginning of the model.
"""
conv_num = conv_num or random.choice(const.mutations.fget()['n_conv']) \
if len(const.mutations.fget()['kernel_size']) == 1 else const.default_n_conv
if len(const.input_shape.fget()) < 3:
return add_dense_drop(base_net)
if not helpers_other.can_add_max_number(base_net.arch):
if const.debug:
print('')
print('add_conv_max - before calling remove_conv_max')
print('Arch: {}'.format(base_net.arch))
print('Max l limit: {}'.format(const.max_layers_limit.fget()))
print('')
return remove_conv_max(base_net)
max_idx = [0]
idx = 1
for l in base_net.arch:
if helpers_other.arch_type(l) == 'max':
max_idx += [idx]
idx += 1
if const.deep_debug:
print('')
print('add_conv_max')
print('max_idx: {}'.format(max_idx))
print('')
idx_add = random.choice(max_idx)
conv_params = (random.choice(const.mutations.fget()['kernel_size']),
random.choice(const.mutations.fget()['conv_filters']))
return __add_conv_max(base_net, idx_add, conv_num, conv_params)
def remove_conv_max(base_net):
# type: (Network) -> Network
"""
Removes a sequence of Convolution layers, followed by MaxOut layer, in a given Network.\n
If no such sequence is found, then it adds one, instead of removing it.
:param base_net: A Network, which copy, with mutations, will be returned.
:return: A Network, based on base_net, but with a sequence of Conv layers and a MaxOut layer removed.
"""
if len(const.input_shape.fget()) < 3:
return remove_dense_drop(base_net)
max_idx = []
idx = 0 # Since Activation layer is always first.
for l in base_net.arch:
if helpers_other.arch_type(l) == 'max':
max_idx += [idx]
idx += 1
if not max_idx:
return add_conv_max(base_net)
if len(max_idx) > 1:
curr_idx = random.randint(1, len(max_idx))
else:
curr_idx = 1
if const.deep_debug:
print('')
print('remove_conv_max')
print('\tmax_idx: {}'.format(max_idx))
print('\tcurr_idx: {}'.format(curr_idx - 1))
print('')
end = max_idx[curr_idx - 1]
if curr_idx == 1:
start = 0
else:
start = max_idx[curr_idx - 2] + 1
return __remove_conv_max(base_net, start, end)
def add_arch_conv_max(base_arch, # type: List[Union[str, int, Tuple[Tuple[int, int], int]]]
conv_num=None # type int
):
# type: (...) -> List[Union[str, int, Tuple[Tuple[int, int], int]]]
"""
:param base_arch: Network, which copy (with added sequence) will be returned.
:param conv_num: Number of convolutional layers in a sequence.
Default value is a random choice from possibilities specified in mutations under 'n_conv' key, if there is only
one kernel size possible, or default number of convolution layers otherwise.
:return: Copy of base_arch with conv_num number of conv layers, followed by max layer
"""
conv_num = conv_num or random.choice(const.mutations.fget()['n_conv']) \
if len(const.mutations.fget()['kernel_size']) == 1 else const.default_n_conv
if len(const.input_shape.fget()) < 3:
return add_arch_dense_drop(base_arch)
if not helpers_other.can_add_max_number(base_arch):
return add_arch_dense_drop(base_arch)
max_idx = [0]
idx = 1
for l in base_arch:
if helpers_other.arch_type(l) == 'max':
max_idx += [idx]
idx += 1
idx = random.choice(max_idx)
conv_params = (random.choice(const.mutations.fget()['kernel_size']),
random.choice(const.mutations.fget()['conv_filters']))
new_arch = base_arch[:]
new_arch = new_arch[:idx] + conv_num * [conv_params] + ['max'] + new_arch[idx:]
return new_arch
def add_dense_drop(base_net):
# type: (Network) -> Network
"""
Adds a sequence of Dense layer, followed by Dropout layer to a copy of a given Network.
:param base_net: Network, which copy (with added sequence) will be returned.
:return: Copy of given network, with additional sequence inserted in a position of a random dropout layer,
or at the beginning of 1D computations in the model.
"""
drop_idx = [helpers_other.find_first_drop_dense_arch(base_net.arch)]
idx = 0
for l in base_net.arch:
if helpers_other.arch_type(l) == 'drop':
drop_idx += [idx + 1] # Since it can be added after
idx += 1
if const.deep_debug:
print('')
print('add_drop_dense')
print('drop_idx: {}'.format(drop_idx))
print('')
idx_add = random.choice(drop_idx)
dense_params = random.choice(const.mutations.fget()['dense_size'])
drop_params = 'drop%.2f' % random.choice(const.mutations.fget()['dropout'])
if const.deep_debug:
print('')
print('add_drop_dense before private call')
print('idx_add: {}'.format(idx_add))
print('dense_params: {}'.format(dense_params))
print('drop_params: {}'.format(drop_params))
print('arch: {}'.format(base_net.arch))
print('')
return __add_dense_drop(base_net, idx_add, dense_params, drop_params)
def __add_dense_drop(base_net, idx, dense_params, drop_params):
# type: (Network, int, int, str) -> Network
if idx < len(base_net.arch) and helpers_other.arch_type(base_net.arch[idx]) == 'drop':
from warnings import warn
warn('Invalid index given to __add_dense_drop. idx cannot point at place dropout layer in base_net.arch.\n'
'Adding 1 to idx, so this check will be satisfied.')
idx += 1
new_arch = base_net.arch[:idx] + [dense_params] + [drop_params] + base_net.arch[idx:]
new_net = Network(
architecture=new_arch,
opt=base_net.opt,
activation=base_net.act,
callbacks=base_net.callbacks
)
if const.debug:
print('')
print('__add_dense_drop: after adding dense')
print('Index of adding sequence: %d' % idx)
print('Old arch: {}'.format(base_net.arch))
print('New arch: {}'.format(new_arch))
print('\n\t BASE MODEL')
print(base_net.model.summary())
print('\n\t NEW MODEL')
print(new_net.model.summary())
print('')
dim_offset = 2 if len(const.input_shape.fget()) > 2 else 1
for i_l, l in enumerate(new_net.model.layers[:idx + dim_offset - 1]):
if const.deep_debug:
print('')
print('__add_dense_drop inside for loop till idx')
print('\tIdx: {}'.format(i_l))
print('\tOld layer type: {}'.format(type(base_net.model.get_layer(index=i_l))))
print('\tNew layer type: {}'.format(type(l)))
print('')
l.set_weights(base_net.model.get_layer(index=i_l).get_weights())
from keras.layers import Dense
if const.debug:
print('')
print('__add_dense_drop: after adding dense')
print('Index of adding sequence: %d' % idx)
print('Old arch: {}'.format(base_net.arch))
print('New arch: {}'.format(new_arch))
print('\n\t BASE MODEL')
print(base_net.model.summary())
print('\n\t NEW MODEL')
print(new_net.model.summary())
print('')
# 1st edge case, one before added sequence. Input matches, but output not necessarily.
if isinstance(new_net.model.get_layer(index=idx + dim_offset - 1), Dense):
w_a = base_net.model.get_layer(index=idx + dim_offset - 1).get_weights()
w_n = new_net.model.get_layer(index=idx + dim_offset - 1).get_weights()
new_weights_0 = np.array(w_a[0][:len(w_n[0])])
if len(w_a[0]) < len(w_n[0]):
if const.deep_debug:
print('\t\t new_weights shape: {}'.format(new_weights_0.shape))
print('\t\t w_n[0] add shape: {}'.format(np.array(w_n[0][len(new_weights_0):]).shape))
new_weights_0 = np.concatenate((new_weights_0, w_n[0][len(new_weights_0):]), axis=0)
# Output can be different, but we can reuse as many weights as possible
new_weights_1 = np.array(w_a[1][:len(w_n[1])])
if len(w_a[1]) < len(w_n[1]):
if const.deep_debug:
print(new_weights_1.shape)
print(np.array(w_n[1][len(new_weights_1):]).shape)
new_weights_1 = np.concatenate((new_weights_1, w_n[1][len(new_weights_1):]), axis=0)
new_net.model.get_layer(index=idx + dim_offset - 1).set_weights(
[new_weights_0, new_weights_1]
)
# 2nd edge case, one after added sequence. Output matches, but Input not necessarily.
if isinstance(new_net.model.get_layer(index=idx + dim_offset + 2), Dense):
w_a = base_net.model.get_layer(index=idx + dim_offset).get_weights()
w_n = new_net.model.get_layer(index=idx + dim_offset + 2).get_weights()
new_weights_0 = np.array(w_a[0][:len(w_n[0])])
if len(w_a[0]) < len(w_n[0]):
if const.deep_debug:
print('\t\t new_weights shape: {}'.format(new_weights_0.shape))
print('\t\t w_n[0] add shape: {}'.format(np.array(w_n[0][len(new_weights_0):]).shape))
new_weights_0 = np.concatenate((new_weights_0, w_n[0][len(new_weights_0):]), axis=0)
# Output can be different, but we can reuse as many weights as possible
new_weights_1 = np.array(w_a[1][:len(w_n[1])])
if len(w_a[1]) < len(w_n[1]):
if const.deep_debug:
print(new_weights_1.shape)
print(np.array(w_n[1][len(new_weights_1):]).shape)
new_weights_1 = np.concatenate((new_weights_1, w_n[1][len(new_weights_1):]), axis=0)
new_net.model.get_layer(index=idx + dim_offset + 2).set_weights(
[new_weights_0, new_weights_1]
)
if const.debug:
print('')
print('__add_dense_drop: after adding dense')
print('Index of adding sequence: %d' % idx)
print('Old arch: {}'.format(base_net.arch))
print('New arch: {}'.format(new_arch))
print('\n\t BASE MODEL')
print(base_net.model.summary())
print('\n\t NEW MODEL')
print(new_net.model.summary())
print('')
for i_l, l in enumerate(new_net.model.layers[idx + dim_offset + 3:], start=idx + dim_offset + 1):
if const.deep_debug:
print('')
print('add_dense_drop inside for loop till end')
print('Idx old: {}'.format(i_l))
print('Idx_new: {}'.format(i_l + 3))
print('Old layer type: {}'.format(base_net.model.get_layer(index=i_l).get_config()))
print('New layer type: {}'.format(l.get_config()))
print('')
l.set_weights(base_net.model.get_layer(index=i_l).get_weights())
return new_net