def build(self, input_layer):
downsampling_factor = int(np.prod(self.downsample_factors))
last_layer = input_layer
input_shape = K.int_shape(last_layer)
if len(input_shape) == 3:
# Add channel dimension if not already present.
last_layer = Reshape(input_shape[1:] + (1,))(last_layer)
per_stage_before_pool = []
for layer_idx in range(self.num_layers + 1):
cur_num_units = int(np.rint(self.num_units*2**layer_idx))
last_layer = Conv2D(cur_num_units, 3,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
if self.with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=self.l2_weight),
gamma_regularizer=L1L2(l2=self.l2_weight))(last_layer)
last_layer = Activation(self.activation)(last_layer)
last_layer = Conv2D(cur_num_units, 3,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
if self.with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=self.l2_weight),
gamma_regularizer=L1L2(l2=self.l2_weight))(last_layer)
last_layer = Activation(self.activation)(last_layer)
per_stage_before_pool.append(last_layer)
if layer_idx != self.num_layers: # Last layer doesn't require max pooling.
last_layer = MaxPooling2D(pool_size=(2, 2))(last_layer)
if self.p_dropout != 0.0:
last_layer = Dropout(self.p_dropout)(last_layer)
start_idx = 0 if downsampling_factor == 1 else int(np.log2(self.downsample_factors[0]))
for layer_idx in reversed(range(start_idx, self.num_layers)):
cur_num_units = int(np.rint(self.num_units*2**layer_idx))
last_layer = UpSampling2D(size=(2, 2))(last_layer)
last_layer = Conv2D(cur_num_units, 2,
padding='same', kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
if self.with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=self.l2_weight),
gamma_regularizer=L1L2(l2=self.l2_weight))(last_layer)
last_layer = Activation(self.activation)(last_layer)
last_layer = concatenate([per_stage_before_pool[layer_idx], last_layer], axis=3)
last_layer = Conv2D(cur_num_units, 3,
padding='same', kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
if self.with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=self.l2_weight),
gamma_regularizer=L1L2(l2=self.l2_weight))(last_layer)
last_layer = Activation(self.activation)(last_layer)
last_layer = Conv2D(cur_num_units, 3,
padding='same', kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
if self.with_bn:
last_layer = BatchNormalization(beta_regularizer=L1L2(l2=self.l2_weight),
gamma_regularizer=L1L2(l2=self.l2_weight))(last_layer)
last_layer = Activation(self.activation)(last_layer)
last_layer = Conv2D(self.num_output_channels, 3,
activation="linear" if self.skip_last_dense else self.activation,
padding='same', kernel_initializer='he_normal',
kernel_regularizer=L1L2(l2=self.l2_weight),
bias_regularizer=L1L2(l2=self.l2_weight),
use_bias=not self.with_bn and self.with_bias)(last_layer)
return last_layer
#------- AE MODEL---------
#encoder
inputs = Input(shape=input_shape, name='encoder_input')
x = inputs
# stack of Conv2D(32)-Conv2D(64)
for filters in layer_filters:
x = Conv2D(filters=filters,
kernel_size=kernel_size,
activation='relu',
strides=2,
padding='same')(x)
# shape info needed to build decoder model so we don't do hand computation
# the input to the decoder's first Conv2DTranspose will have this shape
# shape is (7, 7, 64) which is processed by the decoder back to (28, 28, 1)
shape = K.int_shape(x)
# generate latent vector
x = Flatten()(x)
latent = Dense(latent_dim, name='latent_vector')(x)
# instantiate encoder model
encoder = Model(inputs, latent, name='encoder')
encoder.summary()
plot_model(encoder, to_file='encoder.png', show_shapes=True)
# DECODER
latent_inputs = Input(shape=(latent_dim, ), name='decoder_input')
# use the shape (7, 7, 64) that was earlier saved
x = Dense(shape[1] * shape[2] * shape[3])(latent_inputs)
# from vector to suitable shape for transposed conv
def compute_output_shape(self, input_shape):
return K.int_shape(self.result)
def call(self,
inputs,
mask=None,
training=None,
initial_state=None,
constants=None):
# input shape: `(samples, time (padded with zeros), input_dim)`
# note that the .build() method of subclasses MUST define
# self.input_spec and self.state_spec with complete input shapes.
if isinstance(inputs, list):
# get initial_state from full input spec
# as they could be copied to multiple GPU.
if self._num_constants is None:
initial_state = inputs[1:]
else:
initial_state = inputs[1:-self._num_constants]
if len(initial_state) == 0:
initial_state = None
inputs = inputs[0]
if initial_state is not None:
pass
elif self.stateful:
initial_state = self.states
else:
initial_state = self.get_initial_state(inputs)
if mask is not None:
ValueError('Mask Not Supported.\n' +
'Perhaps you want the original RNN class from tf.keras.layers')
if len(initial_state) != len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_state)) +
' initial states.')
input_shape = int_shape(inputs)
timesteps = input_shape[1]
if self.unroll and timesteps in [None, 1]:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined or equal to 1. \n'
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer. If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.\n'
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
kwargs = {}
if has_arg(self.cell.call, 'training'):
kwargs['training'] = training
if constants:
if not has_arg(self.cell.call, 'constants'):
raise ValueError('RNN cell does not support constants')
def step(inputs, states):
constants = states[-self._num_constants:]
states = states[:-self._num_constants]
return self.cell.call(inputs, states, constants=constants,
**kwargs)
else:
def step(inputs, states):
return self.cell.call(inputs, states, **kwargs)
last_output, outputs, states, last_lf, lfs = lemol_rnn(
step,
inputs,
initial_state,
constants=constants,
go_backwards=self.go_backwards,
mask=mask,
unroll=self.unroll,
input_length=timesteps
)
if self.stateful:
updates = []
for i in range(len(states)):
updates.append((self.states[i], states[i]))
self.add_update(updates, inputs)
if self.return_sequences:
output = outputs
lf = lfs
else:
output = last_output
lf = last_lf
# Properly set learning phase
if getattr(last_output, '_uses_learning_phase', False):
output._uses_learning_phase = True
lf._uses_learning_phase = True
for state in states:
state._uses_learning_phase = True
if self.return_state:
if isinstance(states, tuple) and not isinstance(states, list):
states = list(states)
else:
states = [states]
return [output, lf] + states
else:
return output, lf
def darknet_conv(x, layer, rf):
filters = int(layer['filters'])
size = int(layer['size'])
stride = int(layer['stride'])
pad = int(layer['pad'])
activation = layer['activation']
batch_normalize = 'batch_normalize' in list(layer.keys())
if pad == 1 and stride == 1:
padding = 'SAME'
else:
padding = 'VALID'
prev_layer_shape = K.int_shape(x)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, prev_layer_shape[-1], size, size)
weights_size = np.product(weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype=np.float32,
buffer=rf.read(filters * 4))
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters),
dtype=np.float32,
buffer=rf.read(filters * 12))
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(shape=darknet_w_shape,
dtype=np.float32,
buffer=rf.read(weights_size * 4))
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
if batch_normalize:
conv_weights = [conv_weights]
else:
conv_weights = [conv_weights, conv_bias]
act_fn = None
if activation in ['leaky', 'relu', 'mish', 'linear']:
pass
else:
raise NotImplementedError(f'{activation} is not implemented')
if stride > 1:
x = ZeroPadding2D(((1, 0), (1, 0)))(x)
x = Conv2D(filters=filters,
kernel_size=size,
strides=stride,
padding=padding,
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
kernel_regularizer=tf.keras.regularizers.l2(0.0005),
kernel_initializer=tf.random_normal_initializer(stddev=0.01),
bias_initializer=tf.constant_initializer(0.0))(x)
if batch_normalize:
x = BatchNormalization(weights=bn_weight_list)(x)
if FLAGS.mode == 'dynamic':
if activation == 'leaky':
x = tf.nn.leaky_relu(x)
elif activation == 'mish':
x = Mish(x)
elif activation == 'relu':
x = Activation('relu')(x)
else:
if activation == 'leaky':
x = Activation('relu')(x)
elif activation == 'mish':
x = Activation('relu')(x)
elif activation == 'relu':
x = Activation('relu')(x)
return x
def call(self, x, mask=None):
if hasattr(x, '_keras_shape'):
input_shape = x._keras_shape
elif hasattr(K, 'int_shape'):
input_shape = K.int_shape(x)
# --------------------------------- #
# 获取输入进来的特征层的宽和高
# 比如38x38
# --------------------------------- #
layer_width = input_shape[self.waxis]
layer_height = input_shape[self.haxis]
# --------------------------------- #
# 获取输入进来的图片的宽和高
# 比如300x300
# --------------------------------- #
img_width = self.img_size[1]
img_height = self.img_size[0]
box_widths = []
box_heights = []
# --------------------------------- #
# self.aspect_ratios一般有两个值
# [1, 1, 2, 1/2]
# [1, 1, 2, 1/2, 3, 1/3]
# --------------------------------- #
for ar in self.aspect_ratios:
# 首先添加一个较小的正方形
if ar == 1 and len(box_widths) == 0:
box_widths.append(self.min_size)
box_heights.append(self.min_size)
# 然后添加一个较大的正方形
elif ar == 1 and len(box_widths) > 0:
box_widths.append(np.sqrt(self.min_size * self.max_size))
box_heights.append(np.sqrt(self.min_size * self.max_size))
# 然后添加长方形
elif ar != 1:
box_widths.append(self.min_size * np.sqrt(ar))
box_heights.append(self.min_size / np.sqrt(ar))
# --------------------------------- #
# 获得所有先验框的宽高1/2
# --------------------------------- #
box_widths = 0.5 * np.array(box_widths)
box_heights = 0.5 * np.array(box_heights)
# --------------------------------- #
# 每一个特征层对应的步长
# --------------------------------- #
step_x = img_width / layer_width
step_y = img_height / layer_height
# --------------------------------- #
# 生成网格中心
# --------------------------------- #
linx = np.linspace(0.5 * step_x, img_width - 0.5 * step_x, layer_width)
liny = np.linspace(0.5 * step_y, img_height - 0.5 * step_y,
layer_height)
centers_x, centers_y = np.meshgrid(linx, liny)
centers_x = centers_x.reshape(-1, 1)
centers_y = centers_y.reshape(-1, 1)
# 每一个先验框需要两个(centers_x, centers_y)，前一个用来计算左上角，后一个计算右下角
num_priors_ = len(self.aspect_ratios)
prior_boxes = np.concatenate((centers_x, centers_y), axis=1)
prior_boxes = np.tile(prior_boxes, (1, 2 * num_priors_))
# 获得先验框的左上角和右下角
prior_boxes[:, ::4] -= box_widths
prior_boxes[:, 1::4] -= box_heights
prior_boxes[:, 2::4] += box_widths
prior_boxes[:, 3::4] += box_heights
# --------------------------------- #
# 将先验框变成小数的形式
# 归一化
# --------------------------------- #
prior_boxes[:, ::2] /= img_width
prior_boxes[:, 1::2] /= img_height
prior_boxes = prior_boxes.reshape(-1, 4)
prior_boxes = np.minimum(np.maximum(prior_boxes, 0.0), 1.0)
num_boxes = len(prior_boxes)
if len(self.variances) == 1:
variances = np.ones((num_boxes, 4)) * self.variances[0]
elif len(self.variances) == 4:
variances = np.tile(self.variances, (num_boxes, 1))
else:
raise Exception('Must provide one or four variances.')
prior_boxes = np.concatenate((prior_boxes, variances), axis=1)
prior_boxes_tensor = K.expand_dims(
tf.cast(prior_boxes, dtype=tf.float32), 0)
pattern = [tf.shape(x)[0], 1, 1]
prior_boxes_tensor = tf.tile(prior_boxes_tensor, pattern)
return prior_boxes_tensor
def _augment(self, model):
# TODO: There is a lot of overlap with the OracleWrapper, merge some
# functionality into a separate function or a parent class
# Extract info from the model
loss_function = model.loss
output_shape = model.layers[-1].get_output_shape_at(0)[1:]
# Create two identical models one with the current weights and one with
# the snapshot of the weights
self.model = model
self._snapshot = clone_model(model)
# Create the target variable and compute the losses and the metrics
inputs = [
Input(shape=K.int_shape(x)[1:])
for x in _tolist(model.layers[0].get_input_at(0))
]
model_output = self.model(inputs)
snapshot_output = self._snapshot(inputs)
y_true = Input(shape=output_shape)
loss = LossLayer(loss_function)([y_true, model_output])
loss_snapshot = LossLayer(loss_function)([y_true, snapshot_output])
metrics = self.model.metrics or []
metrics = [
MetricLayer(metric)([y_true, model_output]) for metric in metrics
]
# Make a set of variables that will be holding the batch gradient of
# the snapshot
self._batch_grad = [
K.zeros(K.int_shape(p)) for p in self.model.trainable_weights
]
# Create an optimizer that computes the variance reduced gradients and
# get the updates
loss_mean = K.mean(loss)
loss_snapshot_mean = K.mean(loss_snapshot)
optimizer, updates = self._get_updates(loss_mean, loss_snapshot_mean,
self._batch_grad)
# Create the training function and gradient computation function
metrics_updates = []
if hasattr(self.model, "metrics_updates"):
metrics_updates = self.model.metrics_updates
learning_phase = []
if loss._uses_learning_phase:
learning_phase.append(K.learning_phase())
inputs = inputs + [y_true] + learning_phase
outputs = [loss_mean, loss] + metrics
train_on_batch = K.function(inputs=inputs,
outputs=outputs,
updates=updates + self.model.updates +
metrics_updates)
evaluate_on_batch = K.function(inputs=inputs,
outputs=outputs,
updates=self.model.state_updates +
metrics_updates)
get_grad = K.function(inputs=inputs,
outputs=K.gradients(
loss_mean, self.model.trainable_weights),
updates=self.model.updates)
self.optimizer = optimizer
self._train_on_batch = train_on_batch
self._evaluate_on_batch = evaluate_on_batch
self._get_grad = get_grad
def make_vae():
latent_dim = 128
inputs = Input(shape=(VOXEL_SHAPE[0], VOXEL_SHAPE[1], VOXEL_SHAPE[2], 1))
x = inputs
# Encoder
x = Conv3D(16, 3, activation='relu', padding='same')(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
x = Dropout(0.2)(x)
x = Conv3D(32, 3, activation='relu', padding='same')(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
x = Dropout(0.2)(x)
x = Conv3D(64, 3, activation='relu', padding='same')(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
x = Dropout(0.2)(x)
x = Conv3D(128, 3, activation='relu', padding='same')(x)
x = MaxPooling3D(pool_size=(2, 2, 2))(x)
x = Dropout(0.2)(x)
# shape info needed to build decoder model
shape = K.int_shape(x)
# generate latent vector Q(z|X)
x = Flatten()(x)
x = Dense(2048, activation='relu')(x)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)
# use reparameterization trick to push the sampling out as input
# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim, ),
name='z')([z_mean, z_log_var])
# instantiate encoder model
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
# build decoder model
latent_inputs = Input(shape=(latent_dim, ), name='z_sampling')
x = Dense(2048, activation='relu')(latent_inputs)
x = Reshape((8, 8, 8, 4))(x)
x = Dropout(0.2)(x)
x = UpSampling3D(size=(2, 2, 2))(x)
x = Dropout(0.2)(x)
x = Conv3D(128, 3, activation='relu', padding='same')(x)
x = Dropout(0.2)(x)
x = UpSampling3D(size=(2, 2, 2))(x)
x = Conv3D(64, 3, activation='relu', padding='same')(x)
x = Dropout(0.2)(x)
x = UpSampling3D(size=(2, 2, 2))(x)
x = Conv3D(32, 3, activation='relu', padding='same')(x)
x = Dropout(0.2)(x)
outputs = Conv3D(1, 3, activation='sigmoid', padding='same')(x)
# instantiate decoder model
decoder = Model(latent_inputs, outputs, name='decoder')
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae')
reconstruction_loss = weighted_bce(K.flatten(inputs), K.flatten(outputs))
reconstruction_loss *= np.prod(VOXEL_SHAPE)
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
return vae
def sampling(args):
mu, sigma = args
batch = K.shape(mu)[0]
dim = K.int_shape(mu)[1]
epsilon = K.random_normal(shape = (batch, dim))
return mu + K.exp(0.5 * sigma) * epsilon
def _sum_per_sample(self, x):
"""Sum across all the dimensions except the batch dim"""
# Instead we might be able to use x.ndims but there have been problems
# with ndims and Keras so I think len(int_shape()) is more reliable
dims = len(K.int_shape(x))
return K.sum(x, axis=list(range(1, dims)))
def _inception_resnet_block(x, scale, block_type, block_idx,
activation='relu'):
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
if block_idx is None:
prefix = None
else:
prefix = '_'.join((block_type, str(block_idx)))
name_fmt = partial(_generate_layer_name, prefix=prefix)
if block_type == 'Block35':
branch_0 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 1))
branch_2 = conv2d_bn(x, 32, 1, name=name_fmt('Conv2d_0a_1x1', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0b_3x3', 2))
branch_2 = conv2d_bn(branch_2,
32,
3,
name=name_fmt('Conv2d_0c_3x3', 2))
branches = [branch_0, branch_1, branch_2]
elif block_type == 'Block17':
branch_0 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 128, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
128, [1, 7],
name=name_fmt('Conv2d_0b_1x7', 1))
branch_1 = conv2d_bn(branch_1,
128, [7, 1],
name=name_fmt('Conv2d_0c_7x1', 1))
branches = [branch_0, branch_1]
elif block_type == 'Block8':
branch_0 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_1x1', 0))
branch_1 = conv2d_bn(x, 192, 1, name=name_fmt('Conv2d_0a_1x1', 1))
branch_1 = conv2d_bn(branch_1,
192, [1, 3],
name=name_fmt('Conv2d_0b_1x3', 1))
branch_1 = conv2d_bn(branch_1,
192, [3, 1],
name=name_fmt('Conv2d_0c_3x1', 1))
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "Block35", "Block17" or "Block8", '
'but got: ' + str(block_type))
mixed = Concatenate(axis=channel_axis,
name=name_fmt('Concatenate'))(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=name_fmt('Conv2d_1x1'))
up = Lambda(scaling,
output_shape=K.int_shape(up)[1:],
arguments={'scale': scale})(up)
x = add([x, up])
if activation is not None:
x = Activation(activation, name=name_fmt('Activation'))(x)
return x
def call(self, inputs):
# [batch,max_steps,dims]
q, k, v, mask = inputs
dims = K.int_shape(q)[-1]
# 计算position embedding
q_position, k_position = generate_position(inputs[:3])
# 将position embedding 添加到原矩阵中
q, k, v = q + q_position, k + k_position, v + k_position
# multi-head
q = K.tile(K.expand_dims(q, 0), [self.heads, 1, 1, 1])
k = K.tile(K.expand_dims(k, 0), [self.heads, 1, 1, 1])
v = K.tile(K.expand_dims(v, 0), [self.heads, 1, 1, 1])
# 添加dropout
q = self.q_dropout(q)
k = self.k_dropout(k)
v = self.v_dropout(v)
# 求取q,k,v
q = self.q_Dense(q)
k = self.k_Dense(k)
v = self.v_Dense(v)
# Q和K点乘
# q:[heads,batch,steps1,embedding_size]
# k:[heads,batch,steps2,embedding_size]
k = K.permute_dimensions(k, (0, 1, 3, 2))
# q_k:[heads,batch,steps1,steps2]
q_k = tf.matmul(q, k)
# 还需要scale操作
q_k = q_k / (dims**(1 / 2))
# 因为mask主要在steps上操作
# 先转换step
# q_k:[heads,batch,steps2,steps1]
q_k = K.permute_dimensions(q_k, (0, 1, 3, 2))
q_k = mask_func(q_k, mask)
# 接下来softmax操作
# 先把维度转回来
# q_k:heads,batch,steps1,steps2
q_k = K.permute_dimensions(q_k, (0, 1, 3, 2))
q_k_soft = keras.layers.softmax(q_k)
# 权重和矩阵进行相乘
q_k_soft = tf.matmul(q_k_soft, v)
# 接下来按header拆开，进行concat
q_k_heads = tf.unstack(q_k_soft)
# [batch,steps1,embedding_size*heads]
q_k_concat = K.concatenate(q_k_heads)
# 将concat结果linear
# [batch, steps1, dims]
dense_result = keras.layers.Dense(dims)(q_k_concat)
# 将输入和dense_result相加
add_result = q + dense_result
# 进行标准化
result = keras.layers.BatchNormalization()(add_result)
return result
out = l.Dropout(0.25)(out) #w/0.8128 #w/o 0.7975
out = l.BatchNormalization()(out)
out = l.LeakyReLU()(out) #w ReLU/ 0.8165 ELU 0.8256 LeakyReLU 0.8458
plus1 = plus = out
out = l.Conv2D(64 * i, kernel_size=(3, 3), strides=1, padding="same")(out)
out = l.Dropout(0.25)(out)
out = l.BatchNormalization()(out)
out = l.LeakyReLU()(out)
out = l.add([plus1, out])
out = l.Conv2D(64 * i, kernel_size=(3, 3), strides=1, padding="same")(out)
out = l.Dropout(0.25)(out)
out = l.BatchNormalization()(out)
out = l.LeakyReLU()(out)
out = l.add([plus, out])
out = l.AveragePooling2D(pool_size=(K.int_shape(out)[1:3]))(out)
out = l.Flatten()(out)
#out = l.Dense(512, activation="relu")(out) #ezzel 79.36
out = l.Dense(10, activation='softmax')(out)
# ... AveragePooling2D, Flatten, Dense w/ softmax
model = Model(inputs=x, outputs=out)
if os.path.exists(PATH):
model.load_weights(PATH)
model.summary()
model.compile(
optimizer=Adam(lr=LEARNING_RATE, decay=DECAY), #, decay=DECAY
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
def call(self, inp):
images, vertexlabel, Js_in = inp
out_dict = {}
images = [
tf.Variable(x, dtype=tf.float32, trainable=False) for x in images
]
vertexlabel = tf.cast(tf.Variable(vertexlabel, trainable=False),
tf.int32)
if FACE:
Js = [Lambda(lambda j: j[:, :25])(J) for J in Js_in]
else:
Js = [
self.flatten(
tf.cast(tf.Variable(x, trainable=False), tf.float32))
for x in Js_in
]
with tf.device('/gpu:0'):
lat_codes = [self.top_([q, j]) for q, j in zip(images, Js)]
latent_code_offset = self.avg([q[0] for q in lat_codes])
latent_code_betas = self.avg([q[1] for q in lat_codes])
latent_code_pose = [
tf.concat([q[1], x], axis=-1) for q, x in zip(lat_codes, Js)
]
with tf.device('/gpu:0'):
latent_code_betas = self.lat_betas(latent_code_betas)
betas = self.betas(latent_code_betas)
latent_code_pose = [self.lat_pose(x) for x in latent_code_pose]
pose_trans_init = tf.tile(tf.expand_dims(self.pose_trans, 0),
(K.int_shape(betas)[0], 1))
poses_ = [
self.lat_pose_layer(x) + pose_trans_init
for x in latent_code_pose
]
trans_ = [self.cut_trans(x) for x in poses_]
trans = [la(i) for la, i in zip(self.trans_layers, trans_)]
poses_ = [self.cut_poses(x) for x in poses_]
poses_ = [self.reshape_pose(x) for x in poses_]
poses = [la(i) for la, i in zip(self.pose_layers, poses_)]
##
out_dict['betas'] = betas
for i in range(NUM):
out_dict['pose_{}'.format(i)] = poses[i]
out_dict['trans_{}'.format(i)] = trans[i]
latent_code_offset_ShapeMerged = self.latent_code_offset_ShapeMerged(
latent_code_offset)
latent_code_offset_ShapeMerged = self.latent_code_offset_ShapeMerged_2(
latent_code_offset_ShapeMerged)
garm_model_outputs = [
fe(latent_code_offset_ShapeMerged) for fe in self.garmentModels
]
garment_verts_all = [fe[0] for fe in garm_model_outputs]
garment_pca = [fe[1] for fe in garm_model_outputs]
garment_pca = tf.stack(garment_pca, axis=1)
##
out_dict['pca_verts'] = garment_pca
lis = []
for go, vs in zip(garment_verts_all, self.scatters):
lis.append(vs(go))
garment_verts_all_scattered = tf.stack(lis, axis=-1)
## Get naked smpl to compute garment offsets
zerooooooo = K.zeros_like(garment_verts_all_scattered[..., 0])
pooooooooo = [K.zeros_like(p) for p in poses]
tooooooooo = [K.zeros_like(p) for p in trans]
smpls_base = []
for i, (p, t) in enumerate(zip(pooooooooo, tooooooooo)):
v, _, n, _ = self.smpl(p, betas, t, zerooooooo)
smpls_base.append(v)
if i == 0:
vertices_naked_ = n
## Append Skin offsets
garment_verts_all_scattered = tf.concat([
K.expand_dims(vertices_naked_, -1),
tf.cast(garment_verts_all_scattered, vertices_naked_.dtype)
],
axis=-1)
garment_verts_all_scattered = tf.transpose(
garment_verts_all_scattered, perm=[0, 1, 3, 2])
clothed_verts = tf.batch_gather(garment_verts_all_scattered,
vertexlabel)
clothed_verts = tf.squeeze(
tf.transpose(clothed_verts, perm=[0, 1, 3, 2]))
offsets_ = clothed_verts - vertices_naked_
smpls = []
for i, (p, t) in enumerate(zip(poses, trans)):
v, t, n, _ = self.smpl(p, betas, t, offsets_)
smpls.append(v)
if i == 0:
vertices_naked = n
vertices_tposed = t
Js = [
jl(self.smpl_J([p, betas, t]))
for jl, p, t in zip(self.J_layers, poses, trans)
]
vertices = tf.concat([
tf.expand_dims(smpl, axis=-1) for i, smpl in enumerate(smpls)
],
axis=-1)
##
out_dict['vertices'] = vertices
out_dict['vertices_tposed'] = vertices_tposed
out_dict['vertices_naked'] = vertices_naked
##
out_dict['vertices'] = vertices
out_dict['vertices_tposed'] = vertices_tposed
out_dict['vertices_naked'] = vertices_naked
out_dict['offsets_h'] = offsets_
for i in range(NUM):
out_dict['J_{}'.format(i)] = Js[i]
vert_cols = tf.reshape(
tf.gather(self.colormap, tf.reshape(vertexlabel, (-1, ))),
(-1, config.NVERTS, 3))
renderered_garms_all = []
for view in range(NUM):
renderered_garms_all.append(
render_colored_batch(
vertices[..., view],
self.faces,
vert_cols, # [bat],
IMG_SIZE,
IMG_SIZE,
FOCAL_LENGTH,
CAMERA_CENTER,
np.zeros(3, dtype=np.float32),
num_channels=3))
renderered_garms_all = tf.transpose(renderered_garms_all,
[1, 2, 3, 4, 0])
out_dict['rendered'] = renderered_garms_all
lap = compute_laplacian_diff(vertices_tposed, vertices_naked,
self.faces)
##
out_dict['laplacian'] = lap
return out_dict
def inception_resnet_block(x, scale, block_type, block_idx, activation='relu'):
"""Adds a Inception-ResNet block.
This function builds 3 types of Inception-ResNet blocks mentioned
in the paper, controlled by the `block_type` argument (which is the
block name used in the official TF-slim implementation):
- Inception-ResNet-A: `block_type='block35'`
- Inception-ResNet-B: `block_type='block17'`
- Inception-ResNet-C: `block_type='block8'`
# Arguments
x: input tensor.
scale: scaling factor to scale the residuals (i.e., the output of
passing `x` through an inception module) before adding them
to the shortcut branch.
Let `r` be the output from the residual branch,
the output of this block will be `x + scale * r`.
block_type: `'block35'`, `'block17'` or `'block8'`, determines
the network structure in the residual branch.
block_idx: an `int` used for generating layer names.
The Inception-ResNet blocks
are repeated many times in this network.
We use `block_idx` to identify
each of the repetitions. For example,
the first Inception-ResNet-A block
will have `block_type='block35', block_idx=0`,
and the layer names will have
a common prefix `'block35_0'`.
activation: activation function to use at the end of the block
(see [activations](../activations.md)).
When `activation=None`, no activation is applied
(i.e., "linear" activation: `a(x) = x`).
# Returns
Output tensor for the block.
# Raises
ValueError: if `block_type` is not one of `'block35'`,
`'block17'` or `'block8'`.
"""
if block_type == 'block35':
branch_0 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(branch_1, 32, 3)
branch_2 = conv2d_bn(x, 32, 1)
branch_2 = conv2d_bn(branch_2, 48, 3)
branch_2 = conv2d_bn(branch_2, 64, 3)
branches = [branch_0, branch_1, branch_2]
elif block_type == 'block17':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 128, 1)
branch_1 = conv2d_bn(branch_1, 160, [1, 7])
branch_1 = conv2d_bn(branch_1, 192, [7, 1])
branches = [branch_0, branch_1]
elif block_type == 'block8':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(branch_1, 224, [1, 3])
branch_1 = conv2d_bn(branch_1, 256, [3, 1])
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "block35", "block17" or "block8", '
'but got: ' + str(block_type))
block_name = block_type + '_' + str(block_idx)
channel_axis = 3
mixed = Concatenate(axis=channel_axis,
name=block_name + '_mixed')(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=block_name + '_conv')
x = Lambda(lambda inputs, scale: inputs[0] + inputs[1] * scale,
output_shape=K.int_shape(x)[1:],
arguments={'scale': scale},
name=block_name)([x, up])
if activation is not None:
x = Activation(activation, name=block_name + '_ac')(x)
return x
def _main(config_path, weights_path, output_path, weights_only=False, plot_model=False):
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(config_path)
assert weights_path.endswith('.weights'), '{} is not a .weights file'.format(weights_path)
assert output_path.endswith('.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
major, minor, revision = np.ndarray(shape=(3,), dtype='int32', buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1,), dtype='int64', buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1,), dtype='int32', buffer=weights_file.read(4))
print('Weights Header: ', major, minor, revision, seen)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = float(cfg_parser['net_0']['decay']) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
out_index = []
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
padding = 'same' if pad == 1 and stride == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn' if batch_normalize else ' ', activation, weights_shape)
conv_bias = np.ndarray(shape=(filters,), dtype='float32', buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(shape=(3, filters), dtype='float32', buffer=weights_file.read(filters * 12))
count += 3 * filters
# (scale gamma, shift beta, running mean, running var)
bn_weight_list = [bn_weights[0], conv_bias, bn_weights[1], bn_weights[2]]
conv_weights = np.ndarray(shape=darknet_w_shape, dtype='float32',
buffer=weights_file.read(weights_size * 4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [conv_weights, conv_bias]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError('Unknown activation function `{}` in section {}'.format(activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(pool_size=(size, size), strides=(stride, stride), padding='same')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('shortcut'):
index = int(cfg_parser[section]['from'])
activation = cfg_parser[section]['activation']
assert activation == 'linear', 'Only linear activation supported.'
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith('upsample'):
stride = int(cfg_parser[section]['stride'])
assert stride == 2, 'Only stride=2 supported.'
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('yolo'):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith('net'):
pass
else:
raise ValueError('Unsupported section header type: {}'.format(section))
for layer in all_layers:
print(layer)
print(len(layer.weights))
pass
a = b2
# Create and save model.
if len(out_index) == 0:
out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer, outputs=[all_layers[i] for i in out_index])
print(model.summary())
if weights_only:
model.save_weights('{}'.format(output_path))
print('Saved Keras weights to {}'.format(output_path))
else:
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(count, count + remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
def transition_layer(x):
x = bn_rl_conv(x, K.int_shape(x)[-1] // 2)
x = AvgPool2D(2, strides=2, padding='same')(x)
return x
def ASPP(tensor):
'''atrous spatial pyramid pooling'''
dims = K.int_shape(tensor)
y_pool = AveragePooling2D(pool_size=(dims[1], dims[2]),
name='average_pooling')(tensor)
y_pool = Conv2D(filters=256,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
name='pool_1x1conv2d',
use_bias=False)(y_pool)
y_pool = BatchNormalization(name=f'bn_1')(y_pool)
y_pool = Activation('relu', name=f'relu_1')(y_pool)
y_pool = Upsample(tensor=y_pool, size=[dims[1], dims[2]])
y_1 = Conv2D(filters=256,
kernel_size=1,
dilation_rate=1,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d1',
use_bias=False)(tensor)
y_1 = BatchNormalization(name=f'bn_2')(y_1)
y_1 = Activation('relu', name=f'relu_2')(y_1)
y_6 = Conv2D(filters=256,
kernel_size=3,
dilation_rate=6,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d6',
use_bias=False)(tensor)
y_6 = BatchNormalization(name=f'bn_3')(y_6)
y_6 = Activation('relu', name=f'relu_3')(y_6)
y_12 = Conv2D(filters=256,
kernel_size=3,
dilation_rate=12,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d12',
use_bias=False)(tensor)
y_12 = BatchNormalization(name=f'bn_4')(y_12)
y_12 = Activation('relu', name=f'relu_4')(y_12)
y_18 = Conv2D(filters=256,
kernel_size=3,
dilation_rate=18,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_d18',
use_bias=False)(tensor)
y_18 = BatchNormalization(name=f'bn_5')(y_18)
y_18 = Activation('relu', name=f'relu_5')(y_18)
y = concatenate([y_pool, y_1, y_6, y_12, y_18], name='ASPP_concat')
y = Conv2D(filters=256,
kernel_size=1,
dilation_rate=1,
padding='same',
kernel_initializer='he_normal',
name='ASPP_conv2d_final',
use_bias=False)(y)
y = BatchNormalization(name=f'bn_final')(y)
y = Activation('relu', name=f'relu_final')(y)
return y
def echo_sample(inputs,
clip=None,
d_max=100,
batch=100,
multiplicative=False,
echo_mc=False,
replace=True,
fx_clip=None,
plus_sx=True,
calc_log=True,
set_batch=True,
return_noise=False,
**kwargs):
# kwargs unused
if isinstance(inputs, list):
fx = inputs[0]
sx = inputs[-1]
else:
fx = inputs
# TO DO : CALC_LOG currently determines both whether to do log space calculations AND whether sx is a log
fx_shape = fx.get_shape()
sx_shape = sx.get_shape()
z_dim = K.int_shape(fx)[-1]
batch_size = batch
batch = K.shape(fx)[0]
# clip is multiplied times s(x) to ensure that sum of truncated terms < machine precision
# clip should be calculated numerically according to App C in paper
# M (r ^ dmax / 1-r ) < precision, SOLVE for r (clipping factor), with M = max magnitude of f(x)
# calculation below is an approximation (ensuring only term d_max + 1 < precision)
if clip is None:
max_fx = fx_clip if fx_clip is not None else 1.0
clip = (2**(-23) / max_fx)**(1.0 / d_max)
# fx_clip can be used to restrict magnitude of f(x), not used in paper
# defaults to no clipping and M = 1 (e.g. with tanh activation for f(x))
if fx_clip is not None:
fx = K.clip(fx, -fx_clip, fx_clip)
if not calc_log:
sx = tf.multiply(clip, sx)
sx = tf.where(tf.abs(sx) < K.epsilon(), K.epsilon() * tf.sign(sx), sx)
else:
# plus_sx based on activation for sx = s(x):
# True for log_sigmoid
# False for softplus
sx = tf.math.log(clip) + (-1 * sx if not plus_sx else sx)
if echo_mc is not None and echo_mc:
# use mean centered fx for noise : performs worse
fx = fx - K.mean(fx, axis=0, keepdims=True)
if replace: # replace doesn't set batch size (using permute_neighbor_indices does)
sx = K.batch_flatten(sx) if len(sx_shape) > 2 else sx
fx = K.batch_flatten(fx) if len(fx_shape) > 2 else fx
# Sampling with replacement
inds = K.reshape(random_indices(batch, d_max), (-1, 1))
select_sx = gather_nd_reshape(sx, inds, (-1, d_max, z_dim))
select_fx = gather_nd_reshape(fx, inds, (-1, d_max, z_dim))
if len(sx_shape) > 2:
select_sx = K.expand_dims(K.expand_dims(select_sx, 2), 2)
sx = K.expand_dims(K.expand_dims(sx, 1), 1)
if len(fx_shape) > 2:
select_fx = K.expand_dims(K.expand_dims(select_fx, 2), 2)
fx = K.expand_dims(K.expand_dims(fx, 1), 1)
else:
# batch x batch x z_dim
# for all i, stack_sx[i, :, :] = sx
repeat = tf.multiply(tf.ones_like(tf.expand_dims(fx, 0)),
tf.ones_like(tf.expand_dims(fx, 1)))
stack_fx = tf.multiply(fx, repeat)
stack_sx = tf.multiply(sx, repeat)
# select a set of dmax examples from original fx / sx for each batch entry
if not set_batch:
inds = indices_without_replacement(batch, d_max)
else:
inds = permute_neighbor_indices(batch_size, d_max, replace=replace)
select_sx = tf.gather_nd(stack_sx, inds)
select_fx = tf.gather_nd(stack_fx, inds)
if calc_log:
sx_echoes = tf.cumsum(select_sx, axis=1, exclusive=True)
else:
sx_echoes = tf.cumprod(select_sx, axis=1, exclusive=True)
# calculates S(x0)S(x1)...S(x_l)*f(x_(l+1))
sx_echoes = tf.exp(sx_echoes) if calc_log else sx_echoes
fx_sx_echoes = tf.multiply(select_fx, sx_echoes)
# performs the sum over dmax terms to calculate noise
noise = tf.reduce_sum(fx_sx_echoes, axis=1)
sx = sx if not calc_log else tf.exp(sx)
if multiplicative:
# unused in paper, not extensively tested : log Z has Echo distribution
output = tf.exp(fx + tf.multiply(sx, noise))
else:
output = fx + tf.multiply(sx, noise)
return output if not return_noise else noise
def get_build_func(OD_tensor, args):
dim = args.dim
delta = tf.constant(1e-7, dtype=tf.float32)
nb_regions = K.int_shape(OD_tensor)[-1]
nb_actions = get_nb_actions(args.action_mode, nb_regions)
mobility_decay = tf.constant(args.mobility_decay, dtype=tf.float32)
OD_mean = Lambda(lambda x: tf.reduce_mean(x, axis=0, keepdims=True))(
OD_tensor) # 1 * 323 * nb_regions
OD_mean_out = Lambda(lambda x: tf.reduce_mean(x, axis=-1))(
OD_mean) # 1 * nb_regions
def get_accumulated_cost(inputs):
OD, OD_, accumlated = inputs
return accumlated * mobility_decay + K.sum(OD - OD_, axis=[1, -1])
def get_feature_input():
print('Layer type', args.layer_type)
if args.layer_type == 'weights':
print('No visible Round')
layer = Lambda(lambda x: tf.concat((tf.math.reduce_sum(
x[:, :, :4], axis=-1, keepdims=True), x[:, :, 2:8]),
axis=-1))
else:
layer = Lambda(lambda x: x[:, :, 2:8])
return layer
aggregate_op = None
if args.layer_type == 'weights':
print('Use OD Layer')
GNN_layer = GraphSageConvOD
elif args.layer_type == 'softmax':
print('Use Softmax Layer')
GNN_layer = GraphSageConvSoftmax
else:
print('Use', args.layer_type)
aggregate_op = args.layer_type
GNN_layer = GraphSageConv
def build():
state_input = Input(shape=(nb_regions, 11), name='state_input')
control_index = np.arange(0, args.period).reshape(4, -1)
def build_actor():
print('-' * 30 + 'Build Actor' + '-' * 30)
time_input = Lambda(lambda x: x[:, :args.period, 8])(
state_input) # None * period
OD_input = Lambda(lambda x: tf.gather(
OD_tensor, tf.cast(x, tf.int32), axis=0))(time_input)
ac_m_input = Lambda(lambda x: K.expand_dims(
K.repeat(x[:, :, 9], nb_regions), axis=-1))(
state_input) # nb_regions * nb_regions * 1
ac_d_input = Lambda(lambda x: K.expand_dims(
K.repeat(x[:, :, 10], nb_regions), axis=-1))(
state_input) # nb_regions * nb_regions * 1
features_input = get_feature_input()(state_input)
features = GNN_layer(
16,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[0])([features_input, OD_input])
features = GNN_layer(32,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[1])([features, OD_input])
features = GNN_layer(dim,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[2])([features, OD_input])
features = GNN_layer(dim,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[3])([features, OD_input])
print('Action mode', args.action_mode)
if args.action_mode == 'edge':
OD_sum = Lambda(lambda x: K.sum(x, 1))(
OD_input) # nb_regions*nb_regions
features = Lambda(cross_concate)(
[features, features]) # nb_regions*nb_regions, 2*dim
OD_ = Permute((2, 3, 1))(OD_input) # nb_regions*nb_regions*4
if args.layer_type == 'weights':
features = Reshape(
(nb_regions, nb_regions, dim * 2 + 2))(features)
else:
features = Reshape(
(nb_regions, nb_regions, dim * 2))(features)
features = Lambda(lambda inputs: tf.concat(
(inputs[0], inputs[1], inputs[2], inputs[3]), -1))(
[ac_m_input, ac_d_input, OD_, features])
features = Dense(dim, activation='relu')(features)
features = BatchNormalization()(
features) # None * nb_regions * nb_regions * dim
actions_ = Dense(1, activation='sigmoid')(features)
actions_ = Reshape((nb_regions, nb_regions))(actions_)
actions = Lambda(lambda x: tf.where(tf.not_equal(x[1], 0), x[
0], tf.ones_like(x[0])))([actions_, OD_sum])
actions = Reshape((nb_regions * nb_regions, ))(actions)
elif args.action_mode == 'node':
# nb_regions actions
OD_sum = Lambda(lambda x: tf.reduce_sum(x, axis=[1, -1]))(
OD_input) # nb_regions*nb_regions
features = Lambda(lambda inputs: tf.concat(
(inputs[0][:, 0], inputs[1][:, 0],
tf.expand_dims(inputs[2], -1), inputs[3]), -1))(
[ac_m_input, ac_d_input, OD_sum, features])
features = Dense(dim, activation='relu')(features)
features = BatchNormalization()(features)
actions = Dense(1, activation='sigmoid')(features)
actions = Reshape((nb_regions, ))(actions)
elif args.action_mode == 'graph':
# 1 action
OD_sum = Lambda(lambda x: tf.reduce_sum(x, axis=[1, -1]))(
OD_input) # nb_regions*nb_regions
features = Lambda(lambda inputs: tf.concat(
(inputs[0][:, 0], inputs[1][:, 0],
tf.expand_dims(inputs[2], -1), inputs[3]), -1))(
[ac_m_input, ac_d_input, OD_sum, features])
features = Flatten()(features)
features = Dense(dim * 8, activation='relu')(features)
features = BatchNormalization()(features)
actions = Dense(1, activation='sigmoid')(features)
else:
print('Wrong action mode')
exit()
actor = Model(state_input, actions)
return actor
def build_critic():
print('-' * 30 + 'Build Critic' + '-' * 30)
action_input = Input(shape=(nb_actions, ),
name='critic/action_input')
ac_m_input = Lambda(lambda x: x[:, :, 9])(state_input)
ac_d_input = Lambda(lambda x: x[:, :, 10])(state_input)
time_input = Lambda(lambda x: x[:, :args.period, 8])(state_input)
OD_input = Lambda(lambda x: tf.gather(
OD_tensor, tf.cast(x, tf.int32), axis=0))(time_input)
features_input = get_feature_input()(state_input)
print('Action mode', args.action_mode)
if args.action_mode == 'edge':
# nb_regions*nb_regions actions
action_ = Reshape((nb_regions, nb_regions))(action_input)
elif args.action_mode == 'node':
# nb_regions actions
action_ = Lambda(lambda x: tf.expand_dims(x, -1))(action_input)
elif args.action_mode == 'graph':
# one action
action_ = action_input
else:
print('Wrong action mode')
exit()
OD_ = Multiply()([action_, OD_input])
ac_m = Lambda(get_accumulated_cost)([OD_input, OD_, ac_m_input])
ac_d = ac_d_input
if 'avg' in args.reward_func:
print('Avg mode')
current_ratio = Lambda(lambda x: tf.math.divide_no_nan(
tf.reduce_sum(x[0], axis=[1, -1]), delta + x[1]))(
[OD_, ac_d])
else:
print('Norm mode')
current_ratio = Lambda(lambda x: tf.math.divide_no_nan(
tf.reduce_sum(x[0], axis=[1, -1]), delta + tf.reduce_sum(
x[1], axis=[1, -1])))([OD_, OD_input])
ac_ratio = Lambda(
lambda x: tf.math.divide_no_nan(x[0], delta + x[1]))(
[ac_m, ac_d])
features = GNN_layer(16,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[0])([features_input, OD_])
features = GNN_layer(32,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[1])([features, OD_])
features = GNN_layer(dim,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[2])([features, OD_])
features = GNN_layer(dim,
aggregate_op=aggregate_op,
BN=BatchNormalization(),
activation='relu',
kernel_regularizer=l2(5e-4),
index=control_index[3])([features, OD_])
if args.pool_type == 'flatten':
print('Graph Pool Type: Flatten')
reward = Flatten()(features)
reward = Lambda(lambda x: tf.concat(
(x[0], x[1], x[2], x[3]), axis=-1))(
[ac_ratio, current_ratio, ac_d, reward])
reward = Dense(dim * 8, activation='relu')(reward)
reward = BatchNormalization()(reward)
reward = Dense(dim * 2, activation='relu')(reward)
reward = BatchNormalization()(reward)
elif args.pool_type == 'weight_sum':
print('Graph Pool Type: WeightSum')
OD_mean_out_ = Lambda(lambda x: tf.gather(
OD_mean_out,
tf.cast(x[:, args.period, 8], tf.int32),
axis=0))(state_input)
features = Lambda(lambda x: tf.concat(
(tf.expand_dims(x[0], -1), tf.expand_dims(x[1], -1),
tf.expand_dims(x[2], -1), x[3]),
axis=-1))([ac_ratio, current_ratio, ac_d, features])
features = Dense(dim, activation='relu')(features)
features = BatchNormalization()(features)
reward = GraphSoftmaxPool()([features, OD_mean_out_])
elif args.pool_type == 'weight_dense':
print('Graph Pool Type: WeightDenseSum')
OD_mean_out_ = Lambda(lambda x: tf.gather(
OD_mean_out,
tf.cast(x[:, args.period, 8], tf.int32),
axis=0))(state_input) # None * nb_regions
features = Lambda(lambda x: tf.concat(
(tf.expand_dims(x[0], -1), tf.expand_dims(x[1], -1),
tf.expand_dims(x[2], -1), x[3]),
axis=-1))([ac_ratio, current_ratio, ac_d, features])
features = Dense(dim, activation='relu')(features)
features = BatchNormalization()(features)
edge = Lambda(lambda x: tf.expand_dims(x, -1))(OD_mean_out_)
edge = Dense(16, activation='relu')(edge)
edge = BatchNormalization()(edge)
edge = Dense(1)(edge)
edge = Flatten()(edge)
reward = GraphSoftmaxPool()([features, edge])
else:
print('Not Supported Pool Type')
reward = Dense(1)(reward)
critic = Model(inputs=[action_input, state_input], outputs=reward)
return critic
return build_actor, build_critic
return build
def call(self, x, mask=None):
if hasattr(x, '_keras_shape'):
input_shape = x._keras_shape
elif hasattr(K, 'int_shape'):
input_shape = K.int_shape(x)
# ------------------ #
# 获取宽和高
# ------------------ #
layer_width = input_shape[self.waxis]
layer_height = input_shape[self.haxis]
img_width = self.img_size[0]
img_height = self.img_size[1]
box_widths = []
box_heights = []
for ar in self.aspect_ratios:
if ar == 1 and len(box_widths) == 0:
box_widths.append(self.min_size)
box_heights.append(self.min_size)
elif ar == 1 and len(box_widths) > 0:
box_widths.append(np.sqrt(self.min_size * self.max_size))
box_heights.append(np.sqrt(self.min_size * self.max_size))
elif ar != 1:
box_widths.append(self.min_size * np.sqrt(ar))
box_heights.append(self.min_size / np.sqrt(ar))
box_widths = 0.5 * np.array(box_widths)
box_heights = 0.5 * np.array(box_heights)
step_x = img_width / layer_width
step_y = img_height / layer_height
linx = np.linspace(0.5 * step_x, img_width - 0.5 * step_x, layer_width)
liny = np.linspace(0.5 * step_y, img_height - 0.5 * step_y,
layer_height)
centers_x, centers_y = np.meshgrid(linx, liny)
centers_x = centers_x.reshape(-1, 1)
centers_y = centers_y.reshape(-1, 1)
num_priors_ = len(self.aspect_ratios)
# 每一个先验框需要两个(centers_x, centers_y)，前一个用来计算左上角，后一个计算右下角
prior_boxes = np.concatenate((centers_x, centers_y), axis=1)
prior_boxes = np.tile(prior_boxes, (1, 2 * num_priors_))
# 获得先验框的左上角和右下角
prior_boxes[:, ::4] -= box_widths
prior_boxes[:, 1::4] -= box_heights
prior_boxes[:, 2::4] += box_widths
prior_boxes[:, 3::4] += box_heights
# 变成小数的形式
prior_boxes[:, ::2] /= img_width
prior_boxes[:, 1::2] /= img_height
prior_boxes = prior_boxes.reshape(-1, 4)
prior_boxes = np.minimum(np.maximum(prior_boxes, 0.0), 1.0)
num_boxes = len(prior_boxes)
if len(self.variances) == 1:
variances = np.ones((num_boxes, 4)) * self.variances[0]
elif len(self.variances) == 4:
variances = np.tile(self.variances, (num_boxes, 1))
else:
raise Exception('Must provide one or four variances.')
prior_boxes = np.concatenate((prior_boxes, variances), axis=1)
prior_boxes_tensor = K.expand_dims(K.variable(prior_boxes), 0)
pattern = [tf.shape(x)[0], 1, 1]
prior_boxes_tensor = tf.tile(prior_boxes_tensor, pattern)
return prior_boxes_tensor
def call(self, x, mask=None):
if hasattr(x, '_keras_shape'):
input_shape = x._keras_shape
elif hasattr(K, 'int_shape'):
if isinstance(x, list):
input_shape = K.int_shape(x[0])
else:
input_shape = K.int_shape(x)
feature_map_width = input_shape[2]
feature_map_height = input_shape[1]
img_width = self.img_size[1]
img_height = self.img_size[0]
box_heights = []
box_widths = []
for ar in self.aspect_ratios:
if ar == 1 and len(box_widths) == 0:
box_widths.append(self.min_size)
box_heights.append(self.min_size)
elif ar == 1 and len(box_widths) > 0:
box_widths.append(np.sqrt(self.min_size * self.max_size))
box_heights.append(np.sqrt(self.min_size * self.max_size))
elif ar != 0:
box_widths.append(self.min_size * np.sqrt(ar))
box_heights.append(self.min_size / np.sqrt(ar))
box_widths = np.array(box_widths) * 0.5
box_heights = np.array(box_heights) * 0.5
# Compute the grid of box center points. They are identical for all aspect ratios.
# Compute the step sizes, i.e. how far apart the anchor box center points will be vertically and horizontally.
step_height = img_height / feature_map_height
step_width = img_width / feature_map_width
linx = np.linspace(0.5 * step_width, img_width - 0.5 * step_width,
feature_map_height)
liny = np.linspace(0.5 * step_height, img_height - 0.5 * step_height,
feature_map_height)
center_x, center_y = np.meshgrid(linx, liny)
center_x = center_x.reshape(-1, 1)
center_y = center_y.reshape(-1, 1)
# Every prior_boxes need two boxes, one is be used (xmin, ymin), the other is be used (xmax, ymax)
num_priors = len(self.aspect_ratios)
prior_boxes = np.concatenate((center_x, center_y), axis=1)
prior_boxes = np.tile(prior_boxes, (1, 2 * num_priors))
# Compute four corners
prior_boxes[:, ::4] -= box_widths
prior_boxes[:, 1::4] -= box_heights
prior_boxes[:, 2::4] += box_widths
prior_boxes[:, 3::4] += box_heights
# Normalize
prior_boxes[:, ::2] /= img_width
prior_boxes[:, 1::2] /= img_height
prior_boxes = prior_boxes.reshape(-1, 4)
prior_boxes = np.minimum(np.maximum(prior_boxes, 0.0), 1.0)
num_boxes = len(prior_boxes)
if len(self.variances) == 1:
variances = np.ones((num_boxes, 4)) * self.variances[0]
elif len(self.variances) == 4:
variances = np.tile(self.variances, (num_boxes, 1))
else:
raise Exception('Must provide one or four variances.')
prior_boxes = np.concatenate((prior_boxes, variances), axis=1)
prior_boxes_tensor = K.expand_dims(
tf.cast(prior_boxes, dtype=tf.float32), 0)
pattern = [tf.shape(x)[0], 1, 1]
prior_boxes_tensor = tf.tile(prior_boxes_tensor, pattern)
return prior_boxes_tensor
def target_layer(x, axis=axis, start_i=cur, end_i=cur+split):
slices = [slice(None, None)] * len(K.int_shape(x))
slices[axis] = slice(start_i, end_i)
return x[tuple(slices)]
def yolov2_loss(detector_mask, matching_true_boxes, class_one_hot, true_boxes_grid, y_pred, info=False):
"""
Calculate YOLO V2 loss from prediction (y_pred) and ground truth tensors (detector_mask,
matching_true_boxes, class_one_hot, true_boxes_grid,)
Parameters
----------
- detector_mask : tensor, shape (batch, size, GRID_W, GRID_H, anchors_count, 1)
1 if bounding box detected by grid cell, else 0
- matching_true_boxes : tensor, shape (batch_size, GRID_W, GRID_H, anchors_count, 5)
Contains adjusted coords of bounding box in YOLO format
- class_one_hot : tensor, shape (batch_size, GRID_W, GRID_H, anchors_count, class_count)
One hot representation of bounding box label
- true_boxes_grid : annotations : tensor (shape : batch_size, max annot, 5)
true_boxes_grid format : x, y, w, h, c (coords unit : grid cell)
- y_pred : prediction from model. tensor (shape : batch_size, GRID_W, GRID_H, anchors count, (5 + labels count)
- info : boolean. True to get some infox about loss value
Returns
-------
- loss : scalar
- sub_loss : sub loss list : coords loss, class loss and conf loss : scalar
"""
# anchors tensor
anchors = np.array(ANCHORS)
anchors = anchors.reshape(len(anchors)//2, 2)
# grid coords tensor ---> GRID_W * GRID*H grid
# tf.tile(input, multiples, name=None)
# left up corner coord , total GRID_W * GRID*H * anchor_count
coord_x = tf.cast(tf.reshape(tf.tile(tf.range(GRID_W), [GRID_H]), (1, GRID_H, GRID_W, 1, 1)), tf.float32)
coord_y = tf.transpose(coord_x, (0,2,1,3,4))
coords = tf.tile(tf.concat([coord_x, coord_y], -1), [y_pred.shape[0], 1, 1, 5, 1])
# coordinate loss
# box regression
# bx = (sigmoid(tx) + cx ) /W
# bw = pw * e^tw
# pw is anchors W, cx is left up of coord , tx and tw are pred offset value, W is feature map width
# in this case, we don't multipy width, because the the coord in matching value also is during 0~16
pred_xy = K.sigmoid(y_pred[:,:,:,:,0:2]) # adjust center coords between 0 and 1
pred_xy = (pred_xy + coords) # add cell coord for comparaison with ground truth. New coords in grid cell unit
pred_wh = K.exp(y_pred[:,:,:,:,2:4]) * anchors # adjust width and height for comparaison with ground truth. New coords in grid cell unit
# pred_wh = (pred_wh * anchors) # unit: grid cell
nb_detector_mask = K.sum(tf.cast(detector_mask>0.0, tf.float32))
xy_loss = LAMBDA_COORD*K.sum(detector_mask*K.square(matching_true_boxes[...,:2] - pred_xy))/(nb_detector_mask + 1e-6) # Non /2
wh_loss = LAMBDA_COORD * K.sum(detector_mask * K.square(K.sqrt(matching_true_boxes[...,2:4])-
K.sqrt(pred_wh))) / (nb_detector_mask + 1e-6)
coord_loss = xy_loss + wh_loss
# class loss
pred_box_class = y_pred[...,5:]
true_box_class = tf.argmax(class_one_hot, -1)
# class_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class)
class_loss = K.sparse_categorical_crossentropy(target=true_box_class, output=pred_box_class, from_logits=True)
class_loss = K.expand_dims(class_loss, -1)*detector_mask
class_loss = LAMBDA_CLASS * K.sum(class_loss) / (nb_detector_mask + 1e-6)
# confidence loss
pred_conf = K.sigmoid(y_pred[..., 4:5]) # only two class : object or background
# for each detector : iou between prediction and ground truth
x1 = matching_true_boxes[...,0]
y1 = matching_true_boxes[...,1]
w1 = matching_true_boxes[...,2]
h1 = matching_true_boxes[...,3]
x2 = pred_xy[...,0]
y2 = pred_xy[...,1]
w2 = pred_wh[...,0]
h2 = pred_wh[...,1]
ious = iou(x1, y1, w1, h1, x2, y2, w2, h2)
ious = K.expand_dims(ious, -1)
# for each detector: best ious between pred and true_boxes
pred_xy = K.expand_dims(pred_xy, 4)
pred_wh = K.expand_dims(pred_wh, 4)
pred_wh_half = pred_wh / 2.
pred_mins = pred_xy - pred_wh_half
pred_maxes = pred_xy + pred_wh_half
true_boxe_shape = K.int_shape(true_boxes_grid)
true_boxes_grid = K.reshape(true_boxes_grid, [true_boxe_shape[0], 1, 1, 1, true_boxe_shape[1], true_boxe_shape[2]])
true_xy = true_boxes_grid[...,0:2]
true_wh = true_boxes_grid[...,2:4]
true_wh_half = true_wh * 0.5
true_mins = true_xy - true_wh_half
true_maxes = true_xy + true_wh_half
intersect_mins = K.maximum(pred_mins, true_mins) # shape : m, GRID_W, GRID_H, BOX, max_annot, 2
intersect_maxes = K.minimum(pred_maxes, true_maxes) # shape : m, GRID_W, GRID_H, BOX, max_annot, 2
intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.) # shape : m, GRID_W, GRID_H, BOX, max_annot, 1
intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] # shape : m, GRID_W, GRID_H, BOX, max_annot, 1
pred_areas = pred_wh[..., 0] * pred_wh[..., 1] # shape : m, GRID_W, GRID_H, BOX, 1, 1
true_areas = true_wh[..., 0] * true_wh[..., 1] # shape : m, GRID_W, GRID_H, BOX, max_annot, 1
union_areas = pred_areas + true_areas - intersect_areas
iou_scores = intersect_areas / union_areas # shape : m, GRID_W, GRID_H, BOX, max_annot, 1
best_ious = K.max(iou_scores, axis=4) # Best IOU scores.
best_ious = K.expand_dims(best_ious) # shape : m, GRID_W, GRID_H, BOX, 1
# no object confidence loss
no_object_detection = K.cast(best_ious < 0.6, K.dtype(best_ious))
noobj_mask = no_object_detection * (1 - detector_mask)
nb_noobj_mask = K.sum(tf.cast(noobj_mask > 0.0, tf.float32))
noobject_loss = LAMBDA_NOOBJECT * K.sum(noobj_mask * K.square(-pred_conf)) / (nb_noobj_mask + 1e-6)
# object confidence loss
object_loss = LAMBDA_OBJECT * K.sum(detector_mask * K.square(ious - pred_conf)) / (nb_detector_mask + 1e-6)
# total confidence loss
conf_loss = noobject_loss + object_loss
# total loss
loss = conf_loss + class_loss + coord_loss
sub_loss = [conf_loss, class_loss, coord_loss]
if info:
print('conf_loss : {:.4f}'.format(conf_loss))
print('class_loss : {:.4f}'.format(class_loss))
print('coord_loss : {:.4f}'.format(coord_loss))
print(' xy_loss : {:.4f}'.format(xy_loss))
print(' wh_loss : {:.4f}'.format(wh_loss))
print('--------------------')
print('total loss : {:.4f}'.format(loss))
# display masks for each anchors
for i in range(len(anchors)):
f, (ax1, ax2, ax3) = plt.subplot(1,3,figsize=(10,5))
# https://blog.csdn.net/Strive_For_Future/article/details/115052014?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522161883865316780262527067%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fall.%2522%257D&request_id=161883865316780262527067&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~all~first_rank_v2~rank_v29-2-115052014.first_rank_v2_pc_rank_v29&utm_term=f.tight_layout&spm=1018.2226.3001.4187
f.tight_layout()
f.suptitle("MASKS FOR ANCHOR {} :".format(anchors[i,...]))
ax1.matshow((K.sum(detector_mask[0,:,:,i], axis=2)), cmap='Greys', vmin=0, vmax=1)
ax1.set_title('detector_mask, count : {}'.format(K.sum(tf.cast(detector_mask[0,:,:,i] > 0., tf.int32))))
ax1.xaxis.set_ticks_position('bottom')
ax2.matshow((K.sum(no_object_detection[0,:,:,i], axis=2)), cmap='Greys', vmin=0, vmax=1)
ax2.set_title('no_object_detection mask')
ax2.xaxis.set_ticks_position('bottom')
ax3.matshow((K.sum(noobj_mask[0,:,:,i], axis=2)), cmap='Greys', vmin=0, vmax=1)
ax3.set_title('noobj_mask')
ax3.xaxis.set_ticks_position('bottom')
return loss, sub_loss
def sampling(args):
z_mean, z_log_var = args
batch = K.shape(z_mean)[0]
dim = K.int_shape(z_mean)[1]
epsilon = K.random_normal(shape=(batch, dim))
return z_mean + K.exp(0.5 * z_log_var) * epsilon
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith(".cfg"), "{} is not a .cfg file".format(
config_path)
assert weights_path.endswith(
".weights"), "{} is not a .weights file".format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
".h5"), "output path {} is not a .h5 file".format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print("Loading weights.")
weights_file = open(weights_path, "rb")
major, minor, revision = np.ndarray(shape=(3, ),
dtype="int32",
buffer=weights_file.read(12))
if (major * 10 + minor) >= 2 and major < 1000 and minor < 1000:
seen = np.ndarray(shape=(1, ),
dtype="int64",
buffer=weights_file.read(8))
else:
seen = np.ndarray(shape=(1, ),
dtype="int32",
buffer=weights_file.read(4))
print("Weights Header: ", major, minor, revision, seen)
print("Parsing Darknet config.")
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print("Creating Keras model.")
input_layer = Input(shape=(None, None, 3))
prev_layer = input_layer
all_layers = []
weight_decay = (float(cfg_parser["net_0"]["decay"])
if "net_0" in cfg_parser.sections() else 5e-4)
count = 0
out_index = []
for section in cfg_parser.sections():
print("Parsing section {}".format(section))
if section.startswith("convolutional"):
filters = int(cfg_parser[section]["filters"])
size = int(cfg_parser[section]["size"])
stride = int(cfg_parser[section]["stride"])
pad = int(cfg_parser[section]["pad"])
activation = cfg_parser[section]["activation"]
batch_normalize = "batch_normalize" in cfg_parser[section]
padding = "same" if pad == 1 and stride == 1 else "valid"
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print("conv2d", "bn" if batch_normalize else " ", activation,
weights_shape)
conv_bias = np.ndarray(shape=(filters, ),
dtype="float32",
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(
shape=(3, filters),
dtype="float32",
buffer=weights_file.read(filters * 12),
)
count += 3 * filters
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2], # running var
]
conv_weights = np.ndarray(
shape=darknet_w_shape,
dtype="float32",
buffer=weights_file.read(weights_size * 4),
)
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = ([conv_weights]
if batch_normalize else [conv_weights, conv_bias])
# Handle activation.
act_fn = None
if activation == "leaky":
pass # Add advanced activation later.
elif activation != "linear":
raise ValueError(
"Unknown activation function `{}` in section {}".format(
activation, section))
# Create Conv2D layer
if stride > 1:
# Darknet uses left and top padding instead of 'same' mode
prev_layer = ZeroPadding2D(((1, 0), (1, 0)))(prev_layer)
conv_layer = (Conv2D(
filters,
(size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding,
))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == "linear":
all_layers.append(prev_layer)
elif activation == "leaky":
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith("route"):
ids = [int(i) for i in cfg_parser[section]["layers"].split(",")]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print("Concatenating route layers:", layers)
concatenate_layer = Concatenate()(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith("maxpool"):
size = int(cfg_parser[section]["size"])
stride = int(cfg_parser[section]["stride"])
all_layers.append(
MaxPooling2D(pool_size=(size, size),
strides=(stride, stride),
padding="same")(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith("shortcut"):
index = int(cfg_parser[section]["from"])
activation = cfg_parser[section]["activation"]
assert activation == "linear", "Only linear activation supported."
all_layers.append(Add()([all_layers[index], prev_layer]))
prev_layer = all_layers[-1]
elif section.startswith("upsample"):
stride = int(cfg_parser[section]["stride"])
assert stride == 2, "Only stride=2 supported."
all_layers.append(UpSampling2D(stride)(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith("yolo"):
out_index.append(len(all_layers) - 1)
all_layers.append(None)
prev_layer = all_layers[-1]
elif section.startswith("net"):
pass
else:
raise ValueError(
"Unsupported section header type: {}".format(section))
# Create and save model.
if len(out_index) == 0:
out_index.append(len(all_layers) - 1)
model = Model(inputs=input_layer,
outputs=[all_layers[i] for i in out_index])
print(model.summary())
if args.weights_only:
model.save_weights("{}".format(output_path))
print("Saved Keras weights to {}".format(output_path))
else:
model.save("{}".format(output_path))
print("Saved Keras model to {}".format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print(
f"Read {count:0.0f} of {count + remaining_weights:0.0f} from Darknet weights."
)
if remaining_weights > 0:
print("Warning: {} unused weights".format(remaining_weights))
if args.plot_model:
plot(model, to_file="{}.png".format(output_root), show_shapes=True)
print("Saved model plot to {}.png".format(output_root))
def CrashNet():
road_map = Input(shape=(80, 200, 3))
x = Conv2D(24, kernel_size=5, strides=2, padding='same',
activation='relu')(road_map)
x = BatchNormalization()(x)
x = Conv2D(36, kernel_size=5, strides=2, padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
x = Conv2D(48, kernel_size=3, strides=2, padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
x = Conv2D(64, kernel_size=3, strides=1, padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
volumeSize = K.int_shape(x)
x = Flatten()(x)
latent = Dense(100)(x)
encoder = Model(inputs=road_map, outputs=latent, name='encoder')
latentInputs = Input(shape=(100, ))
x = Dense(np.prod(volumeSize[1:]))(latentInputs)
x = Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x)
x = Conv2DTranspose(64,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
x = Conv2DTranspose(48,
kernel_size=3,
strides=2,
padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
x = Conv2DTranspose(36,
kernel_size=5,
strides=2,
padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
x = Conv2DTranspose(24,
kernel_size=5,
strides=2,
padding='same',
activation='relu')(x)
x = BatchNormalization()(x)
outputs = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(x)
decoder = Model(latentInputs, outputs, name='decoder')
autoencoder = Model(road_map,
decoder(encoder(road_map)),
name='autoencoder')
return encoder, decoder, autoencoder
def conditional_vae(latent=50, dims=128, kernal_size=3, beta=1):
#define model
latent_size = latent
original_dims = dims * dims
input_shape = (dims, dims, 1)
#encoder input
X = Input(shape=input_shape)
cond = Input(shape=(2, ))
cond_layer = Dense(original_dims)(cond)
# reshape to additional channel
cond_img = Reshape((dims, dims, 1))(cond_layer)
merge = concatenate([X, cond_img])
#downsampling/encoder
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(merge)
x = BatchNormalization()(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x)
x = BatchNormalization()(x)
x = MaxPooling2D((2, 2), padding='same')(x)
# get shape info for later
shape = K.int_shape(x)
#latent space vector
x = Flatten()(x)
z_mean = Dense(latent_size)(x)
z_log_var = Dense(latent_size)(x)
#z layer layer
z = Lambda(sampling, output_shape=(latent_size, ),
name='z')([z_mean, z_log_var])
z = concatenate([z, cond], axis=1)
#instantiate encoder model
encoder = Model([X, cond], [z_mean, z_log_var, z], name='encoder')
encoder.summary()
#decoder input
latent_inputs = Input(shape=(latent_size + 2, ), name='z_sampling')
# #upsampling/decoder
x2 = Dense(shape[1] * shape[2] * shape[3])(latent_inputs)
x2 = Reshape((shape[1], shape[2], shape[3]))(x2)
#decoder layers
x2 = Conv2D(1024, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2D(1024, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2DTranspose(8, (kernal_size, kernal_size),
strides=(2, 2),
padding='same')(x2)
x2 = Conv2D(512, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2D(512, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2DTranspose(8, (kernal_size, kernal_size),
strides=(2, 2),
padding='same')(x2)
x2 = Conv2D(256, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2D(256, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2DTranspose(8, (kernal_size, kernal_size),
strides=(2, 2),
padding='same')(x2)
#extra for 256 dims
x2 = Conv2D(128, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2D(128, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2DTranspose(8, (kernal_size, kernal_size),
strides=(2, 2),
padding='same')(x2)
x2 = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2D(64, (kernal_size, kernal_size),
activation='relu',
padding='same')(x2)
x2 = BatchNormalization()(x2)
x2 = Conv2DTranspose(64, (kernal_size, kernal_size),
strides=(2, 2),
padding='same')(x2)
decoder_outputs = Conv2D(1, (kernal_size, kernal_size),
activation='sigmoid',
padding='same')(x2)
# instantiate decoder model
decoder = Model(latent_inputs, decoder_outputs, name='decoder')
decoder.summary()
# instantiate VAE model
outputs = decoder(encoder([X, cond])[2])
vae = Model([X, cond], outputs, name='vae')
#define losses
reconstruction_loss = mse(K.flatten(X), K.flatten(outputs))
reconstruction_loss *= dims * dims
kl_loss = (1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)) * beta
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
return encoder, decoder, vae
def _build(self):
### THE ENCODER
encoder_input = Input(shape=self.input_dim, name='encoder_input')
x = encoder_input
for i in range(self.n_layers_encoder):
conv_layer = Conv2D(filters=self.encoder_conv_filters[i],
kernel_size=self.encoder_conv_kernel_size[i],
strides=self.encoder_conv_strides[i],
padding='same',
name='encoder_conv_' + str(i))
x = conv_layer(x)
if self.use_batch_norm:
x = BatchNormalization()(x)
x = LeakyReLU()(x)
if self.use_dropout:
x = Dropout(rate=0.25)(x)
shape_before_flattening = K.int_shape(x)[1:]
x = Flatten()(x)
self.mu = Dense(self.z_dim, name='mu')(x)
self.log_var = Dense(self.z_dim, name='log_var')(x)
self.encoder_mu_log_var = Model(encoder_input, (self.mu, self.log_var))
def sampling(args):
mu, log_var = args
epsilon = K.random_normal(shape=K.shape(mu), mean=0., stddev=1.)
return mu + K.exp(log_var / 2) * epsilon
encoder_output = Lambda(sampling,
name='encoder_output')([self.mu, self.log_var])
self.encoder = Model(encoder_input, encoder_output)
### THE DECODER
decoder_input = Input(shape=(self.z_dim, ), name='decoder_input')
x = Dense(np.prod(shape_before_flattening))(decoder_input)
x = Reshape(shape_before_flattening)(x)
for i in range(self.n_layers_decoder):
conv_t_layer = Conv2DTranspose(
filters=self.decoder_conv_t_filters[i],
kernel_size=self.decoder_conv_t_kernel_size[i],
strides=self.decoder_conv_t_strides[i],
padding='same',
name='decoder_conv_t_' + str(i))
x = conv_t_layer(x)
if i < self.n_layers_decoder - 1:
if self.use_batch_norm:
x = BatchNormalization()(x)
x = LeakyReLU()(x)
if self.use_dropout:
x = Dropout(rate=0.25)(x)
else:
x = Activation('sigmoid')(x)
decoder_output = x
self.decoder = Model(decoder_input, decoder_output)
### THE FULL VAE
model_input = encoder_input
model_output = self.decoder(encoder_output)
self.model = Model(model_input, model_output)
def _initialize_params(
self,
model: KERAS_MODEL_TYPE,
use_logits: bool,
input_layer: int,
output_layer: int,
):
"""
Initialize most parameters of the classifier. This is a convenience function called by `__init__` and
`__setstate__` to avoid code duplication.
:param model: Keras model
:param use_logits: True if the output of the model are logits.
:param input_layer: Which layer to consider as the Input when the model has multiple input layers.
:param output_layer: Which layer to consider as the Output when the model has multiple output layers.
"""
# pylint: disable=E0401
if self.is_tensorflow:
import tensorflow as tf # lgtm [py/repeated-import]
if tf.executing_eagerly(): # pragma: no cover
raise ValueError("TensorFlow is executing eagerly. Please disable eager execution.")
import tensorflow.keras as keras
import tensorflow.keras.backend as k
self._losses = keras.losses
else:
import keras # lgtm [py/repeated-import]
import keras.backend as k
if hasattr(keras.utils, "losses_utils"):
self._losses = keras.utils.losses_utils
else:
self._losses = None
if hasattr(model, "inputs"):
self._input_layer = input_layer
self._input = model.inputs[input_layer]
else:
self._input = model.input
self._input_layer = 0
if hasattr(model, "outputs"):
self._output = model.outputs[output_layer]
self._output_layer = output_layer
else:
self._output = model.output
self._output_layer = 0
_, nb_classes = k.int_shape(self._output)
# Check for binary classification
if nb_classes == 1:
nb_classes = 2
self.nb_classes = nb_classes
self._input_shape = k.int_shape(self._input)[1:]
logger.debug(
"Inferred %i classes and %s as input shape for Keras classifier.",
self.nb_classes,
str(self.input_shape),
)
self._use_logits = use_logits
# Get loss function
if not hasattr(self._model, "loss"):
logger.warning("Keras model has no loss set. Classifier tries to use `k.sparse_categorical_crossentropy`.")
loss_function = k.sparse_categorical_crossentropy
else:
self._orig_loss = self._model.loss
if isinstance(self._model.loss, six.string_types):
loss_function = getattr(k, self._model.loss)
elif "__name__" in dir(self._model.loss) and self._model.loss.__name__ in [
"categorical_hinge",
"categorical_crossentropy",
"sparse_categorical_crossentropy",
"binary_crossentropy",
"kullback_leibler_divergence",
]:
if self._model.loss.__name__ in [
"categorical_hinge",
"kullback_leibler_divergence",
]:
loss_function = getattr(keras.losses, self._model.loss.__name__)
else:
loss_function = getattr(keras.backend, self._model.loss.__name__)
elif isinstance(
self._model.loss,
(
keras.losses.CategoricalHinge,
keras.losses.CategoricalCrossentropy,
keras.losses.SparseCategoricalCrossentropy,
keras.losses.BinaryCrossentropy,
keras.losses.KLDivergence,
),
):
loss_function = self._model.loss
else:
loss_function = getattr(k, self._model.loss.__name__)
# Check if loss function is an instance of loss function generator, the try is required because some of the
# modules are not available in older Keras versions
try:
flag_is_instance = isinstance(
loss_function,
(
keras.losses.CategoricalHinge,
keras.losses.CategoricalCrossentropy,
keras.losses.BinaryCrossentropy,
keras.losses.KLDivergence,
),
)
except AttributeError: # pragma: no cover
flag_is_instance = False
# Check if the labels have to be reduced to index labels and create placeholder for labels
if (
"__name__" in dir(loss_function)
and loss_function.__name__
in [
"categorical_hinge",
"categorical_crossentropy",
"binary_crossentropy",
"kullback_leibler_divergence",
]
) or flag_is_instance:
self._reduce_labels = False
label_ph = k.placeholder(shape=self._output.shape)
elif (
"__name__" in dir(loss_function) and loss_function.__name__ in ["sparse_categorical_crossentropy"]
) or isinstance(loss_function, keras.losses.SparseCategoricalCrossentropy):
self._reduce_labels = True
label_ph = k.placeholder(
shape=[
None,
]
)
else: # pragma: no cover
raise ValueError("Loss function not recognised.")
# Define the loss using the loss function
if "__name__" in dir(loss_function,) and loss_function.__name__ in [
"categorical_crossentropy",
"sparse_categorical_crossentropy",
"binary_crossentropy",
]:
loss_ = loss_function(label_ph, self._output, from_logits=self._use_logits)
elif "__name__" in dir(loss_function) and loss_function.__name__ in [
"categorical_hinge",
"kullback_leibler_divergence",
]:
loss_ = loss_function(label_ph, self._output)
elif isinstance(
loss_function,
(
keras.losses.CategoricalHinge,
keras.losses.CategoricalCrossentropy,
keras.losses.SparseCategoricalCrossentropy,
keras.losses.KLDivergence,
keras.losses.BinaryCrossentropy,
),
):
loss_ = loss_function(label_ph, self._output)
# Define loss gradients
loss_gradients = k.gradients(loss_, self._input)
if k.backend() == "tensorflow":
loss_gradients = loss_gradients[0]
elif k.backend() == "cntk": # pragma: no cover
raise NotImplementedError("Only TensorFlow is supported as backend for Keras.")
# Set loss, gradients and prediction functions
self._predictions_op = self._output
self._loss_function = loss_function
self._loss = loss_
self._loss_gradients = k.function([self._input, label_ph, k.learning_phase()], [loss_gradients])
# Get the internal layer
self._layer_names = self._get_layers()
