def decode_not_split_input(self, batch_container):
""" Decodes the given encoded input and returns the raw voxelgrid. """
indices, path = self.sess.run([self.eval_latent_mask, self.eval_path],
feed_dict={K.learning_phase(): 0})
number_of_ambiguous_blocks = (indices == 0).sum()
print("Calc " + str(number_of_ambiguous_blocks) + " blocks")
output = None
for i in range(
int(
ceil(
float(number_of_ambiguous_blocks) /
self.dataset.batch_size))):
batch_output = self.sess.run(self.eval_preds_from_latent,
feed_dict={K.learning_phase(): 0})
if output is None:
output = batch_output
else:
output = np.concatenate((output, batch_output), 0)
batch_container[indices == 1] = self.dataset.truncation_threshold
batch_container[indices == -1] = -self.dataset.truncation_threshold
batch_container[indices == 0] = output
output = batch_container
for i, splits in enumerate([
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size()
]):
output = np.concatenate(np.split(output, splits, 0), i + 1)
return output, path
def __init__(self, layer):
self.layer = layer
weights = layer.get_weights()
W = weights[0]
b = weights[1]
filters = W.shape[3]
up_row = W.shape[0]
up_col = W.shape[1]
input_img = keras.layers.Input(shape=layer.input_shape[1:])
output = keras.layers.Conv2D(
filters, (up_row, up_col),
kernel_initializer=tf.constant_initializer(W),
bias_initializer=tf.constant_initializer(b),
padding='same')(input_img)
self.up_func = K.function([input_img, K.learning_phase()], [output])
# Deconv filter (exchange no of filters and depth of each filter)
W = np.transpose(W, (0, 1, 3, 2))
# Reverse columns and rows
W = W[::-1, ::-1, :, :]
down_filters = W.shape[3]
down_row = W.shape[0]
down_col = W.shape[1]
b = np.zeros(down_filters)
input_d = keras.layers.Input(shape=layer.output_shape[1:])
output = keras.layers.Conv2D(
down_filters, (down_row, down_col),
kernel_initializer=tf.constant_initializer(W),
bias_initializer=tf.constant_initializer(b),
padding='same')(input_d)
self.down_func = K.function([input_d, K.learning_phase()], [output])
def __init__(self, layer):
self.layer = layer
weights = layer.get_weights()
W = weights[0]
b = weights[1]
# Up method
input = Input(shape=layer.input_shape[1:])
output = keras.layers.Dense(
layer.output_shape[1],
kernel_initializer=tf.constant_initializer(W),
bias_initializer=tf.constant_initializer(b))(input)
self.up_func = K.function([input, K.learning_phase()], [output])
# Transpose W for down method
W = W.transpose()
self.input_shape = layer.input_shape
self.output_shape = layer.output_shape
b = np.zeros(self.input_shape[1])
flipped_weights = [W, b]
input = Input(shape=self.output_shape[1:])
output = keras.layers.Dense(
self.input_shape[1:],
kernel_initializer=tf.constant_initializer(W),
bias_initializer=tf.constant_initializer(b))(input)
self.down_func = K.function([input, K.learning_phase()], [output])
def __init__(self, layer):
'''
# Arguments
layer: an instance of Dense layer, whose configuration
will be used to initiate DDense(input_shape,
output_shape, weights)
'''
self.layer = layer
weights = layer.get_weights()
W = weights[0]
b = weights[1]
#Set up_func for DDense
input = Input(shape=layer.input_shape[1:])
output = Dense(output_dim=layer.output_shape[1], weights=[W, b])(input)
self.up_func = K.function([input, K.learning_phase()], output)
#Transpose W and set down_func for DDense
W = W.transpose()
self.input_shape = layer.input_shape
self.output_shape = layer.output_shape
b = np.zeros(self.input_shape[1])
flipped_weights = [W, b]
input = Input(shape=self.output_shape[1:])
output = Dense(output_dim=self.input_shape[1],
weights=flipped_weights)(input)
self.down_func = K.function([input, K.learning_phase()], output)
def get_FPS(self, image, test_interval):
#---------------------------------------------------------#
# 给图像增加灰条，实现不失真的resize
#---------------------------------------------------------#
new_image_size = (self.model_image_size[1], self.model_image_size[0])
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
#---------------------------------------------------------#
# 添加上batch_size维度
#---------------------------------------------------------#
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
#---------------------------------------------------------#
# 将图像输入网络当中进行预测！
#---------------------------------------------------------#
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.get_pred(
image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
t1 = time.time()
for _ in range(test_interval):
#---------------------------------------------------------#
# 将图像输入网络当中进行预测！
#---------------------------------------------------------#
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'),
0)
out_boxes, out_scores, out_classes = self.get_pred(
image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
t2 = time.time()
tact_time = (t2 - t1) / test_interval
return tact_time
def predict_not_split_input(self, return_label=False):
""" Encodes and decodes the given input and returns the result.
Args:
input: A numpy array which contains a "whole"/"non-split" voxelmap.
focused_blocks: The number of focused blocks which should be encoded per forward pass. The higher, the less redundant computation has to be done.
Returns:
The output as a numpy array. Has the same resolution as the given input.
"""
window_positions = self.dataset.possible_input_window_positions()
if len(window_positions) % self.dataset.eval_batch_size != 0:
raise SystemError()
output = None
label = None
for i in range(len(window_positions) // self.dataset.eval_batch_size):
if return_label:
batch_output, batch_label = self.sess.run(
[self.eval_preds, self.eval_label],
feed_dict={K.learning_phase(): 0})
if label is None:
label = batch_label
else:
label = np.concatenate((label, batch_label), 0)
else:
batch_output = self.sess.run(self.eval_preds,
feed_dict={K.learning_phase(): 0})
if output is None:
output = batch_output
else:
output = np.concatenate((output, batch_output), 0)
for i, splits in enumerate([
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size()
]):
output = np.concatenate(np.split(output, splits, 0), i + 1)
if return_label:
for i, splits in enumerate([
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size(),
self.dataset.data_size // self.dataset.output_size()
]):
label = np.concatenate(np.split(label, splits, 0), i + 1)
if return_label:
return output, label
else:
return output
def run_graph(self, images, outputs, image_metas=None):
"""Runs a sub-set of the computation graph that computes the given
outputs.
image_metas: If provided, the images are assumed to be already
molded (i.e. resized, padded, and normalized)
outputs: List of tuples (name, tensor) to compute. The tensors are
symbolic TensorFlow tensors and the names are for easy tracking.
Returns an ordered dict of results. Keys are the names received in the
input and values are Numpy arrays.
"""
model = self.keras_model
# Organize desired outputs into an ordered dict
outputs = OrderedDict(outputs)
for o in outputs.values():
assert o is not None
# Build a Keras function to run parts of the computation graph
inputs = model.inputs
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
inputs += [K.learning_phase()]
kf = K.function(model.inputs, list(outputs.values()))
# Prepare inputs
if image_metas is None:
molded_images, image_metas, _ = self.mold_inputs(images)
else:
molded_images = images
image_shape = molded_images[0].shape
# Anchors
anchors = self.get_anchors(image_shape)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
anchors = np.broadcast_to(anchors,
(self.config.BATCH_SIZE, ) + anchors.shape)
model_in = [molded_images, image_metas, anchors]
# Run inference
if model.uses_learning_phase and not isinstance(
K.learning_phase(), int):
model_in.append(0.)
outputs_np = kf(model_in)
# Pack the generated Numpy arrays into a a dict and log the results.
outputs_np = OrderedDict([(k, v)
for k, v in zip(outputs.keys(), outputs_np)])
for k, v in outputs_np.items():
log(k, v)
return outputs_np
def __init__(self, layer, linear=False):
self.layer = layer
self.linear = linear
self.activation = layer.activation
input = K.placeholder(shape=layer.output_shape)
output = self.activation(input)
# According to the original paper,
# In forward pass and backward pass, do the same activation(relu)
# Up method
self.up_func = K.function([input, K.learning_phase()], [output])
# Down method
self.down_func = K.function([input, K.learning_phase()], [output])
def saliency_maps(temp_model,
grid_train_temp,
range_top,
range_ex,
batch_size,
layer_id=[-1, -2]):
# allocation
top_examples = np.empty(
(range_top[1] - range_top[0], range_ex[1] - range_ex[0]),
dtype=int) # indices of (sorted neurons, sorted samples)
# array that contains gradients, size: top neuron, top exp, size of gradients on the input end
top_gradients = np.empty((
range_top[1] - range_top[0],
range_ex[1] - range_ex[0],
) + grid_train_temp.shape[1:])
batch_i = list(range(0, grid_train_temp.shape[0], batch_size)) + [
grid_train_temp.shape[0]
] # batch samples
# get weight from the output end
weights = temp_model.layers[layer_id[0]].get_weights()[0].ravel()
top_neurons = weights.argsort()[::-1][
range_top[0]:range_top[1]] # most activated neurals
# loop over neurons
print('Sorted order | neuron index | neuron weights')
for n, neuron in enumerate(top_neurons):
print(' {} | {} | {}'.format(
n, neuron, weights[neuron])) # order, index of neuron, weights
# define the activation of neurons as a backend function (for sorting the top examples)
act_func = K.function(
[temp_model.input, K.learning_phase()],
[temp_model.layers[layer_id[1]].output[:, neuron]])
# loss = a monotonic function that takes neurons' final output
loss = (temp_model.layers[layer_id[1]].output[:, neuron] - 4)**2
# calculate gradients from loss (output end) to input end
grads = K.gradients(loss, temp_model.input)[0]
# standardizing gradients
grads /= K.maximum(K.std(grads), K.epsilon())
# define gradients calculation as a backend function
grad_func = K.function([temp_model.input, K.learning_phase()], [grads])
# allocation activation array
act_values = np.zeros(grid_train_temp.shape[0])
# loop over samples by batch
for b in range(len(batch_i) - 1):
act_values[batch_i[b]:batch_i[b + 1]] = act_func(
[grid_train_temp[batch_i[b]:batch_i[b + 1]], 0])[0]
# sort activation values and reteave examples index / gradients
top_examples[n] = act_values.argsort()[::-1][range_ex[0]:range_ex[1]]
top_gradients[n, ...] = -grad_func(
[grid_train_temp[top_examples[n]], 0])[0]
return top_neurons, top_examples, top_gradients
def get_FPS(self, image, test_interval):
if self.letterbox_image:
boxed_image = letterbox_image(
image, (self.model_image_size[1], self.model_image_size[0]))
else:
boxed_image = image.convert('RGB')
boxed_image = boxed_image.resize(
(self.model_image_size[1], self.model_image_size[0]),
Image.BICUBIC)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0)
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.get_pred(
image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
t1 = time.time()
for _ in range(test_interval):
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'),
0)
out_boxes, out_scores, out_classes = self.get_pred(
image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
t2 = time.time()
tact_time = (t2 - t1) / test_interval
return tact_time
def loss_fn(y_true, y_pred, x, model):
if FLAGS.lm_target:
return melt.losses.sampled_bilm_loss(y_true, model.his_embs, model.softmax_loss_function)
if FLAGS.label_smoothing_rate:
prob = tf.random.uniform([], minval=0., maxval=FLAGS.label_smoothing_rate)
cond = tf.cast(tf.not_equal(y_true, 1), tf.float32)
y_true = cond * prob + (1. - cond)
kwargs = {}
use_weight = FLAGS.use_weight and K.learning_phase()
if FLAGS.loss_type == 'sigmoid':
kwargs['reduction'] = 'none'
loss = tf.compat.v1.losses.sigmoid_cross_entropy(y_true, y_pred, **kwargs)
elif FLAGS.loss_type == 'roc_auc':
loss = global_objectives.roc_auc_loss(y_true, y_pred, scope='roc_auc_loss', **kwargs)[0]
elif FLAGS.loss_type == 'pr_auc':
loss = global_objectives.precision_recall_auc_loss(y_true, y_pred, reuse=tf.AUTO_REUSE, scope='pr_auc_loss', **kwargs)[0]
else:
raise ValueError(FLAGS.loss_type)
if use_weight:
loss = tf.reduce_sum(loss) / tf.reduce_sum(x['weight'])
else:
loss = tf.reduce_mean(loss)
main_loss = loss
if FLAGS.aux_loss_rate:
aux_loss = melt.losses.sampled_bilm_loss(x['history'], model.his_embs, model)
loss = loss + aux_loss * FLAGS.aux_loss_rate
if not tf.executing_eagerly():
tag = 'train' if K.learning_phase() else 'valid'
tf.compat.v1.summary.scalar(f'loss/{tag}/main', main_loss)
tf.compat.v1.summary.scalar(f'info/{tag}/his_max', tf.reduce_max(model.history_len))
tf.compat.v1.summary.scalar(f'info/{tag}/his_min', tf.reduce_min(model.history_len))
tf.compat.v1.summary.scalar(f'info/{tag}/his_mean', tf.reduce_mean(model.history_len))
tf.compat.v1.summary.scalar(f'info/{tag}/impression_max', tf.reduce_max(model.impression_len))
tf.compat.v1.summary.scalar(f'info/{tag}/impression_min', tf.reduce_min(model.impression_len))
tf.compat.v1.summary.scalar(f'info/{tag}/impression_mean', tf.reduce_mean(model.impression_len))
tf.compat.v1.summary.scalar(f'info/{tag}/click_ratio', tf.reduce_mean(tf.cast(y_true, tf.float32)))
tf.compat.v1.summary.scalar(f'info/{tag}/pred_mean', tf.reduce_mean(tf.sigmoid(y_pred)))
if use_weight:
tf.compat.v1.summary.scalar(f'info/{tag}/weight_mean', tf.reduce_mean(x['weight']))
if FLAGS.aux_loss_rate or FLAGS.lm_target:
tf.compat.v1.summary.scalar(f'loss/{tag}/aux', aux_loss)
return loss
def predict_on_image(self, image: np.ndarray) -> Tuple[str, float]:
softmax_output_fn = K.function(
[self.network.get_layer('image').input,
K.learning_phase()],
[self.network.get_layer('softmax_output').output])
if image.dtype == np.uint8:
image = (image / 255).astype(np.float32)
# Get the prediction and confidence using softmax_output_fn, passing the right input into it.
##### Your code below (Lab 3)
input_image = np.expand_dims(image, 0)
softmax_output = softmax_output_fn([input_image, 0])[0]
input_length = np.array([softmax_output.shape[1]])
decoded, log_prob = K.ctc_decode(softmax_output,
input_length,
greedy=True)
pred_raw = K.eval(decoded[0])[0]
pred = ''.join(self.data.mapping[label] for label in pred_raw).strip()
neg_sum_logit = K.eval(log_prob)[0][0]
conf = np.exp(-neg_sum_logit)
##### Your code above (Lab 3)
return pred, conf
def saliency(interface, model, input, processed_input, output, processed_output):
with interface.graph.as_default():
with interface.sess.as_default():
output = output.argmax()
input_tensors = [model.layers[0].input, K.learning_phase()]
saliency_input = model.layers[1].input
saliency_output = model.layers[-1].output[:, output]
gradients = model.optimizer.get_gradients(saliency_output, saliency_input)
compute_gradients = K.function(inputs=input_tensors, outputs=gradients)
saliency_graph = compute_gradients(processed_input.reshape(1, 500))[0]
saliency_graph = saliency_graph.reshape(500, 32)
saliency_graph = np.abs(saliency_graph).sum(axis=1)
normalized_saliency = (saliency_graph - saliency_graph.min()) / \
(saliency_graph.max() - saliency_graph.min())
start_idx = np.where(processed_input[0] == START_TOKEN)[0][0]
heat_map = []
counter = 0
words = input.split(" ")
for i in range(start_idx + 1, 500):
heat_map.extend([normalized_saliency[i]] * len(words[counter]))
heat_map.append(0) # zero saliency value assigned to the spaces between words
counter += 1
return np.array(heat_map)
def detect_mc_dropout(in_org_model, in_model, in_mix_id_odd):
nb_MC_samples = 100
MC_output = K.function([in_model.layers[0].input,
K.learning_phase()], [in_model.layers[-1].output])
learning_phase = True # use dropout at test time
batch_size = 2000
for i_d in range(in_mix_id_odd.shape[0] // batch_size):
images_batch = in_mix_id_odd[i_d * batch_size:(i_d + 1) * batch_size]
MC_samples = [
MC_output([images_batch, learning_phase])[0]
for _ in range(nb_MC_samples)
]
MC_samples = np.array(MC_samples)
if i_d == 0:
features = MC_samples
else:
features = np.column_stack((features, MC_samples))
if (i_d + 1) * batch_size < in_mix_id_odd.shape[0]:
images_batch = in_mix_id_odd[(i_d + 1) *
batch_size:in_mix_id_odd.shape[0]]
MC_samples = [
MC_output([images_batch, learning_phase])[0]
for _ in range(nb_MC_samples)
]
MC_samples = np.array(MC_samples)
if i_d == 0:
features = MC_samples
else:
features = np.column_stack((features, MC_samples))
variance = np.mean(features, axis=0)
scores = -np.max(variance, axis=1)
return scores
def get_layer_outputs(model,
input_data,
layer_name=None,
layer_idx=None,
layer=None,
learning_phase=0):
"""Retrieves layer outputs given input data and layer info.
Arguments:
model: keras.Model/tf.keras.Model.
input_data: np.ndarray & supported formats(1). Data w.r.t. which loss is
to be computed for the gradient. Only for mode=='grads'.
labels: np.ndarray & supported formats. Labels w.r.t. which loss is
to be computed for the gradient. Only for mode=='grads'
layer_idx: int. Index of layer to fetch, via model.layers[layer_idx].
layer_name: str. Substring of name of layer to be fetched. Returns
earliest match if multiple found.
layer: keras.Layer/tf.keras.Layer. Layer whose gradients to return.
Overrides `layer_idx` and `layer_name`
(1): tf.data.Dataset, generators, .tfrecords, & other supported TensorFlow
input data formats
"""
_validate_args(layer_name, layer_idx, layer)
layer = get_layer(model, layer_name, layer_idx)
if TF_KERAS:
layers_fn = K.function([model.input], [layer.output])
else:
layers_fn = K.function([model.input, K.learning_phase()],
[layer.output])
return layers_fn([input_data, learning_phase])[0]
def inference(self, image):
image = Image.fromarray(image)
img_height = image.size[1]
img_width = image.size[0]
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = self.letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
# add batch dimension
image_data = np.expand_dims(image_data, 0)
boxes, scores, classes = self._sess.run(
[self._boxes, self._scores, self._classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [img_height, img_width],
K.learning_phase(): 0
})
boxes = [normalize_box(box, img_height, img_width) for box in boxes]
return scores, boxes
def _init_class_gradients(self, label: Optional[Union[int, List[int], np.ndarray]] = None) -> None:
# pylint: disable=E0401
if self.is_tensorflow:
import tensorflow.keras.backend as k
else:
import keras.backend as k
if len(self._output.shape) == 2:
nb_outputs = self._output.shape[1]
else: # pragma: no cover
raise ValueError("Unexpected output shape for classification in Keras model.")
if label is None:
logger.debug("Computing class gradients for all %i classes.", self.nb_classes)
if not hasattr(self, "_class_gradients"):
class_gradients = [k.gradients(self._predictions_op[:, i], self._input)[0] for i in range(nb_outputs)]
self._class_gradients = k.function([self._input], class_gradients)
else:
if isinstance(label, int):
unique_labels = [label]
else:
unique_labels = np.unique(label)
logger.debug("Computing class gradients for classes %s.", str(unique_labels))
if not hasattr(self, "_class_gradients_idx"):
self._class_gradients_idx = [None for _ in range(nb_outputs)]
for current_label in unique_labels:
if self._class_gradients_idx[current_label] is None:
class_gradients = [k.gradients(self._predictions_op[:, current_label], self._input)[0]]
self._class_gradients_idx[current_label] = k.function(
[self._input, k.learning_phase()], class_gradients
)
def _deconv(self, feature_map, f, k, s):
"""
The deconvolution operation to upsample the average feature map downstream
f = # of filters from previous leaky layer (int)
k = size of kernel from previous leaky layer
s = amount of stride from previous leaky layer
"""
x = Input(shape=(None, None, 1))
y = Conv2DTranspose(
filters=1,
kernel_size=(3, 3),
strides=(2, 2),
padding='same',
kernel_initializer=Ones(), # set all weights to 1
bias_initializer=Zeros() # set all biases to 0
)(x)
deconv_model = Model(inputs=[x], outputs=[y])
inps = [deconv_model.input, K.learning_phase()] # input placeholder
outs = [deconv_model.layers[-1].output] # output placeholder
deconv_func = K.function(inps, outs) # evaluation function
return deconv_func([feature_map, 0])[0]
def call(
self,
x: tf.Tensor,
pad_mask: Optional[tf.Tensor] = None,
training: Optional[Union[tf.Tensor, bool]] = None,
) -> Tuple[tf.Tensor, tf.Tensor]:
"""Apply transformer encoder layer.
Arguments:
x: A tensor with shape [batch_size, length, units].
pad_mask: Float tensor with shape broadcastable
to (..., length, length). Defaults to None.
training: A bool, whether in training mode or not.
Returns:
Transformer encoder layer output with shape [batch_size, length, units]
"""
if training is None:
training = K.learning_phase()
x_norm = self._layer_norm(x) # (batch_size, length, units)
attn_out, attn_weights = self._mha(x_norm,
x_norm,
pad_mask=pad_mask,
training=training)
attn_out = self._dropout(attn_out, training=training)
x += attn_out
ffn_out = x # (batch_size, length, units)
for layer in self._ffn_layers:
ffn_out = layer(ffn_out, training=training)
x += ffn_out
# (batch_size, length, units), (batch_size, num_heads, length, length)
return x, attn_weights
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
if self.validation_data and self.freq:
if epoch % self.freq == 0:
# TODO: implement batched calls to sess.run
# (current call will likely go OOM on GPU)
tensors = self.model.inputs + self.gt_outputs + self.model.sample_weights
if self.model.uses_learning_phase:
tensors += [K.learning_phase()]
val_data = list(v[:self.n_images]
for v in self.validation_data[:-1])
val_data += self.validation_data[-1:]
else:
val_data = list(v[:self.n_images]
for v in self.validation_data)
feed_dict = dict(zip(tensors, val_data))
result = self.sess.run([self.merged], feed_dict=feed_dict)
summary_str = result[0]
self.writer.add_summary(summary_str, epoch)
for name, value in logs.items():
if name in ['batch', 'size']:
continue
summary = tf.compat.v1.Summary()
summary_value = summary.value.add()
summary_value.simple_value = float(value)
summary_value.tag = name
self.writer.add_summary(summary, epoch)
self.writer.flush()
def call(self, inputs, training):
if training is None:
training = K.learning_phase()
masked_inputs = inputs
if training:
if self.noise_std > 0:
masked_inputs = GaussianNoise(self.noise_std)(masked_inputs)
if self.swap_prob > 0:
masked_inputs = SwapNoiseMasker(probs=[self.swap_prob] * self.input_dim,
seed=[self.seed] * 2)(masked_inputs)
if self.mask_prob > 0:
masked_inputs = ZeroNoiseMasker(probs=[self.mask_prob] * self.input_dim,
seed=[self.seed] * 2)(masked_inputs)
x = masked_inputs
encoded_list = []
for encoder in self.encoders:
x = encoder(x)
encoded_list.append(x)
encoded = Concatenate()(encoded_list) if len(encoded_list) > 1 else encoded_list[0]
decoded = self.decoder(encoded)
rec_loss = K.mean(mean_squared_error(inputs, decoded))
self.add_loss(rec_loss)
return encoded, decoded
def compute_model_uncertainty(self,
subset: Union[None,
pd.core.series.Series] = None,
n_iterations: int = 200) -> np.ndarray:
"""Predict with dropout as proposed by Gal and Ghahramani (2015).
See https://arxiv.org/abs/1506.02142.
Args:
subset: A Boolean Series that is True for observations for which
predictions will be produced. If None, default to all
observations.
n_iterations: Number of random dropout specifications to obtain
predictions from.
Returns:
A numpy array of predictions by observation, lead length, and
iteration.
"""
subset = survival_modeler.default_subset_to_all(subset, self.data)
model_inputs = split_categorical_features(
self.data[subset], self.categorical_features,
self.numeric_features) + [1.0]
predict_with_dropout = K.function(
self.model.inputs + [K.learning_phase()], self.model.outputs)
predictions = np.dstack([
predict_with_dropout(model_inputs)[0] for i in range(n_iterations)
])
return predictions
def call(self, x, training=None):
"""The output of the previous layer is given as an input to this layer.
Training phase: for 98% of the cases the output will be the argmax of the linear activations;
for the remaining 2% a class is selected by passing the output through a maxboltzmann controller.
Test phase: the output will be the argmax of the linear activations.
"""
if training is None:
training = K.learning_phase()
def exploration():
batch_size, n_classes = tf.shape(x)[0], self.output_dim
argmax_vector = tf.math.argmax(activations, axis=-1)
random_vector = tf.random.uniform([batch_size], minval=0, maxval=1)
multinomial_vector = tf.reshape(
tf.random.categorical(activations, 1), [-1])
selected_classes = tf.where(
tf.greater(random_vector, self.epsilon), argmax_vector,
multinomial_vector)
selected_classes_onehot = tf.one_hot(selected_classes, n_classes)
y_pred = tf.multiply(selected_classes_onehot, activations)
return y_pred
activations = K.dot(x, self.kernel)
output = tf_utils.smart_cond(
training, #if training==1 do exploration, else do lambda
exploration,
lambda: tf.one_hot(tf.math.argmax(activations, axis=-1), self.
output_dim))
return output
def __init__(self, model_name, model_path):
if self.is_tf_gpu_version() is True:
print('use tf GPU version,', tf.__version__)
else:
print('use tf CPU version,', tf.__version__)
# these three parameters are no need to modify
self.model_name = model_name
self.model_path = os.path.join(os.path.dirname(__file__),
'trained_weights_final.h5')
self.input_image_key = 'images'
self.anchors_path = os.path.join(os.path.dirname(__file__),
'yolo_anchors.txt')
self.classes_path = os.path.join(os.path.dirname(__file__),
'train_classes.txt')
# self.score = 0.01####0.6026
self.score = 0.01
self.iou = 0.45
self.model_image_size = (416, 416)
self.label_map = parse_classify_rule(
os.path.join(os.path.dirname(__file__), 'classify_rule.json'))
self.class_names = self._get_class()
self.anchors = self._get_anchors()
self.sess = K.get_session()
with self.sess.as_default():
with self.sess.graph.as_default():
self.K_learning_phase = K.learning_phase()
self.boxes, self.scores, self.classes = self.generate()
print('load weights file success')
def keras_feed_dict(model,
x=None,
y=None,
feed_dict={},
learning_phase=KERAS_LEARNING_PHASE_TEST):
"""Return a feed dict with inputs and labels suitable for Keras.
Args:
model: A Keras Model
x: Model inputs, or None if inputs are not fed
y: Model targets (labels), or None if targets are not fed
feed_dict: Additional feed_dict to merge with (if given, updated in
place)
learning_phase: 0 for TEST, 1 for TRAIN
Returns:
The new feed_dict (equal to feed_dict if that was provided).
"""
new_feed_dict = dict(feed_dict)
if x is not None:
new_feed_dict[model.inputs[0]] = x
new_feed_dict[model.sample_weights[0]] = np.ones(x.shape[0])
if y is not None:
new_feed_dict[model.targets[0]] = y
new_feed_dict[K.learning_phase()] = learning_phase # TEST phase
return new_feed_dict
def deartifacts_A2A(self, s, pin=None, useNoise=False, clip=False):
s_abs = np.abs(s)
s_phase = np.angle(s)
s_norm, s_abs_min, s_abs_max = to_double_all(s_abs)
s_norm = s_norm * np.exp(1j * s_phase)
stemp = s_norm.transpose([2, 0, 1])
if len(s.shape) == 3:
# reshape
num_real, num_imag = [1, 1]
stemp = np.expand_dims(np.expand_dims(stemp, axis=0), axis=4)
stemp_multi = np.concatenate(
(stemp.real * num_real, stemp.imag * num_imag), axis=4)
xtemp = self.sess.run(self.recons,
feed_dict={
self.x: stemp_multi,
k.learning_phase(): 0
})
else:
print('Incorrect s.shape')
exit()
xtemp = xtemp.squeeze().transpose([1, 2, 0, 3])
xtemp = (xtemp[..., 0] / num_real) + 1j * (xtemp[..., 1] / num_imag)
if useNoise:
noise = xtemp
else:
xtemp_phase = np.angle(xtemp)
xtemp = (np.abs(xtemp) * s_abs_max + s_abs_min) * np.exp(
1j * xtemp_phase)
noise = (s - xtemp)
return noise
def get_activations(model, model_inputs, layer_name=None):
activations = []
inp = model.input
model_multi_inputs_cond = True
if not isinstance(inp, list):
# only one input! let's wrap it in a list.
inp = [inp]
model_multi_inputs_cond = False
outputs = [
layer.output for layer in model.layers
if layer.name == layer_name or layer_name is None
] # all layer outputs
# we remove the placeholders (Inputs node in Keras). Not the most elegant though..
outputs = [output for output in outputs if 'input_' not in output.name]
funcs = [K.function(inp + [K.learning_phase()], [out])
for out in outputs] # evaluation functions
if model_multi_inputs_cond:
list_inputs = []
list_inputs.extend(model_inputs)
list_inputs.append(0.)
else:
list_inputs = [model_inputs, 0.]
# Learning phase. 0 = Test mode (no dropout or batch normalization)
# layer_outputs = [func([model_inputs, 0.])[0] for func in funcs]
activations = [func(list_inputs)[0] for func in funcs]
layer_names = [output.name for output in outputs]
result = dict(list(zip(layer_names, activations)))
return result
def compile_saliency_function(model, activation_layer='block5_conv3'):
input_img = model.input
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_output = layer_dict[activation_layer].output
max_output = K.max(layer_output, axis=3)
saliency = K.gradients(K.sum(max_output), input_img)[0]
return K.function([input_img, K.learning_phase()], [saliency])
def _get_training_value(self, training=None):
if training is None:
training = K.learning_phase()
if isinstance(training, int):
training = bool(training)
training = math_ops.logical_and(training, self.trainable)
return training
def _log_gradients(self):
if (not self.training_data):
raise ValueError(
"Need to pass in training data if logging gradients")
X_train = self.training_data[0]
y_train = self.training_data[1]
metrics = {}
weights = self.model.trainable_weights # weight tensors
# filter down weights tensors to only ones which are trainable
weights = [
weight for weight in weights
if self.model.get_layer(weight.name.split('/')[0]).trainable
]
gradients = self.model.optimizer.get_gradients(
self.model.total_loss, weights) # gradient tensors
input_tensors = [
self.model.inputs[0], # input data
# how much to weight each sample by
self.model.sample_weights[0],
self.model.targets[0], # labels
K.learning_phase(), # train or test mode
]
get_gradients = K.function(inputs=input_tensors, outputs=gradients)
grads = get_gradients([X_train, np.ones(len(y_train)), y_train])
for (weight, grad) in zip(weights, grads):
metrics["gradients/" + weight.name.split(':')[0] +
".gradient"] = wandb.Histogram(grad)
return metrics
