def compute_activation(self, lidx, test):
if lidx < 0:
return test
inp = self.model.input
functor = K.function([inp, K.learning_phase()],
[self.model.layers[lidx].output])
return functor([test])[0]
def detect_image(self, image):
print("detect_image")
# 图像调整
boxed_image = self._letterbox_image(
image, tuple(reversed(self.model_image_size)))
image_data = np.array(boxed_image, dtype=np.float)
# print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
# 识别
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
# 数据段值
return_boxes = []
return_scores = []
return_class_names = []
return_class_color = []
for i, class_id in enumerate(out_classes):
y1, x1, y2, x2 = np.array(out_boxes[i], dtype=np.int32)
return_boxes.append([x1, y1, (x2 - x1), (y2 - y1)])
return_scores.append(out_scores[i])
return_class_names.append(self.class_names[class_id])
return_class_color.append(self.colors[class_id])
return return_boxes, return_scores, return_class_names, return_class_color
def pure_detect_image(self, image):
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
# print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
return out_boxes, out_scores, out_classes
def eval_cleverhans():
# Set test phase
learning_phase = K.learning_phase()
K.set_learning_phase(0)
# Pre-process images
images_tf = images.astype(K.floatx())
images_tf /= 255.
# Wrapper for the Keras model
model_wrap = KerasModelWrapper(model)
# Initialize attack
if attack_params_dict['attack'] == 'fgsm':
attack = FastGradientMethod(model_wrap, sess=K.get_session())
attack_params = {'eps': attack_params_dict['eps'], 'clip_min': 0.,
'clip_max': 1.}
elif attack_params_dict['attack'] == 'deepfool':
attack = DeepFool(model_wrap, sess=K.get_session())
attack_params = {'clip_min': 0., 'clip_max': 1.}
elif attack_params_dict['attack'] == 'madry':
attack = ProjectedGradientDescent(model_wrap, sess=K.get_session())
attack_params = {'clip_min': 0., 'clip_max': 1.}
elif attack_params_dict['attack'] == 'carlini':
attack = CarliniWagnerL2(model_wrap, sess=K.get_session())
attack_params = {'clip_min': 0., 'clip_max': 1.}
else:
raise NotImplementedError()
# Define input TF placeholder
x = tf.placeholder(K.floatx(), shape=(None,) + images.shape[1:])
y = tf.placeholder(K.floatx(), shape=(None,) + (labels.shape[-1],))
# Define adversarial predictions symbolically
x_adv = attack.generate(x, **attack_params)
x_adv = tf.stop_gradient(x_adv)
predictions_adv = model(x_adv)
# Evaluate the accuracy of the model on adversarial examples
eval_par = {'batch_size': batch_size}
# feed_dict = {K.learning_phase(): attack_params_dict['learning_phase']}
# acc_adv = model_eval(K.get_session(), x, y, predictions_adv, images,
# labels, feed=feed_dict, args=eval_par)
acc_adv = model_eval(K.get_session(), x, y, predictions_adv, images_tf,
labels, args=eval_par)
print('Aversarial accuracy against %s: %.4f\n' %
(attack_params_dict['attack'], acc_adv))
# Set original phase
K.set_learning_phase(learning_phase)
return acc_adv
def detect_image(self, image):
if self.is_fixed_size:
assert self.model_image_size[0]%32 == 0, 'Multiples of 32 required'
assert self.model_image_size[1]%32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
# print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
return_boxes = []
return_scores = []
return_class_names = []
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
if predicted_class != 'Head': # Modify to detect other classes.
continue
box = out_boxes[i]
score = out_scores[i]
x = int(box[1])
y = int(box[0])
w = int(box[3] - box[1])
h = int(box[2] - box[0])
# h = int(round(box[2] - box[0])) #0: start Y, 1: start X, 2 end Y, 3 end X
if h < 24: ### too small head
continue
if x < 0:
w = w + x
x = 0
if y < 0:
h = h + y
y = 0
return_boxes.append([x, y, w, h])
return_scores.append(score)
return_class_names.append(predicted_class)
return return_boxes, return_scores, return_class_names
def predict_stochastic(neural_net):
"""
Have the neural network make predictions with the Dropout layers on, resulting in stochastic behavior from the
neural net itself.
Args:
neural_net:
data:
Returns:
"""
input = neural_net.input
output = neural_net.output
pred_func = K.function(input + [K.learning_phase()], [output])
return pred_func
def inject(self, model):
"""Inject the Lookahead algorithm for the given model.
The following code is modified from keras's _make_train_function method.
See: https://github.com/keras-team/keras/blob/master/keras/engine/training.py#L497
"""
if not hasattr(model, 'train_function'):
raise RuntimeError('You must compile your model before using it.')
model._check_trainable_weights_consistency()
if model.train_function is None:
inputs = (model._feed_inputs + model._feed_targets +
model._feed_sample_weights)
if model._uses_dynamic_learning_phase():
inputs += [K.learning_phase()]
fast_params = model._collected_trainable_weights
with K.name_scope('training'):
with K.name_scope(model.optimizer.__class__.__name__):
training_updates = model.optimizer.get_updates(
params=fast_params, loss=model.total_loss)
slow_params = [K.variable(p) for p in fast_params]
fast_updates = (model.updates + training_updates +
model.metrics_updates)
slow_updates, copy_updates = [], []
for p, q in zip(fast_params, slow_params):
slow_updates.append(K.update(q, q + self.alpha * (p - q)))
copy_updates.append(K.update(p, q))
# Gets loss and metrics. Updates weights at each call.
fast_train_function = K.function(inputs, [model.total_loss] +
model.metrics_tensors,
updates=fast_updates,
name='fast_train_function',
**model._function_kwargs)
def F(inputs):
self.count += 1
R = fast_train_function(inputs)
if self.count % self.k == 0:
K.batch_get_value(slow_updates)
K.batch_get_value(copy_updates)
return R
model.train_function = F
def detect_image(self, image):
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
print('THE functionality is not implemented!')
image_data = np.expand_dims(boxed_image, 0) # Add batch dimension.
out_boxes, out_scores, out_classes, polygons = self.sess.run(
[self.boxes, self.scores, self.classes, self.polygons],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.shape[0], image.shape[1]],
K.learning_phase(): 0
})
for b in range(0, out_boxes.shape[0]):
cy = (out_boxes[b, 0] + out_boxes[b, 2]) // 2
cx = (out_boxes[b, 1] + out_boxes[b, 3]) // 2
diagonal = np.sqrt(
np.power(out_boxes[b, 3] - out_boxes[b, 1], 2.0) +
np.power(out_boxes[b, 2] - out_boxes[b, 0], 2.0))
for i in range(0, NUM_ANGLES):
x1 = cx - math.cos(
math.radians(
(polygons[b, i + NUM_ANGLES] + i) / NUM_ANGLES *
360)) * polygons[b, i] * diagonal # scale[1]
y1 = cy - math.sin(
math.radians(
(polygons[b, i + NUM_ANGLES] + i) / NUM_ANGLES *
360)) * polygons[b, i] * diagonal # scale[0]
polygons[b, i] = x1
polygons[b, i + NUM_ANGLES] = y1
return out_boxes, out_scores, out_classes, polygons
def detect_image(self, image_id, image):
f = open("./input/detection-results/" + image_id + ".txt", "w")
# 调整图片使其符合输入要求
new_image_size = self.model_image_size
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.yolo_model.predict(
[image_data, input_image_shape])
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
for i, c in enumerate(out_classes):
predicted_class = self.class_names[int(c)]
try:
score = str(out_scores[i].numpy())
except:
score = str(out_scores[i])
top, left, bottom, right = out_boxes[i]
f.write("%s %s %s %s %s %s\n" %
(predicted_class, score[:6], str(int(left)), str(
int(top)), str(int(right)), str(int(bottom))))
f.close()
return
def __init__(self, model_path=None):
self.model_path = model_path
self.model_path_start = "/".join(model_path.split("/")[:-1])
self.model_config = model_path.split("/")[-1].split("_")[2]
self.model = load_model(self.model_path,
custom_objects={
"Interpolate1D": Interpolate1D,
"ConcreteDropout": ConcreteDropout,
"Split1D": Split1D,
"Scale": Scale,
"AutoScale": AutoScale
})
self.pred_func = K.function(self.model.input + [K.learning_phase()],
[self.model.output])
self.x_scaling_file = join(
self.model_path_start,
"gan_X_scaling_values_{0}.csv".format(self.model_config))
self.y_scaling_file = join(
self.model_path_start,
"gan_Y_scaling_values_{0}.csv".format(self.model_config))
self.y_scaling_values = pd.read_csv(self.y_scaling_file,
index_col="Channel")
self.x_scaling_values = pd.read_csv(self.x_scaling_file,
index_col="Channel")
def predict(sess, image_file):
image, image_data = preprocess_image(image_file,
model_image_size=(608, 608))
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],
feed_dict={
yolo_model.input:
image_data,
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
colors = generate_colors(class_names)
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
image.save(os.path.join("out", image_file), quality=90)
output_image = plt.imread(os.path.join("out", image_file))
imshow(output_image)
return out_scores, out_boxes, out_classes
def detect_image(self, image):
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = image_preporcess(
np.copy(image), tuple(reversed(self.model_image_size)))
image_data = boxed_image
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape:
[image.shape[0],
image.shape[1]], #[image.size[1], image.size[0]],
K.learning_phase(): 0
})
#print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
thickness = (image.shape[0] + image.shape[1]) // 600
fontScale = 1
ObjectsList = []
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
#label = '{}'.format(predicted_class)
scores = '{:.2f}'.format(score)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.shape[0],
np.floor(bottom + 0.5).astype('int32'))
right = min(image.shape[1], np.floor(right + 0.5).astype('int32'))
mid_h = (bottom - top) / 2 + top
mid_v = (right - left) / 2 + left
# put object rectangle
cv2.rectangle(image, (left, top), (right, bottom), self.colors[c],
thickness)
# get text size
(test_width, text_height), baseline = cv2.getTextSize(
label, cv2.FONT_HERSHEY_SIMPLEX, thickness / self.text_size, 1)
# put text rectangle
cv2.rectangle(image, (left, top),
(left + test_width, top - text_height - baseline),
self.colors[c],
thickness=cv2.FILLED)
# put text above rectangle
cv2.putText(image, label, (left, top - 2),
cv2.FONT_HERSHEY_SIMPLEX, thickness / self.text_size,
(0, 0, 0), 1)
# add everything to list
ObjectsList.append(
[top, left, bottom, right, mid_v, mid_h, label, scores])
print('Class:', predicted_class, 'Score:', score) # Add on
return image, ObjectsList
def test(images, labels, batch_size, model, model_adv, image_params_dict,
attack_params_dict, output_file=None, do_print=True):
"""
Tests the performance of a model on adversarial images. The adversarial
images are computed according to the attack specified in the arguments.
Parameters
----------
images : dask array
The set of images
labels : dask array
The ground truth labels
batch_size : int
Batch size
model : Keras Model
The model
model_adv : Keras Model
The model used to generate adversarial examples
image_params_dict : dict
Dictionary of data augmentation parameters
attack_params_dict : dict
Dictionary of the attack parameters
output_file : str or None
The outfile to write the results
do_print : bool
Whether to print the adversarial accuracy and MSE
Returns
-------
results_dict : dict
Dictionary containing some performance metrics
"""
# Set test phase and get session
sess = K.get_session()
if isinstance(K.learning_phase(), int):
learning_phase = K.learning_phase()
K.set_learning_phase(0)
# Initialize adversarial attack
attack, attack_params, bs = init_attack(model_adv, attack_params_dict,
sess)
if bs:
batch_size = bs
# Create batch generator
image_gen = data_input.get_generator(images, **image_params_dict)
batch_gen = batch_generator(image_gen, images, labels, batch_size,
aug_per_im=1, shuffle=False)
n_batches_per_epoch = int(np.ceil(float(images.shape[0]) / batch_size))
# Define input TF placeholder
if image_params_dict['crop_size']:
image_shape = image_params_dict['crop_size']
else:
image_shape = images.shape[1:]
x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape))
y = tf.placeholder(K.floatx(), shape=(bs,) + (labels.shape[-1],))
# Define adversarial predictions symbolically
x_adv = attack.generate(x, **attack_params)
x_adv = tf.stop_gradient(x_adv)
predictions_adv = model(x_adv)
# Define accuracy symbolically
correct_preds = tf.equal(tf.argmax(y, axis=-1),
tf.argmax(predictions_adv, axis=-1))
acc_value = tf.reduce_mean(tf.to_float(correct_preds))
# Define mean squared error symbolically
mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv)))
# Init results variables
accuracy = 0.0
mse = 0.0
# Initialize matrix to store the predictions.
# predictions = np.zeros([images.shape[0], labels.shape[-1]])
with sess.as_default():
init = 0
for _ in tqdm(range(n_batches_per_epoch)):
batch = next(batch_gen())
this_batch_size = batch[0].shape[0]
# Evaluate accuracy
if isinstance(batch[1], (list, )):
yy = batch[1][0]
else:
yy = batch[1]
batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy})
# Evaluate MSE
batch_mse = mse_value.eval(feed_dict={x: batch[0]})
# Adversarial predictions
# predictions[init:init+this_batch_size, :] = \
# predictions_adv.eval(feed_dict={x: batch[0]})
# Update accuracy and MSE
accuracy += (this_batch_size * batch_acc)
mse += (this_batch_size * batch_mse)
init += this_batch_size
accuracy /= images.shape[0]
mse /= images.shape[0]
# Compute accuracy
# acc_post = np.divide(np.sum(np.argmax(predictions, axis=1) == \
# np.argmax(labels, axis=1)),
# float(images.shape[0]))
if do_print:
print('Aversarial accuracy against %s: %.4f' %
(attack_params_dict['attack'], accuracy))
# print('Aversarial accuracy against %s: %.4f' %
# (attack_params_dict['attack'], acc_post))
print('MSE between %s adversaries and originals: %.4f\n' %
(attack_params_dict['attack'], mse))
if output_file is not None:
_write_results(accuracy, output_file)
# Set original phase
if isinstance(K.learning_phase(), int):
K.set_learning_phase(learning_phase)
return accuracy, mse
def detect_image(self, image):
start = timer()
#---------------------------------------------------------#
# 给图像增加灰条，实现不失真的resize
#---------------------------------------------------------#
new_image_size = (self.model_image_size[1], self.model_image_size[0])
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
#---------------------------------------------------------#
# 添加上batch_size维度
#---------------------------------------------------------#
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
#---------------------------------------------------------#
# 将图像输入网络当中进行预测！
#---------------------------------------------------------#
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.get_pred(
image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
# --------------------------------------------------------- #
# 设置字体
# --------------------------------------------------------- #
font = ImageFont.truetype(font='font/simhei.ttf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = max((image.size[0] + image.size[1]) // 300, 1)
person_face = image
for i, c in list(enumerate(out_classes)):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
top, left, bottom, right = box
top = top - 5
left = left - 5
bottom = bottom + 5
right = right + 5
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
filename = 'temp/' + str(int(round((time.time()) * 1000))) + '.jpg'
if predicted_class == 'person':
person_face = np.array(person_face)
person_face = person_face[top + 10:bottom + 10,
left + 10:right + 10]
person_face = cv2.cvtColor(person_face, cv2.COLOR_RGB2BGR)
cv2.imwrite(filename, person_face)
# 人脸识别
# face_image = image
# face_image = np.array(face_image.convert('RGB'))
#
# face_image = face_recognition.face_encodings(face_image[top+10:bottom+10, left+10:right+10])
# images = os.listdir('img')
# name = ''
# if face_image:
# face_image = face_image[0]
# for img in images:
# current_image = face_recognition.face_encodings(face_recognition.load_image_file('img/' + img))[0]
# result = face_recognition.compare_faces([face_image], current_image, 0.6)
#
# if result[0]:
# print("匹配: " + img)
# name = img.split('.')[0]
# 画框框
label = '{} {:.2f} '.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
label = label.encode('utf-8')
print(label, top, left, bottom, right)
# pil_image = Image.fromarray(face_image)
# pil_image.show()
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# 标记框
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
# 标记框上面的一个背景框
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin,
str(label, 'UTF-8'),
fill=(0, 0, 0),
font=font)
del draw
end = timer()
print(end - start)
return image
def detect_image(self, image):
# start = timer()
rects = []
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
# print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
# tf.Session.run(fetches, feed_dict=None)
# Runs the operations and evaluates the tensors in fetches.
#
# Args:
# fetches: A single graph element, or a list of graph elements(described above).
#
# feed_dict: A dictionary that maps graph elements to values(described above).
#
# Returns:Either a single value if fetches is a single graph element, or a
# list of values if fetches is a list(described above).
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
# print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
# font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
# size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))
# thickness = (image.size[0] + image.size[1]) // 500
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
# label = '{} {:.2f}'.format(predicted_class, score)
# draw = ImageDraw.Draw(image)
# label_size = draw.textsize(label, font)
y1, x1, y2, x2 = box
y1 = max(0, np.floor(y1 + 0.5).astype('float32'))
x1 = max(0, np.floor(x1 + 0.5).astype('float32'))
y2 = min(image.size[1], np.floor(y2 + 0.5).astype('float32'))
x2 = min(image.size[0], np.floor(x2 + 0.5).astype('float32'))
# print(label, (x1, y1), (x2, y2))
bbox = dict([("score", str(score)), ("class", predicted_class),
("x1", str(x1)), ("y1", str(y1)), ("x2", str(x2)),
("y2", str(y2))])
rects.append(bbox)
# if y1 - label_size[1] >= 0:
# text_origin = np.array([x1, y1 - label_size[1]])
# else:
# text_origin = np.array([x1, y1 + 1])
#
# # My kingdom for a good redistributable image drawing library.
# for i in range(thickness):
# draw.rectangle(
# [x1 + i, y1 + i, x2 - i, y2 - i],
# outline=self.colors[c])
# draw.rectangle(
# [tuple(text_origin), tuple(text_origin + label_size)],
# fill=self.colors[c])
# draw.text(text_origin, label, fill=(0, 0, 0), font=font)
# del draw
#
# end = timer()
# print(str(end - start))
return rects
def activations_norm(images, labels, batch_size, model, layer_regex,
daug_params, norms=['fro']):
"""
Computes the norm of the activation of all feature maps
Parameters
----------
images : h5py Dataset
The set of images
labels : h5py Dataset
The ground truth labels
batch_size : int
Batch size
model : Keras Model
The model
daug_params : dict
Dictionary of data augmentation parameters
Returns
-------
results_dict : dict
Dictionary containing some performance metrics
"""
def _update_stats(mean_norm, std_norm, norm):
mean_norm_batch = np.mean(norm, axis=0)
std_norm_batch = np.std(norm, axis=0)
mean_norm = init / float(end) * mean_norm + \
batch_size / float(end) * mean_norm_batch
std_norm = init / float(end) * std_norm ** 2 + \
batch_size / float(end) * std_norm_batch ** 2 + \
(init * batch_size) / float(end ** 2) * \
(mean_norm - mean_norm_batch) ** 2
std_norm = np.sqrt(std_norm)
return mean_norm, std_norm
def _frobenius_norm(activations):
norm = np.linalg.norm(
activations, ord='fro',
axis=tuple(range(1, len(activations.shape) - 1)))
return norm
def _inf_norm(activations):
norm = np.max(np.abs(activations),
axis=tuple(range(1, len(activations.shape) - 1)))
return norm
n_images = images.shape[0]
n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size))
# Create batch generator
image_gen = get_generator(images, **daug_params)
batch_gen = batch_generator(image_gen, images, labels, batch_size,
aug_per_im=1, shuffle=False)
# Initialize list to store the mean norm of the activations
results_dict = {'activations_norm': {}, 'summary': {}}
# Iterate over the layers
model = del_extra_nodes(model)
for layer in model.layers:
if re.match(layer_regex, layer.name):
layer_name = layer.name.encode('utf-8')
print('\nLayer {}'.format(layer_name))
output = model.get_layer(layer_name)\
.outbound_nodes[0].input_tensors[0]
get_output = K.function([model.input, K.learning_phase()],
[output])
n_channels = K.int_shape(output)[-1]
results_dict['activations_norm'].update({layer_name:
{n: {'mean': np.zeros(n_channels),
'std': np.zeros(n_channels)} for n in norms}})
layer_dict = results_dict['activations_norm'][layer_name]
init = 0
batch_gen.image_gen.reset()
for _ in tqdm(range(n_batches_per_epoch)):
batch_images, _ = next(batch_gen())
batch_size = batch_images.shape[0]
end = init + batch_size
activations = get_output([batch_images, 0])[0]
for norm_key in norms:
mean_norm = layer_dict[norm_key]['mean']
std_norm = layer_dict[norm_key]['std']
if norm_key == 'fro':
norm = _frobenius_norm(activations)
elif norm_key == 'inf':
norm = _inf_norm(activations)
else:
raise NotImplementedError('Implemented norms are fro '
'and inf')
mean_norm, std_norm = _update_stats(mean_norm, std_norm,
norm)
layer_dict[norm_key]['mean'] = mean_norm
layer_dict[norm_key]['std'] = std_norm
init = end
# Compute summary statistics across the channels
for layer, layer_dict in results_dict['activations_norm'].items():
results_dict['summary'].update({layer: {}})
for norm_key, norm_dict in layer_dict.items():
results_dict['summary'][layer].update({norm_key: {
'mean': np.mean(norm_dict['mean']),
'std': np.mean(norm_dict['std'])}})
return results_dict
def __init__(self,
mean_inputs=1,
hidden_layers=2,
hidden_neurons=8,
activation="selu",
l2_weight=0.01,
learning_rate=0.001,
mean_loss="mse",
res_loss="kullback_leibler_divergence",
noise_sd=1,
beta_1=0.9,
model_path=None,
dropout_alpha=0.5,
num_epochs=10,
batch_size=1024,
val_split=0.5,
verbose=0,
model_config=0):
self.config = dict(mean_inputs=mean_inputs,
hidden_layers=hidden_layers,
hidden_neurons=hidden_neurons,
activation=activation,
l2_weight=l2_weight,
learning_rate=learning_rate,
mean_loss=mean_loss,
res_loss=res_loss,
noise_sd=noise_sd,
beta_1=beta_1,
dropout_alpha=dropout_alpha,
model_path=model_path,
num_epochs=num_epochs,
batch_size=batch_size,
verbose=verbose,
model_config=model_config,
val_split=val_split)
mean_model_file = join(
model_path, "annres_config_{0:04d}_mean.nc".format(
self.config["model_config"]))
res_model_file = join(
model_path,
"annres_config_{0:04d}_res.nc".format(self.config["model_config"]))
self.x_scaling_file = join(
model_path,
"annres_scaling_values_{0:04d}.csv".format(model_config))
if model_path is None or not exists(mean_model_file):
nn_input = Input((mean_inputs, ))
nn_model = nn_input
for h in range(hidden_layers):
nn_model = Dense(hidden_neurons,
kernel_regularizer=l2(l2_weight))(nn_model)
if activation == "leaky":
nn_model = LeakyReLU(0.1)(nn_model)
else:
nn_model = Activation(activation)(nn_model)
nn_model = Dense(1)(nn_model)
nn_res_input_x = Input((mean_inputs, ))
nn_res_input_res = Input((1, ))
nn_res = concatenate([nn_res_input_x, nn_res_input_res])
for h in range(hidden_layers):
nn_res = Dense(hidden_neurons,
kernel_regularizer=l2(l2_weight))(nn_res)
if activation == "leaky":
nn_res = LeakyReLU(0.1)(nn_res)
else:
nn_res = Activation(activation)(nn_res)
nn_res = Dropout(dropout_alpha)(nn_res)
nn_res = GaussianNoise(noise_sd)(nn_res)
nn_res = Dense(1)(nn_res)
self.mean_model = Model(nn_input, nn_model)
self.mean_model.compile(Adam(lr=learning_rate, beta_1=beta_1),
loss=mean_loss)
self.res_model = Model([nn_res_input_x, nn_res_input_res], nn_res)
self.res_model.compile(Adam(lr=learning_rate, beta_1=beta_1),
loss=res_loss)
self.x_scaling_values = None
elif type(model_path) == str:
self.mean_model = load_model(mean_model_file)
self.res_model = load_model(res_model_file)
self.x_scaling_values = pd.read_csv(self.x_scaling_file,
index_col="Channel")
self.res_predict = K.function(
self.res_model.input + [K.learning_phase()],
[self.res_model.output])
def predict(predict_var,
x_unlabeled,
inputs,
y_true,
batch_sizes,
x_labeled=None,
y_labeled=None):
"""Evaluates predict_var, batchwise, over all points in x_unlabeled and x_labeled.
Args:
predict_var: list of tensors to evaluate and return
x_unlabeled: unlabeled input data
inputs: dictionary containing input_types and input_placeholders
as key, value pairs, respectively
y_true: true labels tensorflow placeholder
batch_sizes: dictionary containing input_types and batch_sizes as
key, value pairs, respectively
x_labeled: labeled input data
y_labeled: labeled input labels
Returns:
a list of length n containing the result of all tensors
in return_var, where n = len(x_unlabeled) + len(x_labeled)
"""
x_unlabeled, x_labeled, y_labeled = check_inputs(x_unlabeled, x_labeled,
y_labeled, y_true)
# combined data
x = np.concatenate((x_unlabeled, x_labeled), 0)
# get shape of y_true
y_shape = y_true.get_shape()[1:K.ndim(y_true)].as_list()
# calculate batches for predict loop
unlabeled_batch_size = batch_sizes.get('Unlabeled', 0)
labeled_batch_size = batch_sizes.get('Labeled', 0)
if 'Labeled' in batch_sizes and 'Unlabeled' in batch_sizes:
assert unlabeled_batch_size == labeled_batch_size
batch_size = min(len(x), max(unlabeled_batch_size, labeled_batch_size))
batches = make_batches(len(x), batch_size)
y_preds = []
# predict over all points
for _, (batch_start, batch_end) in enumerate(batches):
feed_dict = {K.learning_phase(): 0}
# feed corresponding input for each input_type
for input_type, input_placeholder in inputs.items():
if input_type == 'Unlabeled':
feed_dict[input_placeholder] = x[batch_start:batch_end]
elif input_type == 'Labeled':
if x_labeled:
batch_ids = np.random.choice(len(x_labeled),
size=min(
batch_sizes[input_type],
len(x_labeled)),
replace=False)
feed_dict[input_placeholder] = x_labeled[batch_ids]
feed_dict[y_true] = y_labeled[batch_ids]
else:
# we have no labeled points, so feed an empty array
feed_dict[input_placeholder] = x[0:0]
feed_dict[y_true] = np.empty([0] + y_shape)
# evaluate the batch
y_pred_batch = np.asarray(K.get_session().run(predict_var,
feed_dict=feed_dict))
y_preds.append(y_pred_batch)
if y_preds[0].shape:
return np.concatenate(y_preds)
else:
return np.sum(y_preds)
def detect_image(self, image):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
print(end - start)
# return out_boxes, out_scores, out_classes
return image
def detect_image(self, image):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required' # 必须为32的倍数
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image,
tuple(reversed(self.model_image_size))) # 填充/缩放图像为默认的416x416
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32') # 将图片转为ndarray？？
print(image_data.shape) # 打印ndarray维度 416x416x3
image_data /= 255. # 转换为0~1
image_data = np.expand_dims(image_data,
0) # Add batch dimension. 添加一个批次的维度
# 这个是核心中的核心。调用模型，返回 out_boxes, out_scores, out_classes 已经是结果了
out_boxes, out_scores, out_classes = self.sess.run( # sess.run是tensorflow的方法，传入的参数：
[self.boxes, self.scores, self.classes], # 盒子、得分、类别
feed_dict={ # 字典：
self.yolo_model.input: image_data, # 输入图像0~1，4维
self.input_image_shape: [image.size[1], image.size[0]], # 原始图像的尺寸
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes),
'img')) # # 检测出的盒子（框）
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32')) # 字体
thickness = (image.size[0] + image.size[1]) // 300 # 厚度？？
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c] # 通过索引去List取出对应分类label
box = out_boxes[i] # 框[]
score = out_scores[i] # 置信度
label = '{} {:.2f}'.format(predicted_class, score) # 预测出的标签
draw = ImageDraw.Draw(image) # 画图
label_size = draw.textsize(label, font) # 图上的标签文字
top, left, bottom, right = box # box的四个点坐标
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top),
(right, bottom)) # boat 1.00 (111, 0) (924, 559)
if top - label_size[1] >= 0: # 标签文字
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
# 画框
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle( # 文字背景
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font) # 文字
del draw
end = timer()
print(end - start) # 打印识别共耗时
return image
def detect_image(self, image):
if self.is_fixed_size:
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
# print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
#print(reversed(list(enumerate(out_classes))))
return_boxes = []
return_scores = []
return_class_names = []
return_plates = []
return_cuts = []
car_num = 0
person_num = 0
plate = ''
cut_name = ''
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
if predicted_class not in ('person', 'car', 'motorbike', 'bicycle',
'bus', 'traffic light'):
continue
elif predicted_class in ('car', 'bus'):
car_num += 1
#检测车牌区域
if (out_scores[i] > 0.7):
cut = image.crop(
(int(out_boxes[i][1]), int(out_boxes[i][0]),
int(out_boxes[i][3]), int(out_boxes[i][2])))
cut.save('cut' + str(i) + '.jpg')
cut_name = 'cut' + str(i) + '.jpg'
#appcode_demo.demo('cut'+str(i)+'.jpg')
cut = numpy.array(cut)
l = HyperLPR_plate_recognition(cut)
#l = pp.SimpleRecognizePlate(cut)
if l.__len__() != 0:
print('l', l)
#if (l[0][1] > 0.85):
plate = [l[0][0], l[0][1]]
# print(plate,l[0][1])
elif predicted_class == 'person':
person_num += 1
elif predicted_class == 'traffic light':
#检测红绿灯颜色
cut = image.crop((int(out_boxes[i][1]), int(out_boxes[i][0]),
int(out_boxes[i][3]), int(out_boxes[i][2])))
cut = numpy.array(cut)
color = self.get_color(cut)
predicted_class = str(color)
print(color)
box = out_boxes[i]
score = out_scores[i]
x = int(box[1])
y = int(box[0])
w = int(box[3] - box[1])
h = int(box[2] - box[0])
if x < 0:
w = w + x
x = 0
if y < 0:
h = h + y
y = 0
return_boxes.append([x, y, w, h])
return_scores.append(score)
return_class_names.append(predicted_class)
return_plates.append(plate)
return_cuts.append(cut_name)
plate = ''
cut_name = ''
return return_boxes, return_scores, return_class_names, return_plates, car_num, person_num, return_cuts
def detect_image(self, image):
#---------------------------------------------------------#
# 给图像增加灰条，实现不失真的resize
# 也可以直接resize进行识别
#---------------------------------------------------------#
if self.letterbox_image:
boxed_image = letterbox_image(image, (self.model_image_size[1],self.model_image_size[0]))
else:
boxed_image = image.convert('RGB')
boxed_image = boxed_image.resize((self.model_image_size[1],self.model_image_size[0]), Image.BICUBIC)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
#---------------------------------------------------------#
# 添加上batch_size维度
#---------------------------------------------------------#
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
#---------------------------------------------------------#
# 将图像输入网络当中进行预测！
#---------------------------------------------------------#
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.get_pred(image_data, input_image_shape)
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
#---------------------------------------------------------#
# 设置字体
#---------------------------------------------------------#
font = ImageFont.truetype(font='font/simhei.ttf',
size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))
thickness = max((image.size[0] + image.size[1]) // 300, 1)
for i, c in list(enumerate(out_classes)):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
top, left, bottom, right = box
top = top - 5
left = left - 5
bottom = bottom + 5
right = right + 5
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
# 画框框
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
label = label.encode('utf-8')
print(label, top, left, bottom, right)
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
for i in range(thickness):
draw.rectangle(
[left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin), tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, str(label,'UTF-8'), fill=(0, 0, 0), font=font)
del draw
return image
def on_train_begin(self, logs={}):
if not os.path.exists(self.cfg['SAVE_DIR']):
print("Making directory", self.cfg['SAVE_DIR'])
os.makedirs(self.cfg['SAVE_DIR'])
# Indexes of the layers which we keep track of. Basically, this will be any layer
# which has a 'kernel' attribute, which is essentially the "Dense" or "Dense"-like layers
self.layerixs = []
# Functions return activity of each layer
self.layerfuncs = []
# Functions return weights of each layer
self.layerweights = []
for lndx, l in enumerate(self.model.layers):
if hasattr(l, 'kernel'):
self.layerixs.append(lndx)
self.layerfuncs.append(
K.function(self.model.inputs, [
l.output,
]))
self.layerweights.append(l.kernel)
# print(self.model._feed_inputs)
# print(self.model._feed_sample_weights)
# print(self.model._feed_targets)
inputs = [
self.model._feed_inputs, self.model._feed_targets,
self.model._feed_sample_weights,
K.learning_phase()
]
# inputs = [self.model.input,
# self.model._feed_targets,
# self.model._feed_sample_weights,
# backend.symbollearning_phase()
# ]
# inputs =(3277,12)
# print(inputs)
# grads_fn = K.function(inputs=[model.inputs[0],
# model._feed_targets[0],
# backend.symbolic_learning_phase()],
# outputs=grads)
# g_learning = grads_fn([x, x, True])
# g_not_learning = grads_fn([x, x, False])
# [model.inputs[0], model._feed_targets[0], model.sample_weights[0]]
# Get gradients of all the relevant layers at once
grads = self.model.optimizer.get_gradients(self.model.total_loss,
self.layerweights)
self.get_gradients = K.function(inputs=inputs, outputs=grads)
# Get cross-entropy loss
self.get_loss = K.function(inputs=inputs,
outputs=[
self.model.total_loss,
])
def activations(images, labels, batch_size, model, layer_regex, nodaug_params,
daug_params, include_input=False, class_invariance=False,
n_daug_rep=0, norms=['fro']):
"""
Computes metrics from the activations, such as the norm of the feature
maps, data augmentation invariance, class invariance, etc.
Parameters
----------
images : h5py Dataset
The set of images
labels : h5py Dataset
The ground truth labels
batch_size : int
Batch size
model : Keras Model
The model
nodaug_params : dict
Dictionary of data augmentation parameters for the baseline
daug_params : dict
Dictionary of data augmentation parameters
include_input : bool
If True, the input layer is considered for the analysis
class_invariance : bool
If True, the class invariance score is computed
n_daug_rep : int
If larger than 0, the data augentation invariance score is computed,
performing n_daug_rep repetitions of random augmentations
norms : list
List of keywords to specify the types of norms to compute on the
activations
Returns
-------
results_dict : dict
Dictionary containing some performance metrics
"""
def _update_stats(mean_norm, std_norm, norm):
mean_norm_batch = np.mean(norm, axis=0)
std_norm_batch = np.std(norm, axis=0)
mean_norm = init / float(end) * mean_norm + \
batch_size / float(end) * mean_norm_batch
std_norm = init / float(end) * std_norm ** 2 + \
batch_size / float(end) * std_norm_batch ** 2 + \
(init * batch_size) / float(end ** 2) * \
(mean_norm - mean_norm_batch) ** 2
std_norm = np.sqrt(std_norm)
return mean_norm, std_norm
def _frobenius_norm(activations):
norm = np.linalg.norm(
activations, ord='fro',
axis=tuple(range(1, len(activations.shape) - 1)))
return norm
def _inf_norm(activations):
norm = np.max(np.abs(activations),
axis=tuple(range(1, len(activations.shape) - 1)))
return norm
model = del_extra_nodes(model)
n_images = images.shape[0]
n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size))
# Get relevant layers
if include_input:
layer_regex = '({}|.*input.*)'.format(layer_regex)
else:
layer_regex = layer_regex
layers = [layer.name for layer in model.layers
if re.compile(layer_regex).match(layer.name)]
# Initialize HDF5 to store the activations
# filename = 'hdf5_aux_{}'.format(time.time())
# activations_hdf5_aux = h5py.File(filename, 'w')
# hdf5_aux = [filename]
#
# grp_activations = activations_hdf5_aux.create_group('activations')
if class_invariance:
# grp_labels = activations_hdf5_aux.create_group('labels')
labels_true_da = []
labels_pred_da = []
predictions_da = []
# labels_true = grp_labels.create_dataset(
# 'labels_true', shape=(n_images, ), dtype=np.uint8)
# labels_pred = grp_labels.create_dataset(
# 'labels_pred', shape=(n_images, ), dtype=np.uint8)
# predictions = grp_labels.create_dataset(
# 'predictions', shape=labels.shape, dtype=K.floatx())
idx_softmax = model.output_names.index('softmax')
store_labels = True
else:
store_labels = False
# Initialize results dictionary
results_dict = {'activations_norm': {}, 'summary': {},
'class_invariance': {}, 'daug_invariance': {}}
# Iterate over the layers
for layer_name in layers:
# Create batch generator
image_gen = get_generator(images, **nodaug_params)
batch_gen = generate_batches(image_gen, images, labels, batch_size,
aug_per_im=1, shuffle=False)
layer = model.get_layer(layer_name)
layer_shape = layer.output_shape[1:]
n_channels = layer_shape[-1]
if re.compile('.*input.*').match(layer_name):
layer_name = 'input'
print('\nLayer {}\n'.format(layer_name))
# Create a Dataset for the activations of the layer
# activations_layer = grp_activations.create_dataset(
# layer_name, shape=(n_images, ) + layer_shape,
# dtype=K.floatx())
# Create dask array for the activations of the layer
activations_layer_da = []
# Initialize placeholders in the results dict for the layer
results_dict['activations_norm'].update({layer_name:
{n: {'mean': np.zeros(n_channels),
'std': np.zeros(n_channels)} for n in norms}})
layer_dict = results_dict['activations_norm'][layer_name]
activation_function = K.function([model.input,
K.learning_phase()],
[layer.output])
# Iterate over the data set in batches
init = 0
for batch_images, batch_labels in tqdm(
batch_gen, total=n_batches_per_epoch):
batch_size = batch_images.shape[0]
end = init + batch_size
# Store labels
if store_labels:
preds = model.predict_on_batch(batch_images)
if isinstance(preds, list):
preds = preds[idx_softmax]
labels_pred_da.append(da.from_array(
np.argmax(preds, axis=1)))
labels_true_da.append(da.from_array(
np.argmax(batch_labels, axis=1)))
predictions_da.append(da.from_array(preds))
# labels_pred[init:end] = np.argmax(preds, axis=1)
# labels_true[init:end] = np.argmax(batch_labels, axis=1)
# predictions[init:end, :] = preds
# Get and store activations
activations = activation_function([batch_images, 0])[0]
activations_layer_da.append(da.from_array(
activations, chunks=activations.shape))
# activations_layer[init:end] = activations
# Compute norms
for norm_key in norms:
mean_norm = layer_dict[norm_key]['mean']
std_norm = layer_dict[norm_key]['std']
if norm_key == 'fro':
norm = _frobenius_norm(activations)
elif norm_key == 'inf':
norm = _inf_norm(activations)
else:
raise NotImplementedError('Implemented norms are fro '
'and inf')
mean_norm, std_norm = _update_stats(mean_norm, std_norm,
norm)
layer_dict[norm_key]['mean'] = mean_norm
layer_dict[norm_key]['std'] = std_norm
init = end
if init == n_images:
store_labels = False
break
# Concatenate dask arrays
activations_layer_da = da.concatenate(activations_layer_da, axis=0)
activations_layer_da = activations_layer_da.reshape((n_images, -1))
d_activations = activations_layer_da.shape[-1]
if class_invariance:
print('\nComputing class invariance\n')
labels_pred_da = da.concatenate(labels_pred_da)
labels_true_da = da.concatenate(labels_true_da)
predictions_da = da.concatenate(predictions_da)
n_classes = len(np.unique(labels_true_da))
# Compute MSE matrix of the activations
r = da.reshape(da.sum(da.square(activations_layer_da),
axis=1), (-1, 1))
mse_matrix_da = (r - 2 * da.dot(activations_layer_da,
da.transpose(activations_layer_da)) \
+ da.transpose(r)) / d_activations
mse_matrix_da = mse_matrix_da.rechunk((mse_matrix_da.chunksize[0],
mse_matrix_da.shape[-1]))
# Compute class invariance
time0 = time()
results_dict['class_invariance'].update({layer_name: {}})
class_invariance_scores_da = []
if class_invariance:
# mse_matrix_mean = da.mean(mse_matrix_da).compute()
for cl in tqdm(range(n_classes)):
labels_cl = labels_pred_da == cl
labels_cl = labels_cl.compute()
mse_class = mse_matrix_da[labels_cl, :][:, labels_cl]
mse_class = mse_class.rechunk((-1, -1))
# mse_class_mean = da.mean(mse_class).compute()
# class_invariance_score = 1. - np.divide(
# mse_class_mean, mse_matrix_mean)
# results_dict['class_invariance'][layer_name].update(
# {cl: class_invariance_score})
class_invariance_scores_da.append(
1. - da.divide(da.mean(mse_class),
da.mean(mse_matrix_da)))
# Compute data augmentation invariance
print('\nComputing data augmentation invariance\n')
mse_daug_da = []
results_dict['daug_invariance'].update({layer_name: {}})
for r in range(n_daug_rep):
print('Repetition {}'.format(r))
image_gen_daug = get_generator(images, **daug_params)
batch_gen_daug = generate_batches(image_gen_daug, images, labels,
batch_size, aug_per_im=1,
shuffle=False)
activations_layer_daug_da = []
# Iterate over the data set in batches to compute activations
init = 0
for batch_images, batch_labels in tqdm(
batch_gen, total=n_batches_per_epoch):
batch_size = batch_images.shape[0]
end = init + batch_size
# Get and store activations
activations = activation_function([batch_images, 0])[0]
activations_layer_daug_da.append(da.from_array(
activations, chunks=activations.shape))
init = end
if init == n_images:
break
activations_layer_daug_da = da.concatenate(
activations_layer_daug_da, axis=0)
activations_layer_daug_da = activations_layer_daug_da.reshape(
(n_images, -1))
activations_layer_daug_da = activations_layer_daug_da.rechunk(
(activations_layer_daug_da.chunksize[0],
activations_layer_daug_da.shape[-1]))
# Compute MSE daug
mse_daug_da.append(da.mean(da.square(activations_layer_da - \
activations_layer_daug_da),
axis=1))
mse_daug_da = da.stack(mse_daug_da, axis=1)
mse_sum = da.repeat(da.reshape(da.sum(mse_matrix_da, axis=1),
(n_images, 1)), n_daug_rep, axis=1)
daug_invariance_score_da = 1 - n_images * da.divide(mse_daug_da, mse_sum)
time1 = time()
# Compute dask results and update results dict
results_dask = da.compute(class_invariance_scores_da,
daug_invariance_score_da)
time2 = time()
results_dict['class_invariance'][layer_name].update(
{cl: cl_inv_score
for cl, cl_inv_score in enumerate(results_dask[0])})
results_dict['daug_invariance'].update({layer_name:
{r: daug_inv_score
for r, daug_inv_score in enumerate(results_dask[1].T)}})
# Compute summary statistics of the norms across the channels
for layer, layer_dict in results_dict['activations_norm'].items():
results_dict['summary'].update({layer: {}})
for norm_key, norm_dict in layer_dict.items():
results_dict['summary'][layer].update({norm_key: {
'mean': np.mean(norm_dict['mean']),
'std': np.mean(norm_dict['std'])}})
return results_dict
def train_step(return_vars,
updates,
x_unlabeled,
inputs,
y_true,
batch_sizes,
x_labeled=None,
y_labeled=None,
batches_per_epoch=100):
"""Performs one training step.
Evaluates the tensors in return_vars and updates, then returns the values of
the tensors in return_vars.
Args:
return_vars: list of tensors to evaluate and return
updates: list of tensors to evaluate only
x_unlabeled: unlabeled input data
inputs: dictionary containing input_types and input_placeholders as key,
value pairs, respectively
y_true: true labels placeholder
batch_sizes: dictionary containing input_types and batch_sizes as key, value
pairs, respectively
x_labeled: labeled input data
y_labeled: labeled input labels
batches_per_epoch: parameter updates per epoch*
Returns:
the evaluated result of all tensors in return_vars, summed
across all epochs
*note: the term epoch is used loosely here, it does not necessarily
refer to one iteration over the entire dataset. instead, it
is just batches_per_epoch parameter updates.
"""
x_unlabeled, x_labeled, y_labeled = check_inputs(x_unlabeled, x_labeled,
y_labeled, y_true)
# combine data
x = np.concatenate((x_unlabeled, x_labeled), 0)
# get shape of y_true
y_shape = y_true.get_shape()[1:K.ndim(y_true)].as_list()
return_vars_ = np.zeros(shape=(len(return_vars)))
# train batches_per_epoch batches
for _ in range(0, batches_per_epoch):
feed_dict = {K.learning_phase(): 1}
# feed corresponding input for each input_type
for input_type, input_placeholder in inputs.items():
if input_type == 'Labeled':
if x_labeled:
batch_ids = np.random.choice(len(x_labeled),
size=min(
batch_sizes[input_type],
len(x_labeled)),
replace=False)
feed_dict[input_placeholder] = x_labeled[batch_ids]
feed_dict[y_true] = y_labeled[batch_ids]
else:
# we have no labeled points, so feed an empty array
feed_dict[input_placeholder] = x[0:0]
feed_dict[y_true] = np.empty([0] + y_shape)
elif input_type == 'Unlabeled':
if x_unlabeled:
batch_ids = np.random.choice(len(x_unlabeled),
size=batch_sizes[input_type],
replace=False)
feed_dict[input_placeholder] = x_unlabeled[batch_ids]
else:
# we have no unlabeled points, so feed an empty array
feed_dict[input_placeholder] = x[0:0]
all_vars = return_vars + updates
return_vars_ += np.asarray(K.get_session().run(
all_vars, feed_dict=feed_dict)[:len(return_vars)])
return return_vars_
def __init__(self,
inputs=1,
hidden_layers=2,
hidden_neurons=8,
activation="selu",
l2_weight=0.01,
learning_rate=0.001,
loss="mse",
noise_sd=1,
beta_1=0.9,
model_path=None,
dropout_alpha=0.5,
num_epochs=10,
batch_size=1024,
verbose=0,
model_config=0,
**kwargs):
self.config = dict(inputs=inputs,
hidden_layers=hidden_layers,
hidden_neurons=hidden_neurons,
activation=activation,
l2_weight=l2_weight,
learning_rate=learning_rate,
loss=loss,
noise_sd=noise_sd,
beta_1=beta_1,
dropout_alpha=dropout_alpha,
model_path=model_path,
num_epochs=num_epochs,
batch_size=batch_size,
verbose=verbose,
model_config=model_config,
corr=1,
res_sd=0)
if model_path is None:
nn_input = Input((inputs, ))
nn_model = nn_input
for h in range(hidden_layers):
nn_model = Dense(hidden_neurons,
kernel_regularizer=l2(l2_weight))(nn_model)
if activation == "leaky":
nn_model = LeakyReLU(0.1)(nn_model)
else:
nn_model = Activation(activation)(nn_model)
if dropout_alpha > 0:
nn_model = Dropout(dropout_alpha)(nn_model)
if noise_sd > 0:
nn_model = GaussianNoise(noise_sd)(nn_model)
nn_model = Dense(1)(nn_model)
self.model = Model(nn_input, nn_model)
self.model.compile(Adam(lr=learning_rate, beta_1=beta_1),
loss=loss)
self.x_scaling_file = None
self.x_scaling_values = None
self.sample_predict = K.function(
[self.model.input, K.learning_phase()], [self.model.output])
elif type(model_path) == str:
model_path_start = join(*model_path.split("/")[:-1])
self.model = load_model(model_path)
self.sample_predict = K.function(
[self.model.input, K.learning_phase()], [self.model.output])
self.x_scaling_file = join(
model_path_start, "ann_config_{0:04d}_scale.csv".format(
self.config["model_config"]))
self.x_scaling_values = pd.read_csv(self.x_scaling_file,
index_col="Channel")
def detect_image(self, image, show_stats=True):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, "Multiples of 32 required"
assert self.model_image_size[
1] % 32 == 0, "Multiples of 32 required"
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (
image.width - (image.width % 32),
image.height - (image.height % 32),
)
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype="float32")
if show_stats:
print(image_data.shape)
image_data /= 255.0
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0,
},
)
if show_stats:
print("Found {} boxes for {}".format(len(out_boxes), "img"))
out_prediction = []
font_path = os.path.join(os.path.dirname(__file__),
"font/FiraMono-Medium.otf")
font = ImageFont.truetype(font=font_path,
size=np.floor(3e-2 * image.size[1] +
0.5).astype("int32"))
thickness = (image.size[0] + image.size[1]) // 300
count_mask = np.count_nonzero(np.asarray(out_classes) == 1)
count_no_mask = np.count_nonzero(np.asarray(out_classes) == 0)
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = "{} {:.2f}".format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype("int32"))
left = max(0, np.floor(left + 0.5).astype("int32"))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype("int32"))
right = min(image.size[0], np.floor(right + 0.5).astype("int32"))
# image was expanded to model_image_size: make sure it did not pick
# up any box outside of original image (run into this bug when
# lowering confidence threshold to 0.01)
if top > image.size[1] or right > image.size[0]:
continue
if show_stats:
print(label, (left, top), (right, bottom))
# output as xmin, ymin, xmax, ymax, class_index, confidence
out_prediction.append([left, top, right, bottom, c, score])
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, bottom])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c],
)
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
if show_stats:
print("Time spent: {:.3f}sec".format(end - start))
return out_prediction, image, count_mask, count_no_mask
def test_adv(images, labels, batch_size, model, adv_model, daug_params,
attack_params):
"""
Tests the performance of a model on adversarial images. The adversarial
images are computed according to the attack specified in the arguments.
Parameters
----------
images : dask array
The set of images
labels : dask array
The ground truth labels
batch_size : int
Batch size
model : Keras Model
The model
adv_model : Keras Model
The model used to generate adversarial examples
daug_params : dict
Dictionary of data augmentation parameters
attack_params : dict
Dictionary of the attack parameters
Returns
-------
results_dict : dict
Dictionary containing some performance metrics
"""
# Get session
sess = K.get_session()
# Initialize adversarial attack
attack, attack_params_cleverhans, bs = init_attack(
adv_model, attack_params, sess)
if bs:
batch_size = bs
n_images = images.shape[0]
n_classes = labels.shape[1]
n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size))
# Create batch generator
image_gen = get_generator(images, **daug_params)
batch_gen = batch_generator(image_gen, images, labels, batch_size,
aug_per_im=1, shuffle=False)
# Define input TF placeholder
if daug_params['crop_size']:
image_shape = daug_params['crop_size']
else:
image_shape = images.shape[1:]
x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape))
y = tf.placeholder(K.floatx(), shape=(bs,) + (n_classes,))
# Define adversarial predictions symbolically
x_adv = attack.generate(x, **attack_params_cleverhans)
x_adv = tf.stop_gradient(x_adv)
predictions_adv = model(x_adv)
# Define accuracy and mean squared error symbolically
correct_preds = tf.equal(tf.argmax(y, axis=-1),
tf.argmax(predictions_adv, axis=-1))
acc_value = tf.reduce_mean(tf.to_float(correct_preds))
mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv)))
# Init results variables
accuracy = 0.0
mse = 0.0
with sess.as_default():
init = 0
for _ in tqdm(range(n_batches_per_epoch)):
batch = next(batch_gen())
this_batch_size = batch[0].shape[0]
# Evaluate accuracy
if isinstance(batch[1], (list, )):
yy = batch[1][0]
else:
yy = batch[1]
# Evaluate accuracy and MSE
batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy,
K.learning_phase(): 0})
accuracy += (this_batch_size * batch_acc)
batch_mse = mse_value.eval(feed_dict={x: batch[0],
K.learning_phase(): 0})
mse += (this_batch_size * batch_mse)
init += this_batch_size
accuracy /= n_images
mse /= n_images
results_dict = {'mean_acc': accuracy,
'mean_mse': mse}
return results_dict
def detect_image(self, image):
start = timer()
# 调整图片使其符合输入要求
new_image_size = (self.model_image_size[1], self.model_image_size[0])
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
if self.eager:
# 预测结果
input_image_shape = np.expand_dims(
np.array([image.size[1], image.size[0]], dtype='float32'), 0)
out_boxes, out_scores, out_classes = self.yolo_model.predict(
[image_data, input_image_shape])
else:
# 预测结果
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
# 设置字体
font = ImageFont.truetype(font='font/simhei.ttf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
small_pic = []
for i, c in list(enumerate(out_classes)):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
top, left, bottom, right = box
top = top - 5
left = left - 5
bottom = bottom + 5
right = right + 5
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
# 画框框
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
label = label.encode('utf-8')
print(label)
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin,
str(label, 'UTF-8'),
fill=(0, 0, 0),
font=font)
del draw
end = timer()
print(end - start)
return image
def detect_image(self, image):
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
print(out_classes)
detected_object = []
scores_of_object = []
coord = []
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(predicted_class, score, left, top, right, bottom)
if (left, top, right, bottom) not in coord:
detected_object.append(predicted_class)
scores_of_object.append(score)
coord.append((left, top, right, bottom))
# 높은 확률부터 출력된다는 전제..
while len(detected_object) > 5:
min_id = scores_of_object.index(min(scores_of_object))
detected_object.pop(min_id)
objects = {
"detected_object": detected_object,
"scores": scores_of_object,
"coord": coord
}
return objects
