def compute_activation(self, lidx, test): if lidx < 0: return test inp = self.model.input functor = K.function([inp, K.learning_phase()], [self.model.layers[lidx].output]) return functor([test])[0]
def detect_image(self, image): print("detect_image") # 图像调整 boxed_image = self._letterbox_image( image, tuple(reversed(self.model_image_size))) image_data = np.array(boxed_image, dtype=np.float) # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. # 识别 out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) # 数据段值 return_boxes = [] return_scores = [] return_class_names = [] return_class_color = [] for i, class_id in enumerate(out_classes): y1, x1, y2, x2 = np.array(out_boxes[i], dtype=np.int32) return_boxes.append([x1, y1, (x2 - x1), (y2 - y1)]) return_scores.append(out_scores[i]) return_class_names.append(self.class_names[class_id]) return_class_color.append(self.colors[class_id]) return return_boxes, return_scores, return_class_names, return_class_color
def pure_detect_image(self, image): if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) return out_boxes, out_scores, out_classes
def eval_cleverhans(): # Set test phase learning_phase = K.learning_phase() K.set_learning_phase(0) # Pre-process images images_tf = images.astype(K.floatx()) images_tf /= 255. # Wrapper for the Keras model model_wrap = KerasModelWrapper(model) # Initialize attack if attack_params_dict['attack'] == 'fgsm': attack = FastGradientMethod(model_wrap, sess=K.get_session()) attack_params = {'eps': attack_params_dict['eps'], 'clip_min': 0., 'clip_max': 1.} elif attack_params_dict['attack'] == 'deepfool': attack = DeepFool(model_wrap, sess=K.get_session()) attack_params = {'clip_min': 0., 'clip_max': 1.} elif attack_params_dict['attack'] == 'madry': attack = ProjectedGradientDescent(model_wrap, sess=K.get_session()) attack_params = {'clip_min': 0., 'clip_max': 1.} elif attack_params_dict['attack'] == 'carlini': attack = CarliniWagnerL2(model_wrap, sess=K.get_session()) attack_params = {'clip_min': 0., 'clip_max': 1.} else: raise NotImplementedError() # Define input TF placeholder x = tf.placeholder(K.floatx(), shape=(None,) + images.shape[1:]) y = tf.placeholder(K.floatx(), shape=(None,) + (labels.shape[-1],)) # Define adversarial predictions symbolically x_adv = attack.generate(x, **attack_params) x_adv = tf.stop_gradient(x_adv) predictions_adv = model(x_adv) # Evaluate the accuracy of the model on adversarial examples eval_par = {'batch_size': batch_size} # feed_dict = {K.learning_phase(): attack_params_dict['learning_phase']} # acc_adv = model_eval(K.get_session(), x, y, predictions_adv, images, # labels, feed=feed_dict, args=eval_par) acc_adv = model_eval(K.get_session(), x, y, predictions_adv, images_tf, labels, args=eval_par) print('Aversarial accuracy against %s: %.4f\n' % (attack_params_dict['attack'], acc_adv)) # Set original phase K.set_learning_phase(learning_phase) return acc_adv
def detect_image(self, image): if self.is_fixed_size: assert self.model_image_size[0]%32 == 0, 'Multiples of 32 required' assert self.model_image_size[1]%32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image(image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) return_boxes = [] return_scores = [] return_class_names = [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] if predicted_class != 'Head': # Modify to detect other classes. continue box = out_boxes[i] score = out_scores[i] x = int(box[1]) y = int(box[0]) w = int(box[3] - box[1]) h = int(box[2] - box[0]) # h = int(round(box[2] - box[0])) #0: start Y, 1: start X, 2 end Y, 3 end X if h < 24: ### too small head continue if x < 0: w = w + x x = 0 if y < 0: h = h + y y = 0 return_boxes.append([x, y, w, h]) return_scores.append(score) return_class_names.append(predicted_class) return return_boxes, return_scores, return_class_names
def predict_stochastic(neural_net): """ Have the neural network make predictions with the Dropout layers on, resulting in stochastic behavior from the neural net itself. Args: neural_net: data: Returns: """ input = neural_net.input output = neural_net.output pred_func = K.function(input + [K.learning_phase()], [output]) return pred_func
def inject(self, model): """Inject the Lookahead algorithm for the given model. The following code is modified from keras's _make_train_function method. See: https://github.com/keras-team/keras/blob/master/keras/engine/training.py#L497 """ if not hasattr(model, 'train_function'): raise RuntimeError('You must compile your model before using it.') model._check_trainable_weights_consistency() if model.train_function is None: inputs = (model._feed_inputs + model._feed_targets + model._feed_sample_weights) if model._uses_dynamic_learning_phase(): inputs += [K.learning_phase()] fast_params = model._collected_trainable_weights with K.name_scope('training'): with K.name_scope(model.optimizer.__class__.__name__): training_updates = model.optimizer.get_updates( params=fast_params, loss=model.total_loss) slow_params = [K.variable(p) for p in fast_params] fast_updates = (model.updates + training_updates + model.metrics_updates) slow_updates, copy_updates = [], [] for p, q in zip(fast_params, slow_params): slow_updates.append(K.update(q, q + self.alpha * (p - q))) copy_updates.append(K.update(p, q)) # Gets loss and metrics. Updates weights at each call. fast_train_function = K.function(inputs, [model.total_loss] + model.metrics_tensors, updates=fast_updates, name='fast_train_function', **model._function_kwargs) def F(inputs): self.count += 1 R = fast_train_function(inputs) if self.count % self.k == 0: K.batch_get_value(slow_updates) K.batch_get_value(copy_updates) return R model.train_function = F
def detect_image(self, image): if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: print('THE functionality is not implemented!') image_data = np.expand_dims(boxed_image, 0) # Add batch dimension. out_boxes, out_scores, out_classes, polygons = self.sess.run( [self.boxes, self.scores, self.classes, self.polygons], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.shape[0], image.shape[1]], K.learning_phase(): 0 }) for b in range(0, out_boxes.shape[0]): cy = (out_boxes[b, 0] + out_boxes[b, 2]) // 2 cx = (out_boxes[b, 1] + out_boxes[b, 3]) // 2 diagonal = np.sqrt( np.power(out_boxes[b, 3] - out_boxes[b, 1], 2.0) + np.power(out_boxes[b, 2] - out_boxes[b, 0], 2.0)) for i in range(0, NUM_ANGLES): x1 = cx - math.cos( math.radians( (polygons[b, i + NUM_ANGLES] + i) / NUM_ANGLES * 360)) * polygons[b, i] * diagonal # scale[1] y1 = cy - math.sin( math.radians( (polygons[b, i + NUM_ANGLES] + i) / NUM_ANGLES * 360)) * polygons[b, i] * diagonal # scale[0] polygons[b, i] = x1 polygons[b, i + NUM_ANGLES] = y1 return out_boxes, out_scores, out_classes, polygons
def detect_image(self, image_id, image): f = open("./input/detection-results/" + image_id + ".txt", "w") # 调整图片使其符合输入要求 new_image_size = self.model_image_size boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. if self.eager: # 预测结果 input_image_shape = np.expand_dims( np.array([image.size[1], image.size[0]], dtype='float32'), 0) out_boxes, out_scores, out_classes = self.yolo_model.predict( [image_data, input_image_shape]) else: # 预测结果 out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) for i, c in enumerate(out_classes): predicted_class = self.class_names[int(c)] try: score = str(out_scores[i].numpy()) except: score = str(out_scores[i]) top, left, bottom, right = out_boxes[i] f.write("%s %s %s %s %s %s\n" % (predicted_class, score[:6], str(int(left)), str( int(top)), str(int(right)), str(int(bottom)))) f.close() return
def __init__(self, model_path=None): self.model_path = model_path self.model_path_start = "/".join(model_path.split("/")[:-1]) self.model_config = model_path.split("/")[-1].split("_")[2] self.model = load_model(self.model_path, custom_objects={ "Interpolate1D": Interpolate1D, "ConcreteDropout": ConcreteDropout, "Split1D": Split1D, "Scale": Scale, "AutoScale": AutoScale }) self.pred_func = K.function(self.model.input + [K.learning_phase()], [self.model.output]) self.x_scaling_file = join( self.model_path_start, "gan_X_scaling_values_{0}.csv".format(self.model_config)) self.y_scaling_file = join( self.model_path_start, "gan_Y_scaling_values_{0}.csv".format(self.model_config)) self.y_scaling_values = pd.read_csv(self.y_scaling_file, index_col="Channel") self.x_scaling_values = pd.read_csv(self.x_scaling_file, index_col="Channel")
def predict(sess, image_file): image, image_data = preprocess_image(image_file, model_image_size=(608, 608)) out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_file)) colors = generate_colors(class_names) draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) image.save(os.path.join("out", image_file), quality=90) output_image = plt.imread(os.path.join("out", image_file)) imshow(output_image) return out_scores, out_boxes, out_classes
def detect_image(self, image): if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = image_preporcess( np.copy(image), tuple(reversed(self.model_image_size))) image_data = boxed_image out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.shape[0], image.shape[1]], #[image.size[1], image.size[0]], K.learning_phase(): 0 }) #print('Found {} boxes for {}'.format(len(out_boxes), 'img')) thickness = (image.shape[0] + image.shape[1]) // 600 fontScale = 1 ObjectsList = [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) #label = '{}'.format(predicted_class) scores = '{:.2f}'.format(score) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.shape[0], np.floor(bottom + 0.5).astype('int32')) right = min(image.shape[1], np.floor(right + 0.5).astype('int32')) mid_h = (bottom - top) / 2 + top mid_v = (right - left) / 2 + left # put object rectangle cv2.rectangle(image, (left, top), (right, bottom), self.colors[c], thickness) # get text size (test_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, thickness / self.text_size, 1) # put text rectangle cv2.rectangle(image, (left, top), (left + test_width, top - text_height - baseline), self.colors[c], thickness=cv2.FILLED) # put text above rectangle cv2.putText(image, label, (left, top - 2), cv2.FONT_HERSHEY_SIMPLEX, thickness / self.text_size, (0, 0, 0), 1) # add everything to list ObjectsList.append( [top, left, bottom, right, mid_v, mid_h, label, scores]) print('Class:', predicted_class, 'Score:', score) # Add on return image, ObjectsList
def test(images, labels, batch_size, model, model_adv, image_params_dict, attack_params_dict, output_file=None, do_print=True): """ Tests the performance of a model on adversarial images. The adversarial images are computed according to the attack specified in the arguments. Parameters ---------- images : dask array The set of images labels : dask array The ground truth labels batch_size : int Batch size model : Keras Model The model model_adv : Keras Model The model used to generate adversarial examples image_params_dict : dict Dictionary of data augmentation parameters attack_params_dict : dict Dictionary of the attack parameters output_file : str or None The outfile to write the results do_print : bool Whether to print the adversarial accuracy and MSE Returns ------- results_dict : dict Dictionary containing some performance metrics """ # Set test phase and get session sess = K.get_session() if isinstance(K.learning_phase(), int): learning_phase = K.learning_phase() K.set_learning_phase(0) # Initialize adversarial attack attack, attack_params, bs = init_attack(model_adv, attack_params_dict, sess) if bs: batch_size = bs # Create batch generator image_gen = data_input.get_generator(images, **image_params_dict) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) n_batches_per_epoch = int(np.ceil(float(images.shape[0]) / batch_size)) # Define input TF placeholder if image_params_dict['crop_size']: image_shape = image_params_dict['crop_size'] else: image_shape = images.shape[1:] x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape)) y = tf.placeholder(K.floatx(), shape=(bs,) + (labels.shape[-1],)) # Define adversarial predictions symbolically x_adv = attack.generate(x, **attack_params) x_adv = tf.stop_gradient(x_adv) predictions_adv = model(x_adv) # Define accuracy symbolically correct_preds = tf.equal(tf.argmax(y, axis=-1), tf.argmax(predictions_adv, axis=-1)) acc_value = tf.reduce_mean(tf.to_float(correct_preds)) # Define mean squared error symbolically mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv))) # Init results variables accuracy = 0.0 mse = 0.0 # Initialize matrix to store the predictions. # predictions = np.zeros([images.shape[0], labels.shape[-1]]) with sess.as_default(): init = 0 for _ in tqdm(range(n_batches_per_epoch)): batch = next(batch_gen()) this_batch_size = batch[0].shape[0] # Evaluate accuracy if isinstance(batch[1], (list, )): yy = batch[1][0] else: yy = batch[1] batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy}) # Evaluate MSE batch_mse = mse_value.eval(feed_dict={x: batch[0]}) # Adversarial predictions # predictions[init:init+this_batch_size, :] = \ # predictions_adv.eval(feed_dict={x: batch[0]}) # Update accuracy and MSE accuracy += (this_batch_size * batch_acc) mse += (this_batch_size * batch_mse) init += this_batch_size accuracy /= images.shape[0] mse /= images.shape[0] # Compute accuracy # acc_post = np.divide(np.sum(np.argmax(predictions, axis=1) == \ # np.argmax(labels, axis=1)), # float(images.shape[0])) if do_print: print('Aversarial accuracy against %s: %.4f' % (attack_params_dict['attack'], accuracy)) # print('Aversarial accuracy against %s: %.4f' % # (attack_params_dict['attack'], acc_post)) print('MSE between %s adversaries and originals: %.4f\n' % (attack_params_dict['attack'], mse)) if output_file is not None: _write_results(accuracy, output_file) # Set original phase if isinstance(K.learning_phase(), int): K.set_learning_phase(learning_phase) return accuracy, mse
def detect_image(self, image): start = timer() #---------------------------------------------------------# # 给图像增加灰条,实现不失真的resize #---------------------------------------------------------# new_image_size = (self.model_image_size[1], self.model_image_size[0]) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') image_data /= 255. #---------------------------------------------------------# # 添加上batch_size维度 #---------------------------------------------------------# image_data = np.expand_dims(image_data, 0) # Add batch dimension. #---------------------------------------------------------# # 将图像输入网络当中进行预测! #---------------------------------------------------------# if self.eager: # 预测结果 input_image_shape = np.expand_dims( np.array([image.size[1], image.size[0]], dtype='float32'), 0) out_boxes, out_scores, out_classes = self.get_pred( image_data, input_image_shape) else: # 预测结果 out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) # --------------------------------------------------------- # # 设置字体 # --------------------------------------------------------- # font = ImageFont.truetype(font='font/simhei.ttf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = max((image.size[0] + image.size[1]) // 300, 1) person_face = image for i, c in list(enumerate(out_classes)): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] top, left, bottom, right = box top = top - 5 left = left - 5 bottom = bottom + 5 right = right + 5 top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) filename = 'temp/' + str(int(round((time.time()) * 1000))) + '.jpg' if predicted_class == 'person': person_face = np.array(person_face) person_face = person_face[top + 10:bottom + 10, left + 10:right + 10] person_face = cv2.cvtColor(person_face, cv2.COLOR_RGB2BGR) cv2.imwrite(filename, person_face) # 人脸识别 # face_image = image # face_image = np.array(face_image.convert('RGB')) # # face_image = face_recognition.face_encodings(face_image[top+10:bottom+10, left+10:right+10]) # images = os.listdir('img') # name = '' # if face_image: # face_image = face_image[0] # for img in images: # current_image = face_recognition.face_encodings(face_recognition.load_image_file('img/' + img))[0] # result = face_recognition.compare_faces([face_image], current_image, 0.6) # # if result[0]: # print("匹配: " + img) # name = img.split('.')[0] # 画框框 label = '{} {:.2f} '.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) label = label.encode('utf-8') print(label, top, left, bottom, right) # pil_image = Image.fromarray(face_image) # pil_image.show() if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # 标记框 for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) # 标记框上面的一个背景框 draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, str(label, 'UTF-8'), fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) return image
def detect_image(self, image): # start = timer() rects = [] if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. # tf.Session.run(fetches, feed_dict=None) # Runs the operations and evaluates the tensors in fetches. # # Args: # fetches: A single graph element, or a list of graph elements(described above). # # feed_dict: A dictionary that maps graph elements to values(described above). # # Returns:Either a single value if fetches is a single graph element, or a # list of values if fetches is a list(described above). out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) # print('Found {} boxes for {}'.format(len(out_boxes), 'img')) # font = ImageFont.truetype(font='font/FiraMono-Medium.otf', # size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) # thickness = (image.size[0] + image.size[1]) // 500 for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] # label = '{} {:.2f}'.format(predicted_class, score) # draw = ImageDraw.Draw(image) # label_size = draw.textsize(label, font) y1, x1, y2, x2 = box y1 = max(0, np.floor(y1 + 0.5).astype('float32')) x1 = max(0, np.floor(x1 + 0.5).astype('float32')) y2 = min(image.size[1], np.floor(y2 + 0.5).astype('float32')) x2 = min(image.size[0], np.floor(x2 + 0.5).astype('float32')) # print(label, (x1, y1), (x2, y2)) bbox = dict([("score", str(score)), ("class", predicted_class), ("x1", str(x1)), ("y1", str(y1)), ("x2", str(x2)), ("y2", str(y2))]) rects.append(bbox) # if y1 - label_size[1] >= 0: # text_origin = np.array([x1, y1 - label_size[1]]) # else: # text_origin = np.array([x1, y1 + 1]) # # # My kingdom for a good redistributable image drawing library. # for i in range(thickness): # draw.rectangle( # [x1 + i, y1 + i, x2 - i, y2 - i], # outline=self.colors[c]) # draw.rectangle( # [tuple(text_origin), tuple(text_origin + label_size)], # fill=self.colors[c]) # draw.text(text_origin, label, fill=(0, 0, 0), font=font) # del draw # # end = timer() # print(str(end - start)) return rects
def activations_norm(images, labels, batch_size, model, layer_regex, daug_params, norms=['fro']): """ Computes the norm of the activation of all feature maps Parameters ---------- images : h5py Dataset The set of images labels : h5py Dataset The ground truth labels batch_size : int Batch size model : Keras Model The model daug_params : dict Dictionary of data augmentation parameters Returns ------- results_dict : dict Dictionary containing some performance metrics """ def _update_stats(mean_norm, std_norm, norm): mean_norm_batch = np.mean(norm, axis=0) std_norm_batch = np.std(norm, axis=0) mean_norm = init / float(end) * mean_norm + \ batch_size / float(end) * mean_norm_batch std_norm = init / float(end) * std_norm ** 2 + \ batch_size / float(end) * std_norm_batch ** 2 + \ (init * batch_size) / float(end ** 2) * \ (mean_norm - mean_norm_batch) ** 2 std_norm = np.sqrt(std_norm) return mean_norm, std_norm def _frobenius_norm(activations): norm = np.linalg.norm( activations, ord='fro', axis=tuple(range(1, len(activations.shape) - 1))) return norm def _inf_norm(activations): norm = np.max(np.abs(activations), axis=tuple(range(1, len(activations.shape) - 1))) return norm n_images = images.shape[0] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Create batch generator image_gen = get_generator(images, **daug_params) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) # Initialize list to store the mean norm of the activations results_dict = {'activations_norm': {}, 'summary': {}} # Iterate over the layers model = del_extra_nodes(model) for layer in model.layers: if re.match(layer_regex, layer.name): layer_name = layer.name.encode('utf-8') print('\nLayer {}'.format(layer_name)) output = model.get_layer(layer_name)\ .outbound_nodes[0].input_tensors[0] get_output = K.function([model.input, K.learning_phase()], [output]) n_channels = K.int_shape(output)[-1] results_dict['activations_norm'].update({layer_name: {n: {'mean': np.zeros(n_channels), 'std': np.zeros(n_channels)} for n in norms}}) layer_dict = results_dict['activations_norm'][layer_name] init = 0 batch_gen.image_gen.reset() for _ in tqdm(range(n_batches_per_epoch)): batch_images, _ = next(batch_gen()) batch_size = batch_images.shape[0] end = init + batch_size activations = get_output([batch_images, 0])[0] for norm_key in norms: mean_norm = layer_dict[norm_key]['mean'] std_norm = layer_dict[norm_key]['std'] if norm_key == 'fro': norm = _frobenius_norm(activations) elif norm_key == 'inf': norm = _inf_norm(activations) else: raise NotImplementedError('Implemented norms are fro ' 'and inf') mean_norm, std_norm = _update_stats(mean_norm, std_norm, norm) layer_dict[norm_key]['mean'] = mean_norm layer_dict[norm_key]['std'] = std_norm init = end # Compute summary statistics across the channels for layer, layer_dict in results_dict['activations_norm'].items(): results_dict['summary'].update({layer: {}}) for norm_key, norm_dict in layer_dict.items(): results_dict['summary'][layer].update({norm_key: { 'mean': np.mean(norm_dict['mean']), 'std': np.mean(norm_dict['std'])}}) return results_dict
def __init__(self, mean_inputs=1, hidden_layers=2, hidden_neurons=8, activation="selu", l2_weight=0.01, learning_rate=0.001, mean_loss="mse", res_loss="kullback_leibler_divergence", noise_sd=1, beta_1=0.9, model_path=None, dropout_alpha=0.5, num_epochs=10, batch_size=1024, val_split=0.5, verbose=0, model_config=0): self.config = dict(mean_inputs=mean_inputs, hidden_layers=hidden_layers, hidden_neurons=hidden_neurons, activation=activation, l2_weight=l2_weight, learning_rate=learning_rate, mean_loss=mean_loss, res_loss=res_loss, noise_sd=noise_sd, beta_1=beta_1, dropout_alpha=dropout_alpha, model_path=model_path, num_epochs=num_epochs, batch_size=batch_size, verbose=verbose, model_config=model_config, val_split=val_split) mean_model_file = join( model_path, "annres_config_{0:04d}_mean.nc".format( self.config["model_config"])) res_model_file = join( model_path, "annres_config_{0:04d}_res.nc".format(self.config["model_config"])) self.x_scaling_file = join( model_path, "annres_scaling_values_{0:04d}.csv".format(model_config)) if model_path is None or not exists(mean_model_file): nn_input = Input((mean_inputs, )) nn_model = nn_input for h in range(hidden_layers): nn_model = Dense(hidden_neurons, kernel_regularizer=l2(l2_weight))(nn_model) if activation == "leaky": nn_model = LeakyReLU(0.1)(nn_model) else: nn_model = Activation(activation)(nn_model) nn_model = Dense(1)(nn_model) nn_res_input_x = Input((mean_inputs, )) nn_res_input_res = Input((1, )) nn_res = concatenate([nn_res_input_x, nn_res_input_res]) for h in range(hidden_layers): nn_res = Dense(hidden_neurons, kernel_regularizer=l2(l2_weight))(nn_res) if activation == "leaky": nn_res = LeakyReLU(0.1)(nn_res) else: nn_res = Activation(activation)(nn_res) nn_res = Dropout(dropout_alpha)(nn_res) nn_res = GaussianNoise(noise_sd)(nn_res) nn_res = Dense(1)(nn_res) self.mean_model = Model(nn_input, nn_model) self.mean_model.compile(Adam(lr=learning_rate, beta_1=beta_1), loss=mean_loss) self.res_model = Model([nn_res_input_x, nn_res_input_res], nn_res) self.res_model.compile(Adam(lr=learning_rate, beta_1=beta_1), loss=res_loss) self.x_scaling_values = None elif type(model_path) == str: self.mean_model = load_model(mean_model_file) self.res_model = load_model(res_model_file) self.x_scaling_values = pd.read_csv(self.x_scaling_file, index_col="Channel") self.res_predict = K.function( self.res_model.input + [K.learning_phase()], [self.res_model.output])
def predict(predict_var, x_unlabeled, inputs, y_true, batch_sizes, x_labeled=None, y_labeled=None): """Evaluates predict_var, batchwise, over all points in x_unlabeled and x_labeled. Args: predict_var: list of tensors to evaluate and return x_unlabeled: unlabeled input data inputs: dictionary containing input_types and input_placeholders as key, value pairs, respectively y_true: true labels tensorflow placeholder batch_sizes: dictionary containing input_types and batch_sizes as key, value pairs, respectively x_labeled: labeled input data y_labeled: labeled input labels Returns: a list of length n containing the result of all tensors in return_var, where n = len(x_unlabeled) + len(x_labeled) """ x_unlabeled, x_labeled, y_labeled = check_inputs(x_unlabeled, x_labeled, y_labeled, y_true) # combined data x = np.concatenate((x_unlabeled, x_labeled), 0) # get shape of y_true y_shape = y_true.get_shape()[1:K.ndim(y_true)].as_list() # calculate batches for predict loop unlabeled_batch_size = batch_sizes.get('Unlabeled', 0) labeled_batch_size = batch_sizes.get('Labeled', 0) if 'Labeled' in batch_sizes and 'Unlabeled' in batch_sizes: assert unlabeled_batch_size == labeled_batch_size batch_size = min(len(x), max(unlabeled_batch_size, labeled_batch_size)) batches = make_batches(len(x), batch_size) y_preds = [] # predict over all points for _, (batch_start, batch_end) in enumerate(batches): feed_dict = {K.learning_phase(): 0} # feed corresponding input for each input_type for input_type, input_placeholder in inputs.items(): if input_type == 'Unlabeled': feed_dict[input_placeholder] = x[batch_start:batch_end] elif input_type == 'Labeled': if x_labeled: batch_ids = np.random.choice(len(x_labeled), size=min( batch_sizes[input_type], len(x_labeled)), replace=False) feed_dict[input_placeholder] = x_labeled[batch_ids] feed_dict[y_true] = y_labeled[batch_ids] else: # we have no labeled points, so feed an empty array feed_dict[input_placeholder] = x[0:0] feed_dict[y_true] = np.empty([0] + y_shape) # evaluate the batch y_pred_batch = np.asarray(K.get_session().run(predict_var, feed_dict=feed_dict)) y_preds.append(y_pred_batch) if y_preds[0].shape: return np.concatenate(y_preds) else: return np.sum(y_preds)
def detect_image(self, image): start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) # return out_boxes, out_scores, out_classes return image
def detect_image(self, image): start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' # 必须为32的倍数 assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) # 填充/缩放图像为默认的416x416 else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') # 将图片转为ndarray?? print(image_data.shape) # 打印ndarray维度 416x416x3 image_data /= 255. # 转换为0~1 image_data = np.expand_dims(image_data, 0) # Add batch dimension. 添加一个批次的维度 # 这个是核心中的核心。调用模型,返回 out_boxes, out_scores, out_classes 已经是结果了 out_boxes, out_scores, out_classes = self.sess.run( # sess.run是tensorflow的方法,传入的参数: [self.boxes, self.scores, self.classes], # 盒子、得分、类别 feed_dict={ # 字典: self.yolo_model.input: image_data, # 输入图像0~1,4维 self.input_image_shape: [image.size[1], image.size[0]], # 原始图像的尺寸 K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) # # 检测出的盒子(框) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) # 字体 thickness = (image.size[0] + image.size[1]) // 300 # 厚度?? for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] # 通过索引去List取出对应分类label box = out_boxes[i] # 框[] score = out_scores[i] # 置信度 label = '{} {:.2f}'.format(predicted_class, score) # 预测出的标签 draw = ImageDraw.Draw(image) # 画图 label_size = draw.textsize(label, font) # 图上的标签文字 top, left, bottom, right = box # box的四个点坐标 top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) # boat 1.00 (111, 0) (924, 559) if top - label_size[1] >= 0: # 标签文字 text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. # 画框 for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( # 文字背景 [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) # 文字 del draw end = timer() print(end - start) # 打印识别共耗时 return image
def detect_image(self, image): if self.is_fixed_size: assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') # print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) #print(reversed(list(enumerate(out_classes)))) return_boxes = [] return_scores = [] return_class_names = [] return_plates = [] return_cuts = [] car_num = 0 person_num = 0 plate = '' cut_name = '' for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] if predicted_class not in ('person', 'car', 'motorbike', 'bicycle', 'bus', 'traffic light'): continue elif predicted_class in ('car', 'bus'): car_num += 1 #检测车牌区域 if (out_scores[i] > 0.7): cut = image.crop( (int(out_boxes[i][1]), int(out_boxes[i][0]), int(out_boxes[i][3]), int(out_boxes[i][2]))) cut.save('cut' + str(i) + '.jpg') cut_name = 'cut' + str(i) + '.jpg' #appcode_demo.demo('cut'+str(i)+'.jpg') cut = numpy.array(cut) l = HyperLPR_plate_recognition(cut) #l = pp.SimpleRecognizePlate(cut) if l.__len__() != 0: print('l', l) #if (l[0][1] > 0.85): plate = [l[0][0], l[0][1]] # print(plate,l[0][1]) elif predicted_class == 'person': person_num += 1 elif predicted_class == 'traffic light': #检测红绿灯颜色 cut = image.crop((int(out_boxes[i][1]), int(out_boxes[i][0]), int(out_boxes[i][3]), int(out_boxes[i][2]))) cut = numpy.array(cut) color = self.get_color(cut) predicted_class = str(color) print(color) box = out_boxes[i] score = out_scores[i] x = int(box[1]) y = int(box[0]) w = int(box[3] - box[1]) h = int(box[2] - box[0]) if x < 0: w = w + x x = 0 if y < 0: h = h + y y = 0 return_boxes.append([x, y, w, h]) return_scores.append(score) return_class_names.append(predicted_class) return_plates.append(plate) return_cuts.append(cut_name) plate = '' cut_name = '' return return_boxes, return_scores, return_class_names, return_plates, car_num, person_num, return_cuts
def detect_image(self, image): #---------------------------------------------------------# # 给图像增加灰条,实现不失真的resize # 也可以直接resize进行识别 #---------------------------------------------------------# if self.letterbox_image: boxed_image = letterbox_image(image, (self.model_image_size[1],self.model_image_size[0])) else: boxed_image = image.convert('RGB') boxed_image = boxed_image.resize((self.model_image_size[1],self.model_image_size[0]), Image.BICUBIC) image_data = np.array(boxed_image, dtype='float32') image_data /= 255. #---------------------------------------------------------# # 添加上batch_size维度 #---------------------------------------------------------# image_data = np.expand_dims(image_data, 0) # Add batch dimension. #---------------------------------------------------------# # 将图像输入网络当中进行预测! #---------------------------------------------------------# if self.eager: # 预测结果 input_image_shape = np.expand_dims(np.array([image.size[1], image.size[0]], dtype='float32'), 0) out_boxes, out_scores, out_classes = self.get_pred(image_data, input_image_shape) else: # 预测结果 out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) #---------------------------------------------------------# # 设置字体 #---------------------------------------------------------# font = ImageFont.truetype(font='font/simhei.ttf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = max((image.size[0] + image.size[1]) // 300, 1) for i, c in list(enumerate(out_classes)): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] top, left, bottom, right = box top = top - 5 left = left - 5 bottom = bottom + 5 right = right + 5 top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) # 画框框 label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) label = label.encode('utf-8') print(label, top, left, bottom, right) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, str(label,'UTF-8'), fill=(0, 0, 0), font=font) del draw return image
def on_train_begin(self, logs={}): if not os.path.exists(self.cfg['SAVE_DIR']): print("Making directory", self.cfg['SAVE_DIR']) os.makedirs(self.cfg['SAVE_DIR']) # Indexes of the layers which we keep track of. Basically, this will be any layer # which has a 'kernel' attribute, which is essentially the "Dense" or "Dense"-like layers self.layerixs = [] # Functions return activity of each layer self.layerfuncs = [] # Functions return weights of each layer self.layerweights = [] for lndx, l in enumerate(self.model.layers): if hasattr(l, 'kernel'): self.layerixs.append(lndx) self.layerfuncs.append( K.function(self.model.inputs, [ l.output, ])) self.layerweights.append(l.kernel) # print(self.model._feed_inputs) # print(self.model._feed_sample_weights) # print(self.model._feed_targets) inputs = [ self.model._feed_inputs, self.model._feed_targets, self.model._feed_sample_weights, K.learning_phase() ] # inputs = [self.model.input, # self.model._feed_targets, # self.model._feed_sample_weights, # backend.symbollearning_phase() # ] # inputs =(3277,12) # print(inputs) # grads_fn = K.function(inputs=[model.inputs[0], # model._feed_targets[0], # backend.symbolic_learning_phase()], # outputs=grads) # g_learning = grads_fn([x, x, True]) # g_not_learning = grads_fn([x, x, False]) # [model.inputs[0], model._feed_targets[0], model.sample_weights[0]] # Get gradients of all the relevant layers at once grads = self.model.optimizer.get_gradients(self.model.total_loss, self.layerweights) self.get_gradients = K.function(inputs=inputs, outputs=grads) # Get cross-entropy loss self.get_loss = K.function(inputs=inputs, outputs=[ self.model.total_loss, ])
def activations(images, labels, batch_size, model, layer_regex, nodaug_params, daug_params, include_input=False, class_invariance=False, n_daug_rep=0, norms=['fro']): """ Computes metrics from the activations, such as the norm of the feature maps, data augmentation invariance, class invariance, etc. Parameters ---------- images : h5py Dataset The set of images labels : h5py Dataset The ground truth labels batch_size : int Batch size model : Keras Model The model nodaug_params : dict Dictionary of data augmentation parameters for the baseline daug_params : dict Dictionary of data augmentation parameters include_input : bool If True, the input layer is considered for the analysis class_invariance : bool If True, the class invariance score is computed n_daug_rep : int If larger than 0, the data augentation invariance score is computed, performing n_daug_rep repetitions of random augmentations norms : list List of keywords to specify the types of norms to compute on the activations Returns ------- results_dict : dict Dictionary containing some performance metrics """ def _update_stats(mean_norm, std_norm, norm): mean_norm_batch = np.mean(norm, axis=0) std_norm_batch = np.std(norm, axis=0) mean_norm = init / float(end) * mean_norm + \ batch_size / float(end) * mean_norm_batch std_norm = init / float(end) * std_norm ** 2 + \ batch_size / float(end) * std_norm_batch ** 2 + \ (init * batch_size) / float(end ** 2) * \ (mean_norm - mean_norm_batch) ** 2 std_norm = np.sqrt(std_norm) return mean_norm, std_norm def _frobenius_norm(activations): norm = np.linalg.norm( activations, ord='fro', axis=tuple(range(1, len(activations.shape) - 1))) return norm def _inf_norm(activations): norm = np.max(np.abs(activations), axis=tuple(range(1, len(activations.shape) - 1))) return norm model = del_extra_nodes(model) n_images = images.shape[0] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Get relevant layers if include_input: layer_regex = '({}|.*input.*)'.format(layer_regex) else: layer_regex = layer_regex layers = [layer.name for layer in model.layers if re.compile(layer_regex).match(layer.name)] # Initialize HDF5 to store the activations # filename = 'hdf5_aux_{}'.format(time.time()) # activations_hdf5_aux = h5py.File(filename, 'w') # hdf5_aux = [filename] # # grp_activations = activations_hdf5_aux.create_group('activations') if class_invariance: # grp_labels = activations_hdf5_aux.create_group('labels') labels_true_da = [] labels_pred_da = [] predictions_da = [] # labels_true = grp_labels.create_dataset( # 'labels_true', shape=(n_images, ), dtype=np.uint8) # labels_pred = grp_labels.create_dataset( # 'labels_pred', shape=(n_images, ), dtype=np.uint8) # predictions = grp_labels.create_dataset( # 'predictions', shape=labels.shape, dtype=K.floatx()) idx_softmax = model.output_names.index('softmax') store_labels = True else: store_labels = False # Initialize results dictionary results_dict = {'activations_norm': {}, 'summary': {}, 'class_invariance': {}, 'daug_invariance': {}} # Iterate over the layers for layer_name in layers: # Create batch generator image_gen = get_generator(images, **nodaug_params) batch_gen = generate_batches(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) layer = model.get_layer(layer_name) layer_shape = layer.output_shape[1:] n_channels = layer_shape[-1] if re.compile('.*input.*').match(layer_name): layer_name = 'input' print('\nLayer {}\n'.format(layer_name)) # Create a Dataset for the activations of the layer # activations_layer = grp_activations.create_dataset( # layer_name, shape=(n_images, ) + layer_shape, # dtype=K.floatx()) # Create dask array for the activations of the layer activations_layer_da = [] # Initialize placeholders in the results dict for the layer results_dict['activations_norm'].update({layer_name: {n: {'mean': np.zeros(n_channels), 'std': np.zeros(n_channels)} for n in norms}}) layer_dict = results_dict['activations_norm'][layer_name] activation_function = K.function([model.input, K.learning_phase()], [layer.output]) # Iterate over the data set in batches init = 0 for batch_images, batch_labels in tqdm( batch_gen, total=n_batches_per_epoch): batch_size = batch_images.shape[0] end = init + batch_size # Store labels if store_labels: preds = model.predict_on_batch(batch_images) if isinstance(preds, list): preds = preds[idx_softmax] labels_pred_da.append(da.from_array( np.argmax(preds, axis=1))) labels_true_da.append(da.from_array( np.argmax(batch_labels, axis=1))) predictions_da.append(da.from_array(preds)) # labels_pred[init:end] = np.argmax(preds, axis=1) # labels_true[init:end] = np.argmax(batch_labels, axis=1) # predictions[init:end, :] = preds # Get and store activations activations = activation_function([batch_images, 0])[0] activations_layer_da.append(da.from_array( activations, chunks=activations.shape)) # activations_layer[init:end] = activations # Compute norms for norm_key in norms: mean_norm = layer_dict[norm_key]['mean'] std_norm = layer_dict[norm_key]['std'] if norm_key == 'fro': norm = _frobenius_norm(activations) elif norm_key == 'inf': norm = _inf_norm(activations) else: raise NotImplementedError('Implemented norms are fro ' 'and inf') mean_norm, std_norm = _update_stats(mean_norm, std_norm, norm) layer_dict[norm_key]['mean'] = mean_norm layer_dict[norm_key]['std'] = std_norm init = end if init == n_images: store_labels = False break # Concatenate dask arrays activations_layer_da = da.concatenate(activations_layer_da, axis=0) activations_layer_da = activations_layer_da.reshape((n_images, -1)) d_activations = activations_layer_da.shape[-1] if class_invariance: print('\nComputing class invariance\n') labels_pred_da = da.concatenate(labels_pred_da) labels_true_da = da.concatenate(labels_true_da) predictions_da = da.concatenate(predictions_da) n_classes = len(np.unique(labels_true_da)) # Compute MSE matrix of the activations r = da.reshape(da.sum(da.square(activations_layer_da), axis=1), (-1, 1)) mse_matrix_da = (r - 2 * da.dot(activations_layer_da, da.transpose(activations_layer_da)) \ + da.transpose(r)) / d_activations mse_matrix_da = mse_matrix_da.rechunk((mse_matrix_da.chunksize[0], mse_matrix_da.shape[-1])) # Compute class invariance time0 = time() results_dict['class_invariance'].update({layer_name: {}}) class_invariance_scores_da = [] if class_invariance: # mse_matrix_mean = da.mean(mse_matrix_da).compute() for cl in tqdm(range(n_classes)): labels_cl = labels_pred_da == cl labels_cl = labels_cl.compute() mse_class = mse_matrix_da[labels_cl, :][:, labels_cl] mse_class = mse_class.rechunk((-1, -1)) # mse_class_mean = da.mean(mse_class).compute() # class_invariance_score = 1. - np.divide( # mse_class_mean, mse_matrix_mean) # results_dict['class_invariance'][layer_name].update( # {cl: class_invariance_score}) class_invariance_scores_da.append( 1. - da.divide(da.mean(mse_class), da.mean(mse_matrix_da))) # Compute data augmentation invariance print('\nComputing data augmentation invariance\n') mse_daug_da = [] results_dict['daug_invariance'].update({layer_name: {}}) for r in range(n_daug_rep): print('Repetition {}'.format(r)) image_gen_daug = get_generator(images, **daug_params) batch_gen_daug = generate_batches(image_gen_daug, images, labels, batch_size, aug_per_im=1, shuffle=False) activations_layer_daug_da = [] # Iterate over the data set in batches to compute activations init = 0 for batch_images, batch_labels in tqdm( batch_gen, total=n_batches_per_epoch): batch_size = batch_images.shape[0] end = init + batch_size # Get and store activations activations = activation_function([batch_images, 0])[0] activations_layer_daug_da.append(da.from_array( activations, chunks=activations.shape)) init = end if init == n_images: break activations_layer_daug_da = da.concatenate( activations_layer_daug_da, axis=0) activations_layer_daug_da = activations_layer_daug_da.reshape( (n_images, -1)) activations_layer_daug_da = activations_layer_daug_da.rechunk( (activations_layer_daug_da.chunksize[0], activations_layer_daug_da.shape[-1])) # Compute MSE daug mse_daug_da.append(da.mean(da.square(activations_layer_da - \ activations_layer_daug_da), axis=1)) mse_daug_da = da.stack(mse_daug_da, axis=1) mse_sum = da.repeat(da.reshape(da.sum(mse_matrix_da, axis=1), (n_images, 1)), n_daug_rep, axis=1) daug_invariance_score_da = 1 - n_images * da.divide(mse_daug_da, mse_sum) time1 = time() # Compute dask results and update results dict results_dask = da.compute(class_invariance_scores_da, daug_invariance_score_da) time2 = time() results_dict['class_invariance'][layer_name].update( {cl: cl_inv_score for cl, cl_inv_score in enumerate(results_dask[0])}) results_dict['daug_invariance'].update({layer_name: {r: daug_inv_score for r, daug_inv_score in enumerate(results_dask[1].T)}}) # Compute summary statistics of the norms across the channels for layer, layer_dict in results_dict['activations_norm'].items(): results_dict['summary'].update({layer: {}}) for norm_key, norm_dict in layer_dict.items(): results_dict['summary'][layer].update({norm_key: { 'mean': np.mean(norm_dict['mean']), 'std': np.mean(norm_dict['std'])}}) return results_dict
def train_step(return_vars, updates, x_unlabeled, inputs, y_true, batch_sizes, x_labeled=None, y_labeled=None, batches_per_epoch=100): """Performs one training step. Evaluates the tensors in return_vars and updates, then returns the values of the tensors in return_vars. Args: return_vars: list of tensors to evaluate and return updates: list of tensors to evaluate only x_unlabeled: unlabeled input data inputs: dictionary containing input_types and input_placeholders as key, value pairs, respectively y_true: true labels placeholder batch_sizes: dictionary containing input_types and batch_sizes as key, value pairs, respectively x_labeled: labeled input data y_labeled: labeled input labels batches_per_epoch: parameter updates per epoch* Returns: the evaluated result of all tensors in return_vars, summed across all epochs *note: the term epoch is used loosely here, it does not necessarily refer to one iteration over the entire dataset. instead, it is just batches_per_epoch parameter updates. """ x_unlabeled, x_labeled, y_labeled = check_inputs(x_unlabeled, x_labeled, y_labeled, y_true) # combine data x = np.concatenate((x_unlabeled, x_labeled), 0) # get shape of y_true y_shape = y_true.get_shape()[1:K.ndim(y_true)].as_list() return_vars_ = np.zeros(shape=(len(return_vars))) # train batches_per_epoch batches for _ in range(0, batches_per_epoch): feed_dict = {K.learning_phase(): 1} # feed corresponding input for each input_type for input_type, input_placeholder in inputs.items(): if input_type == 'Labeled': if x_labeled: batch_ids = np.random.choice(len(x_labeled), size=min( batch_sizes[input_type], len(x_labeled)), replace=False) feed_dict[input_placeholder] = x_labeled[batch_ids] feed_dict[y_true] = y_labeled[batch_ids] else: # we have no labeled points, so feed an empty array feed_dict[input_placeholder] = x[0:0] feed_dict[y_true] = np.empty([0] + y_shape) elif input_type == 'Unlabeled': if x_unlabeled: batch_ids = np.random.choice(len(x_unlabeled), size=batch_sizes[input_type], replace=False) feed_dict[input_placeholder] = x_unlabeled[batch_ids] else: # we have no unlabeled points, so feed an empty array feed_dict[input_placeholder] = x[0:0] all_vars = return_vars + updates return_vars_ += np.asarray(K.get_session().run( all_vars, feed_dict=feed_dict)[:len(return_vars)]) return return_vars_
def __init__(self, inputs=1, hidden_layers=2, hidden_neurons=8, activation="selu", l2_weight=0.01, learning_rate=0.001, loss="mse", noise_sd=1, beta_1=0.9, model_path=None, dropout_alpha=0.5, num_epochs=10, batch_size=1024, verbose=0, model_config=0, **kwargs): self.config = dict(inputs=inputs, hidden_layers=hidden_layers, hidden_neurons=hidden_neurons, activation=activation, l2_weight=l2_weight, learning_rate=learning_rate, loss=loss, noise_sd=noise_sd, beta_1=beta_1, dropout_alpha=dropout_alpha, model_path=model_path, num_epochs=num_epochs, batch_size=batch_size, verbose=verbose, model_config=model_config, corr=1, res_sd=0) if model_path is None: nn_input = Input((inputs, )) nn_model = nn_input for h in range(hidden_layers): nn_model = Dense(hidden_neurons, kernel_regularizer=l2(l2_weight))(nn_model) if activation == "leaky": nn_model = LeakyReLU(0.1)(nn_model) else: nn_model = Activation(activation)(nn_model) if dropout_alpha > 0: nn_model = Dropout(dropout_alpha)(nn_model) if noise_sd > 0: nn_model = GaussianNoise(noise_sd)(nn_model) nn_model = Dense(1)(nn_model) self.model = Model(nn_input, nn_model) self.model.compile(Adam(lr=learning_rate, beta_1=beta_1), loss=loss) self.x_scaling_file = None self.x_scaling_values = None self.sample_predict = K.function( [self.model.input, K.learning_phase()], [self.model.output]) elif type(model_path) == str: model_path_start = join(*model_path.split("/")[:-1]) self.model = load_model(model_path) self.sample_predict = K.function( [self.model.input, K.learning_phase()], [self.model.output]) self.x_scaling_file = join( model_path_start, "ann_config_{0:04d}_scale.csv".format( self.config["model_config"])) self.x_scaling_values = pd.read_csv(self.x_scaling_file, index_col="Channel")
def detect_image(self, image, show_stats=True): start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, "Multiples of 32 required" assert self.model_image_size[ 1] % 32 == 0, "Multiples of 32 required" boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = ( image.width - (image.width % 32), image.height - (image.height % 32), ) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype="float32") if show_stats: print(image_data.shape) image_data /= 255.0 image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0, }, ) if show_stats: print("Found {} boxes for {}".format(len(out_boxes), "img")) out_prediction = [] font_path = os.path.join(os.path.dirname(__file__), "font/FiraMono-Medium.otf") font = ImageFont.truetype(font=font_path, size=np.floor(3e-2 * image.size[1] + 0.5).astype("int32")) thickness = (image.size[0] + image.size[1]) // 300 count_mask = np.count_nonzero(np.asarray(out_classes) == 1) count_no_mask = np.count_nonzero(np.asarray(out_classes) == 0) for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = "{} {:.2f}".format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype("int32")) left = max(0, np.floor(left + 0.5).astype("int32")) bottom = min(image.size[1], np.floor(bottom + 0.5).astype("int32")) right = min(image.size[0], np.floor(right + 0.5).astype("int32")) # image was expanded to model_image_size: make sure it did not pick # up any box outside of original image (run into this bug when # lowering confidence threshold to 0.01) if top > image.size[1] or right > image.size[0]: continue if show_stats: print(label, (left, top), (right, bottom)) # output as xmin, ymin, xmax, ymax, class_index, confidence out_prediction.append([left, top, right, bottom, c, score]) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, bottom]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c], ) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() if show_stats: print("Time spent: {:.3f}sec".format(end - start)) return out_prediction, image, count_mask, count_no_mask
def test_adv(images, labels, batch_size, model, adv_model, daug_params, attack_params): """ Tests the performance of a model on adversarial images. The adversarial images are computed according to the attack specified in the arguments. Parameters ---------- images : dask array The set of images labels : dask array The ground truth labels batch_size : int Batch size model : Keras Model The model adv_model : Keras Model The model used to generate adversarial examples daug_params : dict Dictionary of data augmentation parameters attack_params : dict Dictionary of the attack parameters Returns ------- results_dict : dict Dictionary containing some performance metrics """ # Get session sess = K.get_session() # Initialize adversarial attack attack, attack_params_cleverhans, bs = init_attack( adv_model, attack_params, sess) if bs: batch_size = bs n_images = images.shape[0] n_classes = labels.shape[1] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Create batch generator image_gen = get_generator(images, **daug_params) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) # Define input TF placeholder if daug_params['crop_size']: image_shape = daug_params['crop_size'] else: image_shape = images.shape[1:] x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape)) y = tf.placeholder(K.floatx(), shape=(bs,) + (n_classes,)) # Define adversarial predictions symbolically x_adv = attack.generate(x, **attack_params_cleverhans) x_adv = tf.stop_gradient(x_adv) predictions_adv = model(x_adv) # Define accuracy and mean squared error symbolically correct_preds = tf.equal(tf.argmax(y, axis=-1), tf.argmax(predictions_adv, axis=-1)) acc_value = tf.reduce_mean(tf.to_float(correct_preds)) mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv))) # Init results variables accuracy = 0.0 mse = 0.0 with sess.as_default(): init = 0 for _ in tqdm(range(n_batches_per_epoch)): batch = next(batch_gen()) this_batch_size = batch[0].shape[0] # Evaluate accuracy if isinstance(batch[1], (list, )): yy = batch[1][0] else: yy = batch[1] # Evaluate accuracy and MSE batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy, K.learning_phase(): 0}) accuracy += (this_batch_size * batch_acc) batch_mse = mse_value.eval(feed_dict={x: batch[0], K.learning_phase(): 0}) mse += (this_batch_size * batch_mse) init += this_batch_size accuracy /= n_images mse /= n_images results_dict = {'mean_acc': accuracy, 'mean_mse': mse} return results_dict
def detect_image(self, image): start = timer() # 调整图片使其符合输入要求 new_image_size = (self.model_image_size[1], self.model_image_size[0]) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. if self.eager: # 预测结果 input_image_shape = np.expand_dims( np.array([image.size[1], image.size[0]], dtype='float32'), 0) out_boxes, out_scores, out_classes = self.yolo_model.predict( [image_data, input_image_shape]) else: # 预测结果 out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) # 设置字体 font = ImageFont.truetype(font='font/simhei.ttf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 small_pic = [] for i, c in list(enumerate(out_classes)): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] top, left, bottom, right = box top = top - 5 left = left - 5 bottom = bottom + 5 right = right + 5 top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) # 画框框 label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) label = label.encode('utf-8') print(label) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, str(label, 'UTF-8'), fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) return image
def detect_image(self, image): if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) print(out_classes) detected_object = [] scores_of_object = [] coord = [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(predicted_class, score, left, top, right, bottom) if (left, top, right, bottom) not in coord: detected_object.append(predicted_class) scores_of_object.append(score) coord.append((left, top, right, bottom)) # 높은 확률부터 출력된다는 전제.. while len(detected_object) > 5: min_id = scores_of_object.index(min(scores_of_object)) detected_object.pop(min_id) objects = { "detected_object": detected_object, "scores": scores_of_object, "coord": coord } return objects