Exemplo n.º 1
0
def test(CAE_model_path, OVR_SVM_path, args, gap=2, score_threshold=0.4):
    # to get the image paths
    image_folder = prefix + args.dataset + '/testing/frames/'
    vids_paths = util.get_vids_paths(image_folder)
    # to set gpu visible
    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    import tensorflow as tf
    # to load the ssd fpn model, and get related tensor
    object_detection_graph = inference.load_frozen_graph()
    with object_detection_graph.as_default():
        ops = object_detection_graph.get_operations()
        all_tensor_names = {output.name for op in ops for output in op.outputs}
        tensor_dict = {}
        for key in [
                'num_detections', 'detection_boxes', 'detection_scores',
                'detection_classes'
        ]:
            tensor_name = key + ':0'
            if tensor_name in all_tensor_names:
                tensor_dict[key] = object_detection_graph.get_tensor_by_name(
                    tensor_name)

        image_tensor = object_detection_graph.get_tensor_by_name(
            'image_tensor:0')

        former_batch = tf.placeholder(dtype=tf.float32,
                                      shape=[1, 64, 64, 1],
                                      name='former_batch')
        gray_batch = tf.placeholder(dtype=tf.float32,
                                    shape=[1, 64, 64, 1],
                                    name='gray_batch')
        back_batch = tf.placeholder(dtype=tf.float32,
                                    shape=[1, 64, 64, 1],
                                    name='back_batch')

        grad1_x, grad1_y = tf.image.image_gradients(former_batch)
        grad1 = tf.concat([grad1_x, grad1_y], axis=-1)
        # grad2_x,grad2_y=tf.image.image_gradients(gray_batch)
        grad3_x, grad3_y = tf.image.image_gradients(back_batch)
        grad3 = tf.concat([grad3_x, grad3_y], axis=-1)

        #grad_dis_1 = tf.sqrt(tf.square(grad1_x) + tf.square(grad1_y))
        #grad_dis_2 = tf.sqrt(tf.square(grad3_x) + tf.square(grad3_y))

        former_feat = CAE_encoder(grad1, 'former', bn=args.bn, training=False)
        gray_feat = CAE_encoder(gray_batch, 'gray', bn=args.bn, training=False)
        back_feat = CAE_encoder(grad3, 'back', bn=args.bn, training=False)
        # [batch_size,3072]
        feat = tf.concat([
            tf.layers.flatten(former_feat),
            tf.layers.flatten(gray_feat),
            tf.layers.flatten(back_feat)
        ],
                         axis=1)

        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                                     scope='former_encoder')
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope='gray_encoder'))
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope='back_encoder'))

        g_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                   scope='former_encoder')
        g_list.extend(
            tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                              scope='gray_encoder'))
        g_list.extend(
            tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                              scope='back_encoder'))
        bn_list = [
            g for g in g_list
            if 'moving_mean' in g.name or 'moving_variance' in g.name
        ]
        var_list += bn_list
        restorer = tf.train.Saver(var_list=var_list)

        (image_height, image_width) = util.image_size_map[args.dataset]
        #image_height,image_width=640,640
        if not args.matlab:
            clf = joblib.load(OVR_SVM_path)
        else:
            import scipy.io as io
            W = io.loadmat('../matlab_files/{}_weights.mat'.format(
                args.dataset))['W']
            B = io.loadmat('../matlab_files/{}_biases.mat'.format(
                args.dataset))['B']
        anomaly_scores_records = []

        timestamp = time.time()
        num_videos = len(vids_paths)
        total = 0

        with tf.Session() as sess:
            if args.bn:
                restorer.restore(sess, CAE_model_path + '_bn')
            else:
                restorer.restore(sess, CAE_model_path)

            for frame_paths in vids_paths:
                anomaly_scores = np.empty(shape=(len(frame_paths), ),
                                          dtype=np.float32)

                for frame_iter in range(gap, len(frame_paths) - gap):
                    img = np.expand_dims(util.data_preprocessing(
                        frame_paths[frame_iter], target_size=640),
                                         axis=0)
                    output_dict = sess.run(tensor_dict,
                                           feed_dict={image_tensor: img})

                    # all outputs are float32 numpy arrays, so convert types as appropriate
                    output_dict['num_detections'] = int(
                        output_dict['num_detections'][0])
                    output_dict['detection_classes'] = output_dict[
                        'detection_classes'][0].astype(np.int8)
                    output_dict['detection_boxes'] = output_dict[
                        'detection_boxes'][0]
                    output_dict['detection_scores'] = output_dict[
                        'detection_scores'][0]

                    _temp_anomaly_scores = []
                    _temp_anomaly_score = 10000.

                    for score, box in zip(output_dict['detection_scores'],
                                          output_dict['detection_boxes']):
                        if score >= score_threshold:
                            box = [
                                int(box[0] * image_height),
                                int(box[1] * image_height),
                                int(box[2] * image_height),
                                int(box[3] * image_width)
                            ]
                            img_gray = util.box_image_crop(
                                frame_paths[frame_iter], box)
                            img_former = util.box_image_crop(
                                frame_paths[frame_iter - gap], box)
                            img_back = util.box_image_crop(
                                frame_paths[frame_iter + gap], box)

                            _feat = sess.run(feat,
                                             feed_dict={
                                                 former_batch:
                                                 np.expand_dims(img_former, 0),
                                                 gray_batch:
                                                 np.expand_dims(img_gray, 0),
                                                 back_batch:
                                                 np.expand_dims(img_back, 0)
                                             })

                            if args.norm != 0:
                                _feat = util.norm_(_feat, l=args.norm)
                            if args.class_add:
                                _temp = np.zeros(90, dtype=np.float32)
                                _temp[output_dict['detection_classes'][0] -
                                      1] = 1
                                result = np.concatenate((_feat[0], _temp),
                                                        axis=0)
                                _feat = np.expand_dims(result, 0)
                            if not args.matlab:
                                scores = clf.decision_function(_feat)
                            else:
                                scores = np.dot(_feat, W) + B
                                # print(scores[0])
                            _temp_anomaly_scores.append(-max(scores[0]))
                    if _temp_anomaly_scores.__len__() != 0:
                        _temp_anomaly_score = max(_temp_anomaly_scores)

                    print(
                        'video = {} / {}, i = {} / {}, score = {:.6f}'.format(
                            frame_paths[0].split('/')[-2], num_videos,
                            frame_iter, len(frame_paths), _temp_anomaly_score))

                    anomaly_scores[frame_iter] = _temp_anomaly_score

                anomaly_scores[:gap] = anomaly_scores[gap]
                anomaly_scores[-gap:] = anomaly_scores[-gap - 1]

                min_score = min(anomaly_scores)
                for i, _s in enumerate(anomaly_scores):
                    if _s == 10000.:
                        anomaly_scores[i] = min_score
                anomaly_scores_records.append(anomaly_scores)
                total += len(frame_paths)

    # use the evaluation functions from github.com/StevenLiuWen/ano_pred_cvpr2018
    result_dict = {
        'dataset': args.dataset,
        'psnr': anomaly_scores_records,
        'flow': [],
        'names': [],
        'diff_mask': []
    }
    used_time = time.time() - timestamp

    print('total time = {}, fps = {}'.format(used_time, total / used_time))

    # TODO specify what's the actual name of ckpt.
    if not args.bn:
        pickle_path = '/home/' + args.machine + '/anomaly_scores/' + args.dataset + '.pkl'
    else:
        pickle_path = '/home/' + args.machine + '/anomaly_scores/' + args.dataset + '_bn' + '.pkl'

    with open(pickle_path, 'wb') as writer:
        pickle.dump(result_dict, writer, pickle.HIGHEST_PROTOCOL)

    results = evaluate.evaluate_all(pickle_path, reverse=True, smoothing=True)
    print(results)
Exemplo n.º 2
0
def test_CAE(CAE_model_path, args, gap=2, score_threshold=0.4):
    image_folder = prefix + args.dataset + "/testing/frames/"
    vids_paths = util.get_vids_paths(image_folder)
    # to set gpu visible
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    import tensorflow as tf

    # to load the ssd fpn model, and get related tensor
    object_detection_graph = inference.load_frozen_graph()
    with object_detection_graph.as_default():
        ops = object_detection_graph.get_operations()
        all_tensor_names = {output.name for op in ops for output in op.outputs}
        tensor_dict = {}
        for key in [
                "num_detections",
                "detection_boxes",
                "detection_scores",
                "detection_classes",
        ]:
            tensor_name = key + ":0"
            if tensor_name in all_tensor_names:
                tensor_dict[key] = object_detection_graph.get_tensor_by_name(
                    tensor_name)

        image_tensor = object_detection_graph.get_tensor_by_name(
            "image_tensor:0")

        former_batch = tf.placeholder(dtype=tf.float32,
                                      shape=[1, 64, 64, 1],
                                      name="former_batch")
        gray_batch = tf.placeholder(dtype=tf.float32,
                                    shape=[1, 64, 64, 1],
                                    name="gray_batch")
        back_batch = tf.placeholder(dtype=tf.float32,
                                    shape=[1, 64, 64, 1],
                                    name="back_batch")

        # grad1_x, grad1_y = tf.image.image_gradients(former_batch)
        # grad1=tf.concat([grad1_x,grad1_y],axis=-1)
        grad1 = tf.math.abs(tf.math.subtract(former_batch, gray_batch))
        # grad2_x,grad2_y=tf.image.image_gradients(gray_batch)
        # grad3_x, grad3_y = tf.image.image_gradients(back_batch)
        # grad3=tf.concat([grad3_x,grad3_y],axis=-1)
        grad3 = tf.math.abs(tf.math.subtract(back_batch, gray_batch))

        # grad_dis_1 = tf.sqrt(tf.square(grad1_x) + tf.square(grad1_y))
        # grad_dis_2 = tf.sqrt(tf.square(grad3_x) + tf.square(grad3_y))

        former_output = CAE(grad1, "former", bn=args.bn, training=False)
        gray_output = CAE(gray_batch, "gray", bn=args.bn, training=False)
        back_output = CAE(grad3, "back", bn=args.bn, training=False)

        outputs = tf.concat([former_output, gray_output, back_output], axis=1)
        targets = tf.concat([grad_dis_1, gray_batch, grad_dis_2], axis=1)
        L2_dis = tf.reduce_sum(tf.square(outputs - targets))

        var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                                     scope="former_encoder")
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope="gray_encoder"))
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope="back_encoder"))
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope="former_decoder"))
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope="gray_decoder"))
        var_list.extend(
            tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                              scope="back_decoder"))

        if args.bn:
            g_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                       scope="former_encoder")
            g_list.extend(
                tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                  scope="gray_encoder"))
            g_list.extend(
                tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                  scope="back_encoder"))
            g_list.extend(
                tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                  scope="former_decoder"))
            g_list.extend(
                tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                  scope="gray_decoder"))
            g_list.extend(
                tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
                                  scope="back_decoder"))
            bn_list = [
                g for g in g_list
                if "moving_mean" in g.name or "moving_variance" in g.name
            ]
            var_list += bn_list
        restorer = tf.train.Saver(var_list=var_list)

        (image_height, image_width) = util.image_size_map[args.dataset]
        # image_height,image_width=640,640
        anomaly_scores_records = []

        timestamp = time.time()
        num_videos = len(vids_paths)
        total = 0

        with tf.Session() as sess:
            if args.bn:
                restorer.restore(sess, CAE_model_path + "_bn")
            else:
                restorer.restore(sess, CAE_model_path)

            for frame_paths in vids_paths:
                anomaly_scores = np.empty(shape=(len(frame_paths), ),
                                          dtype=np.float32)

                for frame_iter in range(gap, len(frame_paths) - gap):
                    img = np.expand_dims(
                        util.data_preprocessing(frame_paths[frame_iter],
                                                target_size=640),
                        axis=0,
                    )
                    output_dict = sess.run(tensor_dict,
                                           feed_dict={image_tensor: img})

                    # all outputs are float32 numpy arrays, so convert types as appropriate
                    output_dict["num_detections"] = int(
                        output_dict["num_detections"][0])
                    output_dict["detection_classes"] = output_dict[
                        "detection_classes"][0].astype(np.int8)
                    output_dict["detection_boxes"] = output_dict[
                        "detection_boxes"][0]
                    output_dict["detection_scores"] = output_dict[
                        "detection_scores"][0]

                    _temp_anomaly_scores = []
                    _temp_anomaly_score = 10000.0

                    for score, box in zip(output_dict["detection_scores"],
                                          output_dict["detection_boxes"]):
                        if score >= score_threshold:
                            box = [
                                int(box[0] * image_height),
                                int(box[1] * image_height),
                                int(box[2] * image_height),
                                int(box[3] * image_width),
                            ]
                            img_gray = util.box_image_crop(
                                frame_paths[frame_iter], box)
                            img_former = util.box_image_crop(
                                frame_paths[frame_iter - gap], box)
                            img_back = util.box_image_crop(
                                frame_paths[frame_iter + gap], box)

                            l2_dis = sess.run(
                                L2_dis,
                                feed_dict={
                                    former_batch:
                                    np.expand_dims(img_former, 0),
                                    gray_batch: np.expand_dims(img_gray, 0),
                                    back_batch: np.expand_dims(img_back, 0),
                                },
                            )

                            _temp_anomaly_scores.append(l2_dis)
                    if _temp_anomaly_scores.__len__() != 0:
                        _temp_anomaly_score = max(_temp_anomaly_scores)

                    print(
                        "video = {} / {}, i = {} / {}, score = {:.6f}".format(
                            frame_paths[0].split("/")[-2],
                            num_videos,
                            frame_iter,
                            len(frame_paths),
                            _temp_anomaly_score,
                        ))

                    anomaly_scores[frame_iter] = _temp_anomaly_score

                anomaly_scores[:gap] = anomaly_scores[gap]
                anomaly_scores[-gap:] = anomaly_scores[-gap - 1]

                min_score = np.min(anomaly_scores)
                for i, _s in enumerate(anomaly_scores):
                    if _s == 10000.0:
                        anomaly_scores[i] = min_score
                anomaly_scores_records.append(anomaly_scores)
                total += len(frame_paths)

            # use the evaluation functions from github.com/StevenLiuWen/ano_pred_cvpr2018
            result_dict = {
                "dataset": args.dataset,
                "psnr": anomaly_scores_records,
                "flow": [],
                "names": [],
                "diff_mask": [],
            }
            used_time = time.time() - timestamp

            print("total time = {}, fps = {}".format(used_time,
                                                     total / used_time))

            # TODO specify what's the actual name of ckpt.
            if not args.bn:
                # pickle_path = '/home/' + args.machine + '/anomaly_scores/' + args.dataset + '_CAE_only' + '.pkl'
                pickle_path = f"{PATHS.get_anomaly_scores_pickle_path()}/{args.dataset}_CAE_only.pkl"
            else:
                # pickle_path = '/home/' + args.machine + '/anomaly_scores/' + args.dataset + '_CAE_only' + '_bn' + '.pkl'
                pickle_path = f"{PATHS.get_anomaly_scores_pickle_path()}/{args.dataset}_CAE_only_bn.pkl"

            with open(pickle_path, "wb") as writer:
                pickle.dump(result_dict, writer, pickle.HIGHEST_PROTOCOL)

            results = evaluate.evaluate_all(pickle_path,
                                            reverse=True,
                                            smoothing=True)
            print(results)