Ejemplo n.º 1
0
def save_descriptors_keypoints(SLIDES_PATH, VID_FRAMES_PATH, SLIDE_START,
                               SLIDE_END, VID_START, VID_END,
                               SLIDES_KP_SAVE_PATH, VID_KP_SAVE_PATH):

    ## getting filenames
    videoFrameNames = [
        filename for filename in os.listdir(VID_FRAMES_PATH)
        if filename.endswith(".png")
    ]
    videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END)

    slideNames = [
        filename for filename in os.listdir(SLIDES_PATH)
        if filename.endswith(".png")
    ]
    slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END)

    ## calculating, saving keypoints
    slideNameKPDesc = dict()
    for slideName in slideNames:
        path = SLIDES_PATH + slideName
        img = cv2.imread(SLIDES_PATH + slideName, 0)
        keypoints, descriptors = calculateKeypoints(img)
        serialized = serializableKeypts(keypoints, descriptors)
        print("Saving " + path)
        slideNameKPDesc[path] = serialized
    pickleOutput = open(SLIDES_KP_SAVE_PATH + "slideKeyptsDesc.pkl", 'wb')
    pickle.dump(slideNameKPDesc, pickleOutput)

    vidNameKPDesc = dict()
    for vidName in videoFrameNames:
        path = VID_FRAMES_PATH + vidName
        img = cv2.imread(VID_FRAMES_PATH + vidName, 0)
        keypoints, descriptors = calculateKeypoints(img)
        print("Saving " + path)
        serialized = serializableKeypts(keypoints, descriptors)
        vidNameKPDesc[path] = serialized
    pickleOutput = open(VID_KP_SAVE_PATH + "vidKeyptsDesc.pkl", 'wb')
    pickle.dump(vidNameKPDesc, pickleOutput)
## where to load/save
slideImgPath = "../slides/"
vidFramePath = "../bb_sampled_video_frames/"

slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl"
vidImgKeyptsFle = "./vid_keypts/vidKeyptsDesc.pkl"

matchesPath = "./desc_matches/"

## getting filenames
videoFrameNames = [
    filename for filename in os.listdir(vidFramePath)
    if filename.endswith(".jpg")
]
videoFrameNames = slidesInRange(videoFrameNames, 0, 28)

slideNames = [
    filename for filename in os.listdir(slideImgPath)
    if filename.endswith(".png")
]
slideNames = slidesInRange(slideNames, 2, 30)

## load dictionary of things
slideImgKeypts = pickle.load(open(slideImgKeyptsFile, "rb"))
vidImgKeypts = pickle.load(open(vidImgKeyptsFle, "rb"))

slideImgKeyptsDesc = dict()
vidImgKeyptsDesc = dict()

## load slide img, keypoints
vidImgKeyptsFle = "./new_vid_keypts/vidKeyptsDesc.pkl"

savePath = "./new_cross_matches_save/"

VID_START = 0
VID_END = 22
SLIDE_START = 0
SLIDE_END = 12
NUM_SLIDES = VID_END - VID_START - 1

## getting filenames
videoFrameNames = [
    filename for filename in os.listdir(vidFramePath)
    if filename.endswith(".png")
]
videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END)

slideNames = [
    filename for filename in os.listdir(slideImgPath)
    if filename.endswith(".png")
]
slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END)

sampleImgSlides = cv2.imread(slideImgPath + slideNames[0], 0)
sampleVidSlides = cv2.imread(vidFramePath + videoFrameNames[0], 0)

## slide and video dimensions
slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1])
vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1])

## load dictionary of things
Ejemplo n.º 4
0
def compute_match_save(SLIDES_PATH, SLIDE_START, SLIDE_END,
                       FRAME_SAMPLE_SAVE_FOLDER,
                       SLIDE_KEYPTS_DESCS_PKL_SAVE_PATH, VID_START, VID_END,
                       VID_KEYPTS_DESCS_PKL_SAVE_PATH,
                       FINAL_FIGURES_RESULTS_SAVE_PATH):

    NUM_SLIDES = SLIDE_END - SLIDE_START
    NUM_FRAMES = VID_END - VID_START
    ## getting filenames
    videoFrameNames = [
        filename for filename in os.listdir(FRAME_SAMPLE_SAVE_FOLDER)
        if filename.endswith(".png")
    ]
    videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END)

    slideNames = [
        filename for filename in os.listdir(SLIDES_PATH)
        if filename.endswith(".png")
    ]
    slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END)

    sampleImgSlides = cv2.imread(slideImgPath + slideNames[0], 0)
    sampleVidSlides = cv2.imread(vidFramePath + videoFrameNames[0], 0)

    ## slide and video dimensions
    slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1])
    vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1])

    ## load dictionary of things
    slideImgKeypts = pickle.load(open(SLIDE_KEYPTS_DESCS_PKL_SAVE_PATH, "rb"))
    vidImgKeypts = pickle.load(open(VID_KEYPTS_DESCS_PKL_SAVE_PATH, "rb"))

    slideImgKeyptsDesc = dict()
    vidImgKeyptsDesc = dict()

    ## load slide img, keypoints
    for sn in slideNames:
        path = slideImgPath + sn
        slideImgKeyptsDesc[sn] = convertPickledToKPDesc(slideImgKeypts[path])

    for vidName in videoFrameNames:
        path = vidFramePath + vidName
        vidImgKeyptsDesc[vidName] = convertPickledToKPDesc(vidImgKeypts[path])

    FLANN_INDEX_KDTREE = 0
    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
    search_params = dict(checks=50)

    ## matches is a list of DMatch objects
    flann = cv2.FlannBasedMatcher(index_params, search_params)

    matchSlideFrameDict = dict()

    matchMetric = dict()
    matchMetricNotNormalized = dict()

    ## cross matching
    matchID = 0
    for slide in range(0, NUM_SLIDES):
        for frame in range(0, NUM_FRAMES):
            print("Slide, Frame", (slide, frame), slideNames[slide],
                  videoFrameNames[frame])
            slidekpts, slideDescs = slideImgKeyptsDesc[slideNames[slide]]
            framekpts, frameDescs = vidImgKeyptsDesc[videoFrameNames[frame]]
            matches = flann.knnMatch(np.asarray(slideDescs, np.float32),
                                     np.asarray(frameDescs, np.float32),
                                     k=2)

            matchPairs = []
            distances = []
            good = []
            slideKeypoints = []
            frameKeypoints = []

            good = []

            for m, n in matches:
                if m.distance < 0.75 * n.distance:
                    slideKeypoints.append(slidekpts[m.queryIdx].pt)
                    frameKeypoints.append(framekpts[m.trainIdx].pt)
                    good.append(m)
            '''
            slideImg = cv2.imread(slideImgPath + slideNames[slide], 0)  
            vidImg = cv2.imread(vidFramePath + videoFrameNames[frame], 0)
            saveMatches(slidekpts, framekpts, slideImg, vidImg, good, savePath + str(slide) + '-' + str(frame) + '-match.jpg')
            '''

            matchMetric[(slide, frame)] = computeSumDiffKeypoints(
                slideKeypoints, frameKeypoints, slideDimensions, vidDimensions)
            matchMetricNotNormalized[(slide, frame)] = computeSumDiffKeypoints(
                slideKeypoints, frameKeypoints, slideDimensions, vidDimensions,
                False)

    ## save visualizations
    mat = visualize(
        matchMetric, NUM_SLIDES, NUM_FRAMES, FINAL_FIGURES_RESULTS_SAVE_PATH +
        "normalized_sumDistanceKeypoints.jpg")
    visualize(matchMetricNotNormalized, NUM_SLIDES, NUM_FRAMES,
              FINAL_FIGURES_RESULTS_SAVE_PATH + "sumDistanceKeypoints.jpg")

    ## save matrix as txt
    np.savetxt(FINAL_FIGURES_RESULTS_SAVE_PATH +
               "slides_by_vidframes_metric.txt",
               mat,
               fmt='%d')

    col_best_prediction = {}
    for col in range(NUM_FRAMES):
        slides = list(mat[:, col])
        best = slides.index(min(slides))
        col_best_prediction[col] = best

    print(json.dumps(col_best_prediction, sort_keys=True))
import cv2
import os
import pickle
from utilities import visualize, drawMatches, predictARGMIN
from utilities import calculateKeypoints, serializableKeypts, slidesInRange, convertPickledToKPDesc, computeRDistanceDifference, computeSumDiffKeypoints
import numpy as np
from matplotlib import pyplot as plt

## plot number of matches
## where to load/save
slideImgPath = "./360-video-10-23/"

## Load slides, compute keypoint matching to following slide
slideNames = [ filename for filename in os.listdir( slideImgPath) if filename.endswith(".jpg")] 
slideNames = slidesInRange(slideNames, 0, 74)

## load dictionary of things
slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl"
slideImgKeypts = pickle.load( open( slideImgKeyptsFile, "rb" ) )

print(slideImgKeypts.keys())

slideImgKeyptsDesc = dict()

## load slide img, keypoints
for sn in slideNames:
    path = slideImgPath + sn
    slideImgKeyptsDesc[sn] = convertPickledToKPDesc(slideImgKeypts[path])


Ejemplo n.º 6
0
import numpy as np
from matplotlib import pyplot as plt

SLIDE_PATH = "./360-video-10-23/"
TARGET_SLIDES = [54, 58, 61, 67, 71, 72]
VIDEO_PATH = "./subsampled_slides_video/cropped_shortened_pres.mp4"
VIDEO_START_SEC = 0
VIDEO_END_SEC = 63
VIDEO_FRAME_PATH = "./vid_frames_sampled/"
SAVE_MATCH_PATH = "./frames_slides_matches/"

slideNames = [
    filename for filename in os.listdir(SLIDE_PATH)
    if filename.endswith(".jpg")
]
slideNames = slidesInRange(slideNames, 0, 74)
videoFrameNames = [
    filename for filename in os.listdir(VIDEO_FRAME_PATH)
    if filename.endswith(".jpg")
]
videoFrameNames = slidesInRange(videoFrameNames, 0, 74)

## slide and video dimensions
sampleImgSlides = cv2.imread(SLIDE_PATH + slideNames[0], 0)
sampleVidSlides = cv2.imread(VIDEO_FRAME_PATH + videoFrameNames[0], 0)
slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1])
vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1])

slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl"
vidImgKeyptsFle = "./vid_keypts/vidKeyptsDesc.pkl"
slideImgKeypts = pickle.load(open(slideImgKeyptsFile, "rb"))