def save_descriptors_keypoints(SLIDES_PATH, VID_FRAMES_PATH, SLIDE_START, SLIDE_END, VID_START, VID_END, SLIDES_KP_SAVE_PATH, VID_KP_SAVE_PATH): ## getting filenames videoFrameNames = [ filename for filename in os.listdir(VID_FRAMES_PATH) if filename.endswith(".png") ] videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END) slideNames = [ filename for filename in os.listdir(SLIDES_PATH) if filename.endswith(".png") ] slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END) ## calculating, saving keypoints slideNameKPDesc = dict() for slideName in slideNames: path = SLIDES_PATH + slideName img = cv2.imread(SLIDES_PATH + slideName, 0) keypoints, descriptors = calculateKeypoints(img) serialized = serializableKeypts(keypoints, descriptors) print("Saving " + path) slideNameKPDesc[path] = serialized pickleOutput = open(SLIDES_KP_SAVE_PATH + "slideKeyptsDesc.pkl", 'wb') pickle.dump(slideNameKPDesc, pickleOutput) vidNameKPDesc = dict() for vidName in videoFrameNames: path = VID_FRAMES_PATH + vidName img = cv2.imread(VID_FRAMES_PATH + vidName, 0) keypoints, descriptors = calculateKeypoints(img) print("Saving " + path) serialized = serializableKeypts(keypoints, descriptors) vidNameKPDesc[path] = serialized pickleOutput = open(VID_KP_SAVE_PATH + "vidKeyptsDesc.pkl", 'wb') pickle.dump(vidNameKPDesc, pickleOutput)
## where to load/save slideImgPath = "../slides/" vidFramePath = "../bb_sampled_video_frames/" slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl" vidImgKeyptsFle = "./vid_keypts/vidKeyptsDesc.pkl" matchesPath = "./desc_matches/" ## getting filenames videoFrameNames = [ filename for filename in os.listdir(vidFramePath) if filename.endswith(".jpg") ] videoFrameNames = slidesInRange(videoFrameNames, 0, 28) slideNames = [ filename for filename in os.listdir(slideImgPath) if filename.endswith(".png") ] slideNames = slidesInRange(slideNames, 2, 30) ## load dictionary of things slideImgKeypts = pickle.load(open(slideImgKeyptsFile, "rb")) vidImgKeypts = pickle.load(open(vidImgKeyptsFle, "rb")) slideImgKeyptsDesc = dict() vidImgKeyptsDesc = dict() ## load slide img, keypoints
vidImgKeyptsFle = "./new_vid_keypts/vidKeyptsDesc.pkl" savePath = "./new_cross_matches_save/" VID_START = 0 VID_END = 22 SLIDE_START = 0 SLIDE_END = 12 NUM_SLIDES = VID_END - VID_START - 1 ## getting filenames videoFrameNames = [ filename for filename in os.listdir(vidFramePath) if filename.endswith(".png") ] videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END) slideNames = [ filename for filename in os.listdir(slideImgPath) if filename.endswith(".png") ] slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END) sampleImgSlides = cv2.imread(slideImgPath + slideNames[0], 0) sampleVidSlides = cv2.imread(vidFramePath + videoFrameNames[0], 0) ## slide and video dimensions slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1]) vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1]) ## load dictionary of things
def compute_match_save(SLIDES_PATH, SLIDE_START, SLIDE_END, FRAME_SAMPLE_SAVE_FOLDER, SLIDE_KEYPTS_DESCS_PKL_SAVE_PATH, VID_START, VID_END, VID_KEYPTS_DESCS_PKL_SAVE_PATH, FINAL_FIGURES_RESULTS_SAVE_PATH): NUM_SLIDES = SLIDE_END - SLIDE_START NUM_FRAMES = VID_END - VID_START ## getting filenames videoFrameNames = [ filename for filename in os.listdir(FRAME_SAMPLE_SAVE_FOLDER) if filename.endswith(".png") ] videoFrameNames = slidesInRange(videoFrameNames, VID_START, VID_END) slideNames = [ filename for filename in os.listdir(SLIDES_PATH) if filename.endswith(".png") ] slideNames = slidesInRange(slideNames, SLIDE_START, SLIDE_END) sampleImgSlides = cv2.imread(slideImgPath + slideNames[0], 0) sampleVidSlides = cv2.imread(vidFramePath + videoFrameNames[0], 0) ## slide and video dimensions slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1]) vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1]) ## load dictionary of things slideImgKeypts = pickle.load(open(SLIDE_KEYPTS_DESCS_PKL_SAVE_PATH, "rb")) vidImgKeypts = pickle.load(open(VID_KEYPTS_DESCS_PKL_SAVE_PATH, "rb")) slideImgKeyptsDesc = dict() vidImgKeyptsDesc = dict() ## load slide img, keypoints for sn in slideNames: path = slideImgPath + sn slideImgKeyptsDesc[sn] = convertPickledToKPDesc(slideImgKeypts[path]) for vidName in videoFrameNames: path = vidFramePath + vidName vidImgKeyptsDesc[vidName] = convertPickledToKPDesc(vidImgKeypts[path]) FLANN_INDEX_KDTREE = 0 index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) search_params = dict(checks=50) ## matches is a list of DMatch objects flann = cv2.FlannBasedMatcher(index_params, search_params) matchSlideFrameDict = dict() matchMetric = dict() matchMetricNotNormalized = dict() ## cross matching matchID = 0 for slide in range(0, NUM_SLIDES): for frame in range(0, NUM_FRAMES): print("Slide, Frame", (slide, frame), slideNames[slide], videoFrameNames[frame]) slidekpts, slideDescs = slideImgKeyptsDesc[slideNames[slide]] framekpts, frameDescs = vidImgKeyptsDesc[videoFrameNames[frame]] matches = flann.knnMatch(np.asarray(slideDescs, np.float32), np.asarray(frameDescs, np.float32), k=2) matchPairs = [] distances = [] good = [] slideKeypoints = [] frameKeypoints = [] good = [] for m, n in matches: if m.distance < 0.75 * n.distance: slideKeypoints.append(slidekpts[m.queryIdx].pt) frameKeypoints.append(framekpts[m.trainIdx].pt) good.append(m) ''' slideImg = cv2.imread(slideImgPath + slideNames[slide], 0) vidImg = cv2.imread(vidFramePath + videoFrameNames[frame], 0) saveMatches(slidekpts, framekpts, slideImg, vidImg, good, savePath + str(slide) + '-' + str(frame) + '-match.jpg') ''' matchMetric[(slide, frame)] = computeSumDiffKeypoints( slideKeypoints, frameKeypoints, slideDimensions, vidDimensions) matchMetricNotNormalized[(slide, frame)] = computeSumDiffKeypoints( slideKeypoints, frameKeypoints, slideDimensions, vidDimensions, False) ## save visualizations mat = visualize( matchMetric, NUM_SLIDES, NUM_FRAMES, FINAL_FIGURES_RESULTS_SAVE_PATH + "normalized_sumDistanceKeypoints.jpg") visualize(matchMetricNotNormalized, NUM_SLIDES, NUM_FRAMES, FINAL_FIGURES_RESULTS_SAVE_PATH + "sumDistanceKeypoints.jpg") ## save matrix as txt np.savetxt(FINAL_FIGURES_RESULTS_SAVE_PATH + "slides_by_vidframes_metric.txt", mat, fmt='%d') col_best_prediction = {} for col in range(NUM_FRAMES): slides = list(mat[:, col]) best = slides.index(min(slides)) col_best_prediction[col] = best print(json.dumps(col_best_prediction, sort_keys=True))
import cv2 import os import pickle from utilities import visualize, drawMatches, predictARGMIN from utilities import calculateKeypoints, serializableKeypts, slidesInRange, convertPickledToKPDesc, computeRDistanceDifference, computeSumDiffKeypoints import numpy as np from matplotlib import pyplot as plt ## plot number of matches ## where to load/save slideImgPath = "./360-video-10-23/" ## Load slides, compute keypoint matching to following slide slideNames = [ filename for filename in os.listdir( slideImgPath) if filename.endswith(".jpg")] slideNames = slidesInRange(slideNames, 0, 74) ## load dictionary of things slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl" slideImgKeypts = pickle.load( open( slideImgKeyptsFile, "rb" ) ) print(slideImgKeypts.keys()) slideImgKeyptsDesc = dict() ## load slide img, keypoints for sn in slideNames: path = slideImgPath + sn slideImgKeyptsDesc[sn] = convertPickledToKPDesc(slideImgKeypts[path])
import numpy as np from matplotlib import pyplot as plt SLIDE_PATH = "./360-video-10-23/" TARGET_SLIDES = [54, 58, 61, 67, 71, 72] VIDEO_PATH = "./subsampled_slides_video/cropped_shortened_pres.mp4" VIDEO_START_SEC = 0 VIDEO_END_SEC = 63 VIDEO_FRAME_PATH = "./vid_frames_sampled/" SAVE_MATCH_PATH = "./frames_slides_matches/" slideNames = [ filename for filename in os.listdir(SLIDE_PATH) if filename.endswith(".jpg") ] slideNames = slidesInRange(slideNames, 0, 74) videoFrameNames = [ filename for filename in os.listdir(VIDEO_FRAME_PATH) if filename.endswith(".jpg") ] videoFrameNames = slidesInRange(videoFrameNames, 0, 74) ## slide and video dimensions sampleImgSlides = cv2.imread(SLIDE_PATH + slideNames[0], 0) sampleVidSlides = cv2.imread(VIDEO_FRAME_PATH + videoFrameNames[0], 0) slideDimensions = (sampleImgSlides.shape[0], sampleImgSlides.shape[1]) vidDimensions = (sampleVidSlides.shape[0], sampleVidSlides.shape[1]) slideImgKeyptsFile = "./slide_keypts/slideKeyptsDesc.pkl" vidImgKeyptsFle = "./vid_keypts/vidKeyptsDesc.pkl" slideImgKeypts = pickle.load(open(slideImgKeyptsFile, "rb"))