def _find_frame_vectors(self):
if self.get_image1_luminance() and self.get_image2_luminance():
klog("Evaluating image with size: %dx%d" %(self.get_image1_luminance().width(), self.get_image2_luminance().height() ))
comp = ImageComparator(self.image_1_luminance)
start_time = time.time()
self.vectors = comp.get_motion_vectors(self.image_2_luminance, self.searcher())
elasped_time = time.time() - start_time
self.ui.searchElapsedTimeLabel.setText("%.2f seconds" %elasped_time)
self._draw_compressed_frame2()
def compared_frames_statuses(self, motion_threshold, MAD_threshold):
holden_frames = [0] #start with the first frame
discarded_frames = []
i = 0
j = 1
while i < self._video.frames_count()-2 and j < self._video.frames_count()-1:
#controllo il frame successivo per vedere se sono sotto la soglia
#si lo sono, posso aggiungere il frame alla lista di quelli da non considerare
#no non lo sono, il frame e necessario
if i is j:
print "CYCLE COMPARISON ERROR"
print "\nComparing frame #%d with frame #%d" %(i, j)
frame_1 = self._video.frames[i].grayscaled_image()
frame_2 = self._video.frames[j].grayscaled_image()
comp = ImageComparator(frame_1)
vectors = comp.get_motion_vectors(frame_2, self._searcher, MAD_threshold)
longest_vector, max_distance = ImageComparator.longest_motion_vector(vectors)
print "Max distance found: %f" %max_distance
print "Longest vector is: "+ str(longest_vector)
if max_distance < motion_threshold:
print "Frame #%d discared.. :-) " %j
discarded_frames.append(j) #the compared frame contains only short motion vectors, so I can discard that frame
j += 1 #the I frame is same, the frame to be compared is the j+1 so the search continue
else:
print "Frame #%d holden... :-(" %j
holden_frames.append(j) #the compared frame contains a very long motion vector, so the frame will be rendered as frame I
i = j
j = i+1
holden_frames.append(self._video.frames_count()-1) #keep the last frame
return holden_frames, discarded_frames
def search(self, image1_pixels, x_start, y_start, image2_pixels):
self.reset_search()
block_size = self.block_size
pass_step = self.pass_step
subimage_1_pixels = ImageConverter.sub_pixels(image1_pixels, x_start, y_start, x_start+block_size, y_start+block_size)
p = pass_step
ds = p/2
s = 1
xs = x_start
ys = y_start
xs_min = -1
ys_min = -1
best_local_MAD = 10000
best_global_MAD = 10000
best_x = -1
best_y = -1
klog("Check block from %d, %d" %(x_start, y_start))
while ds >= 1:
for x in [xs, xs+ds, xs-ds]:
for y in [ys, ys+ds, ys-ds]:
if not ImageComparator.is_valid_coordinate(x, y, block_size, image2_pixels):
continue
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x, y, x+block_size, y+block_size)
if MAD < best_local_MAD:
best_local_MAD = MAD
xs_min = x
ys_min = y
#Check if the local MAD is the best global MAD
if MAD < best_global_MAD:
best_global_MAD = MAD
best_x = x
best_y = y
s += 1
ds -= 1
xs = xs_min
ys = ys_min
print "-"
return best_x, best_y, best_global_MAD, self._MAD_checks_count
def calculate_MAD(self, subimage_1_pixels, image2_pixels, x, y, to_x, to_y):
"""
subimage_1_pixels is the the image to be found
image2_pixels is the pixels of the image that should contain sub_pixels_1
x, y is the upper coordinate of pixels_2 to be checked
x+block_size, y+block_size is the lower coordinate of pixels_2 to be checked
"""
MAD = self._get_saved_MAD(x, y)
if not MAD:
subimage_2_pixels = ImageConverter.sub_pixels(image2_pixels, x, y, to_x, to_y)
#Calculate the MAD
MAD = ImageComparator.calculate_MAD_v2(subimage_1_pixels, subimage_2_pixels)
self._coordinates_checked["%dx%d" %(x,y)] = MAD
self._MAD_checks_count += 1
#klog("%d,%d\t\t\t\tMAD-> %f" %(x,y, MAD))
return MAD
def _draw_motion_vectors(self):
sze = 3
scene = self.ui.frame2GraphicsView.scene()
scene.clear()
self._draw_frame(self.image_2, self.ui.frame2GraphicsView)
pen = QPen(Qt.red, 1, Qt.SolidLine)
for v in self.vectors:
x = int(v["x"])
y = int(v["y"])
to_x = int(v["to_x"])
to_y = int(v["to_y"])
MAD = v["MAD"]
klog( "(%d, %d) => (%d, %d)" % (x,y, to_x, to_y) )
if scene:
if MAD < self.ui.MADThresholdSpingBox.value() and (x != to_x or y != to_y):
scene.addLine(x,y,to_x, to_y, pen)
M_PI = math.pi
curr_x = x - to_x
curr_y = y - to_y
if curr_x != 0 or curr_y != 0:#altrimenti la linea e lunga 0!!!
alpha = math.atan2 (curr_y, curr_x)
pa_x = sze * math.cos (alpha + M_PI / 7) + to_x
pa_y = sze * math.sin (alpha + M_PI / 7) + to_y
pb_x = sze * math.cos (alpha - M_PI / 7) + to_x
pb_y = sze * math.sin (alpha - M_PI / 7) + to_y
#scene.addLine(to_x, to_y,pa_x, pa_y)
#scene.addLine(to_x, to_y,pb_x, pb_y)
polygon = QPolygonF([QPointF(to_x-sze * math.cos (alpha), to_y-sze * math.sin (alpha)),QPointF(pa_x, pa_y),QPointF(pb_x, pb_y)])
scene.addPolygon(polygon, pen)
longest_vector, max_distance = ImageComparator.longest_motion_vector(self.vectors)
print "Max motion vector distance: %f" % max_distance
def _interpolateVideo(self):
print "Interpolating video..."
dialog = QMessageBox(self)
dialog.setText("Attendi mentre ricostruisco il video...")
dialog.show()
start_time = time.time()
interpolator = VideoInterpolator(self.video)
#self._discared_frames = [1, 2, 3] #TODO: remove me
frames_path = interpolator.interpolate(self._discared_frames, self.searcher(), self.ui.blockSizeSpinBox.value(), self.ui.MADThresholdSpingBox.value())
klog("L'interpolazione del video ha impiegato: %.2f secondi" % (time.time()-start_time))
interpolated_video = Video(frames_path=frames_path)
#interpolated_video = Video(frames_path="/tmp/pallone.mov.interpolated")
interpolated_video.load()
dialog.close()
self.ui.interpolatedFramesTimelineListView.setModel( QFramesTimelineListModel( interpolated_video ) )
self.ui.interpolatedFramesTimelineListView.setItemDelegate( QFramesTimelineDelegate() )
klog("Calculating PSNR for frames...")
summed_PSNR = 0
for i in xrange(interpolated_video.frames_count()):
original_frame = self.video.frames[i]
new_frame = interpolated_video.frames[i]
PSNR = ImageComparator.calculate_PSNR( original_frame.image(), new_frame.image(), interpolated_video.width(), interpolated_video.height() )
klog("Frame %d\t\tPSNR:%d" %(i, PSNR))
if PSNR > 1000:
PSNR = 50
summed_PSNR += PSNR
PSNR = summed_PSNR/interpolated_video.frames_count()
klog("The interpolated video has a PSNR of %f" %PSNR)
self.ui.interpolatedVideoPSNRLabel.setText("%f" %PSNR )
klog("Saving the interpolated video...")
FFMpegWrapper.generate_video(frames_path, "interpolated_video.mp4")
def interpolate(self, discarded_frames, searcher, block_size, MAD_threshold):
new_video_path = "/tmp/%s.interpolated/" % self._oldvideo.videoname()
if os.path.exists(new_video_path):
shutil.rmtree(new_video_path)
os.makedirs(new_video_path)
i=0
while i < self._oldvideo.frames_count():
if not i in discarded_frames:
#The frame must me copied in the folder
frame = self._oldvideo.frames[i]
shutil.copyfile(frame.path(), "%s/Frame_%07d.png" %(new_video_path, i))
i+=1
else:
#the frame must be interpolated
#check which other adiacent frames must me interpolated
new_frames = [i]
for j in range(i+1, self._oldvideo.frames_count()):
if j in discarded_frames:
new_frames.append(j)
else:
#The adiacent frame is an I frame, so stop searching
i = j
break
#interpolate the frames
print "I have to interpolate these frames: "+ str(new_frames)
left_frame_index = new_frames[0] - 1
right_frame_index = new_frames[-1] + 1
print "Left I frame: %s" %self._oldvideo.frames[left_frame_index].path()
print "Right I frame: %s" %self._oldvideo.frames[right_frame_index].path()
frame_1 = self._oldvideo.frames[left_frame_index] #the frame on the LEFT of the first frame to be interpolated
frame_2 = self._oldvideo.frames[right_frame_index] #the frame on the RIGHT of the last frame to be interpolated
comp = ImageComparator(frame_1.grayscaled_image())
vectors = comp.get_motion_vectors(frame_2.grayscaled_image(), searcher, MAD_threshold)
subvectors = VideoInterpolator.sub_motion_vectors(vectors, len(new_frames))
#TODO: check len(subvectors) MUST be equal to len(new_frames)
for k in range(len(new_frames)):
frame_num = new_frames[k]
frame_path = "%s/Frame_%07d.png" % (new_video_path, frame_num)
print "Devo interpolare %d" % frame_num
image1 = frame_1.image()
image_new = image1.copy()
#image_new.paste((0,0,0))
#First copy the background from the frame_2 to the image_new
#image2 = frame_2.image()
#for v in subvectors[k]:
# background_block_image = image2.crop( (v["x"], v["y"], v["x"]+block_size, v["y"]+block_size) )
# image_new.paste(background_block_image, (v["x"], v["y"], v["x"]+block_size, v["y"]+block_size) )
#Then copy the moved blocks
print "------------"
for v in subvectors[k]:
#print "Copying the block (%d, %d) from frame: %d to (%d, %d) to the new frame %d" %(v["x"], v["y"], left_frame_index, v["to_x"], v["to_y"], frame_num)
moved_block_image = image1.crop( (v["x"], v["y"], v["x"]+block_size, v["y"]+block_size) )
image_new.paste(moved_block_image, (v["to_x"], v["to_y"], v["to_x"]+block_size, v["to_y"]+block_size) )
#image_new.paste((255, 0, 0), (v["to_x"], v["to_y"], v["to_x"]+block_size, v["to_y"]+block_size) )
image_new.save(frame_path)
#print "Saved new frame: "+ frame_path
#print "---"
return new_video_path[:len(new_video_path)-1] #remove the trailing slash
def _draw_compressed_frame2(self):
if len(self.vectors) > 0:
self._draw_motion_vectors()
zero_vectors_blocks_count = 0
new_blocks_count = 0
moved_vectors_blocks_count = 0
self._draw_frame(self.image_2, self.ui.frame2CompressedGraphicsView)
scene = self.ui.frame2CompressedGraphicsView.scene()
image2_new = QImage(self.image_2)
sameBlockPen = QPen(Qt.black, 1, Qt.SolidLine)
movedBlockPen = QPen(Qt.green, 1, Qt.SolidLine)
for v in self.vectors:
x = int(v["x"])
y = int(v["y"])
to_x = int(v["to_x"])
to_y = int(v["to_y"])
MAD = v["MAD"]
if x == to_x and y == to_y:
#The block is the same of the previous frame. Transmit a zero vector
zero_vectors_blocks_count += 1
scene.addRect(x, y, self.get_block_size(), self.get_block_size(), sameBlockPen, QBrush(Qt.SolidPattern))
else:
if MAD < self.ui.MADThresholdSpingBox.value():
#The block is moved
moved_vectors_blocks_count += 1
scene.addRect(x, y, self.get_block_size(), self.get_block_size(), movedBlockPen, QBrush(Qt.green, Qt.SolidPattern))
moved_block_image = self.image_1.copy(x,y, self.get_block_size(), self.get_block_size())
ImageConverter.draw_image_into_image(moved_block_image, image2_new, to_x, to_y)
else:
#A new block of the frame is needed
new_blocks_count += 1
#Draw the reconstructed Frame
scene = QGraphicsScene()
scene.addPixmap(QPixmap.fromImage(image2_new))
self.ui.frame2ReconstructedGraphicsView.setScene(scene)
#Show the statistics
zero_vectors_blocks_percent = (zero_vectors_blocks_count*100/len(self.vectors))
new_blocks_percent = (new_blocks_count*100/len(self.vectors))
moved_vectors_blocks_percent = (moved_vectors_blocks_count*100/len(self.vectors))
compression_ratio = 100 - new_blocks_percent - moved_vectors_blocks_percent/3
self.ui.zeroVectorsPercentLabel.setText("%d %%" %zero_vectors_blocks_percent )
self.ui.newBlocksPercentLabel.setText("%d %%" % new_blocks_percent)
self.ui.movedVectorsPercentLabel.setText("%d %%" % moved_vectors_blocks_percent)
self.ui.compressionRatioLabel.setText("%d %%" %compression_ratio)
total_mads_checked = 0
for v in self.vectors:
total_mads_checked += v["MAD_checks_count"]
self.ui.MADsCheckedLabel.setText("%d" %total_mads_checked)
#Calculate the PSNR
PSNR = ImageComparator.calculate_PSNR(ImageConverter.qtimage_to_pil_image(self.image_2), ImageConverter.qtimage_to_pil_image(image2_new), self.image_2.width(), self.image_2.height())
self.ui.psnrFrame2Label.setText("%d" % PSNR)
def search(self, image1_pixels, x_start, y_start, image2_pixels):
self.reset_search()
block_size = self.block_size
pass_step = self.pass_step
subimage_1_pixels = ImageConverter.sub_pixels(image1_pixels, x_start, y_start, x_start+block_size, y_start+block_size)
p = pass_step
ds = p/2+1
s = 1
xs = x_start
ys = y_start
best_local_MAD = 10000
xs_min = -1
ys_min = -1
best_local_a_MAD = 10000
xs_a_min = -1
ys_a_min = -1
best_global_MAD = 10000
best_x = -1
best_y = -1
#klog("Check block from %d, %d" %(x_start, y_start))
while True:
for (x,y) in [ (xs,ys), (xs-ds,ys), (xs+ds,ys) ]:
if not ImageComparator.is_valid_coordinate(x, y, block_size, image2_pixels):
continue
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x, y, x+block_size, y+block_size)
if MAD < best_local_a_MAD:
best_local_a_MAD = MAD
xs_a_min = x
ys_a_min = y
#Check if the local MAD is the best global MAD
if MAD < best_global_MAD:
best_global_MAD = MAD
best_x = x
best_y = y
for (x,y) in [ (xs_a_min,ys_a_min), (xs_a_min,ys_a_min-ds), (xs_a_min, ys_a_min+ds) ]:
if not ImageComparator.is_valid_coordinate(x, y, block_size, image2_pixels):
continue
#Calculate the MAD
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x, y, x+block_size, y+block_size)
if MAD < best_local_MAD:
best_local_MAD = MAD
xs_min = x
ys_min = y
#Check if the local MAD is the best global MAD
if MAD < best_global_MAD:
best_global_MAD = MAD
best_x = x
best_y = y
if ds == 1:
break
s += 1
ds = math.ceil( ds/2 )
# klog("ds: %d" %(ds))
xs = xs_min
ys = ys_min
best_local_MAD = 10000 #reset
best_local_a_MAD = 10000 #reset
#klog("-")
#klog("MADs check count: %d" %MAD_checks_count)
return best_x, best_y, best_global_MAD, self._MAD_checks_count
def search(self, image1_pixels, x_start, y_start, image2_pixels):
self.reset_search()
block_size = self.block_size
pass_step = self.pass_step
#subimage_1 = image1.copy(x_start, y_start, block_size, block_size)
subimage_1_pixels = ImageConverter.sub_pixels(image1_pixels, x_start, y_start, x_start+block_size, y_start+block_size)
p = pass_step
ds = math.pow(2, math.floor( math.log(p,2))-1 )
s = 1
xs = x_start
ys = y_start
xs_min = -1
ys_min = -1
best_local_MAD = 10000
best_global_MAD = 10000
best_x = -1
best_y = -1
while ds >= 1:
if ds != 1:
for (x,y) in [(xs, ys), (xs-ds,ys), (xs+ds, ys), (xs, ys+ds), (xs,ys-ds)]:
if not ImageComparator.is_valid_coordinate(x, y, block_size, image2_pixels):
continue
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x, y, x+block_size, y+block_size)
if MAD < best_local_MAD:
best_local_MAD = MAD
xs_min = x
ys_min = y
#Check if the local MAD is the best global MAD
if MAD < best_global_MAD:
best_global_MAD = MAD
best_x = x
best_y = y
s += 1
#Check if the xs_min and ys_min are the central point of the fives checked
if xs_min == xs and ys_min == ys:
ds /= 2
xs = xs_min
ys = ys_min
else: #ds==1 last step
for x in [xs, xs+ds, xs-ds]:
for y in [ys, ys+ds, ys-ds]:
#TODO: CODE HERE IS COPY-PASTED!! THIS SUCKS! secondo me va bene!
if not ImageComparator.is_valid_coordinate(x, y, block_size, image2_pixels):
continue
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x, y, x+block_size, y+block_size)
if MAD < best_local_MAD:
best_local_MAD = MAD
xs_min = x
ys_min = y
#Check if the local MAD is the best global MAD
if MAD < best_global_MAD:
best_global_MAD = MAD
best_x = x
best_y = y
ds = 0
s += 1
print "ds: %d" %ds
return best_x, best_y, best_global_MAD, self._MAD_checks_count
def search(self, image1_pixels, x_start, y_start, image2_pixels):
self.reset_search()
block_size = self.block_size
margin_size = self.margin_size
best_MAD = 1000000
best_x = None
best_y = None
subimage_1_pixels = ImageConverter.sub_pixels(image1_pixels, x_start, y_start, x_start+block_size, y_start+block_size)
#Start with the center
if ImageComparator.is_valid_coordinate(x_start, y_start, block_size, image2_pixels):
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, x_start, y_start, x_start+block_size, y_start+block_size)
if MAD < best_MAD:
#klog("Best MAD found: %f, at (%f,%f)" % (MAD, px, py))
best_MAD = MAD
best_x = x_start
best_y = y_start
if best_MAD == 0:
return best_x, best_y, best_MAD, self._MAD_checks_count
for py in range(y_start-margin_size, y_start+margin_size):
if py < 0:
continue #il blocco esce in su dall'immagine, avanza con il prossimo py incrementato
#CHECK!!
if not ImageComparator.is_valid_coordinate(0, py, block_size, image2_pixels):
break #il blocco esce in giu dall'immagine, esci
for px in range(x_start-margin_size, x_start+margin_size):
if px < 0:
continue #il blocco esce a sinistra dall'immagine, avanza con il prossimo px incrementato
#CHECK!!
if not ImageComparator.is_valid_coordinate(px, py, block_size, image2_pixels):
break #il blocco esce in giu dall'immagine, esci
#if px+block_size > image2.width():
# break #il blocco esce a destra dall'immagine, esci
#klog("Valuating block (%f,%f)" %(px, py))
MAD = self.calculate_MAD(subimage_1_pixels, image2_pixels, px, py, px+block_size, py+block_size)
if MAD < best_MAD:
#klog("Best MAD found: %f, at (%f,%f)" % (MAD, px, py))
best_MAD = MAD
best_x = px
best_y = py
if best_MAD == 0:
return best_x, best_y, best_MAD, self._MAD_checks_count
return best_x, best_y, best_MAD, self._MAD_checks_count