def task(candidate_gop): from vfs.gop import Gop closest_gops = Descriptor.closest_match(epoch, candidate_gop) if candidate_gop else (None, None) overlap_gop = None with cls.lock: for gop_id, matches in closest_gops: if VFS.instance().database.execute( 'SELECT examined FROM gops WHERE id = ? AND joint = 0', gop_id).fetchone()[0] <= epoch: overlap_gop = Gop.get(gop_id) VFS.instance().database.execute( 'UPDATE gops SET examined = ? WHERE id = ?', (epoch + 1, overlap_gop.id)).close() break if not overlap_gop is None: #VFS.instance().database.execute( # 'UPDATE gops SET examined = ? WHERE id in (?, ?)', (epoch + 10, candidate_gop.id, overlap_gop.id)).close() #threading.Thread(target=cls.co_compress, args=(candidate_gop, overlap_gop, matches)).start() return cls.co_compress(candidate_gop, overlap_gop, matches) #return True, candidate_gop, pool.submit(cls.co_compress, candidate_gop, overlap_gop, matches) #return True, candidate_gop, future #bytes_saved elif not candidate_gop is None: logging.info("Deferring joint compression for gop %d-%d", candidate_gop.physical_id, candidate_gop.id) VFS.instance().database.execute( 'UPDATE gops SET examined = ? WHERE id = ?', (epoch + 1, candidate_gop.id)).close() return 0 #False, candidate_gop, None else: return 0 #False, None, None
def budget(self, value): VFS.instance().database.execute( 'UPDATE logical_videos SET budget = ? WHERE id = ?', (value, self.id)) self._budget = value logging.debug('Logical video %s has budget %dMB', self.name, value // (1000 * 1000))
def deduplicate(cls, gop1, gop2): logging.info("Merging duplicate GOPs %d and %d", gop1.id, gop2.id) VFS.instance().database.execute( 'UPDATE gops SET examined=9999999, filename=? WHERE id = ?', (gop1.filename, gop2.id)) #TODO os.remove(gop2.filename) gop2.filename = gop1.filename return path.getsize(gop1.filename)
def addmany(cls, physical_video, data): #TODO SQL injection :( VFS.instance().database.executebatch( ('INSERT INTO gops(physical_id, filename, start_time, end_time, size, fps, mse, estimated_mse, parent_id, original_size) ' "VALUES ({}, '{}', {}, {}, {}, {}, {}, {}, {}, {})".format( physical_video.id, filename, start_time, end_time, size, fps, mse if mse is not None else 'NULL', estimated_mse if estimated_mse is not None else 'NULL', parent_id if parent_id is not None else 'NULL', size) for filename, start_time, end_time, size, fps, mse, estimated_mse, parent_id in data))
def reconstruct_gop(gop, temp_path, times, resolution, codec, roi, fps): #, filenames, cache_sequences): with log_runtime( f'Reconstruct GOP {gop.video().id}.{gop.id} ({gop.video().width}x{gop.video().height}, {gop.video().codec}, t={gop.start_time:0.2f}-{gop.end_time:0.2f})' ): if not gop.joint: input_filename = gop.filename else: input_filename = path.join( temp_path, path.basename(gop.filename.format('original'))) # Code to correct codec/resolution to skip transcode VFS.instance().compression.co_decompress(gop, input_filename) gop_times = (max(times[0] - gop.start_time, 0), (min(gop.end_time, times[1]) - gop.start_time)) if times else None if (gop.video().resolution() != resolution or gop.video().codec != codec or #TODO (roi is not None and roi != (0, 0, *resolution)) or (times is None or 0 < gop_times[0] < gop_times[1] < gop.end_time - gop.start_time)): if gop.zstandard: logging.debug("Reconstruction: decompressing raw GOP %d", gop.id) vfs.rawcompression.decompress(gop) container = '.mp4' if encoded[codec] else '' resize_filename = path.join( temp_path, 'resize-{}.{}{}'.format(path.basename(input_filename), codec, container)) new_mse = vfs.videoio.reformat( input_filename, resize_filename, input_resolution=gop.video().resolution(), output_resolution=resolution, input_codec=gop.video().codec, output_codec=codec, input_fps=gop.fps, output_fps=fps, roi=roi, times=gop_times if gop_times != (gop.start_time, gop.end_time) else None) return resize_filename, os.path.getsize(resize_filename), ( gop, resize_filename, new_mse) else: logging.info(f"Cache hit for GOP {gop.id}") return input_filename if gop.zstandard is None else compressed_filename( input_filename), gop.original_size, []
def vacuum(): # Clean broken physical videos for p in PhysicalVideo.get_all(): if not any(p.gops()): print(f'Vacuum {p.id}') PhysicalVideo.delete(p) for g in p.gops(): if (not os.path.exists(g.filename) or os.path.getsize(g.filename) == 0) and not g.joint: print(f'Vacuum {p.id}.{g.id} {g.filename}') VFS.instance().database.execute( "DELETE FROM gops WHERE physical_id = ?", p.id) VFS.instance().database.execute( "DELETE FROM physical_videos WHERE id = ?", p.id)
def load(cls, logical_video, filename, resolution=None, codec=None, fps=None): from vfs.gop import Gop if resolution is None and codec is None: resolution, codec, fps = get_shape_and_codec(filename) elif codec is None: resolution = get_shape(filename) assert(resolution is not None) assert(codec is not None) assert(fps is not None) physical = cls.add(logical_video, *resolution, codec=codec) output_filename_template = os.path.join( VFS.instance().path, cls._gop_filename_template(logical_video, physical, '%04d')) #'{}-{}-%04d.{}'.format(logical_video.name, physical.id, extensions[physical.codec])) gop_filenames = split_video(filename, output_filename_template, resolution, codec, fps) Gop.addmany(physical, [(filename, start_time, end_time, os.path.getsize(filename), fps, 0, 0, None) for (filename, start_time, end_time) in gop_filenames]) logging.info('Ingested physical video %s-%d', logical_video.name, physical.id) return physical
def _update_matcher(cls, epoch, gop_id, cluster_id, codec, fps): matcher = cls._matchers[cluster_id][0] old_descriptors = matcher.getTrainDescriptors() old_physical_map = cls._get_physical_map(cluster_id) offset = len(old_descriptors) cluster = VFS.instance().database.execute( 'SELECT row_number() OVER(ORDER BY gops.id) + ? - 1, physical_id, gops.id, descriptors ' 'FROM gops ' 'INNER JOIN physical_videos ' ' ON gops.physical_id = physical_videos.id ' 'WHERE cluster_id = ? AND (examined <= ? or gops.id = ?) AND codec = ? AND fps = ?' 'ORDER BY physical_id, gops.id', (offset, cluster_id, epoch, gop_id, codec, fps)).fetchall() new_physical_map = { pid: [rowid for rowid, pid, gid, descriptor in rows] for pid, rows in groupby(cluster, lambda c: c[1]) } index_map = {rowid: gid for rowid, pid, gid, descriptor in cluster} physical_map = { key: old_physical_map.get(key, []) + new_physical_map.get(key, []) for key in set(old_physical_map) | set(new_physical_map) } new_descriptors = [c[3].astype(np.float32) for c in cluster] if new_descriptors: matcher.add(new_descriptors) matcher.train() cls._matchers[cluster_id] = matcher, physical_map, index_map return cls._matchers[cluster_id][0]
def add(cls, physical_video, filename, start_time, end_time, size, fps): gop = Gop(None, physical_video.id, filename, start_time, end_time, None, False, False, None, None, None, size, None, fps, None, None, None) gop.id = VFS.instance().database.execute( 'INSERT INTO gops(physical_id, filename, start_time, end_time, size, fps, original_size) ' 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (gop.physical_id, gop.filename, gop.start_time, gop.end_time, gop.size, gop.fps, gop.mse, gop.estimated_mse, gop.parent_id, gop.size)).lastrowid return gop
def duration(self): if self._duration is None: self._duration = VFS.instance().database.execute( 'SELECT MAX(end_time) ' 'FROM physical_videos ' 'INNER JOIN gops ON physical_videos.id = gops.physical_id ' 'WHERE logical_id = ?', self.id).fetchone()[0] return self._duration
def videos(self): if self._videos is None: self._videos = list( map( lambda args: PhysicalVideo(*args), VFS.instance().database.execute( 'SELECT id, logical_id, height, width, codec FROM physical_videos WHERE logical_id = ?', self.id).fetchall())) return self._videos
def delete(cls, video): from vfs.gop import Gop logging.info('Deleting Physical Video %d', video.id) gops = video.gops() references = {filename: count for (filename, count) in VFS.instance().database.execute( 'SELECT filename, COUNT(*) ' 'FROM gops ' 'WHERE filename IN (SELECT filename FROM gops WHERE physical_id = ?) ' 'GROUP BY filename', video.id).fetchall()} VFS.instance().database.execute('DELETE FROM gops WHERE physical_id = ?', video.id) VFS.instance().database.execute('DELETE FROM physical_videos WHERE id = ?', video.id) video.id = None for gop in gops: Gop.delete(gop, references=references.get(gop.filename, 0))
def gops(self): from vfs.gop import Gop if self._gops is None: self._gops = list(map(lambda args: Gop(*args), VFS.instance().database.execute( 'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, ' ' histogram, descriptors, keypoints, size, zstandard, fps, mse, estimated_mse, parent_id, original_size ' 'FROM gops WHERE physical_id = ? ORDER BY id', self.id).fetchall())) return self._gops
def get_candidate(cls, epoch): from vfs.gop import Gop smallest_cluster = (VFS.instance().database.execute( 'SELECT cluster_id, COUNT(*) FROM gops candidate ' 'WHERE examined <= ? AND joint = 0 AND ' ' cluster_id > 0 ' # What did this do? #' physical_id < (SELECT MAX(physical_id) FROM gops WHERE examined <= ? AND joint = 0 AND cluster_id = candidate.cluster_id) ' 'GROUP BY cluster_id ' 'ORDER BY COUNT(*) ASC ' 'LIMIT 1', (epoch)).fetchone() or [None])[0] #(epoch, epoch) if smallest_cluster is not None: return Gop.get(*VFS.instance().database.execute( 'SELECT MIN(id) FROM gops ' 'WHERE cluster_id = ? AND examined <= ? AND joint = 0', (smallest_cluster, epoch)).fetchone()) else: return None
def delete(cls, gop, references=None): references = (references or VFS.instance().database.execute( 'SELECT COUNT(*) FROM gops WHERE filename = ?', gop.filename).fetchone()[0]) logging.info('Deleting GOP %d (underlying data has %d references)', gop.id, references) if gop.id: VFS.instance().database.execute('DELETE FROM gops WHERE id = ?', gop.id) gop.id = None if not gop.joint: #logging.warning("Skipping deleting physical GOP file in debug mode") os.remove(gop.filename) #pass elif references <= 1: logging.info("Last joint compressed candidate deleted; removing physical file") #logging.warning("Skipping deleting physical GOP file in debug mode") os.remove(gop.filename.format(JointCompression.LEFT)) #os.remove(gop.filename.format(JointCompression.RIGHT)) #os.remove(gop.filename.format(JointCompression.OVERLAP)) else: logging.info("Not removing physical file; other references exist")
def cache_reconstructions(logical, resolution, codec, times, fps, cache_sequences): for sequence in (sequence for sequence in cache_sequences if sequence): physical = PhysicalVideo.add(logical, *resolution, codec) new_gop_data = [] transitive_estimated_mses = VFS.instance().database.execute( 'WITH RECURSIVE' ' error(id, estimated_mse) AS ' ' (SELECT id, estimated_mse FROM gops child WHERE id IN ({}) '. format(','.join(str(gop.id) for gop, _, _ in sequence)) + ' UNION ALL ' ' SELECT id, estimated_mse FROM gops parent WHERE parent.id = id)' 'SELECT id, SUM(estimated_mse) FROM error GROUP BY id').fetchall() for (_, current_estimated_mse), (index, (gop, filename, new_mse)) in zip( transitive_estimated_mses, enumerate(sequence)): new_filename = path.join( VFS.instance().path, PhysicalVideo._gop_filename_template(logical, physical, index)) new_gop_data.append( (new_filename, max(gop.start_time, times[0]), min(gop.end_time, times[1]), os.path.getsize(filename), fps, None, 2 * (current_estimated_mse + (new_mse or 0)), gop.id)) #TODO remove guard #duplicates = VFS.instance().database.execute("SELECT * FROM gops, physical_videos WHERE start_time = ? AND end_time = ? and height = ? and width = ? and codec = ? and gops.physical_id = physical_videos.id", # (new_gop_data[-1][0], new_gop_data[-1][1], resolution[0], resolution[1], codec)).fetchall() #if len(duplicates) > 0: # print("Duplicate detected???") os.rename(filename, new_filename) Gop.addmany(physical, new_gop_data) logging.info('Cached physical video %s-%d', logical.name, physical.id) return new_gop_data
def cluster_all(cls): data = VFS.instance().database.execute( 'SELECT id, histogram FROM gops WHERE NOT histogram IS NULL AND examined != 9999999' ).fetchall() if not data or len(data) == 1: return 0 ids, histograms = zip(*data) id_array = np.array(ids) scaled_histograms = maxabs_scale(np.vstack(histograms)) clusters = Birch(n_clusters=len(histograms)).fit(scaled_histograms) cluster_count = 0 for cluster_id in range(max(clusters.labels_) + 1): rows = id_array[clusters.labels_ == cluster_id] if len(rows) > 1: cluster_count += 1 VFS.instance().database.executebatch( 'UPDATE gops SET cluster_id = {} where id = {}'.format( int(cluster_id) + 1, int(row_id)) for row_id in rows) return cluster_count
def solve_exact(logical, resolution, roi, t, fps, codec): #if roi is not None: # return None gop_id = VFS.instance().database.execute( """ SELECT gops.id FROM gops, physical_videos WHERE height = ? AND width = ? AND start_time <= ? AND end_time >= ? AND codec = ? AND gops.physical_id = physical_videos.id AND logical_id = ?""", (resolution[0], resolution[1], t[0], t[1], codec, logical.id)).fetchone() return [Gop.get(gop_id[0])] if gop_id else None
def co_decompress(cls, gop, filename): logging.info('Joint decompress %s-%d-%d', gop.video().logical().name, gop.video().id, gop.id) assert(gop.joint) left_filename = gop.filename.format(cls.LEFT) right_filename = gop.filename.format(cls.RIGHT) overlap_filename = gop.filename.format(cls.OVERLAP) codec = gop.video().codec H, shapes, is_left = VFS.instance().database.execute( 'SELECT homography, shapes, is_left FROM gops WHERE id = ?', gop.id).fetchone() left_shape, overlap_shape, right_shape = shapes[0], shapes[1], shapes[2] frame = np.zeros(gop.video().shape(), dtype=np.uint8) if H.shape[1] > 1: # Current code assumes H=3x3 matrix, but actually list of hstack[frameid, 3x3.flatten] matrices raise NotImplementedError("Need to add support for reprojection") else: H = H[1:, 0].reshape((3, 3)) with VideoReader(left_filename, left_shape, codec) if is_left else NullReader() as leftreader, \ VideoReader(overlap_filename, overlap_shape, codec) as overlapreader, \ VideoReader(right_filename, right_shape, codec) if not is_left else NullReader() as rightreader, \ VideoWriter(filename, gop.video().shape(), codec) as writer: while not leftreader.eof or not overlapreader.eof or not rightreader.eof: if is_left: left, overlap = leftreader.read(), overlapreader.read() if overlap is not None: top = overlap.shape[0]//2 - frame.shape[0]//2 bottom = top + frame.shape[0] np.copyto(frame[:, left.shape[1]:], overlap[top:bottom]) if left is not None: np.copyto(frame[:, :left.shape[1]], left) else: overlap, right = overlapreader.read(), rightreader.read() if overlap is not None: cv2.warpPerspective(overlap, H, dsize=tuple(reversed(frame.shape[:2])), dst=frame) if right is not None: np.copyto(frame[:, -right.shape[1]:], right) writer.write(frame) pass
def get_all(cls): return map( lambda args: LogicalVideo(*args), VFS.instance().database.execute( 'SELECT id, name FROM logical_videos').fetchall())
def get_all(cls, ids): return (Gop(*args) for args in VFS.instance().database.execute( 'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, ' ' histogram, keypoints, descriptors, size, zstandard, fps, mse, estimated_mse, parent_id, original_size ' 'FROM gops WHERE id IN ({})'.format(','.join(map(str, ids)))).fetchall())
def get(cls, id): return Gop(*VFS.instance().database.execute( 'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, ' ' histogram, keypoints, descriptors, size, zstandard, fps, mse, estimated_mse, parent_id, original_size ' 'FROM gops WHERE id = ?', id).fetchone())
def _prepare(logical, resolution, t): """ physical_videos = {physical.id: {'id': physical.id, 'start': physical.start_time(), 'end': physical.end_time(), 'format': physical.codec, 'clock': 0, 'size': physical.size(), 'time': None, 'resolution': physical.resolution(), 'frames': sum(gop.fps for gop in physical.gops()) } for physical in logical.videos() if t is None or (physical.start_time() is not None and physical.end_time() is not None and physical.start_time() < t[1] and physical.end_time() >= t[0]) and #(physical.start_time() or 0) <= t[0] and #(physical.end_time() or t[1]) >= t[1]) and #physical.start_time() is not None and physical.resolution()[0] >= resolution[0] and physical.resolution()[1] >= resolution[1]} """ """ fragments = [{'id': gop.id, 'source': gop.physical_id, 'start': gop.start_time, 'end': gop.end_time, 'fps': gop.fps} for physical in logical.videos() for gop in physical.gops() if physical.id in physical_videos and (t is None or (gop.start_time <= t[0] and gop.end_time >= t[1]) or (gop.start_time <= t[0] < gop.end_time) or (gop.start_time < t[1] <= gop.end_time) or (gop.start_time >= t[0] and gop.end_time <= t[1]))] """ physical_videos = { physical_id: { 'id': physical_id, 'start': start_time, 'end': end_time, 'format': codec, 'clock': 0, 'size': size, 'time': None, 'resolution': (height, width), 'frames': (end_time - start_time) * fps } for (physical_id, start_time, end_time, codec, height, width, size, fps) in VFS.instance().database.execute( 'SELECT physical_videos.id, MIN(start_time), MAX(end_time), codec, height, width, SUM(size), MAX(fps) ' 'FROM gops, physical_videos ' 'WHERE logical_id = ? AND ' ' height >= ? AND width >= ? AND ' ' start_time < ? AND end_time >= ? AND ' ' gops.physical_id = physical_videos.id ' 'GROUP BY physical_videos.id', ( logical.id, resolution[0], resolution[1], t[1] if t is not None else 9999999, t[0] if t is not None else -1)).fetchall() #for physical in logical.videos() if t is None or (start_time is not None and end_time is not None and start_time < t[1] and end_time >= t[0]) and #(physical.start_time() or 0) <= t[0] and #(physical.end_time() or t[1]) >= t[1]) and #physical.start_time() is not None and height >= resolution[0] and width >= resolution[1] } fragments = [{ 'id': gop_id, 'source': physical_id, 'start': start_time, 'end': end_time, 'fps': fps } for (gop_id, physical_id, start_time, end_time, fps) in VFS.instance().database.execute( 'SELECT gops.id, physical_id, start_time, end_time, fps ' 'FROM gops, physical_videos ' 'WHERE logical_id = ? AND ' ' physical_id IN ({}) AND ' ' gops.physical_id = physical_videos.id AND ' ' (? = -1 OR (' ' (start_time <= ? AND end_time >= ?) OR ' ' (start_time <= ? AND ? < end_time) OR ' ' (start_time < ? AND ? <= end_time) OR ' ' (start_time >= ? AND end_time <= ?)))'.format( ','.join(map(str, physical_videos.keys()))), (logical.id, -1 if t is None else 0, t[0] if t is not None else -1, t[1] if t is not None else -1, t[0] if t is not None else -1, t[0] if t is not None else -1, t[1] if t is not None else -1, t[1] if t is not None else -1, t[0] if t is not None else -1, t[1] if t is not None else -1)).fetchall()] return physical_videos.values(), fragments
def budget(self): if self._budget is None: self._budget = VFS.instance().database.execute( 'SELECT budget FROM logical_videos WHERE id = ?', self.id).fetchone()[0] return self._budget
def co_compress(cls, gop1, gop2, matches, abort_psnr_threshold=25, dryrun=False, gops_reversed=False): #cls.output.write('%s,%d,%d,%s,%d,%d\n' % (gop1.video().logical().name, gop1.video().id, gop1.id, # gop2.video().logical().name, gop2.video().id, gop2.id)) #cls.output.flush() logging.info('Joint compress %s-%d-%d and %s-%d-%d (%d matches, %s, %s)', gop1.video().logical().name, gop1.video().id, gop1.id, gop2.video().logical().name, gop2.video().id, gop2.id, len(matches), gop1.filename, gop2.filename) assert(gop1.id != gop2.id) assert(gop1.video().codec == gop2.video().codec) assert(not gop1.joint) assert(not gop2.joint) H, Hi, left, right, overlap, overlap_subframe, recovered_frame2, frame2_overlap_yoffset, inverted_homography, has_homography = cls.estimate_homography(gop1, gop2, matches=None, fast=False) #matches) #TODO fast, matches Hs = np.hstack([[0], H.flatten()]) #H, Hi = homography.project(gop1.keypoints, gop2.keypoints, matches) #frame2_overlap_yoffset = -int(round(np.dot([0, gop1.video().height, 1], Hi)[0])) #frame1_left_width = roundeven(Hi.dot([0,0,1])[0]) #frame2_right_width = roundeven(gop2.video().width - H.dot([gop1.video().width, 0, 1])[0] / H.dot([gop1.video().width, 0, 1])[2]) #overlap_height = gop2.video().height + 2 * frame2_overlap_yoffset #overlap_width = gop1.video().width - frame1_left_width if has_homography and inverted_homography and not gops_reversed: return cls.co_compress(gop2, gop1, matches, gops_reversed=True) # Are videos identical? elif not inverted_homography and left.shape[1] == 0 and right.shape[1] == 0: #if frame1_left_width == 0 and frame2_right_width == 0: return cls.deduplicate(gop1, gop2) # Are left/right frames too small to encode? elif 0 <= left.shape[1] < 32 or 0 <= right.shape[1] < 32 or 0 <= overlap.shape[1] < 32: logging.info('Joint compression aborted; left/right/overlap frames too small (%d, %d)', gop1.id, gop2.id) VFS.instance().database.execute( 'UPDATE gops SET examined=9999998 ' # 9999998 = possibly examine again? 'WHERE id in (?, ?)', (gop1.id, gop2.id)).close() return 0 # return cls.deduplicate(gop1, gop2) else: #pretransform_points = np.float32([ # [frame1_left_width, 0], # [gop1.video().width, 0], # [frame1_left_width, gop1.video().height], # [gop1.video().width, gop1.video().height]]) #posttransform_points = np.float32( # [[0, 0 + frame2_overlap_yoffset], # [gop1.video().width - frame1_left_width + 1, 0 + frame2_overlap_yoffset], # [0, gop1.video().height + frame2_overlap_yoffset], # [gop1.video().width - frame1_left_width + 1, gop1.video().height + frame2_overlap_yoffset]]) #transform = cv2.getPerspectiveTransform(pretransform_points, posttransform_points) #inverse_transform = cv2.getPerspectiveTransform(posttransform_points, pretransform_points) ##Hi = transform.dot(Hi) #Ho = H #H, Hi = H.dot(inverse_transform), transform.dot(Hi) #left = cls._create_image(gop1.video(), width=frame1_left_width) #right = cls._create_image(gop2.video(), width=frame2_right_width) #overlap = cls._create_image(height=overlap_height, width=overlap_width) #recovered_frame2 = np.empty(gop2.video().shape(), dtype=np.uint8) filenametemplate = '{}-{{}}{}'.format(*path.splitext(gop1.filename)) abort = False frame_index = 0 total_left_psnr, total_right_psnr = 0, 0 codec = gop1.video().codec # if frame1_left_width else NullWriter() as leftwriter, \ #if frame2_right_width else NullWriter() as rightwriter, \ with log_runtime('Joint compression:'): with VideoReader(gop1.filename, gop1.video().shape(), codec) as reader1, \ VideoReader(gop2.filename, gop2.video().shape(), codec) as reader2, \ VideoWriter(filenametemplate.format(cls.LEFT), left.shape, codec) \ if left.shape[1] else NullWriter() as leftwriter, \ VideoWriter(filenametemplate.format(cls.RIGHT), right.shape, codec) \ if right.shape[1] else NullWriter() as rightwriter, \ VideoWriter(filenametemplate.format(cls.OVERLAP), overlap.shape, codec) as overlapwriter: while (not reader1.eof or not reader2.eof) and not abort: attempts = 0 frame_index += 1 frame1, frame2 = reader1.read(), reader2.read() while attempts < 2: attempts += 1 if frame1 is not None and frame2 is not None: pass elif frame1 is not None and frame2 is None: frame2 = np.zeros(gop2.video().shape(), dtype=np.uint8) elif frame2 is not None and frame1 is None: frame1 = np.zeros(gop1.video().shape(), dtype=np.uint8) if frame1 is not None or frame2 is not None: # Create and write overlap cv2.warpPerspective(frame2, Hi, dsize=tuple(reversed(overlap.shape[:2])), dst=overlap) # Left join """cv2.imwrite('frame1.png', frame1) cv2.imwrite('frame2.png', frame2) cv2.imwrite('overlap.png', overlap) print(gop1.filename) print(gop2.filename) print(gop1.video().shape()) print(left.shape) print(right.shape) print(overlap.shape) print(frame2_overlap_yoffset, frame2_overlap_yoffset + gop1.video().height) print(left.shape[1]) print(Hi)""" #tmp = frame2_overlap_yoffset #if frame2_overlap_yoffset < 0: # frame2_overlap_yoffset = 0 # Mean join #cls.mean_join(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], overlap_subframe, frame1[:, left.shape[1]:]) #np.copyto(overlap_subframe, overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]) #overlap_subframe = (overlap_subframe + frame1[:, left.shape[1]:]) // 2 #np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], overlap_subframe) #mean_subframe = np.copy(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]).astype(np.uint16) // 2 ##mean_subframe = np.copy(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]).astype(np.uint16) #mean_subframe = (mean_subframe + frame1[:, left.shape[1]:]) // 2 ##mean_subframe //= 2 #np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], # mean_subframe.astype(np.uint8)) # Left join cls.left_join(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], frame1[:, left.shape[1]:]) #np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], # frame1[:, left.shape[1]:]) #frame2_overlap_yoffset = tmp # Mean join (has a bug in non-overlapping left) #overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height] //= 2 #overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height] += frame1[:, left.shape[1]:] // 2 if left.shape[1] != 0: np.copyto(left, frame1[:, :left.shape[1]]) if right.shape[1] != 0: np.copyto(right, frame2[:, -right.shape[1]:]) right_psnr = cls.recovered_right_psnr(H, overlap, frame2, recovered_frame2, right) left_psnr = cls.recovered_left_psnr(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], frame1, left) #cv2.warpPerspective(overlap, H, dsize=tuple(reversed(recovered_frame2.shape[:2])), dst=recovered_frame2) #np.copyto(recovered_frame2[:, -right.shape[1]:], frame2[:, -right.shape[1]:]) #psnr = vfs.utilities.psnr(frame2, recovered_frame2) if right_psnr < abort_psnr_threshold: if attempts == 2: abort = True break else: logging.debug(f"Recomputing homography ({gop1.id} <-> {gop2.id}) PSNR {right_psnr:0.1f}") #TODO save new homography H, Hi, left, right, overlap, overlap_subframe, recovered_frame2, frame2_overlap_yoffset, inverted_homography, has_homography = cls.estimate_homography( gop1, gop2, frame1=frame1, frame2=frame2, matches=None, fast=True) #matches) np.vstack([Hs, np.hstack([[frame_index], H.flatten()])]) #Hs.append(np.hstack([[frame_index], H.flatten()])) else: total_right_psnr += right_psnr total_left_psnr += left_psnr attempts = 999 #break leftwriter.write(left) rightwriter.write(right) overlapwriter.write(overlap) if abort: #cv2.imwrite('abortframe1_%d.png' % gop1.id, frame1) #cv2.imwrite('abortframe2_%d.png' % gop1.id, frame2) #cv2.imwrite('abortleft_%d.png' % gop1.id, left) #cv2.imwrite('abortright_%d.png' % gop1.id, right) cv2.imwrite('abortoverlap.png', overlap) cv2.imwrite('abortrecovered1.png', np.hstack([left, overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]])) cv2.imwrite('abortrecovered2.png', recovered_frame2) logging.info('Joint compression aborted; quality threshold violated %d < %d (%d vs %d)', right_psnr, abort_psnr_threshold, gop1.id, gop2.id) #ssim = compare_ssim(frame2, recovered_frame2, multichannel=True) os.remove(filenametemplate.format(cls.LEFT)) os.remove(filenametemplate.format(cls.OVERLAP)) os.remove(filenametemplate.format(cls.RIGHT)) VFS.instance().database.executebatch([ f'INSERT INTO gop_joint_aborted(gop1, gop2) VALUES ({gop1.id}, {gop2.id})', f'INSERT INTO gop_joint_aborted(gop1, gop2) VALUES ({gop2.id}, {gop1.id})', f'UPDATE gops SET examined=examined + 1 WHERE id in ({gop1.id}, {gop2.id})']) return 0 elif not dryrun: original_size = path.getsize(gop1.filename) + path.getsize(gop2.filename) VFS.instance().database.execute( 'UPDATE gops SET examined=9999999, joint=1, original_filename=filename, filename=?, homography=?, shapes=?, is_left=(id=?) ' 'WHERE id in (?, ?)', (filenametemplate, np.vstack(Hs), np.vstack([left.shape, overlap.shape, right.shape]), gop1.id, gop1.id, gop2.id)).close() os.remove(gop1.filename) os.remove(gop2.filename) bytes_saved = (original_size - (path.getsize(filenametemplate.format(cls.LEFT)) + path.getsize(filenametemplate.format(cls.OVERLAP)) + path.getsize(filenametemplate.format(cls.RIGHT)))) logging.info('Joint compression saved %dKB (%d%%), %d frames, PSNR left=%d, right=%d', bytes_saved // 1000, (bytes_saved * 100) // original_size, frame_index-1, total_left_psnr // (frame_index-1), total_right_psnr // (frame_index-1)) with open('joint.csv', 'a') as f: f.write(f'{gop1.id},{gop2.id},{frame_index-1},{total_right_psnr // (frame_index-1)},{total_left_psnr // (frame_index-1)}\n') return bytes_saved else: return 0
def exists_by_name(cls, name): return VFS.instance().database.execute( 'SELECT 1 FROM logical_videos WHERE name = ? LIMIT 1', name).fetchone() is not None
def add(cls, name): return LogicalVideo( VFS.instance().database.execute( 'INSERT INTO logical_videos(name) VALUES (?)', name).lastrowid, name)
def get(cls, id): return LogicalVideo(*VFS.instance().database.execute( 'SELECT id, name FROM logical_videos WHERE id = ?', id).fetchone())
def closest_match(cls, epoch, gop, matches_required=DEFAULT_MATCHES_REQUIRED, radius=400): #400): from vfs.gop import Gop with log_runtime("Joint compression candidate selection"): cluster_gops = VFS.instance().database.execute( "SELECT id, filename, descriptors FROM gops WHERE cluster_id = ? AND physical_id != ? AND descriptors IS NOT NULL AND joint = 0 AND examined <= ? AND NOT EXISTS (SELECT id FROM gop_joint_aborted WHERE gop1 = ? AND gop2 = id)", (gop.cluster_id, gop.physical_id, epoch, gop.id)).fetchall() candidate_gops = [] for gop2_id, gop2_filename, gop2_descriptors in cluster_gops: success, matches = cls.adhoc_match(gop.descriptors, gop2_descriptors) if success and len(matches) > matches_required: candidate_gops.append( (gop.filename.split('-')[-1] == gop2_filename.split( '-')[-1], len(matches), gop2_id, matches)) if candidate_gops[-1][1] > 400 or candidate_gops[-1][ 0]: # Break on "good enough" match to try out break candidate_gop = sorted(candidate_gops, reverse=True)[0] if candidate_gops else None return [(candidate_gop[2], candidate_gop[3])] matcher = cls._get_matcher(epoch, gop) physical_map = cls._get_physical_map(gop.cluster_id).get( gop.physical_id, None) index_map = cls._get_index_map(gop.cluster_id) all_matches = matcher.radiusMatch(queryDescriptors=gop.descriptors, maxDistance=radius) good_matches = defaultdict(lambda: []) first_matches = {} complete = set() # For each frame/descriptor pair, find matches that pass the Lowe threshold test #all_matches = all_matches[:5000] filtered_matches = ( m for d in all_matches for m in d if index_map[m.imgIdx] > gop.id and m.imgIdx not in physical_map and not (m.imgIdx, m.queryIdx) in complete) #for descriptor_matches in all_matches: # for match in descriptor_matches: with log_runtime("Lowes test"): for match in filtered_matches: #if match.imgIdx not in physical_map and \ # index_map[match.imgIdx] > gop.id and \ #if not (match.imgIdx, match.queryIdx) in complete: # First match if (match.imgIdx, match.queryIdx) not in first_matches: first_matches[match.imgIdx, match.queryIdx] = match # Second match else: if first_matches[ match.imgIdx, match. queryIdx].distance < cls.LOWE_THRESHOLD * match.distance: good_matches[match.imgIdx].append( first_matches[match.imgIdx, match.queryIdx]) del first_matches[match.imgIdx, match.queryIdx] complete.add((match.imgIdx, match.queryIdx)) # Some matches may not have a second match to apply Lowe's threshold on. # Check to see if we should have seen it and count it if so. for first_match in first_matches.values(): if first_match.distance / cls.LOWE_THRESHOLD < radius: good_matches[first_match.imgIdx].append(first_match) ignore_ids = set(VFS.instance().database.execute( 'SELECT gop2 FROM gop_joint_aborted WHERE gop1 = ?', gop.id).fetchall()) best_indexes = [ index for index, matches in good_matches.items() if len(matches) >= matches_required and index_map[index] not in ignore_ids ] #best_ids = [index_map[index] for index in best_indexes if index_map[index] not in ignore_ids] #best_id = VFS.instance().database.execute( # 'SELECT MIN(id) FROM gops WHERE joint=0 AND id in ({})'.format(','.join(map(str, best_ids)))).fetchone()[0] #best_index = max((index for index, matches in good_matches.items() if len(matches) >= matches_required), # default=None) #best = sorted([(index_map[index], good_matches[index]) for index in best_indexes], key=lambda pair: len(pair[1]), reverse=True) gops = Gop.get_all(index_map[index] for index in best_indexes) best = [(gop, good_matches[index]) for gop, index in zip(gops, best_indexes)] best = sorted(best, key=lambda pair: (pair[0].filename.split('-')[-1] == gop. filename.split('-')[-1], len(pair[1])), reverse=True) best = best[:len(best) // 20 + 1] # Keep top 5% best = [(mgop.id, matches) for mgop, matches in best] #best = sorted([(mgop.id, cv2.compareHist(gop.histogram, mgop.histogram, cv2.HISTCMP_CHISQR), gop.filename, mgop.filename, matches) for (mgop, matches) in best], key=lambda pair: (len(pair[2]), cv2.compareHist(gop.histogram, pair[0].histogram, cv2.HISTCMP_CHISQR), -pair[0].id), reverse=True) #best = sorted([(id, cv2.compareHist(gop.histogram, Gop.get(id).histogram, cv2.HISTCMP_CHISQR), gop.filename, Gop.get(id).filename, matches) for (id, matches) in best], key=lambda pair: (len(pair[2]), cv2.compareHist(gop.histogram, Gop.get(pair[0]).histogram, cv2.HISTCMP_CHISQR), -pair[0]), reverse=True) return best if best_id is not None: return Gop.get(best_id), good_matches[best_indexes[best_ids.index( best_id)]] #best_index] #if best_index is not None: # return Gop.get(index_map[best_index]), good_matches[best_index] #return Gop.get(VFS.instance().database.execute( # '''SELECT id FROM gops # WHERE cluster_id = ? AND joint = 0 # LIMIT 1 # OFFSET ?''', (gop.cluster_id, best_index)).fetchone()[0]), good_matches[best_index] else: return None, None
def get_by_name(cls, name): return LogicalVideo(*VFS.instance().database.execute( 'SELECT id, name FROM logical_videos WHERE name = ?', name).fetchone())