def task(candidate_gop):
from vfs.gop import Gop
closest_gops = Descriptor.closest_match(epoch, candidate_gop) if candidate_gop else (None, None)
overlap_gop = None
with cls.lock:
for gop_id, matches in closest_gops:
if VFS.instance().database.execute(
'SELECT examined FROM gops WHERE id = ? AND joint = 0', gop_id).fetchone()[0] <= epoch:
overlap_gop = Gop.get(gop_id)
VFS.instance().database.execute(
'UPDATE gops SET examined = ? WHERE id = ?', (epoch + 1, overlap_gop.id)).close()
break
if not overlap_gop is None:
#VFS.instance().database.execute(
# 'UPDATE gops SET examined = ? WHERE id in (?, ?)', (epoch + 10, candidate_gop.id, overlap_gop.id)).close()
#threading.Thread(target=cls.co_compress, args=(candidate_gop, overlap_gop, matches)).start()
return cls.co_compress(candidate_gop, overlap_gop, matches)
#return True, candidate_gop, pool.submit(cls.co_compress, candidate_gop, overlap_gop, matches)
#return True, candidate_gop, future #bytes_saved
elif not candidate_gop is None:
logging.info("Deferring joint compression for gop %d-%d", candidate_gop.physical_id, candidate_gop.id)
VFS.instance().database.execute(
'UPDATE gops SET examined = ? WHERE id = ?', (epoch + 1, candidate_gop.id)).close()
return 0 #False, candidate_gop, None
else:
return 0 #False, None, None
def budget(self, value):
VFS.instance().database.execute(
'UPDATE logical_videos SET budget = ? WHERE id = ?',
(value, self.id))
self._budget = value
logging.debug('Logical video %s has budget %dMB', self.name,
value // (1000 * 1000))
def deduplicate(cls, gop1, gop2):
logging.info("Merging duplicate GOPs %d and %d", gop1.id, gop2.id)
VFS.instance().database.execute(
'UPDATE gops SET examined=9999999, filename=? WHERE id = ?', (gop1.filename, gop2.id))
#TODO
os.remove(gop2.filename)
gop2.filename = gop1.filename
return path.getsize(gop1.filename)
def addmany(cls, physical_video, data):
#TODO SQL injection :(
VFS.instance().database.executebatch(
('INSERT INTO gops(physical_id, filename, start_time, end_time, size, fps, mse, estimated_mse, parent_id, original_size) '
"VALUES ({}, '{}', {}, {}, {}, {}, {}, {}, {}, {})".format(
physical_video.id, filename, start_time, end_time, size, fps,
mse if mse is not None else 'NULL',
estimated_mse if estimated_mse is not None else 'NULL',
parent_id if parent_id is not None else 'NULL',
size)
for filename, start_time, end_time, size, fps, mse, estimated_mse, parent_id in data))
def reconstruct_gop(gop, temp_path, times, resolution, codec, roi,
fps): #, filenames, cache_sequences):
with log_runtime(
f'Reconstruct GOP {gop.video().id}.{gop.id} ({gop.video().width}x{gop.video().height}, {gop.video().codec}, t={gop.start_time:0.2f}-{gop.end_time:0.2f})'
):
if not gop.joint:
input_filename = gop.filename
else:
input_filename = path.join(
temp_path, path.basename(gop.filename.format('original')))
# Code to correct codec/resolution to skip transcode
VFS.instance().compression.co_decompress(gop, input_filename)
gop_times = (max(times[0] - gop.start_time, 0),
(min(gop.end_time, times[1]) -
gop.start_time)) if times else None
if (gop.video().resolution() != resolution
or gop.video().codec != codec or
#TODO (roi is not None and roi != (0, 0, *resolution)) or
(times is None or
0 < gop_times[0] < gop_times[1] < gop.end_time - gop.start_time)):
if gop.zstandard:
logging.debug("Reconstruction: decompressing raw GOP %d",
gop.id)
vfs.rawcompression.decompress(gop)
container = '.mp4' if encoded[codec] else ''
resize_filename = path.join(
temp_path,
'resize-{}.{}{}'.format(path.basename(input_filename), codec,
container))
new_mse = vfs.videoio.reformat(
input_filename,
resize_filename,
input_resolution=gop.video().resolution(),
output_resolution=resolution,
input_codec=gop.video().codec,
output_codec=codec,
input_fps=gop.fps,
output_fps=fps,
roi=roi,
times=gop_times if gop_times !=
(gop.start_time, gop.end_time) else None)
return resize_filename, os.path.getsize(resize_filename), (
gop, resize_filename, new_mse)
else:
logging.info(f"Cache hit for GOP {gop.id}")
return input_filename if gop.zstandard is None else compressed_filename(
input_filename), gop.original_size, []
def vacuum():
# Clean broken physical videos
for p in PhysicalVideo.get_all():
if not any(p.gops()):
print(f'Vacuum {p.id}')
PhysicalVideo.delete(p)
for g in p.gops():
if (not os.path.exists(g.filename)
or os.path.getsize(g.filename) == 0) and not g.joint:
print(f'Vacuum {p.id}.{g.id} {g.filename}')
VFS.instance().database.execute(
"DELETE FROM gops WHERE physical_id = ?", p.id)
VFS.instance().database.execute(
"DELETE FROM physical_videos WHERE id = ?", p.id)
def load(cls, logical_video, filename, resolution=None, codec=None, fps=None):
from vfs.gop import Gop
if resolution is None and codec is None:
resolution, codec, fps = get_shape_and_codec(filename)
elif codec is None:
resolution = get_shape(filename)
assert(resolution is not None)
assert(codec is not None)
assert(fps is not None)
physical = cls.add(logical_video, *resolution, codec=codec)
output_filename_template = os.path.join(
VFS.instance().path,
cls._gop_filename_template(logical_video, physical, '%04d'))
#'{}-{}-%04d.{}'.format(logical_video.name, physical.id, extensions[physical.codec]))
gop_filenames = split_video(filename, output_filename_template, resolution, codec, fps)
Gop.addmany(physical, [(filename, start_time, end_time, os.path.getsize(filename), fps, 0, 0, None)
for (filename, start_time, end_time) in gop_filenames])
logging.info('Ingested physical video %s-%d', logical_video.name, physical.id)
return physical
def _update_matcher(cls, epoch, gop_id, cluster_id, codec, fps):
matcher = cls._matchers[cluster_id][0]
old_descriptors = matcher.getTrainDescriptors()
old_physical_map = cls._get_physical_map(cluster_id)
offset = len(old_descriptors)
cluster = VFS.instance().database.execute(
'SELECT row_number() OVER(ORDER BY gops.id) + ? - 1, physical_id, gops.id, descriptors '
'FROM gops '
'INNER JOIN physical_videos '
' ON gops.physical_id = physical_videos.id '
'WHERE cluster_id = ? AND (examined <= ? or gops.id = ?) AND codec = ? AND fps = ?'
'ORDER BY physical_id, gops.id',
(offset, cluster_id, epoch, gop_id, codec, fps)).fetchall()
new_physical_map = {
pid: [rowid for rowid, pid, gid, descriptor in rows]
for pid, rows in groupby(cluster, lambda c: c[1])
}
index_map = {rowid: gid for rowid, pid, gid, descriptor in cluster}
physical_map = {
key: old_physical_map.get(key, []) + new_physical_map.get(key, [])
for key in set(old_physical_map) | set(new_physical_map)
}
new_descriptors = [c[3].astype(np.float32) for c in cluster]
if new_descriptors:
matcher.add(new_descriptors)
matcher.train()
cls._matchers[cluster_id] = matcher, physical_map, index_map
return cls._matchers[cluster_id][0]
def add(cls, physical_video, filename, start_time, end_time, size, fps):
gop = Gop(None, physical_video.id, filename, start_time, end_time, None, False, False, None, None, None, size, None, fps, None, None, None)
gop.id = VFS.instance().database.execute(
'INSERT INTO gops(physical_id, filename, start_time, end_time, size, fps, original_size) '
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(gop.physical_id, gop.filename, gop.start_time, gop.end_time, gop.size, gop.fps, gop.mse, gop.estimated_mse, gop.parent_id, gop.size)).lastrowid
return gop
def duration(self):
if self._duration is None:
self._duration = VFS.instance().database.execute(
'SELECT MAX(end_time) '
'FROM physical_videos '
'INNER JOIN gops ON physical_videos.id = gops.physical_id '
'WHERE logical_id = ?', self.id).fetchone()[0]
return self._duration
def videos(self):
if self._videos is None:
self._videos = list(
map(
lambda args: PhysicalVideo(*args),
VFS.instance().database.execute(
'SELECT id, logical_id, height, width, codec FROM physical_videos WHERE logical_id = ?',
self.id).fetchall()))
return self._videos
def delete(cls, video):
from vfs.gop import Gop
logging.info('Deleting Physical Video %d', video.id)
gops = video.gops()
references = {filename: count for (filename, count) in
VFS.instance().database.execute(
'SELECT filename, COUNT(*) '
'FROM gops '
'WHERE filename IN (SELECT filename FROM gops WHERE physical_id = ?) '
'GROUP BY filename', video.id).fetchall()}
VFS.instance().database.execute('DELETE FROM gops WHERE physical_id = ?', video.id)
VFS.instance().database.execute('DELETE FROM physical_videos WHERE id = ?', video.id)
video.id = None
for gop in gops:
Gop.delete(gop, references=references.get(gop.filename, 0))
def gops(self):
from vfs.gop import Gop
if self._gops is None:
self._gops = list(map(lambda args: Gop(*args),
VFS.instance().database.execute(
'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, '
' histogram, descriptors, keypoints, size, zstandard, fps, mse, estimated_mse, parent_id, original_size '
'FROM gops WHERE physical_id = ? ORDER BY id', self.id).fetchall()))
return self._gops
def get_candidate(cls, epoch):
from vfs.gop import Gop
smallest_cluster = (VFS.instance().database.execute(
'SELECT cluster_id, COUNT(*) FROM gops candidate '
'WHERE examined <= ? AND joint = 0 AND '
' cluster_id > 0 '
# What did this do?
#' physical_id < (SELECT MAX(physical_id) FROM gops WHERE examined <= ? AND joint = 0 AND cluster_id = candidate.cluster_id) '
'GROUP BY cluster_id '
'ORDER BY COUNT(*) ASC '
'LIMIT 1', (epoch)).fetchone() or [None])[0] #(epoch, epoch)
if smallest_cluster is not None:
return Gop.get(*VFS.instance().database.execute(
'SELECT MIN(id) FROM gops '
'WHERE cluster_id = ? AND examined <= ? AND joint = 0',
(smallest_cluster, epoch)).fetchone())
else:
return None
def delete(cls, gop, references=None):
references = (references or VFS.instance().database.execute(
'SELECT COUNT(*) FROM gops WHERE filename = ?', gop.filename).fetchone()[0])
logging.info('Deleting GOP %d (underlying data has %d references)', gop.id, references)
if gop.id:
VFS.instance().database.execute('DELETE FROM gops WHERE id = ?', gop.id)
gop.id = None
if not gop.joint:
#logging.warning("Skipping deleting physical GOP file in debug mode")
os.remove(gop.filename)
#pass
elif references <= 1:
logging.info("Last joint compressed candidate deleted; removing physical file")
#logging.warning("Skipping deleting physical GOP file in debug mode")
os.remove(gop.filename.format(JointCompression.LEFT))
#os.remove(gop.filename.format(JointCompression.RIGHT))
#os.remove(gop.filename.format(JointCompression.OVERLAP))
else:
logging.info("Not removing physical file; other references exist")
def cache_reconstructions(logical, resolution, codec, times, fps,
cache_sequences):
for sequence in (sequence for sequence in cache_sequences if sequence):
physical = PhysicalVideo.add(logical, *resolution, codec)
new_gop_data = []
transitive_estimated_mses = VFS.instance().database.execute(
'WITH RECURSIVE'
' error(id, estimated_mse) AS '
' (SELECT id, estimated_mse FROM gops child WHERE id IN ({}) '.
format(','.join(str(gop.id)
for gop, _, _ in sequence)) + ' UNION ALL '
' SELECT id, estimated_mse FROM gops parent WHERE parent.id = id)'
'SELECT id, SUM(estimated_mse) FROM error GROUP BY id').fetchall()
for (_, current_estimated_mse), (index, (gop, filename,
new_mse)) in zip(
transitive_estimated_mses,
enumerate(sequence)):
new_filename = path.join(
VFS.instance().path,
PhysicalVideo._gop_filename_template(logical, physical, index))
new_gop_data.append(
(new_filename, max(gop.start_time, times[0]),
min(gop.end_time, times[1]), os.path.getsize(filename), fps,
None, 2 * (current_estimated_mse + (new_mse or 0)), gop.id))
#TODO remove guard
#duplicates = VFS.instance().database.execute("SELECT * FROM gops, physical_videos WHERE start_time = ? AND end_time = ? and height = ? and width = ? and codec = ? and gops.physical_id = physical_videos.id",
# (new_gop_data[-1][0], new_gop_data[-1][1], resolution[0], resolution[1], codec)).fetchall()
#if len(duplicates) > 0:
# print("Duplicate detected???")
os.rename(filename, new_filename)
Gop.addmany(physical, new_gop_data)
logging.info('Cached physical video %s-%d', logical.name, physical.id)
return new_gop_data
def cluster_all(cls):
data = VFS.instance().database.execute(
'SELECT id, histogram FROM gops WHERE NOT histogram IS NULL AND examined != 9999999'
).fetchall()
if not data or len(data) == 1:
return 0
ids, histograms = zip(*data)
id_array = np.array(ids)
scaled_histograms = maxabs_scale(np.vstack(histograms))
clusters = Birch(n_clusters=len(histograms)).fit(scaled_histograms)
cluster_count = 0
for cluster_id in range(max(clusters.labels_) + 1):
rows = id_array[clusters.labels_ == cluster_id]
if len(rows) > 1:
cluster_count += 1
VFS.instance().database.executebatch(
'UPDATE gops SET cluster_id = {} where id = {}'.format(
int(cluster_id) + 1, int(row_id)) for row_id in rows)
return cluster_count
def solve_exact(logical, resolution, roi, t, fps, codec):
#if roi is not None:
# return None
gop_id = VFS.instance().database.execute(
"""
SELECT gops.id FROM gops, physical_videos
WHERE height = ? AND width = ? AND
start_time <= ? AND end_time >= ? AND
codec = ? AND
gops.physical_id = physical_videos.id AND
logical_id = ?""", (resolution[0], resolution[1], t[0], t[1],
codec, logical.id)).fetchone()
return [Gop.get(gop_id[0])] if gop_id else None
def co_decompress(cls, gop, filename):
logging.info('Joint decompress %s-%d-%d',
gop.video().logical().name, gop.video().id, gop.id)
assert(gop.joint)
left_filename = gop.filename.format(cls.LEFT)
right_filename = gop.filename.format(cls.RIGHT)
overlap_filename = gop.filename.format(cls.OVERLAP)
codec = gop.video().codec
H, shapes, is_left = VFS.instance().database.execute(
'SELECT homography, shapes, is_left FROM gops WHERE id = ?', gop.id).fetchone()
left_shape, overlap_shape, right_shape = shapes[0], shapes[1], shapes[2]
frame = np.zeros(gop.video().shape(), dtype=np.uint8)
if H.shape[1] > 1:
# Current code assumes H=3x3 matrix, but actually list of hstack[frameid, 3x3.flatten] matrices
raise NotImplementedError("Need to add support for reprojection")
else:
H = H[1:, 0].reshape((3, 3))
with VideoReader(left_filename, left_shape, codec) if is_left else NullReader() as leftreader, \
VideoReader(overlap_filename, overlap_shape, codec) as overlapreader, \
VideoReader(right_filename, right_shape, codec) if not is_left else NullReader() as rightreader, \
VideoWriter(filename, gop.video().shape(), codec) as writer:
while not leftreader.eof or not overlapreader.eof or not rightreader.eof:
if is_left:
left, overlap = leftreader.read(), overlapreader.read()
if overlap is not None:
top = overlap.shape[0]//2 - frame.shape[0]//2
bottom = top + frame.shape[0]
np.copyto(frame[:, left.shape[1]:], overlap[top:bottom])
if left is not None:
np.copyto(frame[:, :left.shape[1]], left)
else:
overlap, right = overlapreader.read(), rightreader.read()
if overlap is not None:
cv2.warpPerspective(overlap, H, dsize=tuple(reversed(frame.shape[:2])), dst=frame)
if right is not None:
np.copyto(frame[:, -right.shape[1]:], right)
writer.write(frame)
pass
def get_all(cls):
return map(
lambda args: LogicalVideo(*args),
VFS.instance().database.execute(
'SELECT id, name FROM logical_videos').fetchall())
def get_all(cls, ids):
return (Gop(*args) for args in VFS.instance().database.execute(
'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, '
' histogram, keypoints, descriptors, size, zstandard, fps, mse, estimated_mse, parent_id, original_size '
'FROM gops WHERE id IN ({})'.format(','.join(map(str, ids)))).fetchall())
def get(cls, id):
return Gop(*VFS.instance().database.execute(
'SELECT id, physical_id, filename, start_time, end_time, cluster_id, joint, examined, '
' histogram, keypoints, descriptors, size, zstandard, fps, mse, estimated_mse, parent_id, original_size '
'FROM gops WHERE id = ?', id).fetchone())
def _prepare(logical, resolution, t):
"""
physical_videos = {physical.id:
{'id': physical.id,
'start': physical.start_time(),
'end': physical.end_time(),
'format': physical.codec,
'clock': 0,
'size': physical.size(),
'time': None,
'resolution': physical.resolution(),
'frames': sum(gop.fps for gop in physical.gops()) }
for physical in logical.videos()
if t is None or
(physical.start_time() is not None and physical.end_time() is not None and
physical.start_time() < t[1] and physical.end_time() >= t[0]) and
#(physical.start_time() or 0) <= t[0] and
#(physical.end_time() or t[1]) >= t[1]) and
#physical.start_time() is not None and
physical.resolution()[0] >= resolution[0] and
physical.resolution()[1] >= resolution[1]}
"""
"""
fragments = [{'id': gop.id,
'source': gop.physical_id,
'start': gop.start_time,
'end': gop.end_time,
'fps': gop.fps}
for physical in logical.videos()
for gop in physical.gops()
if physical.id in physical_videos and
(t is None or
(gop.start_time <= t[0] and gop.end_time >= t[1]) or
(gop.start_time <= t[0] < gop.end_time) or
(gop.start_time < t[1] <= gop.end_time) or
(gop.start_time >= t[0] and gop.end_time <= t[1]))]
"""
physical_videos = {
physical_id: {
'id': physical_id,
'start': start_time,
'end': end_time,
'format': codec,
'clock': 0,
'size': size,
'time': None,
'resolution': (height, width),
'frames': (end_time - start_time) * fps
}
for (physical_id, start_time, end_time, codec, height, width, size,
fps) in
VFS.instance().database.execute(
'SELECT physical_videos.id, MIN(start_time), MAX(end_time), codec, height, width, SUM(size), MAX(fps) '
'FROM gops, physical_videos '
'WHERE logical_id = ? AND '
' height >= ? AND width >= ? AND '
' start_time < ? AND end_time >= ? AND '
' gops.physical_id = physical_videos.id '
'GROUP BY physical_videos.id', (
logical.id, resolution[0], resolution[1],
t[1] if t is not None else 9999999,
t[0] if t is not None else -1)).fetchall()
#for physical in logical.videos()
if t is None or (start_time is not None and end_time is not None
and start_time < t[1] and end_time >= t[0]) and
#(physical.start_time() or 0) <= t[0] and
#(physical.end_time() or t[1]) >= t[1]) and
#physical.start_time() is not None and
height >= resolution[0] and width >= resolution[1]
}
fragments = [{
'id': gop_id,
'source': physical_id,
'start': start_time,
'end': end_time,
'fps': fps
} for (gop_id, physical_id, start_time, end_time,
fps) in VFS.instance().database.execute(
'SELECT gops.id, physical_id, start_time, end_time, fps '
'FROM gops, physical_videos '
'WHERE logical_id = ? AND '
' physical_id IN ({}) AND '
' gops.physical_id = physical_videos.id AND '
' (? = -1 OR ('
' (start_time <= ? AND end_time >= ?) OR '
' (start_time <= ? AND ? < end_time) OR '
' (start_time < ? AND ? <= end_time) OR '
' (start_time >= ? AND end_time <= ?)))'.format(
','.join(map(str, physical_videos.keys()))),
(logical.id, -1 if t is None else 0,
t[0] if t is not None else -1, t[1] if t is not None else -1,
t[0] if t is not None else -1, t[0] if t is not None else -1,
t[1] if t is not None else -1, t[1] if t is not None else -1,
t[0] if t is not None else -1,
t[1] if t is not None else -1)).fetchall()]
return physical_videos.values(), fragments
def budget(self):
if self._budget is None:
self._budget = VFS.instance().database.execute(
'SELECT budget FROM logical_videos WHERE id = ?',
self.id).fetchone()[0]
return self._budget
def co_compress(cls, gop1, gop2, matches, abort_psnr_threshold=25, dryrun=False, gops_reversed=False):
#cls.output.write('%s,%d,%d,%s,%d,%d\n' % (gop1.video().logical().name, gop1.video().id, gop1.id,
# gop2.video().logical().name, gop2.video().id, gop2.id))
#cls.output.flush()
logging.info('Joint compress %s-%d-%d and %s-%d-%d (%d matches, %s, %s)',
gop1.video().logical().name, gop1.video().id, gop1.id,
gop2.video().logical().name, gop2.video().id, gop2.id,
len(matches),
gop1.filename, gop2.filename)
assert(gop1.id != gop2.id)
assert(gop1.video().codec == gop2.video().codec)
assert(not gop1.joint)
assert(not gop2.joint)
H, Hi, left, right, overlap, overlap_subframe, recovered_frame2, frame2_overlap_yoffset, inverted_homography, has_homography = cls.estimate_homography(gop1, gop2, matches=None, fast=False) #matches) #TODO fast, matches
Hs = np.hstack([[0], H.flatten()])
#H, Hi = homography.project(gop1.keypoints, gop2.keypoints, matches)
#frame2_overlap_yoffset = -int(round(np.dot([0, gop1.video().height, 1], Hi)[0]))
#frame1_left_width = roundeven(Hi.dot([0,0,1])[0])
#frame2_right_width = roundeven(gop2.video().width - H.dot([gop1.video().width, 0, 1])[0] / H.dot([gop1.video().width, 0, 1])[2])
#overlap_height = gop2.video().height + 2 * frame2_overlap_yoffset
#overlap_width = gop1.video().width - frame1_left_width
if has_homography and inverted_homography and not gops_reversed:
return cls.co_compress(gop2, gop1, matches, gops_reversed=True)
# Are videos identical?
elif not inverted_homography and left.shape[1] == 0 and right.shape[1] == 0:
#if frame1_left_width == 0 and frame2_right_width == 0:
return cls.deduplicate(gop1, gop2)
# Are left/right frames too small to encode?
elif 0 <= left.shape[1] < 32 or 0 <= right.shape[1] < 32 or 0 <= overlap.shape[1] < 32:
logging.info('Joint compression aborted; left/right/overlap frames too small (%d, %d)', gop1.id, gop2.id)
VFS.instance().database.execute(
'UPDATE gops SET examined=9999998 ' # 9999998 = possibly examine again?
'WHERE id in (?, ?)',
(gop1.id, gop2.id)).close()
return 0
# return cls.deduplicate(gop1, gop2)
else:
#pretransform_points = np.float32([
# [frame1_left_width, 0],
# [gop1.video().width, 0],
# [frame1_left_width, gop1.video().height],
# [gop1.video().width, gop1.video().height]])
#posttransform_points = np.float32(
# [[0, 0 + frame2_overlap_yoffset],
# [gop1.video().width - frame1_left_width + 1, 0 + frame2_overlap_yoffset],
# [0, gop1.video().height + frame2_overlap_yoffset],
# [gop1.video().width - frame1_left_width + 1, gop1.video().height + frame2_overlap_yoffset]])
#transform = cv2.getPerspectiveTransform(pretransform_points, posttransform_points)
#inverse_transform = cv2.getPerspectiveTransform(posttransform_points, pretransform_points)
##Hi = transform.dot(Hi)
#Ho = H
#H, Hi = H.dot(inverse_transform), transform.dot(Hi)
#left = cls._create_image(gop1.video(), width=frame1_left_width)
#right = cls._create_image(gop2.video(), width=frame2_right_width)
#overlap = cls._create_image(height=overlap_height, width=overlap_width)
#recovered_frame2 = np.empty(gop2.video().shape(), dtype=np.uint8)
filenametemplate = '{}-{{}}{}'.format(*path.splitext(gop1.filename))
abort = False
frame_index = 0
total_left_psnr, total_right_psnr = 0, 0
codec = gop1.video().codec
# if frame1_left_width else NullWriter() as leftwriter, \
#if frame2_right_width else NullWriter() as rightwriter, \
with log_runtime('Joint compression:'):
with VideoReader(gop1.filename, gop1.video().shape(), codec) as reader1, \
VideoReader(gop2.filename, gop2.video().shape(), codec) as reader2, \
VideoWriter(filenametemplate.format(cls.LEFT), left.shape, codec) \
if left.shape[1] else NullWriter() as leftwriter, \
VideoWriter(filenametemplate.format(cls.RIGHT), right.shape, codec) \
if right.shape[1] else NullWriter() as rightwriter, \
VideoWriter(filenametemplate.format(cls.OVERLAP), overlap.shape, codec) as overlapwriter:
while (not reader1.eof or not reader2.eof) and not abort:
attempts = 0
frame_index += 1
frame1, frame2 = reader1.read(), reader2.read()
while attempts < 2:
attempts += 1
if frame1 is not None and frame2 is not None:
pass
elif frame1 is not None and frame2 is None:
frame2 = np.zeros(gop2.video().shape(), dtype=np.uint8)
elif frame2 is not None and frame1 is None:
frame1 = np.zeros(gop1.video().shape(), dtype=np.uint8)
if frame1 is not None or frame2 is not None:
# Create and write overlap
cv2.warpPerspective(frame2, Hi, dsize=tuple(reversed(overlap.shape[:2])), dst=overlap)
# Left join
"""cv2.imwrite('frame1.png', frame1)
cv2.imwrite('frame2.png', frame2)
cv2.imwrite('overlap.png', overlap)
print(gop1.filename)
print(gop2.filename)
print(gop1.video().shape())
print(left.shape)
print(right.shape)
print(overlap.shape)
print(frame2_overlap_yoffset, frame2_overlap_yoffset + gop1.video().height)
print(left.shape[1])
print(Hi)"""
#tmp = frame2_overlap_yoffset
#if frame2_overlap_yoffset < 0:
# frame2_overlap_yoffset = 0
# Mean join
#cls.mean_join(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], overlap_subframe, frame1[:, left.shape[1]:])
#np.copyto(overlap_subframe, overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height])
#overlap_subframe = (overlap_subframe + frame1[:, left.shape[1]:]) // 2
#np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], overlap_subframe)
#mean_subframe = np.copy(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]).astype(np.uint16) // 2
##mean_subframe = np.copy(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]).astype(np.uint16)
#mean_subframe = (mean_subframe + frame1[:, left.shape[1]:]) // 2
##mean_subframe //= 2
#np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height],
# mean_subframe.astype(np.uint8))
# Left join
cls.left_join(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], frame1[:, left.shape[1]:])
#np.copyto(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height],
# frame1[:, left.shape[1]:])
#frame2_overlap_yoffset = tmp
# Mean join (has a bug in non-overlapping left)
#overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height] //= 2
#overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height] += frame1[:, left.shape[1]:] // 2
if left.shape[1] != 0:
np.copyto(left, frame1[:, :left.shape[1]])
if right.shape[1] != 0:
np.copyto(right, frame2[:, -right.shape[1]:])
right_psnr = cls.recovered_right_psnr(H, overlap, frame2, recovered_frame2, right)
left_psnr = cls.recovered_left_psnr(overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height], frame1, left)
#cv2.warpPerspective(overlap, H, dsize=tuple(reversed(recovered_frame2.shape[:2])), dst=recovered_frame2)
#np.copyto(recovered_frame2[:, -right.shape[1]:], frame2[:, -right.shape[1]:])
#psnr = vfs.utilities.psnr(frame2, recovered_frame2)
if right_psnr < abort_psnr_threshold:
if attempts == 2:
abort = True
break
else:
logging.debug(f"Recomputing homography ({gop1.id} <-> {gop2.id}) PSNR {right_psnr:0.1f}")
#TODO save new homography
H, Hi, left, right, overlap, overlap_subframe, recovered_frame2, frame2_overlap_yoffset, inverted_homography, has_homography = cls.estimate_homography(
gop1, gop2, frame1=frame1, frame2=frame2, matches=None, fast=True) #matches)
np.vstack([Hs, np.hstack([[frame_index], H.flatten()])])
#Hs.append(np.hstack([[frame_index], H.flatten()]))
else:
total_right_psnr += right_psnr
total_left_psnr += left_psnr
attempts = 999
#break
leftwriter.write(left)
rightwriter.write(right)
overlapwriter.write(overlap)
if abort:
#cv2.imwrite('abortframe1_%d.png' % gop1.id, frame1)
#cv2.imwrite('abortframe2_%d.png' % gop1.id, frame2)
#cv2.imwrite('abortleft_%d.png' % gop1.id, left)
#cv2.imwrite('abortright_%d.png' % gop1.id, right)
cv2.imwrite('abortoverlap.png', overlap)
cv2.imwrite('abortrecovered1.png', np.hstack([left, overlap[frame2_overlap_yoffset:frame2_overlap_yoffset + gop1.video().height]]))
cv2.imwrite('abortrecovered2.png', recovered_frame2)
logging.info('Joint compression aborted; quality threshold violated %d < %d (%d vs %d)',
right_psnr, abort_psnr_threshold, gop1.id, gop2.id)
#ssim = compare_ssim(frame2, recovered_frame2, multichannel=True)
os.remove(filenametemplate.format(cls.LEFT))
os.remove(filenametemplate.format(cls.OVERLAP))
os.remove(filenametemplate.format(cls.RIGHT))
VFS.instance().database.executebatch([
f'INSERT INTO gop_joint_aborted(gop1, gop2) VALUES ({gop1.id}, {gop2.id})',
f'INSERT INTO gop_joint_aborted(gop1, gop2) VALUES ({gop2.id}, {gop1.id})',
f'UPDATE gops SET examined=examined + 1 WHERE id in ({gop1.id}, {gop2.id})'])
return 0
elif not dryrun:
original_size = path.getsize(gop1.filename) + path.getsize(gop2.filename)
VFS.instance().database.execute(
'UPDATE gops SET examined=9999999, joint=1, original_filename=filename, filename=?, homography=?, shapes=?, is_left=(id=?) '
'WHERE id in (?, ?)',
(filenametemplate, np.vstack(Hs), np.vstack([left.shape, overlap.shape, right.shape]),
gop1.id, gop1.id, gop2.id)).close()
os.remove(gop1.filename)
os.remove(gop2.filename)
bytes_saved = (original_size -
(path.getsize(filenametemplate.format(cls.LEFT)) +
path.getsize(filenametemplate.format(cls.OVERLAP)) +
path.getsize(filenametemplate.format(cls.RIGHT))))
logging.info('Joint compression saved %dKB (%d%%), %d frames, PSNR left=%d, right=%d', bytes_saved // 1000, (bytes_saved * 100) // original_size, frame_index-1, total_left_psnr // (frame_index-1), total_right_psnr // (frame_index-1))
with open('joint.csv', 'a') as f:
f.write(f'{gop1.id},{gop2.id},{frame_index-1},{total_right_psnr // (frame_index-1)},{total_left_psnr // (frame_index-1)}\n')
return bytes_saved
else:
return 0
def exists_by_name(cls, name):
return VFS.instance().database.execute(
'SELECT 1 FROM logical_videos WHERE name = ? LIMIT 1',
name).fetchone() is not None
def add(cls, name):
return LogicalVideo(
VFS.instance().database.execute(
'INSERT INTO logical_videos(name) VALUES (?)', name).lastrowid,
name)
def get(cls, id):
return LogicalVideo(*VFS.instance().database.execute(
'SELECT id, name FROM logical_videos WHERE id = ?', id).fetchone())
def closest_match(cls,
epoch,
gop,
matches_required=DEFAULT_MATCHES_REQUIRED,
radius=400): #400):
from vfs.gop import Gop
with log_runtime("Joint compression candidate selection"):
cluster_gops = VFS.instance().database.execute(
"SELECT id, filename, descriptors FROM gops WHERE cluster_id = ? AND physical_id != ? AND descriptors IS NOT NULL AND joint = 0 AND examined <= ? AND NOT EXISTS (SELECT id FROM gop_joint_aborted WHERE gop1 = ? AND gop2 = id)",
(gop.cluster_id, gop.physical_id, epoch, gop.id)).fetchall()
candidate_gops = []
for gop2_id, gop2_filename, gop2_descriptors in cluster_gops:
success, matches = cls.adhoc_match(gop.descriptors,
gop2_descriptors)
if success and len(matches) > matches_required:
candidate_gops.append(
(gop.filename.split('-')[-1] == gop2_filename.split(
'-')[-1], len(matches), gop2_id, matches))
if candidate_gops[-1][1] > 400 or candidate_gops[-1][
0]: # Break on "good enough" match to try out
break
candidate_gop = sorted(candidate_gops,
reverse=True)[0] if candidate_gops else None
return [(candidate_gop[2], candidate_gop[3])]
matcher = cls._get_matcher(epoch, gop)
physical_map = cls._get_physical_map(gop.cluster_id).get(
gop.physical_id, None)
index_map = cls._get_index_map(gop.cluster_id)
all_matches = matcher.radiusMatch(queryDescriptors=gop.descriptors,
maxDistance=radius)
good_matches = defaultdict(lambda: [])
first_matches = {}
complete = set()
# For each frame/descriptor pair, find matches that pass the Lowe threshold test
#all_matches = all_matches[:5000]
filtered_matches = (
m for d in all_matches for m in d
if index_map[m.imgIdx] > gop.id and m.imgIdx not in physical_map
and not (m.imgIdx, m.queryIdx) in complete)
#for descriptor_matches in all_matches:
# for match in descriptor_matches:
with log_runtime("Lowes test"):
for match in filtered_matches:
#if match.imgIdx not in physical_map and \
# index_map[match.imgIdx] > gop.id and \
#if not (match.imgIdx, match.queryIdx) in complete:
# First match
if (match.imgIdx, match.queryIdx) not in first_matches:
first_matches[match.imgIdx, match.queryIdx] = match
# Second match
else:
if first_matches[
match.imgIdx, match.
queryIdx].distance < cls.LOWE_THRESHOLD * match.distance:
good_matches[match.imgIdx].append(
first_matches[match.imgIdx, match.queryIdx])
del first_matches[match.imgIdx, match.queryIdx]
complete.add((match.imgIdx, match.queryIdx))
# Some matches may not have a second match to apply Lowe's threshold on.
# Check to see if we should have seen it and count it if so.
for first_match in first_matches.values():
if first_match.distance / cls.LOWE_THRESHOLD < radius:
good_matches[first_match.imgIdx].append(first_match)
ignore_ids = set(VFS.instance().database.execute(
'SELECT gop2 FROM gop_joint_aborted WHERE gop1 = ?',
gop.id).fetchall())
best_indexes = [
index for index, matches in good_matches.items()
if len(matches) >= matches_required
and index_map[index] not in ignore_ids
]
#best_ids = [index_map[index] for index in best_indexes if index_map[index] not in ignore_ids]
#best_id = VFS.instance().database.execute(
# 'SELECT MIN(id) FROM gops WHERE joint=0 AND id in ({})'.format(','.join(map(str, best_ids)))).fetchone()[0]
#best_index = max((index for index, matches in good_matches.items() if len(matches) >= matches_required),
# default=None)
#best = sorted([(index_map[index], good_matches[index]) for index in best_indexes], key=lambda pair: len(pair[1]), reverse=True)
gops = Gop.get_all(index_map[index] for index in best_indexes)
best = [(gop, good_matches[index])
for gop, index in zip(gops, best_indexes)]
best = sorted(best,
key=lambda pair: (pair[0].filename.split('-')[-1] == gop.
filename.split('-')[-1], len(pair[1])),
reverse=True)
best = best[:len(best) // 20 + 1] # Keep top 5%
best = [(mgop.id, matches) for mgop, matches in best]
#best = sorted([(mgop.id, cv2.compareHist(gop.histogram, mgop.histogram, cv2.HISTCMP_CHISQR), gop.filename, mgop.filename, matches) for (mgop, matches) in best], key=lambda pair: (len(pair[2]), cv2.compareHist(gop.histogram, pair[0].histogram, cv2.HISTCMP_CHISQR), -pair[0].id), reverse=True)
#best = sorted([(id, cv2.compareHist(gop.histogram, Gop.get(id).histogram, cv2.HISTCMP_CHISQR), gop.filename, Gop.get(id).filename, matches) for (id, matches) in best], key=lambda pair: (len(pair[2]), cv2.compareHist(gop.histogram, Gop.get(pair[0]).histogram, cv2.HISTCMP_CHISQR), -pair[0]), reverse=True)
return best
if best_id is not None:
return Gop.get(best_id), good_matches[best_indexes[best_ids.index(
best_id)]] #best_index]
#if best_index is not None:
# return Gop.get(index_map[best_index]), good_matches[best_index]
#return Gop.get(VFS.instance().database.execute(
# '''SELECT id FROM gops
# WHERE cluster_id = ? AND joint = 0
# LIMIT 1
# OFFSET ?''', (gop.cluster_id, best_index)).fetchone()[0]), good_matches[best_index]
else:
return None, None
def get_by_name(cls, name):
return LogicalVideo(*VFS.instance().database.execute(
'SELECT id, name FROM logical_videos WHERE name = ?',
name).fetchone())
