def generate_summary(seg_score, cps, n_frames, nfps, proportion=0.15):
""" Generate keyshot-based video summary i.e. a binary vector
seg_score: predicted segment importance scores.
cps: change points, 2D matrix, each row contains a segment.
n_frames: original number of frames.
nfps: number of frames per segment.
proportion: length of video summary (compared to original video length).
"""
n_segs = cps.shape[0]
seg_score = list(seg_score)
limits = int(math.floor(n_frames * proportion))
picks = knapsack_dp(seg_score, nfps, n_segs, limits)
summary = np.zeros((1), dtype=np.float32) # this element should be deleted
for seg_idx in xrange(n_segs):
nf = nfps[seg_idx]
if seg_idx in picks:
tmp = np.ones((nf), dtype=np.float32)
else:
tmp = np.zeros((nf), dtype=np.float32)
summary = np.concatenate((summary, tmp))
summary = np.delete(summary, 0) # delete the first element
return summary
def generate_summary(ypred, cps, n_frames, nfps, positions, proportion=0.15, method='knapsack'):
"""Generate keyshot-based video summary i.e. a binary vector.
Args:
---------------------------------------------
- ypred: predicted importance scores.
- cps: change points, 2D matrix, each row contains a segment.
- n_frames: original number of frames.
- nfps: number of frames per segment.
- positions: positions of subsampled frames in the original video.
- proportion: length of video summary (compared to original video length).
- method: defines how shots are selected, ['knapsack', 'rank'].
"""
n_segs = cps.shape[0]
frame_scores = np.zeros((n_frames), dtype=np.float32)
if positions.dtype != int:
positions = positions.astype(np.int32)
if positions[-1] != n_frames:
positions = np.concatenate([positions, [n_frames]])
for i in range(len(positions) - 1):
pos_left, pos_right = positions[i], positions[i+1]
if i == len(ypred):
frame_scores[pos_left:pos_right] = 0
else:
frame_scores[pos_left:pos_right] = ypred[i]
seg_score = []
for seg_idx in range(n_segs):
start, end = int(cps[seg_idx,0]), int(cps[seg_idx,1]+1)
scores = frame_scores[start:end]
seg_score.append(float(scores.mean()))
limits = int(math.floor(n_frames * proportion))
if method == 'knapsack':
picks = knapsack_dp(seg_score, nfps, n_segs, limits)
elif method == 'rank':
order = np.argsort(seg_score)[::-1].tolist()
picks = []
total_len = 0
for i in order:
if total_len + nfps[i] < limits:
picks.append(i)
total_len += nfps[i]
else:
raise KeyError("Unknown method {}".format(method))
summary = np.zeros((1), dtype=np.float32) # this element should be deleted
for seg_idx in range(n_segs):
nf = nfps[seg_idx]
if seg_idx in picks:
tmp = np.ones((nf), dtype=np.float32)
else:
tmp = np.zeros((nf), dtype=np.float32)
summary = np.concatenate((summary, tmp))
summary = np.delete(summary, 0) # delete the first element
return summary
def generate_summary_ky(ypred,
cps,
n_frames,
nfps,
positions,
proportion=0.15):
""" Generate keyshot-based video summary i.e. a binary vector
ypred: predicted importance scores.
cps: change points, 2D matrix, each row contains a segment.
n_frames: original number of frames.
nfps: number of frames per segment.
positions: positions of subsampled frames in the original video.
proportion: length of video summary (compared to original video length).
"""
n_segs = cps.shape[0]
frame_scores = np.zeros((n_frames), dtype=np.float32)
if positions.dtype != int:
positions = positions.astype(np.int32)
if positions[-1] != n_frames:
positions = np.concatenate([positions, [n_frames]])
for i in xrange(len(positions) - 1):
pos_left, pos_right = positions[i], positions[i + 1]
frame_scores[pos_left:pos_right] = ypred[i]
seg_score = []
for seg_idx in xrange(n_segs):
start, end = int(cps[seg_idx, 0]), int(cps[seg_idx, 1] + 1)
scores = frame_scores[start:end]
seg_score.append(float(scores.mean()))
limits = int(math.floor(n_frames * proportion))
picks = knapsack_dp(seg_score, nfps, n_segs, limits)
summary = np.zeros((1), dtype=np.float32) # this element should be deleted
for seg_idx in xrange(n_segs):
nf = nfps[seg_idx]
if seg_idx in picks:
tmp = np.ones((nf), dtype=np.float32)
else:
tmp = np.zeros((nf), dtype=np.float32)
summary = np.concatenate((summary, tmp))
summary = np.delete(summary, 0) # delete the first element
return summary
def generate_summary(ypred, video_info, positions, proportion=0.15, method='knapsack'):
"""
Generate keyshot-based video summary. i.e. a binary vector
Args:
ypred: predicted importance scores.
cps: change points, 2D matrix, each row contains a segment.
n_frames: original number of frames.
nfps: number of frames per segment.
positions: positions of subsampled frames in the original video.
proportion: length of video summary (compared to original video length).
method: defines how shots are selected, ['knapsack', 'rank'].
"""
if 'n_frames' in video_info.keys():
n_frames = video_info['n_frames'][()]
else:
n_frames = video_info['length'][()]
nfps_seg = video_info['n_frame_per_seg'][...].tolist()
# positions = video_info['picks'][...]
cps = video_info['change_points'][...]
n_segs = cps.shape[0]
# ypred = ypred.data.cpu().numpy()
if ypred.shape[0] > n_frames-2: # for FCSN_up network, no need for upsampling
frame_scores = ypred
else:
frame_scores = upsample_scores(ypred, n_frames, positions)
# frame_scores = upsample_scores_interp(ypred, n_frames, positions)
# Segment Score
seg_score = []
for seg_idx in range(n_segs):
pos_start, pos_end = int(cps[seg_idx, 0]), int(cps[seg_idx, 1]+1)
scores = frame_scores[pos_start: pos_end]
seg_score.append(float(scores.mean()))
limits = int(math.floor(n_frames * proportion))
if method == 'knapsack':
picks = knapsack_dp(seg_score, nfps_seg, n_segs, limits)
elif method == 'rank':
order = np.argsort(seg_score)[::-1].tolist()
picks = []
total_len = 0
for idx in order:
if total_len + nfps_seg[idx] < limits:
picks.append(idx)
total_len += nfps_seg[idx]
else:
raise KeyError("Unknown method {}".format(method))
summary = np.zeros((1), dtype=np.float32) # this element should be deleted
for seg_idx in range(n_segs):
nf = nfps_seg[seg_idx] + 1 # =1 added by tianrui to make sure summary has same length of the n_frames
if seg_idx in picks:
tmp = np.ones((nf), dtype=np.float32)
else:
tmp = np.zeros((nf), dtype=np.float32)
summary = np.concatenate((summary, tmp))
summary = np.delete(summary, 0) # delete the first element
return summary, frame_scores, picks