def test_cps(model, evals, model_type, scaler, features_used):
for i in range(int(len(files)*0.8), len(files)):
current_file_x = np.load(files[i])
fileName = files[i].split('/')[-1].split('.')[0]
current_fps = int(np.round(sio.loadmat(mat_files+fileName+'.mat').get('FPS')[0][0]))
normalised_features = scaler.transform(current_file_x)
K = np.dot(normalised_features, normalised_features.T)
m = get_cps_number(current_file_x.shape[0], current_fps) # avergae 5sec cps
cps, scores = cpd_auto(K, m, 1)
# cps = np.arange(0,current_file_x.shape[0], current_fps*5, dtype=int)
X, _ = create_cuboids(normalised_features, np.zeros([normalised_features.shape[0]]), window_size)
preds = model.predict(X)
# np.save('../../saved_numpy_arrays/predictions/I3D/MLP/'+fileName+'_'+features_used+'.npy', preds)
if model_type =='categorical':
preds = np.argmax(preds, axis=1)
print(fileName)
targets, ks_targets = create_key_shots(preds, cps)
np.save('../../saved_numpy_arrays/predictions/I3D/MLP/'+fileName+'_greedy_'+features_used+'.npy', targets)
np.save('../../saved_numpy_arrays/predictions/I3D/MLP/'+fileName+'_ks_'+features_used+'.npy', ks_targets)
evals.evaluateSumMe(targets, fileName)
evals.evaluateSumMe(ks_targets, fileName)
def vid2shots(imSeq, maxShots=5, vmax=0.6, colBins=40):
"""
Convert a given video into number of shots
imSeq: (n,h,w,c): 0-255: np.uint8: RGB
shotIdx: (k,): start Index of shot: 0-indexed
shotScore: (k,): First change ../lib/kts/cpd_auto.py return value to
scores2 instead of costs (a bug)
"""
X = np.zeros((imSeq.shape[0], compute_features(imSeq[0], colBins).size))
print('Feature Matrix shape:', X.shape)
for i in range(imSeq.shape[0]):
X[i] = compute_features(imSeq[i], colBins)
K = np.dot(X, X.T)
shotIdx, _ = cpd_auto(K, maxShots - 1, vmax)
shotIdx = np.concatenate(([0], shotIdx))
return shotIdx
def test():
for i in range(int(len(features) * 0.8), len(features)):
current_file_x = np.load(features[i])
fileName = features[i].split('/')[-1].split('.')[0]
normalised_features = np.array([power_norm(e) for e in current_file_x])
preds = model.predict(normalised_features)
K = np.dot(current_file_x, current_file_x.T)
cps, scores = cpd_auto(K, 2 * m, 1)
for i in range(len(cps) - 1):
if any(c > 0.3 for c in preds[cps[i]:cps[i + 1]]):
preds[cps[i]:cps[i + 1]] = max(preds[cps[i]:cps[i + 1]])
print(fileName)
evals.evaluateSumMe(preds, fileName)
return preds
def test_cps(model):
for i in range(int(len(features) * 0.8), len(features)):
current_file_x = np.load(features[i])
fileName = features[i].split('/')[-1].split('.')[0]
current_fps = sio.loadmat(mat_files_path + fileName +
'.mat').get('FPS')[0][0]
normalised_features = np.array([power_norm(e) for e in current_file_x])
K = np.dot(normalised_features, normalised_features.T)
m = get_cps_number(current_file_x.shape[0],
current_fps) # avergae 5sec cps
cps, scores = cpd_auto(K, m, 1)
preds = model.predict(normalised_features)
print(fileName)
targets = create_key_shots(preds, cps)
evals.evaluateSumMe(targets, fileName)
def test_cps(model, model_type, scaler, features_used, test_files):
all_cps = []
all_fileNames = []
all_ks_targets = []
for i in range(len(test_files)):
current_file_x = np.load(test_files[i])
fileName = test_files[i].split('/')[-1].split('.')[0]
current_fps = getFPS_tvSum50(fileName)
normalised_features = scaler.transform(current_file_x)
K = np.dot(normalised_features, normalised_features.T)
m = get_cps_number(current_file_x.shape[0],
current_fps) # avergae 5sec cp
cps, scores = cpd_auto(K, m, 1)
# cps = np.arange(0,current_file_x.shape[0], current_fps*5, dtype=int)
X, _ = create_cuboids(normalised_features,
np.zeros([normalised_features.shape[0]]),
window_size)
preds = model.predict(X)
# np.save('../../saved_numpy_arrays/predictions/I3D/MLP/'+fileName+'_'+features_used+'.npy', preds)
if model_type == 'categorical':
preds = np.argmax(preds, axis=1)
print(fileName)
targets, ks_targets = create_key_shots(preds, cps)
all_fileNames.append(fileName)
all_cps.append(cps)
all_ks_targets.append(ks_targets)
sio.savemat(
'../../saved_numpy_arrays/TvSum50/predictions/I3D/MLP/' +
fileName + '_' + features_used + '.mat', dict(x=preds))
sio.savemat(
'../../saved_numpy_arrays/TvSum50/predictions/I3D/MLP/AllData_' +
features_used + '_.mat',
dict(fileName=all_fileNames,
shot_boundaries=all_cps,
targets=all_ks_targets))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--dataset', type=str, required=True)
args = parser.parse_args()
h5in = h5py.File(args.dataset, 'r')
h5out = h5py.File(args.dataset + '.custom', 'w')
for video_name, video_file in h5in.items():
features = video_file['features'][...].astype(np.float32)
gtscore = video_file['gtscore'][...].astype(np.float32)
gtsummary = video_file['gtsummary'][...].astype(np.float32)
seq_len = gtscore.size
n_frames = seq_len * 15 - 1
picks = np.arange(0, seq_len) * 15
kernel = np.matmul(features, features.T)
change_points, _ = cpd_auto(kernel, seq_len - 1, 1)
change_points *= 15
change_points = np.hstack((0, change_points, n_frames))
begin_frames = change_points[:-1]
end_frames = change_points[1:]
change_points = np.vstack((begin_frames, end_frames - 1)).T
n_frame_per_seg = end_frames - begin_frames
h5out.create_dataset(video_name + '/features', data=features)
h5out.create_dataset(video_name + '/gtscore', data=gtscore)
# h5out.create_dataset(name + '/user_summary', data=data_of_name)
h5out.create_dataset(video_name + '/change_points', data=change_points)
h5out.create_dataset(video_name + '/n_frame_per_seg',
data=n_frame_per_seg)
h5out.create_dataset(video_name + '/n_frames', data=n_frames)
h5out.create_dataset(video_name + '/picks', data=picks)
# h5out.create_dataset(video_name + '/n_steps', data=data_of_name)
h5out.create_dataset(video_name + '/gtsummary', data=gtsummary)
# h5out.create_dataset(name + '/video_name', data=data_of_name)
h5in.close()
h5out.close()
def test_cps(model, evals, model_type, scaler):
f_score1 = []
f_score2 = []
for i in range(int(len(files) * 0.8), len(files)):
current_file_x = np.load(files[i])
fileName = files[i].split('/')[-1].split('.')[0]
current_fps = int(
np.round(
sio.loadmat(mat_files + fileName + '.mat').get('FPS')[0][0]))
# normalised_features = np.array([power_norm(e) for e in current_file_x])
normalised_features = scaler.transform(current_file_x)
K = np.dot(normalised_features, normalised_features.T)
m = get_cps_number(current_file_x.shape[0],
current_fps) # avergae 5sec cps
cps, scores = cpd_auto(K, m, 1)
# cps = np.arange(0,current_file_x.shape[0], current_fps*5, dtype=int)
X, _ = create_cuboids(normalised_features,
np.zeros([normalised_features.shape[0]]),
window_size)
preds = model.predict(X)
# pdb.set_trace()
if model_type == 'categorical':
preds = np.argmax(preds, axis=1)
print(fileName)
targets, ks_targets = create_key_shots(preds, cps)
# pdb.set_trace()
# print(np.count_nonzero(targets)/float(current_file_x.shape[0]))
# print(np.count_nonzero(ks_targets)/float(current_file_x.shape[0]))
f1_max, f1_mean = evals.evaluateSumMe(targets, fileName)
f2_max, f2_mean = evals.evaluateSumMe(ks_targets, fileName)
f_score1.append(f1_max)
f_score2.append(f2_max)
print "greedy mean: ", np.mean(f_score1)
print "ks mean: ", np.mean(f_score2)
plt.figure('Test 2: multidimensional signal')
n = 1000
m = 20
(X, cps_gt) = gen_data(n, m, d=50)
print('Ground truth:', cps_gt)
plt.plot(X)
K = np.dot(X, X.T)
cps, scores = cpd_nonlin(K, m, lmin=1, lmax=10000)
print('Estimated:', cps)
mi = np.min(X)
ma = np.max(X)
for cp in cps:
plt.plot([cp, cp], [mi, ma], 'r')
plt.show()
print('=' * 79)
print('Test 3: automatic selection of the number of change-points')
plt.figure('Test 3: automatic selection of the number of change-points')
(X, cps_gt) = gen_data(n, m)
print('Ground truth: (m=%d)' % m, cps_gt)
plt.plot(X)
K = np.dot(X, X.T)
cps, scores = cpd_auto(K, 2 * m, 1)
print('Estimated: (m=%d)' % len(cps), cps)
mi = np.min(X)
ma = np.max(X)
for cp in cps:
plt.plot([cp, cp], [mi, ma], 'r')
plt.show()
print('=' * 79)
def test_cps(model_rgb, model_flow, enc_model_rgb, enc_model_flow, evals,
model_type, scaler_rgb, scaler_flow, files_rgb, files_flow,
mat_files, window_size):
rgb = []
flow = []
comb_r = []
comb_f = []
for i in range(int(len(files_rgb) * 0.8), len(files_rgb)):
current_file_x_rgb = np.load(files_rgb[i])
current_file_x_flow = np.load(files_flow[i])
fileName = files_rgb[i].split('/')[-1].split('.')[0]
current_fps = int(
np.round(
sio.loadmat(mat_files + fileName + '.mat').get('FPS')[0][0]))
print(fileName)
normalised_features_flow = scaler_flow.transform(current_file_x_flow)
normalised_features_rgb = scaler_rgb.transform(current_file_x_rgb)
K = np.dot(normalised_features_flow, normalised_features_flow.T)
m = get_cps_number(current_file_x_flow.shape[0],
current_fps) # avergae 5sec cps
cps, scores = cpd_auto(K, m, 1)
cps = add_cps(cps, current_fps)
K_rgb = np.dot(normalised_features_rgb, normalised_features_rgb.T)
m_rgb = get_cps_number(current_file_x_rgb.shape[0],
current_fps) # avergae 5sec cps
cps_rgb, scores_rgb = cpd_auto(K_rgb, m_rgb, 1)
cps_rgb = add_cps(cps_rgb, current_fps)
# cps = np.arange(0,current_file_x.shape[0], current_fps*5, dtype=int)
x_rgb, _ = create_cuboids(normalised_features_rgb,
np.zeros([normalised_features_rgb.shape[0]]),
window_size)
x_flow, _ = create_cuboids(
normalised_features_flow,
np.zeros([normalised_features_flow.shape[0]]), window_size)
if model_type == 'AE':
x_rgb = enc_model_rgb.predict(x_rgb)
x_flow = enc_model_flow.predict(x_flow)
preds_rgb = model_rgb.predict(x_rgb)
preds_flow = model_flow.predict(x_flow)
preds = (preds_rgb + preds_flow) / 2.
targets, ks_targets_rgb = create_key_shots(preds_rgb, cps_rgb)
targets, ks_targets_flow = create_key_shots(preds_flow, cps)
targets, ks_targets_comb_r = create_key_shots(preds_flow, cps_rgb)
targets, ks_targets_comb_f = create_key_shots(preds, cps)
# np.save('../../saved_numpy_arrays/predictions/I3D/Conv_RNN/'+fileName+'_greedy_'+'.npy', targets)
np.save(
'../../saved_numpy_arrays/predictions/I3D/Conv_RNN/' + fileName +
'_ks_' + '.npy', ks_targets_comb_f)
# pdb.set_trace()
# evals.evaluateSumMe(targets, fileName)
r_max, _ = evals.evaluateSumMe(ks_targets_rgb, fileName)
f_max, _ = evals.evaluateSumMe(ks_targets_flow, fileName)
comb_r_max, _ = evals.evaluateSumMe(ks_targets_comb_r, fileName)
comb_f_max, _ = evals.evaluateSumMe(ks_targets_comb_f, fileName)
rgb.append(r_max)
flow.append(f_max)
comb_r.append(comb_r_max)
comb_f.append(comb_f_max)
print('rgb: ', np.mean(rgb))
print('flow: ', np.mean(flow))
print('comb rgb - cps: ', np.mean(comb_r))
print('comb flow - cps: ', np.mean(comb_f))
def test_cps(model_rgb, model_flow, enc_model_rgb, enc_model_flow, model_type, scaler_rgb, scaler_flow, files_rgb, files_flow, window_size):
all_cps_flow=[]
all_cps_rgb=[]
all_cps_comb=[]
all_fileNames = []
all_ks_targets = []
for i in range(len(files_rgb)):
current_file_x_rgb = np.load(files_rgb[i])
current_file_x_flow = np.load(files_flow[i])
fileName = files_rgb[i].split('/')[-1].split('.')[0]
current_fps = getFPS_tvSum50(fileName)
print(fileName)
normalised_features_flow = scaler_flow.transform(current_file_x_flow)
normalised_features_rgb = scaler_rgb.transform(current_file_x_rgb)
normalised_features_comb = np.concatenate((normalised_features_flow, normalised_features_rgb))
K = np.dot(normalised_features_flow, normalised_features_flow.T)
m = get_cps_number(current_file_x_flow.shape[0], current_fps) # avergae 5sec cps
cps, scores = cpd_auto(K, m, 1)
K_r = np.dot(normalised_features_rgb, normalised_features_rgb.T)
m_r = get_cps_number(current_file_x_rgb.shape[0], current_fps) # avergae 5sec cps
cps_r, scores_r = cpd_auto(K_r, m_r, 1)
# K_comb = np.dot(normalised_features_comb, normalised_features_comb.T)
# m_comb = get_cps_number(current_file_x_rgb.shape[0], current_fps) # avergae 5sec cps
# cps_comb, scores_comb = cpd_auto(K_comb, m_comb, 1)
# cps = add_cps(cps, current_fps)
# cps = np.arange(0,current_file_x.shape[0], current_fps*5, dtype=int)
x_rgb, _ = create_cuboids(normalised_features_rgb, np.zeros([normalised_features_rgb.shape[0]]), window_size)
x_flow, _ = create_cuboids(normalised_features_flow, np.zeros([normalised_features_flow.shape[0]]), window_size)
if model_type=='AE':
x_rgb = enc_model_rgb.predict(x_rgb)
x_flow = enc_model_flow.predict(x_flow)
preds_rgb = model_rgb.predict(x_rgb)
preds_flow = model_flow.predict(x_flow)
if preds_rgb.shape[0] > preds_flow.shape[0]:
preds_rgb = preds_rgb[:preds_flow.shape[0]]
preds = (preds_rgb+preds_flow) / 2.
targets, ks_targets = create_key_shots(preds, cps)
# pdb.set_trace()
all_fileNames.append(fileName)
all_cps_flow.append(cps)
all_cps_rgb.append(cps_r)
# all_cps_comb.append(cps_comb)
all_ks_targets.append(ks_targets)
sio.savemat('../../saved_numpy_arrays/TvSum50/predictions/I3D/3D_AE/' + fileName + '_rgb.mat', dict(x=preds_rgb))
sio.savemat('../../saved_numpy_arrays/TvSum50/predictions/I3D/3D_AE/' + fileName + '_flow.mat', dict(x=preds_flow))
sio.savemat('../../saved_numpy_arrays/TvSum50/predictions/I3D/3D_AE/' + fileName + '_combined.mat', dict(x=preds))
sio.savemat('../../saved_numpy_arrays/TvSum50/predictions/I3D/3D_AE/AllData_flow_.mat', dict(fileName = all_fileNames, shot_boundaries=all_cps_flow, targets=all_ks_targets))
sio.savemat('../../saved_numpy_arrays/TvSum50/predictions/I3D/3D_AE/AllData_rgb_.mat', dict(fileName = all_fileNames, shot_boundaries=all_cps_rgb, targets=all_ks_targets))
