def plot_detections_over_time(data_dir, out_dir, fps, smooth=False):
frame_files = glob.glob(os.path.join(data_dir, '*.jpg'))
frame_files.sort()
result_files = [
file.replace(data_dir, out_dir).replace('.jpg', '.txt')
for file in frame_files
]
det_confs = [[], []]
time = []
for frame, result in zip(frame_files, result_files):
num = fps * (int(frame[frame.rindex('_') + 1:frame.rindex('.')]) -
1) + fps / 2.
time.append(num / 60.)
# time.append(int(frame[frame.rindex('_')+1:frame.rindex('.')]))
dets = util.readLinesFromFile(result)
if len(dets) == 0:
[det_confs[idx].append(0) for idx in range(2)]
else:
class_conf = []
for det in dets:
det = det.split(' ')
det = [int(det[0]), float(det[1])]
class_conf.append(det)
class_conf = np.array(class_conf)
for class_curr in range(2):
rel_dets = class_conf[class_conf[:, 0] == class_curr, :]
if rel_dets.size == 0:
max_conf = 0
else:
max_conf = np.max(rel_dets[:, 1])
det_confs[class_curr].append(max_conf)
if smooth:
smooth_window = int(round(60 / float(fps)))
# print smooth_window
for idx, det in enumerate(det_confs):
box = np.ones((smooth_window, )) / float(smooth_window)
det_confs[idx] = np.convolve(np.array(det), box, mode='same')
labels = ['side', 'front']
xlabel = 'Video Time (min)'
ylabel = 'Detection Confidence'
title = 'Face Detections Over Time'
out_file = os.path.join(out_dir, 'detections_over_time.jpg')
visualize.plotSimple([(time, det_confs[0]), (time, det_confs[1])],
out_file=out_file,
xlabel=xlabel,
ylabel=ylabel,
title=title,
legend_entries=labels)
print 'DETECTION GRAPH:', out_file
def plot_graph_size():
graph_size = [1, 2, 3, 4, 8, 16, 32]
xticks = [str(val) for val in graph_size]
overlap_str = ['%.1f' % val for val in np.arange(0.1, 0.6, 0.1)]
graph_vals = [[63.7, 57.0, 47.7, 36.9,
26.2], [63.2, 56.7, 47.9, 37.1, 26.6],
[64.0, 58.2, 49.8, 39.2,
28.2], [63.7, 57.3, 48.2, 36.7, 25.5],
[62.7, 56.5, 47.0, 36.0,
23.9], [62.7, 55.5, 46.6, 35.3, 24.0],
[61.1, 54.1, 44.6, 32.8, 21.9]]
graph_vals = np.array(graph_vals)
graph_vals = graph_vals.T
graph_size = [1, 2, 4, 8, 16, 32]
xticks = [str(val) for val in graph_size]
overlap_str = ['%.1f' % val for val in np.arange(0.1, 0.6, 0.1)]
# graph_vals = [[ 63.7 , 57.0 , 47.7 , 36.9 , 26.2],
# [ 63.2 , 56.7 , 47.9 , 37.1 , 26.6],
# [ 64.0 , 58.2 , 49.8 , 39.2 , 28.2],
# [ 63.7 , 57.3 , 48.2 , 36.7 , 25.5],
# [ 62.7 , 56.5 , 47.0 , 36.0 , 23.9],
# [ 62.7 , 55.5 , 46.6 , 35.3 , 24.0],
# [ 61.1 , 54.1 , 44.6 , 32.8 , 21.9]]
# graph_vals = np.array(graph_vals)
# graph_vals = graph_vals.T
out_file = '../scratch/qualitative_figs_wacv/graph_size_anet.jpg'
util.mkdir('../scratch/qualitative_figs_wacv')
ylabel = 'Average Precision'
xlabel = 'Graph Video Size'
xAndYs = []
legend_entries = []
# graph_val = [63.65, 62.96, 63.16, 62.9, 61.58, 60.13]
# title = 'THUMOS'
graph_val = [29.44, 30.05, 30.53, 29.25, 29.85, 28.56]
overlap_str = ['%.1f' % val for val in np.arange(0.5, 1.0, 0.2)]
title = 'ActivityNet'
# for idx_graph_val,graph_val in enumerate(graph_vals):
# if idx_graph_val<4:
# continue
x = range(len(graph_val))
xAndYs.append((x, graph_val))
legend_entries.append(overlap_str[0] + ' Overlap')
visualize.plotSimple(xAndYs,
out_file=out_file,
title=title,
xlabel=xlabel,
ylabel=ylabel,
legend_entries=legend_entries,
xticks=[x, xticks])
def get_models_accuracy():
out_dir_meta = '../experiments_dropout'
exp_name = 'bp4d_norecon'
wdecay = 0
lr = [0.001, 0.001]
route_iter = 3
folds_all = [0, 1]
model_name = 'khorrami_capsule_7_3_gray'
epoch_stuff = [15, 15]
gpu_id = 0
# folds =
# dropout =
commands_all = []
params_arr = [(0, [2], 0), (0, [2], 1)]
# (0,[0,1],0),(0.5,[0,1],1),(0.5,[2],2)]
xAndYs = []
legend_strs = []
for dropout in [0., 0.5]:
for aug_more in [True, False]:
val_arr_all = []
for fold in range(3):
params = dict(wdecay=wdecay,
lr=lr,
route_iter=route_iter,
model_name=model_name,
epoch_stuff=epoch_stuff,
gpu_id=gpu_id,
fold=fold,
dropout=dropout,
aug_more=aug_more)
out_dir_train, pre_pend, post_pend = get_out_dir_train_name(
**params)
log_file = os.path.join(out_dir_train, 'log.txt')
assert os.path.exists(log_file)
val_arr = get_log_val_arr(log_file)
val_arr_all.append(val_arr)
val_arr_all = np.array(val_arr_all)
val_arr_all = np.mean(val_arr_all, 0)
xAndYs.append((range(len(val_arr_all)), val_arr_all))
legend_strs.append(' '.join(
[str(val) for val in [dropout, aug_more]]))
out_file = os.path.join(out_dir_meta, exp_name + '.jpg')
visualize.plotSimple(xAndYs,
out_file,
title='Dropout',
xlabel='Epoch',
ylabel='Val Accuracy',
legend_entries=legend_strs,
outside=True)
def save_graphs_to_look_at(model_file, graph_nums):
out_dir_meta = model_file[:model_file.rindex('.')]
out_dir_meta_meta = out_dir_meta + '_graph_etc'
out_dir_viz = out_dir_meta_meta + '_viz'
util.mkdir(out_dir_viz)
for graph_num in graph_nums:
out_dir_meta = out_dir_meta_meta + '_' + str(graph_num)
assert os.path.exists(out_dir_meta)
vid_files = glob.glob(os.path.join(out_dir_meta, '*test*.npz'))
for vid_file in vid_files:
npz_data = np.load(vid_file)
vid_file = os.path.split(vid_file)[1]
affinity = npz_data['affinity']
gt_vecs = npz_data['gt_vecs']
gt_classes = npz_data['gt_classes']
x_all = npz_data['x_all']
plotter = []
legend_entries = []
for gt_idx, gt_class in enumerate(gt_classes):
gt_vec = gt_vecs[gt_idx]
val_rel = x_all[0, :, gt_class]
gt_vec = gt_vec / np.max(gt_vec)
gt_vec = gt_vec * np.max(val_rel)
# (gt_idx+1)
x_axis = range(gt_vec.size)
plotter.append((x_axis, gt_vec))
plotter.append((x_axis, val_rel))
legend_entries.append(class_names[gt_class])
legend_entries.append(class_names[gt_class] + ' pred')
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_gt.jpg')
visualize.plotSimple(plotter,
out_file=out_file,
xlabel='time',
ylabel='',
legend_entries=legend_entries,
outside=True)
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_' +
str(graph_num) + '.jpg')
visualize.saveMatAsImage(affinity, out_file)
visualize.writeHTMLForFolder(out_dir_viz)
def save_sim_viz_mean(vid_name, out_shape_curr, sim_mat, class_idx, out_dir):
gt_vals, det_times = get_gt_vector(vid_name, out_shape_curr, class_idx)
out_dir_curr = os.path.join(out_dir, class_names[class_idx])
util.mkdir(out_dir_curr)
idx_pos = gt_vals > 0
idx_neg = gt_vals < 1
sim_pos_all = []
sim_neg_all = []
for idx_idx_curr, idx_curr in enumerate(np.where(idx_pos)[0]):
sim_pos = sim_mat[idx_curr, idx_pos]
sim_neg = sim_mat[idx_curr, idx_neg]
sim_pos_all.append(sim_pos[np.newaxis, :])
sim_neg_all.append(sim_neg[np.newaxis, :])
sim_pos_all = np.concatenate(sim_pos_all, axis=0)
sim_neg_all = np.concatenate(sim_neg_all, axis=0)
sim_pos_mean = np.mean(sim_pos_all, axis=0)
sim_neg_mean = np.mean(sim_neg_all, axis=0)
pos_vals = np.zeros(gt_vals.shape)
pos_vals[gt_vals > 0] = sim_pos_mean
neg_vals = np.zeros(gt_vals.shape)
neg_vals[gt_vals < 1] = sim_neg_mean
max_val = max(np.max(pos_vals), np.max(neg_vals))
gt_vals = gt_vals * max_val
arr_plot = [(det_times, curr_arr)
for curr_arr in [gt_vals, pos_vals, neg_vals]]
legend_entries = ['gt', 'pos', 'neg']
out_file_curr = os.path.join(out_dir_curr, vid_name + '.jpg')
title = vid_name
visualize.plotSimple(arr_plot,
out_file=out_file_curr,
title=title,
xlabel='time',
ylabel='max sim',
legend_entries=legend_entries)
print out_file_curr
def graph_overfitting():
log_file_us = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_True_MultiCrossEntropyMultiBranchWithL1_CASL_500_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_0.50__noLimit/log_det.txt'
log_file_them = '../experiments/graph_multi_video_with_L1_retW/graph_multi_video_with_L1_retW_aft_nonlin_RL_L2_non_lin_None_sparsify_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_True_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_0.50__caslexp/log_det.txt'
log_file_us = '../experiments/graph_multi_video_with_L1_retF_tanh/graph_multi_video_with_L1_retF_tanh_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_1_sigmoid_True_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_forplot_0/log_det.txt'
log_file_them = '../experiments/graph_multi_video_with_L1_retW_new/graph_multi_video_with_L1_retW_new_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_1_sigmoid_True_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_actuallytanh_0/log_det.txt'
files = [log_file_us, log_file_them]
x_vals = range(0, 251, 10)
xAndYs = []
for file_curr in files:
lines = util.readLinesFromFile(file_curr)
lines = lines[:26]
print lines[0]
det_vals = []
# for line in lines:
# line = [val for val in line.split(' ') if val is not '']
# print line
# det_vals.append(float(line[-1]))
# raw_input()
det_vals = [float(line.split('\t')[-1]) for line in lines]
# det_vals = det_vals[::5]
xAndYs.append((x_vals, det_vals))
out_file = '../scratch/qualitative_figs_wacv/graph_overfitting.jpg'
util.mkdir('../scratch/qualitative_figs_wacv')
legend_entries = ['Ours-MCASL', 'CASL-Graph']
xlabel = 'Training Epoch'
ylabel = 'Detection Accuracy'
title = 'Detection Accuracy at 0.5 Overlap'
visualize.plotSimple(xAndYs,
out_file=out_file,
xlabel=xlabel,
ylabel=ylabel,
legend_entries=legend_entries,
title=title)
def viz_overlap_multi(out_dir_meta,
det_conf_all_dict,
out_shapes,
fps_stuff,
title=None,
dataset='ucf'):
# print 'HELLO'
# fps_stuff = 1./10.
# activitynet = False
print dataset
raw_input()
if dataset == 'activitynet':
class_names = globals.class_names_activitynet
gt_vid_names_all, gt_class_names, gt_time_intervals_all = load_activitynet_gt(
False)
elif dataset == 'activitynet_select':
class_names = globals.class_names_activitynet_select
gt_vid_names_all, gt_class_names, gt_time_intervals_all = load_activitynet_gt(
False, True)
elif dataset == 'multithumos':
class_names = globals.class_names_multithumos
gt_vid_names_all, gt_class_names, gt_time_intervals_all = load_multithumos_gt(
False)
else:
class_names = [
'BaseballPitch', 'BasketballDunk', 'Billiards', 'CleanAndJerk',
'CliffDiving', 'CricketBowling', 'CricketShot', 'Diving',
'FrisbeeCatch', 'GolfSwing', 'HammerThrow', 'HighJump',
'JavelinThrow', 'LongJump', 'PoleVault', 'Shotput',
'SoccerPenalty', 'TennisSwing', 'ThrowDiscus', 'VolleyballSpiking'
]
gt_vid_names_all, gt_class_names, gt_time_intervals_all = load_ucf_gt(
False)
class_names.sort()
aps = np.zeros((len(class_names) + 1, 5))
overlap_thresh_all = np.arange(0.1, 0.6, 0.1)
for idx_class_name, class_name in enumerate(class_names):
# if idx_class_name<6:
# continue
out_dir = os.path.join(out_dir_meta, class_name)
util.mkdir(out_dir)
# if not dataset.startswith('activitynet'):
# mat_file = os.path.join('../TH14evalkit','mat_files', class_name+'_test.mat')
# loaded = scipy.io.loadmat(mat_file)
# gt_vid_names_all = loaded['gtvideonames'][0]
# gt_class_names = loaded['gt_events_class'][0]
# gt_time_intervals = loaded['gt_time_intervals'][0]
# arr_meta = [gt_vid_names_all, gt_class_names]
# arr_out = []
# for arr_curr in arr_meta:
# arr_curr = [str(a[0]) for a in arr_curr]
# arr_out.append(arr_curr)
# [gt_vid_names_all, gt_class_names] = arr_out
# gt_time_intervals_all = np.array([a[0] for a in gt_time_intervals])
gt_vid_names = list(
np.unique(
np.array(gt_vid_names_all)[np.array(gt_class_names) ==
class_name]))
for gt_vid_name in gt_vid_names:
gt_time_intervals = gt_time_intervals_all[np.logical_and(
np.array(gt_vid_names_all) == gt_vid_name,
np.array(gt_class_names) == class_name)]
if gt_vid_name not in list(out_shapes.keys()):
continue
out_shape_curr = out_shapes[gt_vid_name]
det_times = np.array(range(0, out_shape_curr + 1)) * fps_stuff
gt_vals = np.zeros(det_times.shape)
plot_arr = []
legend_entries = []
max_det_conf = None
for k in det_conf_all_dict.keys():
[
det_conf_curr, det_time_intervals_all,
det_events_class_all, det_vid_names
] = det_conf_all_dict[k]
bin_keep = det_vid_names == gt_vid_name
# print det_vid_names[0], gt_vid_names, np.sum(bin_keep)
bin_keep = np.logical_and(
bin_keep, det_events_class_all == idx_class_name)
if np.sum(bin_keep) == 0:
# print 'Continuing'
continue
# print 'not Continuing'
# det_conf_curr = det_conf_all_dict[k][0]
det_time_intervals_merged = det_time_intervals_all[bin_keep, :]
det_conf_curr = det_conf_curr[bin_keep]
# print k, det_time_intervals_merged
if max_det_conf is None:
max_det_conf = np.max(det_conf_curr)
else:
max_det_conf = max(max_det_conf, np.max(det_conf_curr))
det_vals = np.zeros(det_times.shape)
for idx_det_time_curr, det_time_curr in enumerate(
det_time_intervals_merged):
idx_start = np.argmin(np.abs(det_times - det_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - det_time_curr[1]))
det_vals[idx_start:idx_end] = det_conf_curr[
idx_det_time_curr]
legend_entries.append(k)
plot_arr.append((det_times, det_vals))
for gt_time_curr in gt_time_intervals:
idx_start = np.argmin(np.abs(det_times - gt_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - gt_time_curr[1]))
gt_vals[idx_start:idx_end] = max_det_conf
plot_arr.append((det_times, gt_vals))
legend_entries.append('GT')
out_file_curr = os.path.join(out_dir, gt_vid_name + '.jpg')
if title is None:
title = 'det conf over time'
# print plot_arr
out_file_first = out_file_curr[:out_file_curr.rindex('.')]
# plot_arr_for_save = {}
for idx_arr in range(len(plot_arr)):
# plot_arr_for_save[legend_entries[idx_arr]]=np.array(list(plot_arr[idx_arr]))
arr_curr = np.array(list(plot_arr[idx_arr]))
np.save(
out_file_first + '_' + legend_entries[idx_arr] + '.npy',
arr_curr)
# print legend_entries[idx_arr],plot_arr_for_save[legend_entries[idx_arr]].shape
visualize.plotSimple(plot_arr,
out_file=out_file_curr,
title=title,
xlabel='Time',
ylabel='Detection Confidence',
legend_entries=legend_entries)
visualize.writeHTMLForFolder(out_dir)
def main():
gt_vec_dir = '../experiments/graph_multi_video_same_F_ens_dll_moredepth/graph_multi_video_same_F_ens_dll_moredepth_aft_nonlin_HT_L2_non_lin_HT_num_graphs_1_sparsify_0.5_graph_size_2_sigmoid_True_deno_0.5_n_classes_20_in_out_2048_256_feat_dim_2048_512_method_cos_zero_self_ucf/all_classes_False_just_primary_False_limit_None_cw_True_MultiCrossEntropy_300_step_300_0.1_0.001_0.001_0.001_ABS_bias_retry/model_299_graph_etc_0'
dir_results = '../experiments/graph_multi_video_same_i3dF/graph_multi_video_same_i3dF_aft_nonlin_HT_l2_sparsify_0.5_non_lin_None_method_cos_zero_self_deno_8_n_classes_20_in_out_2048_2_graph_size_2_ucf/all_classes_False_just_primary_False_limit_None_cw_True_MultiCrossEntropy_100_step_100_0.1_0.001_0.01_ABS_bias_wb/results_model_99_0_0.5'
dir_results = '../experiments/graph_multi_video_same_i3dF/graph_multi_video_same_i3dF_aft_nonlin_HT_l2_non_lin_None_sparsify_False_graph_size_2_method_cos_zero_self_deno_8_n_classes_20_in_out_2048_2_graph_sum_True_ucf/all_classes_False_just_primary_False_limit_None_cw_True_MultiCrossEntropyMultiBranchWithL1_100_step_100_0.1_0.001_0.001_lw_1.00_0.50_ABS_bias_wb/results_model_99_0_0.5'
feat_dir = os.path.join(dir_results, 'outf')
out_dir = os.path.join(dir_results, 'viz_outf')
util.mkdir(out_dir)
class_idx_keep = range(10, 16)
plot_classes(feat_dir, gt_vec_dir, out_dir, class_idx_keep)
return
g_str = 'graph_2_nononlin_b'
lin_str = 'lin_2_nononlin'
graph_feat_dir = os.path.join('../scratch', g_str)
lin_feat_dir = os.path.join('../scratch', lin_str)
out_dir = '../scratch/comparing_features'
util.mkdir(out_dir)
vid_names = glob.glob(os.path.join(graph_feat_dir, '*.npy'))
vid_names = [os.path.split(file_curr)[1][:-4] for file_curr in vid_names]
graph_features = [[] for i in range(20)]
lin_features = [[] for i in range(20)]
for vid_name in vid_names:
gt_file = os.path.join(gt_vec_dir, vid_name + '.npz')
npz_data = np.load(gt_file)
gt_vecs = npz_data['gt_vecs']
gt_classes = npz_data['gt_classes']
graph_data = np.load(os.path.join(graph_feat_dir, vid_name + '.npy'))
lin_data = np.load(os.path.join(lin_feat_dir, vid_name + '.npy'))
if gt_classes.size > 1:
continue
bin_curr = gt_vecs[0] > 0
graph_data_rel = graph_data[bin_curr, :]
lin_data_rel = lin_data[bin_curr, :]
print graph_data_rel.shape
print lin_data_rel.shape
class_curr = int(gt_classes[0])
lin_features[class_curr].append(lin_data_rel)
graph_features[class_curr].append(graph_data_rel)
# lin_features = [np.concatenate(lin_feat,axis = 0) for lin_feat in lin_features]
# graph_features = [np.concatenate(graph_feat,axis = 0) for graph_feat in graph_features]
titles = [lin_str, g_str]
class_names_curr = class_names[10:16]
lin_features = lin_features[10:16]
graph_features = graph_features[10:16]
for idx_features, features in enumerate([lin_features, graph_features]):
class_names_keep = [
class_names_curr[idx] for idx, f in enumerate(features)
if len(f) > 0
]
features = [np.concatenate(f, axis=0) for f in features if len(f) > 0]
xAndYs = [(f[:, 0], f[:, 1]) for f in features]
title = titles[idx_features]
out_file = os.path.join(out_dir, title + '.jpg')
xlabel = 'x'
ylabel = 'y'
legend_entries = class_names_keep
visualize.plotSimple(xAndYs,
out_file=out_file,
title=title,
xlabel=xlabel,
ylabel=ylabel,
legend_entries=legend_entries,
outside=True,
noline=True)
print out_file
def viz_overlap(out_dir_meta, det_vid_names, det_conf_all,
det_time_intervals_all, det_events_class_all, out_shapes):
class_names = [
'BaseballPitch', 'BasketballDunk', 'Billiards', 'CleanAndJerk',
'CliffDiving', 'CricketBowling', 'CricketShot', 'Diving',
'FrisbeeCatch', 'GolfSwing', 'HammerThrow', 'HighJump', 'JavelinThrow',
'LongJump', 'PoleVault', 'Shotput', 'SoccerPenalty', 'TennisSwing',
'ThrowDiscus', 'VolleyballSpiking'
]
class_names.sort()
aps = np.zeros((len(class_names) + 1, 5))
overlap_thresh_all = np.arange(0.1, 0.6, 0.1)
for idx_class_name, class_name in enumerate(class_names):
# if idx_class_name<6:
# continue
out_dir = os.path.join(out_dir_meta, class_name)
util.mkdir(out_dir)
mat_file = os.path.join('../TH14evalkit', 'mat_files',
class_name + '_test.mat')
loaded = scipy.io.loadmat(mat_file)
gt_vid_names_all = loaded['gtvideonames'][0]
gt_class_names = loaded['gt_events_class'][0]
gt_time_intervals = loaded['gt_time_intervals'][0]
arr_meta = [gt_vid_names_all, gt_class_names]
arr_out = []
for arr_curr in arr_meta:
arr_curr = [str(a[0]) for a in arr_curr]
arr_out.append(arr_curr)
[gt_vid_names_all, gt_class_names] = arr_out
gt_time_intervals_all = np.array([a[0] for a in gt_time_intervals])
gt_vid_names = list(
np.unique(
np.array(gt_vid_names_all)[np.array(gt_class_names) ==
class_name]))
det_vid_names = np.array(det_vid_names)
# print len(det_vid_names)
# print np.unique(det_vid_names).shape
# print gt_vid_name
# print len(gt_vid_names)
for gt_vid_name in gt_vid_names:
bin_keep = det_vid_names == gt_vid_name
# if np.sum(bin_keep):
# print idx_class_name, np.sum(bin_keep)
# print gt_vid_name
# print np.sum(det_events_class_all==idx_class_name)
# raw_input()
bin_keep = np.logical_and(bin_keep,
det_events_class_all == idx_class_name)
if np.sum(bin_keep) == 0:
print 'Continuing'
continue
gt_time_intervals = gt_time_intervals_all[np.array(
gt_vid_names_all) == gt_vid_name]
det_conf = det_conf_all[bin_keep]
det_time_intervals = det_time_intervals_all[bin_keep, :]
out_shape_curr = out_shapes[bin_keep]
assert len(np.unique(out_shape_curr)) == 1
out_shape_curr = np.unique(out_shape_curr)[0]
# print out_shape_curr
det_time_intervals_merged = det_time_intervals
det_times = det_time_intervals[:, 0]
det_times = np.array(range(0, out_shape_curr + 1)) * 16. / 25.
gt_vals = np.zeros(det_times.shape)
# print gt_time_intervals.shape
# print det_times.shape
for gt_time_curr in gt_time_intervals:
idx_start = np.argmin(np.abs(det_times - gt_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - gt_time_curr[1]))
gt_vals[idx_start:idx_end] = np.max(det_conf)
det_vals = np.zeros(det_times.shape)
for idx_det_time_curr, det_time_curr in enumerate(
det_time_intervals_merged):
idx_start = np.argmin(np.abs(det_times - det_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - det_time_curr[1]))
det_vals[idx_start:idx_end] = det_conf[idx_det_time_curr]
out_file_curr = os.path.join(out_dir, gt_vid_name + '.jpg')
visualize.plotSimple([(det_times, det_vals), (det_times, gt_vals)],
out_file=out_file_curr,
title='det conf over time',
xlabel='time',
ylabel='det conf',
legend_entries=['Det', 'GT'])
visualize.writeHTMLForFolder(out_dir)
def script_debug_old_testing():
print 'hello'
train = False
det_file = '../scratch/debug_det.npz'
out_dir_meta = '../scratch/seeing_dets'
util.mkdir(out_dir_meta)
det_data = np.load(det_file)
det_vid_names = det_data['det_vid_names']
det_conf_all = det_data['det_conf']
det_time_intervals_all = det_data['det_time_intervals']
print det_vid_names.shape, det_vid_names[0], det_vid_names[-1]
print det_conf_all.shape, np.min(det_conf_all), np.max(det_conf_all)
print det_time_intervals_all.shape, np.min(det_time_intervals_all), np.max(
det_time_intervals_all)
class_names = [
'BaseballPitch', 'BasketballDunk', 'Billiards', 'CleanAndJerk',
'CliffDiving', 'CricketBowling', 'CricketShot', 'Diving',
'FrisbeeCatch', 'GolfSwing', 'HammerThrow', 'HighJump', 'JavelinThrow',
'LongJump', 'PoleVault', 'Shotput', 'SoccerPenalty', 'TennisSwing',
'ThrowDiscus', 'VolleyballSpiking'
]
class_names.sort()
aps = np.zeros((len(class_names) + 1, 5))
overlap_thresh_all = np.arange(0.1, 0.6, 0.1)
for idx_class_name, class_name in enumerate(class_names):
# if idx_class_name<6:
# continue
out_dir = os.path.join(out_dir_meta, class_name)
util.mkdir(out_dir)
if train:
mat_file = os.path.join('../TH14evalkit', class_name + '.mat')
else:
mat_file = os.path.join('../TH14evalkit', 'mat_files',
class_name + '_test.mat')
loaded = scipy.io.loadmat(mat_file)
gt_vid_names_all = loaded['gtvideonames'][0]
gt_class_names = loaded['gt_events_class'][0]
gt_time_intervals = loaded['gt_time_intervals'][0]
arr_meta = [gt_vid_names_all, gt_class_names]
arr_out = []
for arr_curr in arr_meta:
arr_curr = [str(a[0]) for a in arr_curr]
arr_out.append(arr_curr)
[gt_vid_names_all, gt_class_names] = arr_out
gt_time_intervals_all = np.array([a[0] for a in gt_time_intervals])
gt_vid_names = list(
np.unique(
np.array(gt_vid_names_all)[np.array(gt_class_names) ==
class_name]))
print class_name, len(gt_vid_names)
for gt_vid_name in gt_vid_names:
print gt_vid_name
bin_keep = det_vid_names == gt_vid_name
gt_time_intervals = gt_time_intervals_all[np.array(
gt_vid_names_all) == gt_vid_name]
# print det_vid_names[bin_keep]
print det_conf_all.shape, idx_class_name, bin_keep.shape
det_conf = det_conf_all[bin_keep, idx_class_name]
det_time_intervals = det_time_intervals_all[bin_keep, :]
thresh = np.max(det_conf) - (np.max(det_conf) -
np.min(det_conf)) * 0.5
bin_second_thresh = det_conf > thresh
det_conf, det_time_intervals_merged = merge_detections(
bin_second_thresh, det_conf, det_time_intervals)
# det_conf[det_conf<thresh]=0
det_times = det_time_intervals[:, 0]
gt_vals = np.zeros(det_times.shape)
for gt_time_curr in gt_time_intervals:
idx_start = np.argmin(np.abs(det_times - gt_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - gt_time_curr[1]))
gt_vals[idx_start:idx_end] = np.max(det_conf)
det_vals = np.zeros(det_times.shape)
for idx_det_time_curr, det_time_curr in enumerate(
det_time_intervals_merged):
idx_start = np.argmin(np.abs(det_times - det_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - det_time_curr[1]))
det_vals[idx_start:idx_end] = det_conf[idx_det_time_curr]
out_file_curr = os.path.join(out_dir,
'dets_' + gt_vid_name + '_merged.jpg')
visualize.plotSimple([(det_times, det_vals), (det_times, gt_vals)],
out_file=out_file_curr,
title='det conf over time',
xlabel='time',
ylabel='det conf',
legend_entries=['Det', 'GT'])
# print out_file_curr
# raw_input()
visualize.writeHTMLForFolder(out_dir)
def plot_classes(feat_dir, gt_vec_dir, out_dir, class_idx_keep):
# g_str = 'graph_2_nononlin_b'
# lin_str = 'lin_2_nononlin'
# graph_feat_dir = os.path.join('../scratch',g_str)
# lin_feat_dir = os.path.join('../scratch',lin_str)
# out_dir = '../scratch/comparing_features'
n_classes = len(class_names)
if class_idx_keep is None:
class_idx_keep = range(n_classes)
util.mkdir(out_dir)
vid_names = glob.glob(os.path.join(feat_dir, '*.npy'))
vid_names = [os.path.split(file_curr)[1][:-4] for file_curr in vid_names]
graph_features = [[] for i in range(n_classes)]
for vid_name in vid_names:
gt_file = os.path.join(gt_vec_dir, vid_name + '.npz')
npz_data = np.load(gt_file)
gt_vecs = npz_data['gt_vecs']
gt_classes = npz_data['gt_classes']
graph_data = np.load(os.path.join(feat_dir, vid_name + '.npy'))
for idx_gt, gt_vec in enumerate(gt_vecs):
class_curr = int(gt_classes[idx_gt])
if class_curr not in class_idx_keep:
continue
bin_curr = gt_vec > 0
graph_data_rel = graph_data[bin_curr, :]
graph_features[class_curr].append(graph_data_rel)
class_names_curr = np.array(class_names)[class_idx_keep]
graph_features = [graph_features[idx] for idx in class_idx_keep]
class_names_keep = [
class_names_curr[idx] for idx, f in enumerate(graph_features)
if len(f) > 0
]
features = [
np.concatenate(f, axis=0) for f in graph_features if len(f) > 0
]
xAndYs = [(f[:, 0], f[:, 1]) for f in features]
title = 'Features'
out_file = '_'.join(class_names_keep) + '.jpg'
out_file = os.path.join(out_dir, out_file)
xlabel = 'x'
ylabel = 'y'
legend_entries = class_names_keep
visualize.plotSimple(xAndYs,
out_file=out_file,
title=title,
xlabel=xlabel,
ylabel=ylabel,
legend_entries=legend_entries,
outside=True,
noline=True)
print out_file
def script_viewing_sim():
dir_files = '../data/ucf101/train_test_files'
n_classes = 20
train_file = os.path.join(dir_files, 'train_just_primary.txt')
test_file = os.path.join(dir_files, 'test_just_primary.txt')
out_dir = '../scratch/debugging_graph_self1'
util.mkdir(out_dir)
train_lines = util.readLinesFromFile(test_file)
train_npy = [line_curr.split(' ') for line_curr in train_lines]
for line_curr in train_lines:
line_curr = line_curr.split(' ')
npy_file = line_curr[0]
anno = [int(val) for val in line_curr[1:]]
anno = np.array(anno)
assert np.sum(anno) == 1
class_idx = np.where(anno)[0][0]
out_dir_curr = os.path.join(out_dir, class_names[class_idx])
util.mkdir(out_dir_curr)
features = np.load(npy_file)
out_shape_curr = features.shape[0]
vid_name = os.path.split(npy_file)[1]
vid_name = vid_name[:vid_name.rindex('.')]
sim_mat = get_similarity(features)
gt_vals, det_times = get_gt_vector(vid_name, out_shape_curr, class_idx)
# idx_pos = np.where(gt_vals>0)[0]
idx_pos = gt_vals > 0
idx_neg = gt_vals < 1
# print idx_pos
sim_pos_all = []
sim_neg_all = []
for idx_idx_curr, idx_curr in enumerate(np.where(idx_pos)[0]):
sim_pos = sim_mat[idx_curr, idx_pos]
sim_neg = sim_mat[idx_curr, idx_neg]
sim_pos_all.append(sim_pos[np.newaxis, :])
sim_neg_all.append(sim_neg[np.newaxis, :])
# idx_pos_leave = np.in1d
# sim_rel = sim_mat[idx_curr, idx_pos]
# print sim_rel.shape
# print sim_rel
# print sim_rel[idx_idx_curr]
# print np.min(sim_rel), np.max(sim_rel), np.mean(sim_rel)
# sim_rel = sim_mat[idx_curr, :]
# print sim_rel.shape
# print np.min(sim_rel), np.max(sim_rel), np.mean(sim_rel)
sim_pos_all = np.concatenate(sim_pos_all, axis=0)
sim_neg_all = np.concatenate(sim_neg_all, axis=0)
# print sim_pos_all.shape
# print sim_neg_all.shape
sim_pos_mean = np.mean(sim_pos_all, axis=0)
sim_neg_mean = np.mean(sim_neg_all, axis=0)
pos_vals = np.zeros(gt_vals.shape)
pos_vals[gt_vals > 0] = sim_pos_mean
neg_vals = np.zeros(gt_vals.shape)
neg_vals[gt_vals < 1] = sim_neg_mean
arr_plot = [(det_times, curr_arr)
for curr_arr in [gt_vals, pos_vals, neg_vals]]
legend_entries = ['gt', 'pos', 'neg']
out_file_curr = os.path.join(out_dir_curr, vid_name + '.jpg')
title = vid_name
visualize.plotSimple(arr_plot,
out_file=out_file_curr,
title=title,
xlabel='time',
ylabel='max sim',
legend_entries=legend_entries)
print out_file_curr
# print sim_pos_mean
# print sim_neg_mean
# break
for class_name in class_names:
out_dir_curr = os.path.join(out_dir, class_name)
visualize.writeHTMLForFolder(out_dir_curr)
def train_model(out_dir_train,
train_file,
test_file,
data_transforms,
batch_size=None,
batch_size_val=None,
num_epochs=100,
save_after=20,
disp_after=1,
plot_after=10,
test_after=1,
lr=0.0001,
dec_after=100,
model_name='alexnet'):
util.mkdir(out_dir_train)
log_file = os.path.join(out_dir_train, 'log.txt')
plot_file = os.path.join(out_dir_train, 'loss.jpg')
log_arr = []
plot_arr = [[], []]
plot_val_arr = [[], []]
train_data = dataset.Horse_Image_Dataset(train_file,
data_transforms['train'])
test_data = dataset.Horse_Image_Dataset(test_file, data_transforms['val'])
if batch_size is None:
batch_size = len(train_data)
if batch_size_val is None:
batch_size_val = len(test_data)
train_dataloader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=True,
num_workers=0)
test_dataloader = torch.utils.data.DataLoader(test_data,
batch_size=batch_size_val,
shuffle=False,
num_workers=0)
class_weights = get_class_weights(util.readLinesFromFile(train_file))
torch.cuda.device(0)
iter_begin = 0
network = models.get('caps_alexnet_simple')
model = network.model.cuda()
# model.train(True)
# criterion = nn.CrossEntropyLoss(weight = torch.FloatTensor(class_weights).cuda())
optimizer = optim.Adam(network.get_lr_list(lr), lr=0)
if dec_after is not None:
exp_lr_scheduler = lr_scheduler.StepLR(optimizer,
step_size=dec_after,
gamma=0.1)
for num_epoch in range(num_epochs):
for num_iter_train, batch in enumerate(train_dataloader):
# print batch['image'].shape,torch.min(batch['image']),torch.max(batch['image'])
# im = np.transpose(batch['image'][0].numpy(),(1,2,0))
# im = batch['image'][0].numpy()
# print im.shape
# scipy.misc.imsave('../scratch/check.jpg',im)
# raw_input()
data = Variable(batch['image'].cuda())
one_hot = models.utils.one_hot_encode(batch['label'], 2)
loss_weights = torch.FloatTensor(
np.tile(
np.array(class_weights)[np.newaxis, :],
(one_hot.shape[0], 1)))
one_hot = torch.mul(one_hot, loss_weights)
labels = Variable(one_hot).cuda()
# labels = Variable(models.utils.one_hot_encode(batch['label'],2)).cuda()
output = model(data) # output from DigitCaps (out_digit_caps)
loss = model.loss(data, output,
labels) # pass in data for image reconstruction
loss.backward()
loss_iter = loss.data[0]
optimizer.step()
num_iter = num_epoch * len(train_dataloader) + num_iter_train
# num_iter +=1
plot_arr[0].append(num_iter)
plot_arr[1].append(loss_iter)
str_display = 'lr: %.6f, iter: %d, loss: %.4f' % (
optimizer.param_groups[-1]['lr'], num_iter, loss_iter)
log_arr.append(str_display)
print str_display
if num_iter % plot_after == 0 and num_iter > 0:
util.writeFile(log_file, log_arr)
if len(plot_val_arr[0]) == 0:
visualize.plotSimple([(plot_arr[0], plot_arr[1])],
out_file=plot_file,
title='Loss',
xlabel='Iteration',
ylabel='Loss',
legend_entries=['Train'])
else:
visualize.plotSimple([(plot_arr[0], plot_arr[1]),
(plot_val_arr[0], plot_val_arr[1])],
out_file=plot_file,
title='Loss',
xlabel='Iteration',
ylabel='Loss',
legend_entries=['Train', 'Val'])
if num_epoch % test_after == 0:
model.eval()
for num_iter_test, batch in enumerate(test_dataloader):
# data = Variable(batch['image'].cuda())
# labels = Variable(torch.LongTensor(batch['label']).cuda())
# loss = criterion(model(data), labels)
# loss_iter = loss.data[0]
data = Variable(batch['image'].cuda())
# labels = Variable(models.utils.one_hot_encode(batch['label'],2)).cuda()
one_hot = models.utils.one_hot_encode(batch['label'], 2)
loss_weights = torch.FloatTensor(
np.tile(
np.array(class_weights)[np.newaxis, :],
(one_hot.shape[0], 1)))
one_hot = torch.mul(one_hot, loss_weights)
labels = Variable(one_hot).cuda()
output = model(data) # output from DigitCaps (out_digit_caps)
loss = model.loss(
data, output,
labels) # pass in data for image reconstruction
# loss.backward()
loss_iter = loss.data[0]
optimizer.step()
# test_epoch = num_epoch/test_after
num_iter = num_epoch * len(train_dataloader) + num_iter_test
# +=1
#
plot_val_arr[0].append(num_iter)
plot_val_arr[1].append(loss_iter)
str_display = 'lr: %.6f, val iter: %d, val loss: %.4f' % (
optimizer.param_groups[-1]['lr'], num_iter, loss_iter)
log_arr.append(str_display)
print str_display
model.train(True)
if num_epoch % save_after == 0:
out_file = os.path.join(out_dir_train,
'model_' + str(num_epoch) + '.pt')
print 'saving', out_file
torch.save(model, out_file)
if dec_after is not None:
exp_lr_scheduler.step()
out_file = os.path.join(out_dir_train, 'model_' + str(num_epoch) + '.pt')
print 'saving', out_file
torch.save(model, out_file)
print plot_arr[0]
util.writeFile(log_file, log_arr)
if len(plot_val_arr[0]) == 0:
visualize.plotSimple([(plot_arr[0], plot_arr[1])],
out_file=plot_file,
title='Loss',
xlabel='Iteration',
ylabel='Loss',
legend_entries=['Train'])
else:
visualize.plotSimple([(plot_arr[0], plot_arr[1]),
(plot_val_arr[0], plot_val_arr[1])],
out_file=plot_file,
title='Loss',
xlabel='Iteration',
ylabel='Loss',
legend_entries=['Train', 'Val'])
def visualizing_attention():
model_file = '../experiments/graph_multi_video_multi_F_joint_train_normalize_True_True_non_lin_HT_sparsify_True_graph_size_2_deno_8_n_classes_20_in_out_2048_64_feat_dim_2048_64_method_cos_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropyMultiBranch_300_step_300_0.1_0.001_0.001_lw_0.5_0.5_ABS/model_299.pt'
model = torch.load(model_file).cuda()
model.eval()
train_data, test_train_data, test_data, n_classes, trim_preds = emb.get_data(
'ucf', 500, False, just_primary=False, gt_vec=False)
# test_data = train_data
# test_bool = False
# test_data.feature_limit = None
batch_size = 1
branch_to_test = 0
out_dir_meta = model_file[:model_file.rindex('.')]
out_dir_meta = out_dir_meta + '_visualizing_attention_max' + str(
branch_to_test)
util.mkdir(out_dir_meta)
print out_dir_meta
anno_file = test_data.anno_file
vid_names, annos = readTrainTestFile(anno_file)
test_dataloader = torch.utils.data.DataLoader(
test_data,
batch_size=batch_size,
collate_fn=test_data.collate_fn,
shuffle=False,
num_workers=1)
import torch.nn.functional as F
preds = []
labels = []
for idx_data, data in enumerate(test_dataloader):
gt_classes = np.where(annos[idx_data])[0]
vid_name = os.path.split(vid_names[idx_data])[1]
vid_name = vid_name[:vid_name.rindex('.')]
# out_dir_curr = os.path.join(out_dir_meta,vid_name)
# util.mkdir(out_dir_curr)
label = data['label'].cpu().data.numpy()
# affinity = model.get_similarity(data['features'],sparsify = True)
x_all, pmf = model(data['features'], branch_to_test=branch_to_test)
assert len(pmf) == 1
x_all = torch.cat([x_all_curr.unsqueeze(0) for x_all_curr in x_all], 0)
x_all = F.softmax(x_all, dim=1)
x_all = x_all.data.cpu().numpy()
# affinity = affinity.data.cpu().numpy()
test_bool = True
max_all = np.max(x_all, axis=1)
#
for gt_class in gt_classes:
class_name_curr = class_names[gt_class]
print class_name_curr
out_dir_curr = os.path.join(out_dir_meta, class_name_curr)
util.mkdir(out_dir_curr)
# affinity_copy = np.array(affinity)
x_rel = x_all[:, gt_class]
thresh = np.max(x_rel) - (np.max(x_rel) - np.min(x_rel)) * 0.5
gt_vec, _ = get_gt_vector(vid_name,
x_rel.shape[0],
gt_class,
test=test_bool)
if np.sum(gt_vec) == 0:
print 'we got an anno problem', vid_name
continue
out_file_curr = os.path.join(out_dir_curr, vid_name + '.jpg')
# bin_keep = gt_vec.astype(int)
# bin_keep_rot = np.roll(bin_keep, 1)
# bin_keep_rot[0] = 0
# diff = bin_keep - bin_keep_rot
# # diff[-3]=1
# idx_start_all = list(np.where(diff==1)[0])
# idx_end_all = list(np.where(diff==-1)[0])
# idx_borders = np.array(idx_start_all+idx_end_all)
# # print idx_borders
# # raw_input()
# affinity_copy[:,idx_borders]=np.max(affinity_copy)
# affinity_copy[idx_borders,:]=np.max(affinity_copy)
gt_vec = gt_vec * np.max(x_rel)
x_axis = range(x_rel.size)
thresh = thresh * np.ones(x_rel.shape)
# out_file_curr = os.path.join(out_dir_curr,'det_confs_'+class_names[gt_class]+'.jpg')
visualize.plotSimple(
[(x_axis, x_rel), (x_axis, gt_vec), (x_axis, thresh),
(x_axis, max_all)],
out_file=out_file_curr,
title=class_names[gt_class],
xlabel='time',
ylabel='det conf',
legend_entries=['Det', 'GT', 'Thresh', 'Attn'])
# out_file_mat = os.path.join(out_dir_curr,'mat_'+class_names[gt_class]+'.jpg')
# visualize.saveMatAsImage(affinity_copy, out_file_mat)
preds.append(F.softmax(pmf[0]).data.cpu().numpy())
labels.append(label)
visualize.writeHTMLForFolder(out_dir_curr)
labels = np.concatenate(labels, axis=0)
preds = np.concatenate(preds, axis=0)
labels[labels > 0] = 1
accuracy = sklearn.metrics.average_precision_score(labels, preds)
print accuracy
def visualizing_threshes():
anno_file = '../data/activitynet/train_test_files/val.txt'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_True_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00__cwOld_MulNumClasses_numSim_64/results_model_249_original_class_0_0.5_-2/outf'
# out_dir = '../scratch/looking_at_anet/viz_pred_gt'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_249_original_class_0_0.5_-2/outf'
# dataset = 'anet'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_static_median_mean_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_249_original_class_0_0.5_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_249_original_class_-1_-4_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_3.00_1.00_1.00_changingSparsityAbs_128/results_model_249_original_class_-1_-4_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_fs_diff_changingSparsityAbs_128/results_model_149_original_class_-1_-4_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_8_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00__classicwithsig_128/results_model_99_original_class_0_-0.1_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_1_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_249_original_class_-1_-0.1_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_self_determination_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_mindeno8_changingSparsityAbs_128/results_model_249_original_class_-1_-2_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_random_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_149_original_class_-1_-0.1_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_random_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_128/results_model_149_original_class_-1_0.5_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_random_n_classes_100_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_activitynet/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_weighted_changingSparsityAbs_128/results_model_49_original_class_-1_-2_-2/outf'
# dataset = 'anet'
anno_file = '../data/ucf101/train_test_files/test.txt'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_0/results_model_249_original_class_0_0.5_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_0/results_model_249_original_class_0.0_0.5_-2/outf'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_500_step_500_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_0/results_model_249_original_class_0.0_0.5_-2/outf'
res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_False_graph_sum_True_deno_8_n_classes_20_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_ucf/all_classes_False_just_primary_False_limit_None_cw_False_BinaryCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_hardtanh01_changingSparsityAbs_0/results_model_249_original_class_-1_-0.1_-2/outf'
dataset = 'ucf'
fps_stuff = 16. / 25.
threshold = 0.75
anno_file = '../data/charades/train_test_files/i3d_charades_both_test.txt'
# res_dir = '../experiments/graph_multi_video_with_L1_retF/graph_multi_video_with_L1_retF_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_8_n_classes_157_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_charades_i3d_charades_both/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_numSim_128/results_model_249_original_class_0_-0.1_-2/outf'
res_dir = '../experiments/graph_multi_video_with_L1_retF_tanh/graph_multi_video_with_L1_retF_tanh_aft_nonlin_RL_L2_non_lin_None_sparsify_percent_0.5_graph_size_2_sigmoid_True_graph_sum_True_deno_8_n_classes_157_in_out_2048_1024_feat_dim_2048_1024_feat_ret_True_method_cos_charades_i3d_charades_both/all_classes_False_just_primary_False_limit_None_cw_False_MultiCrossEntropyMultiBranchWithL1_CASL_250_step_250_0.1_0.001_0.001_0.001_lw_1.00_1.00_1.00_changingSparsityAbs_numSim_128/results_model_249_original_class_justraw_0_-0.9_-2/outf'
dataset = 'charades'
# fps_stuff = 1./6.
# out_dir = '../scratch/looking_at_'+dataset+'/viz_pred_gt_percent_abs_0.5_thresh_ztest_per_class_all'
# out_dir = '../scratch/looking_at_'+dataset+'/viz_pred_gt_percent_abs_0.5_thresh_otsu_individual'
# out_dir = '../scratch/looking_at_'+dataset+'/viz_pred_median_mean_thresh_otsu_individual'
out_dir = '../scratch/looking_at_' + dataset + '/viz_pred_percent_0.5_mce_tanh'
# _bce_3_1_1_249'
util.makedirs(out_dir)
# anno_npz = '../data/activitynet/gt_npys/val_pruned.npz'
out_files = glob.glob(os.path.join(res_dir, '*.npy'))
out_files = np.array(out_files)
# class_thresholds = get_z_test_all(out_files, threshold)
anno_files, labels = read_anno_file(anno_file)
anno_jnames = np.array(
[os.path.split(anno_file)[1] for anno_file in anno_files])
out_jnames = np.array(
[os.path.split(out_file)[1] for out_file in out_files])
num_classes = labels.shape[1]
bin_out_files = []
for class_idx in range(num_classes):
rel_bin = labels[:, class_idx] > 0
rel_anno_jnames = anno_jnames[rel_bin]
rel_bin_out_files = np.in1d(out_jnames, rel_anno_jnames)
bin_out_files.append(rel_bin_out_files)
# print len(bin_out_files), bin_out_files[0].shape, np.sum(bin_out_files[0])
if dataset == 'anet':
gt_vid_names, gt_class_names, gt_time_intervals = et.load_activitynet_gt(
False)
class_names = globals.class_names_activitynet
elif dataset == 'ucf':
gt_vid_names, gt_class_names, gt_time_intervals = et.load_ucf_gt(False)
class_names = globals.class_names
elif dataset == 'charades':
# gt_vid_names, gt_class_names, gt_time_intervals = et.load_ucf_gt(False)
# class_names = globals.class_names
class_names = globals.class_names_charades
gt_vid_names, gt_class_names, gt_time_intervals = et.load_charades_gt(
False)
# overlap_thresh_all = np.arange(0.1,0.2,0.1)
# aps = np.zeros((len(class_names)+1,1))
# fps_stuff = 16./25.
gt_vid_names = np.array(gt_vid_names)
gt_class_names = np.array(gt_class_names)
gt_time_intervals = np.array(gt_time_intervals)
# get threshold for each class
# class_thresholds = get_min_max_all(out_files, threshold)
# print class_thresholds.shape
# return
# return
# class_thresholds = get_threshold_val(out_files, bin_out_files, threshold = threshold)
n_bins = 10
for class_idx, bin_class in enumerate(bin_out_files):
rel_files = out_files[bin_class]
rel_class_name = class_names[class_idx]
out_dir_curr = os.path.join(out_dir, rel_class_name)
util.mkdir(out_dir_curr)
bin_class = gt_class_names == rel_class_name
for rel_file in rel_files:
pred_vals = np.load(rel_file)[:, class_idx]
max_det_conf = np.max(pred_vals)
min_det_conf = np.min(pred_vals)
old_thresh = min_det_conf + (max_det_conf - min_det_conf) / 2.
# hist, bin_edges = np.histogram(pred_vals, n_bins)
# otsu_val = otsu_method(hist)
# print bin_edges, min_det_conf, max_det_conf
# print otsu_val, np.argmax(otsu_val)
# idx_max = np.argmax(otsu_val)
# print bin_edges[idx_max]
# new_thresh = (bin_edges[idx_max]+bin_edges[idx_max+1])/2
# print new_thresh
# raw_input()
# # bin_edges = bin_edges[1:]
# print hist.shape, bin_edges.shape, otsu_val.shape
# new_thresh = bin_edges[np.argmax(otsu_val)]
new_thresh = get_otsu_thresh(pred_vals, n_bins)
# new_thresh = get_z_test(pred_vals)
# new_thresh = class_thresholds[class_idx]
out_shape_curr = len(pred_vals)
rel_name = os.path.split(rel_file)[1][:-4]
bin_vid = gt_vid_names == rel_name
rel_gt_time = gt_time_intervals[np.logical_and(bin_vid, bin_class)]
# print pred_vals.shape
# print rel_gt_time
# print rel_name
# raw_input()
det_times = np.array(range(0, out_shape_curr)) * fps_stuff
gt_vals = np.zeros(det_times.shape)
for gt_time_curr in rel_gt_time:
idx_start = np.argmin(np.abs(det_times - gt_time_curr[0]))
idx_end = np.argmin(np.abs(det_times - gt_time_curr[1]))
gt_vals[idx_start:idx_end] = max_det_conf
gt_vals[gt_vals == 0] = min_det_conf
out_file_viz = os.path.join(out_dir_curr, rel_name + '.jpg')
out_file_hist = os.path.join(out_dir_curr, rel_name + '_hist.jpg')
plot_arr = [(det_times, pred_vals), (det_times, gt_vals)]
plot_arr += [(det_times, np.ones(det_times.shape) * old_thresh),
(det_times, np.ones(det_times.shape) * new_thresh)]
legend_entries = ['Pred', 'GT']
legend_entries += ['Old', 'New ']
title = 'det conf over time'
# print out_file_viz
visualize.hist(pred_vals,
out_file_hist,
bins=n_bins,
normed=True,
xlabel='Value',
ylabel='Frequency',
title=title)
visualize.plotSimple(plot_arr,
out_file=out_file_viz,
title=title,
xlabel='Time',
ylabel='Detection Confidence',
legend_entries=legend_entries)
# raw_input()
visualize.writeHTMLForFolder(out_dir_curr)
print out_dir_curr
def main():
out_dir_figures = '../experiments/figures/mmi'
util.makedirs(out_dir_figures)
dir_exp_meta = '../experiments/khorrami_capsule_7_3_bigclass3'
pre_split = 'mmi_96_'
post_split = '_reconstruct_True_True_all_aug_margin_False_wdecay_0_300_exp_0.96_350_1e-06_0.001_0.001_0.001_lossweights_1.0_100.0'
# '_reconstruct_True_True_all_aug_margin_False_wdecay_0_300_exp_0.96_350_1e-06_0.001_0.001_0.001'
str_titles = ['Hard Training', 'Easy Training']
range_splits = [0, 1]
str_titles = ['Easy Training']
range_splits = [1]
range_tests = range(0, 300, 10) + [299]
test_post = ''
accus_us = []
accus_hard = get_values_from_logs(dir_exp_meta, pre_split, post_split,
range_splits, range_tests, test_post)
test_post = '_easy'
accus_easy = get_values_from_logs(dir_exp_meta, pre_split, post_split,
range_splits, range_tests, test_post)
accus_us = [accus_hard, accus_easy]
dir_exp_meta = '../experiments/khorrami_ck_96_caps_bl'
# 0_train_test_files_khorrami_ck_96_300_exp_0.96_350_1e-06_0.001_0.001/
# '../experiments/khorrami_capsule_7_3_bigclass3'
pre_split = 'mmi_96_'
# 'mmi_96_'
# /mmi_96_1
post_split = '_train_test_files_khorrami_ck_96_300_exp_0.96_350_1e-06_0.001_0.001'
test_post = ''
accus_hard = get_values_from_logs(dir_exp_meta, pre_split, post_split,
range_splits, range_tests, test_post)
test_post = '_easy'
accus_easy = get_values_from_logs(dir_exp_meta, pre_split, post_split,
range_splits, range_tests, test_post)
accus_them = [accus_hard, accus_easy]
# print accus_hard
# print accus_easy
# raw_input()
str_tests = ['Hard Test', 'Easy Test']
for fold_curr, str_title in enumerate(str_titles):
plot_vals = []
legend_entries = []
for idx_ease, str_test in enumerate(str_tests):
# for idx_us,(us,them) in enumerate( zip(accus_us,accus_them)):
plot_vals.append((range_tests, accus_us[idx_ease][fold_curr]))
plot_vals.append((range_tests, accus_them[idx_ease][fold_curr]))
legend_entries.extend(['Ours ' + str_test, 'BL ' + str_test])
figure_name = str_title.lower().replace(' ', '_') + '.jpg'
out_file_curr = os.path.join(out_dir_figures, figure_name)
visualize.plotSimple(plot_vals,
out_file=out_file_curr,
title=str_title,
xlabel='Epoch',
ylabel='Accuracy',
legend_entries=legend_entries)
print out_file_curr
return
dir_exp_meta = '../experiments/khorrami_capsule_7_33'
pre_split = 'ck_96_'
post_split = '_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001'
range_splits = range(10)
print_accuracy(dir_exp_meta,
pre_split,
post_split,
range_splits,
log='log.txt')
return
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu'
pre_split = 'oulu_single_'
post_split = '_all_aug_nopool_300_step_150_0.1_0.001'
num_splits = 10
model_num = 299
dir_exp_meta = '../experiments/caps_heavy_48'
pre_split = 'oulu_single_'
post_split = '_all_aug_nopool_200_step_100_0.1_0.001'
num_splits = 10
model_num = 199
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_class_weights'
# pre_split = 'oulu_single_im_'
# post_split = '_all_aug_nopool_300_step_150_0.1_0.01'
# num_splits = 1
# model_num = 299
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_class_weights'
# pre_split = 'oulu_single_im_'
# post_split = '_all_aug_nopool_300_step_150_0.1_0.0005'
# num_splits = 1
# model_num = 299
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2'
pre_split = 'oulu_single_im_'
post_split = '_all_aug_nopool_300_step_150_0.1_0.001'
num_splits = 1
model_num = 299
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_1_init/'
pre_split = 'oulu_three_im_no_neutral_just_strong_'
post_split = '_all_aug_max_300_step_300_0.1_0.001'
num_splits = 10
model_num = 299
num_classes = 6
class_labels = [
'Anger', 'Disgust', 'Fear', 'Happiness', 'Sadness', 'Surprise'
]
######
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_3_init_correct_out'
num_splits = range(10)
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_2_init_correct_out'
num_splits = [0, 1]
dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_1_init_correct_out'
num_splits = range(10)
pre_split = 'oulu_three_im_no_neutral_just_strong_'
post_split = '_all_aug_max_300_step_300_0.1_0.001'
model_num = 299
######
dir_exp_meta = '../experiments/oulu_r3_hopeful'
num_splits = range(10)
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_2_init_correct_out'
# num_splits = [0,1]
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_1_init_correct_out'
# num_splits = range(10)
pre_split = 'oulu_three_im_no_neutral_just_strong_True_'
post_split = '_wdecay_all_aug_max_500_step_500_0.1_0.0001'
# for model_num in ['499','499_center']:
# ,'bestVal','bestVal_center']:
# print 'MODEL TYPE',model_num
# log = 'log_test_center.txt' if model_num.endswith('_center') else 'log_test.txt'
# get_per_label_accuracy(dir_exp_meta,pre_split,post_split,num_splits,model_num,class_labels)
# print_accuracy(dir_exp_meta,pre_split,post_split,num_splits,log = log)
# view_loss_curves(dir_exp_meta,pre_split,post_split,num_splits,model_num)
####
dir_exp_meta = '../experiments/oulu_vgg_r1_noinit_preprocessed'
num_splits = range(10)
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_2_init_correct_out'
# num_splits = [0,1]
# dir_exp_meta = '../experiments/khorrami_caps_k7_s3_oulu_spread_0.2_vl_gray_r_1_init_correct_out'
# num_splits = range(10)
pre_splits = ['oulu_three_im_no_neutral_just_strong_False_'] * 3
model_names = [
'vgg_capsule_disfa', 'vgg_capsule_disfa_bigprimary',
'vgg_capsule_disfa_bigclass'
]
post_splits = [
'_' + model_name + '_all_aug_wdecay_0_50_step_50_0.1_1e-05_0.0001'
for model_name in model_names
]
for pre_split, post_split in zip(pre_splits, post_splits):
for model_num in ['49']:
print 'MODEL TYPE', model_num
get_per_label_accuracy(dir_exp_meta, pre_split, post_split,
num_splits, model_num, class_labels)
print_accuracy(dir_exp_meta, pre_split, post_split, num_splits)
view_loss_curves(dir_exp_meta, pre_split, post_split, num_splits,
model_num)
def get_distance_from_perfect(model_file, graph_num):
out_dir_meta = model_file[:model_file.rindex('.')]
out_dir_meta_meta = out_dir_meta + '_graph_etc'
out_dir_meta = out_dir_meta_meta + '_' + str(graph_num)
out_dir_viz = out_dir_meta + '_dist_perfectG'
print out_dir_viz
util.mkdir(out_dir_viz)
assert os.path.exists(out_dir_meta)
vid_files = glob.glob(os.path.join(out_dir_meta, '*validation*.npz'))
class_collations = [[] for idx in range(len(class_names))]
class_collations_pos = [[] for idx in range(len(class_names))]
viz = True
threshes = np.arange(0.1, 1.1, 0.1)
print threshes
for vid_file in vid_files:
print vid_file
npz_data = np.load(vid_file)
vid_file = os.path.split(vid_file)[1]
affinity = npz_data['affinity']
gt_vecs = npz_data['gt_vecs']
gt_classes = npz_data['gt_classes']
if viz:
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_' +
str(graph_num) + '.jpg')
visualize.saveMatAsImage(affinity, out_file)
plotter = []
legend_entries = []
for gt_idx, gt_class in enumerate(gt_classes):
gt_vec = gt_vecs[gt_idx]
gt_vec = gt_vec / np.max(gt_vec)
gt_vec = gt_vec * (gt_idx + 1)
x_axis = range(gt_vec.size)
plotter.append((x_axis, gt_vec))
legend_entries.append(class_names[gt_class])
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_gt.jpg')
visualize.plotSimple(plotter,
out_file=out_file,
xlabel='time',
ylabel='',
legend_entries=legend_entries)
for idx_gt, gt_vec in enumerate(gt_vecs):
gt_class = gt_classes[idx_gt]
class_name = class_names[gt_class]
gt_vec = gt_vec[:, np.newaxis]
perfectG = np.dot(gt_vec, gt_vec.T)
aff = np.array(affinity)
aff_just_pos = aff * perfectG
diff = get_l2_diff(aff, perfectG, threshes)
diff_pos = get_l2_diff(aff_just_pos, perfectG, threshes)
class_collations[gt_class].append(diff)
class_collations_pos[gt_class].append(diff_pos)
plotter = [(threshes, diff), (threshes, diff_pos)]
legend_entries = ['All', 'Pos']
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_' +
class_name + '_diff.jpg')
visualize.plotSimple(plotter,
out_file=out_file,
xlabel='Thresh',
ylabel='Diff',
legend_entries=legend_entries)
if viz:
out_file = os.path.join(
out_dir_viz, vid_file[:vid_file.rindex('.')] + '_' +
class_name + '_perfectG.jpg')
visualize.saveMatAsImage(perfectG, out_file)
visualize.writeHTMLForFolder(out_dir_viz)
for idx_class in range(len(class_names)):
class_name = class_names[idx_class]
cc = np.array(class_collations[idx_class])
ccp = np.array(class_collations_pos[idx_class])
cc = np.mean(cc, axis=0)
ccp = np.mean(ccp, axis=0)
plotter = [(threshes, cc), (threshes, ccp)]
legend_entries = ['All', 'Pos']
out_file = os.path.join(out_dir_viz,
'average_' + class_name + '_diff.jpg')
visualize.plotSimple(plotter,
out_file=out_file,
title=class_name,
xlabel='Thresh',
ylabel='Diff',
legend_entries=legend_entries)
visualize.writeHTMLForFolder(out_dir_viz)
def overfitting():
out_dir = '../experiments/figures/overfitting'
util.makedirs(out_dir)
# dirs = []
# dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_31'
# dir_curr = os.path.join(dir_meta,'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001')
# dirs.append(dir_curr)
# dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_33'
# dir_r3 = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001'
# dir_r3_lw_eq = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001'
# dir_r3_lw_b = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_100.0'
dirs = []
# dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_31'
# dir_curr = os.path.join(dir_meta,'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001')
# dirs.append(dir_curr)
dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_33'
dir_r3 = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001'
dir_r3_lw_eq = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_1.0'
dir_r3_lw_b = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_100.0'
dir_r3_do = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_3_with_dropout3/oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.5'
dirs_to_pend = [dir_r3, dir_r3_lw_eq, dir_r3_lw_b]
for dir_curr in dirs_to_pend:
dir_curr = os.path.join(dir_meta, dir_curr)
dirs.append(dir_curr)
dirs.append(dir_r3_do)
window = 10
val_lim = 600
epoch_range = range(window - 1, val_lim)
out_file = os.path.join(out_dir, 'val_accuracy_9_do.png')
xAndYs = []
legend_entries = ['R3+0', 'R3+1e-7', 'R3+1e-5', 'R3+DO']
for dir_curr in dirs:
log_file_curr = os.path.join(dir_curr, 'log.txt')
val_losses = [
line_curr for line_curr in util.readLinesFromFile(log_file_curr)
if 'val accuracy' in line_curr
]
val_losses = [
float(line_curr.split(' ')[-1]) for line_curr in val_losses
]
val_losses = val_losses[:val_lim]
print len(val_losses)
val_losses = np.convolve(val_losses,
np.ones((window, )) / window,
mode='valid')
xAndYs.append((epoch_range, val_losses))
visualize.plotSimple(xAndYs,
out_file=out_file,
xlabel='Epoch',
ylabel='Validation Accuracy',
legend_entries=legend_entries,
ylim=[0.6, 0.8],
outside=True)
def trying_it_out():
# dir_graphs = '../experiments/graph_multi_video_multi_F_joint_train_normalize_True_True_non_lin_HT_sparsify_True_graph_size_2_deno_8_n_classes_20_in_out_2048_64_feat_dim_2048_64_method_cos_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropyMultiBranch_300_step_300_0.1_0.001_0.001_lw_0.5_0.5_ABS/model_299_graph_etc'
# out_dir_meta = '../scratch/spanning_shenanigans'
# util.mkdir(out_dir_meta)
model_file = '../experiments/graph_multi_video_multi_F_joint_train_normalize_True_True_non_lin_HT_sparsify_True_graph_size_2_deno_8_n_classes_20_in_out_2048_64_64_feat_dim_2048_64_64_64_gk_2_method_cos_ucf/all_classes_False_just_primary_False_limit_500_cw_True_MultiCrossEntropyMultiBranch_300_step_300_0.1_0.001_0.001_lw_1_1_1_ABS/model_299.pt'
graph_num = 1
dir_graphs = model_file[:model_file.rindex('.')]
dir_graphs = dir_graphs + '_graph_etc'
if graph_num is not None:
dir_graphs += '_' + str(graph_num)
# out_dir_meta = '../scratch/spanning_shenanigans'
out_dir_meta = os.path.join(
os.path.split(dir_graphs)[0], 'spanning_shenanigans')
util.mkdir(out_dir_meta)
vid_name = 'video_test_0000273'
# oracle = False
# idx = 1
# thresh = 0.85
oracle = False
thresh = 0.9999
branch_pred = 1
idx = 0
str_dir = '_'.join([str(val) for val in [vid_name, oracle, idx, thresh]])
out_dir_curr = os.path.join(out_dir_meta, str_dir)
util.mkdir(out_dir_curr)
npz_file = os.path.join(dir_graphs, vid_name + '.npz')
npz_data = np.load(npz_file)
gt_vecs = npz_data['gt_vecs']
affinity = npz_data['affinity']
x_all = npz_data['x_all']
gt_classes = npz_data['gt_classes']
gt_vec = gt_vecs[idx]
gt_vec[gt_vec > 0] = 1
gt_class = gt_classes[idx]
nodes_kept, edge_threshes, gt_count, _ = get_spanned_nodes(
gt_class,
x_all[branch_pred],
affinity,
thresh,
deno=8,
oracle=oracle,
gt_vec=gt_vec)
precision_all = []
recall_all = []
for idx_n in range(len(nodes_kept)):
idx_str = str(idx_n)
idx_str = '0' * (4 - len(idx_str)) + idx_str
out_file_curr = os.path.join(out_dir_curr, idx_str + '.jpg')
nodes_kept_curr = nodes_kept[:idx_n + 1]
pred = np.zeros(gt_vec.shape)
pred[nodes_kept_curr] = 1
precision = sklearn.metrics.precision_score(gt_vec, pred, labels=[1])
recall = sklearn.metrics.recall_score(gt_vec, pred, labels=[1])
# print precision, recall
# raw_input()
precision_all.append(precision)
recall_all.append(recall)
x_axis = np.array(range(gt_vec.size))
title_curr = ' '.join(
['prec', '%.2f' % precision, 'rec',
'%.2f' % recall])
visualize.plotSimple([(x_axis, pred), (x_axis, gt_vec)],
out_file=out_file_curr,
title=title_curr,
xlabel='time',
ylabel='det conf',
legend_entries=['Det', 'GT'])
out_file_curr = os.path.join(out_dir_curr, 'prec_rec.jpg')
visualize.plotSimple([(recall_all, precision_all)],
out_file=out_file_curr,
title='prec_rec',
xlabel='Recall',
ylabel='Precision')
visualize.writeHTMLForFolder(out_dir_curr)
print out_dir_curr
def overfitting_do():
out_dir = '../experiments_dropout/figures/overfitting'
util.makedirs(out_dir)
dir_meta_r1 = '../experiments_dropout/khorrami_capsule_7_3_bigclass_with_dropout_1'
dir_meta_r3 = '../experiments_dropout/khorrami_capsule_7_3_bigclass_with_dropout_3'
dirs_post = [
'oulu_96_train_test_files_preprocess_vl_9_reconstruct_False_none_600_step_600_0.1_0.001_0.001_0.0',
'oulu_96_train_test_files_preprocess_vl_9_reconstruct_False_none_600_step_600_0.1_0.001_0.001_0.5',
'oulu_96_train_test_files_preprocess_vl_9_reconstruct_False_hs_flip_600_step_600_0.1_0.001_0.001_0.0',
'oulu_96_train_test_files_preprocess_vl_9_reconstruct_False_hs_flip_600_step_600_0.1_0.001_0.001_0.5'
]
meta_legend_entries = ['R1', 'R3']
sub_legend_entries = ['None 0', 'None 0.5', 'HS+Flip 0', 'HS+Flip 0.5']
dirs = [
os.path.join(dir_meta_curr, dir_curr)
for dir_meta_curr in [dir_meta_r1, dir_meta_r3]
for dir_curr in dirs_post
]
legend_entries = [
meta_legend + ' ' + sub_legend for meta_legend in meta_legend_entries
for sub_legend in sub_legend_entries
]
# # dirs = []
# # dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_31'
# # dir_curr = os.path.join(dir_meta,'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001')
# # dirs.append(dir_curr)
# # dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_33'
# # dir_r3 = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001'
# # dir_r3_lw_eq = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001'
# # dir_r3_lw_b = 'oulu_96_three_im_no_neutral_just_strong_False_0_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_100.0'
# dirs = []
# # dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_31'
# # dir_curr = os.path.join(dir_meta,'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001')
# # dirs.append(dir_curr)
# dir_meta = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_33'
# dir_r3 = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001'
# dir_r3_lw_eq = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_1.0'
# dir_r3_lw_b = 'oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_True_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.001_lossweights_1.0_100.0'
# dir_r3_do = '../experiments/showing_overfitting_justhflip_khorrami_capsule_7_3_with_dropout3/oulu_96_three_im_no_neutral_just_strong_False_9_reconstruct_False_True_all_aug_margin_False_wdecay_0_600_step_600_0.1_0.001_0.001_0.5'
# dirs_to_pend = [dir_r3,dir_r3_lw_eq,dir_r3_lw_b]
# for dir_curr in dirs_to_pend:
# dir_curr = os.path.join(dir_meta, dir_curr)
# dirs.append(dir_curr)
# dirs.append(dir_r3_do)
window = 10
val_lim = 600
epoch_range = range(window - 1, val_lim)
dirs = dirs[:4] + dirs[-2:]
out_file = os.path.join(out_dir, 'val_accuracy_9_do.png')
xAndYs = []
# legend_entries = ['R3+0','R3+1e-7','R3+1e-5','R3+DO']
for dir_curr in dirs:
log_file_curr = os.path.join(dir_curr, 'log.txt')
val_losses = [
line_curr for line_curr in util.readLinesFromFile(log_file_curr)
if 'val accuracy' in line_curr
]
val_losses = [
float(line_curr.split(' ')[-1]) for line_curr in val_losses
]
val_losses = val_losses[:val_lim]
print dir_curr, len(val_losses)
val_losses = np.convolve(val_losses,
np.ones((window, )) / window,
mode='valid')
xAndYs.append((epoch_range, val_losses))
visualize.plotSimple(xAndYs,
out_file=out_file,
xlabel='Epoch',
ylabel='Validation Accuracy',
legend_entries=legend_entries,
ylim=[0.6, 0.8],
outside=True)
def train_model(out_dir_train,
train_data,
test_data,
batch_size = None,
batch_size_val =None,
num_epochs = 100,
save_after = 20,
disp_after = 1,
plot_after = 10,
test_after = 1,
lr = 0.0001,
dec_after = 100,
model_name = 'alexnet',
criterion = nn.CrossEntropyLoss(),
gpu_id = 0,
num_workers = 0,
model_file = None,
epoch_start = 0):
util.mkdir(out_dir_train)
log_file = os.path.join(out_dir_train,'log.txt')
plot_file = os.path.join(out_dir_train,'loss.jpg')
log_arr = []
plot_arr = [[],[]]
plot_val_arr = [[],[]]
network = models.get(model_name)
# data_transforms = network.data_transforms
if model_file is not None:
# model = network.model
# else:
network.model = torch.load(model_file)
model = network.model
# train_data = dataset(train_file,data_transforms['train'])
# test_data = dataset(test_file,data_transforms['val'])
if batch_size is None:
batch_size = len(train_data)
if batch_size_val is None:
batch_size_val = len(test_data)
train_dataloader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=True,
num_workers=0)
test_dataloader = torch.utils.data.DataLoader(test_data,
batch_size=batch_size_val,
shuffle=False,
num_workers=num_workers)
torch.cuda.device(gpu_id)
model = model.cuda()
model.train(True)
optimizer = optim.SGD(network.get_lr_list(lr), lr=0, momentum=0.9)
if dec_after is not None:
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=dec_after, gamma=0.1)
for num_epoch in range(epoch_start,num_epochs):
for num_iter_train,batch in enumerate(train_dataloader):
data = Variable(batch['image'].cuda())
labels = Variable(torch.LongTensor(batch['label']).cuda())
optimizer.zero_grad()
loss = criterion(model(data), labels)
loss_iter = loss.data[0]
loss.backward()
optimizer.step()
num_iter = num_epoch*len(train_dataloader)+num_iter_train
plot_arr[0].append(num_iter); plot_arr[1].append(loss_iter)
str_display = 'lr: %.6f, iter: %d, loss: %.4f' %(optimizer.param_groups[-1]['lr'],num_iter,loss_iter)
log_arr.append(str_display)
print str_display
if num_iter % plot_after== 0 and num_iter>0:
util.writeFile(log_file, log_arr)
if len(plot_val_arr[0])==0:
visualize.plotSimple([(plot_arr[0],plot_arr[1])],out_file = plot_file,title = 'Loss',xlabel = 'Iteration',ylabel = 'Loss',legend_entries=['Train'])
else:
visualize.plotSimple([(plot_arr[0],plot_arr[1]),(plot_val_arr[0],plot_val_arr[1])],out_file = plot_file,title = 'Loss',xlabel = 'Iteration',ylabel = 'Loss',legend_entries=['Train','Val'])
if num_epoch % test_after == 0 :
model.eval()
predictions = []
labels_all = []
for num_iter_test,batch in enumerate(test_dataloader):
labels_all.append(batch['label'].numpy())
data = Variable(batch['image'].cuda())
labels = Variable(torch.LongTensor(batch['label']).cuda())
output = model(data)
out = output.data.cpu().numpy()
predictions.append(np.argmax(out,1))
loss = criterion(output, labels)
loss_iter = loss.data[0]
num_iter = num_epoch*len(train_dataloader)+num_iter_test
plot_val_arr[0].append(num_iter); plot_val_arr[1].append(loss_iter)
str_display = 'lr: %.6f, val iter: %d, val loss: %.4f' %(optimizer.param_groups[-1]['lr'],num_iter,loss_iter)
log_arr.append(str_display)
print str_display
labels_all = np.concatenate(labels_all)
predictions = np.concatenate(predictions)
accuracy = np.sum(predictions==labels_all)/float(labels_all.size)
str_display = 'val accuracy: %.4f' %(accuracy)
log_arr.append(str_display)
print str_display
model.train(True)
if num_epoch % save_after == 0:
out_file = os.path.join(out_dir_train,'model_'+str(num_epoch)+'.pt')
print 'saving',out_file
torch.save(model,out_file)
if dec_after is not None:
exp_lr_scheduler.step()
out_file = os.path.join(out_dir_train,'model_'+str(num_epoch)+'.pt')
print 'saving',out_file
torch.save(model,out_file)
# print plot_arr[0]
util.writeFile(log_file, log_arr)
if len(plot_val_arr[0])==0:
visualize.plotSimple([(plot_arr[0],plot_arr[1])],out_file = plot_file,title = 'Loss',xlabel = 'Iteration',ylabel = 'Loss',legend_entries=['Train'])
else:
visualize.plotSimple([(plot_arr[0],plot_arr[1]),(plot_val_arr[0],plot_val_arr[1])],out_file = plot_file,title = 'Loss',xlabel = 'Iteration',ylabel = 'Loss',legend_entries=['Train','Val'])
def viz_pascal():
filename = '../contextlocnet-master/data/with_det_scores_test.h5'
# out_dir = '../scratch/voc_2007_test_scores/det_viz'
# out_dir = '../scratch/voc_2007_test_scores/det_viz_smallest'
out_dir = '../scratch/voc_2007_test_scores/det_viz_biggest'
det_dir = '../scratch/voc_2007_test_scores/output_softmax'
util.mkdir(out_dir)
f = h5py.File(filename, 'r')
labels = f['labels']
label_strs = [
'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person',
'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'
]
legend_entries = [
'608x800', '496x656', '400x544', '720x960', '864x1152', '608x800-h',
'496x656-h', '400x544-h', '720x960-h', '864x1152-h'
]
legend_entries_meta = [
'608x800', '608x800-h', '496x656', '496x656-h', '400x544', '400x544-h',
'720x960', '720x960-h', '864x1152', '864x1152-h'
]
# '496x656-h',
# '400x544-h',
# '720x960-h',
# '864x1152-h']
scales_to_keep = [9]
for idx_test, label in enumerate(labels):
gt_classes = np.where(label > 0)[0]
filename = str(idx_test + 1)
det_file = os.path.join(det_dir, filename + '.npy')
dets_curr = np.load(det_file)
for gt_class in gt_classes:
label_str = label_strs[gt_class]
print label_str
out_dir_curr = os.path.join(out_dir, label_str)
util.mkdir(out_dir_curr)
x = range(dets_curr.shape[2])
xAndYs = [(x, dets_curr[scale_idx, gt_class, :])
for scale_idx in scales_to_keep]
# range(0,dets_curr.shape[0],2)]
out_file = os.path.join(
out_dir_curr, '0' * (5 - len(filename)) + filename + '.jpg')
print out_file
legend_entries = [
legend_entries_meta[idx] for idx in scales_to_keep
]
visualize.plotSimple(xAndYs,
out_file=out_file,
title='Det Branch Output Multiscale',
xlabel='ROIs',
ylabel='Det Conf',
legend_entries=legend_entries)
# print np.sum(dets_curr[:,gt_class,:],axis=1)
# print det_curr.shape
# raw_input()
for label_str in label_strs:
out_dir_curr = os.path.join(out_dir, label_str)
visualize.writeHTMLForFolder(out_dir_curr)
