dsets_per_type = 1
out_result_dir = os.path.join(rd.root_dir, 'plots_results')
# Steps
make_plots_metrics = False
make_tables = False
make_frames = True
# Run plots, metrics, etc..
if (make_plots_metrics):
for key in rd.res_dirs_dict_ksp.keys():
for dset in range(len(rd.res_dirs_dict_ksp[key])):
confs = [rd.confs_dict_ksp[key][dset][g] for g in range(5)]
logger = logging.getLogger('test_self_learning')
if (rd.out_dirs_dict_ksp[key][dset] is not None):
out_dir = os.path.join(rd.root_dir, 'learning_exps',
rd.out_dirs_dict_ksp[key][dset])
else:
now = datetime.datetime.now()
dateTime = now.strftime("%Y-%m-%d_%H-%M-%S")
out_dir = os.path.join(
confs[0].dataOutRoot, 'learning_exps',
'learning_multigaze_' + confs[0].seq_type + '_' + dateTime)
if (not os.path.exists(out_dir)):
os.mkdir(out_dir)
utls.setup_logging(out_dir)
pksp.main(out_dir, confs, logger=logger)
def make_training_patches_and_fit(arg_cfg, out_dir=None):
data = dict()
# Update config
cfg_dict = cfg.cfg()
arg_cfg['seq_type'] = cfg.datasetdir_to_type(arg_cfg['ds_dir'])
cfg_dict.update(arg_cfg)
conf = cfg.Bunch(cfg_dict)
conf.out_dir_prefix = 'exp_vilar'
# Write config to result dir
if (out_dir is None):
if ('dataOutDir' not in arg_cfg):
conf.dataOutDir = utls.getDataOutDir(conf.dataOutRoot, conf.ds_dir,
conf.resultDir,
conf.out_dir_prefix,
conf.testing)
else:
conf.dataOutDir = out_dir
conf.myGaze_fg = utls.readCsv(
os.path.join(conf.root_path, conf.ds_dir, conf.locs_dir,
conf.csvFileName_fg))
# Set logger
utls.setup_logging(conf.dataOutDir)
logger = logging.getLogger('vilar')
logger.info('---------------------------')
logger.info('Extracting training a patches and fit on: ' + conf.ds_dir)
logger.info('type of sequence: ' + conf.seq_type)
logger.info('gaze filename: ' + conf.csvFileName_fg)
logger.info('Result dir:')
logger.info(conf.dataOutDir)
logger.info('n_frames : ' + str(conf.vilar_n_frames))
logger.info('(gamma,C) = (' + str(conf.gamma) + ', ' + str(conf.C) + ')')
logger.info(conf.dataOutDir)
logger.info('---------------------------')
# Make frame file names
gt_dir = os.path.join(conf.root_path, conf.ds_dir, conf.truth_dir)
gtFileNames = utls.makeFrameFileNames(conf.frame_prefix, conf.frameDigits,
conf.truth_dir, conf.root_path,
conf.ds_dir, conf.frame_extension)
conf.frameFileNames = utls.makeFrameFileNames(conf.frame_prefix,
conf.frameDigits,
conf.frameDir,
conf.root_path, conf.ds_dir,
conf.frame_extension)
# conf.myGaze_fg = utls.readCsv(conf.csvName_fg)
conf.myGaze_fg = utls.readCsv(
os.path.join(conf.root_path, conf.ds_dir, conf.locs_dir,
conf.csvFileName_fg))
# conf.myGaze_bg = utls.readCsv(conf.csvName_bg)
gt_positives = utls.getPositives(gtFileNames)
conf.precomp_desc_path = os.path.join(conf.dataOutRoot, conf.ds_dir,
conf.feats_dir)
my_dataset = ds.DatasetVilar(conf)
with open(os.path.join(conf.dataOutDir, 'cfg.yml'), 'w') as outfile:
yaml.dump(conf, outfile, default_flow_style=True)
# Extract seen and unseen patches (to fit SVC)
if (os.path.exists(
os.path.join(conf.dataOutDir, 'vilar', 'vilar_seen_patches_df.p'))
& os.path.exists(
os.path.join(conf.dataOutDir, 'vilar',
'vilar_unseen_patches_df.p'))):
logger.info('seen and unseen patches already computed. Skipping.')
else:
my_dataset.calc_training_patches(save=True)
if (not os.path.exists(os.path.join(conf.dataOutDir, 'clf.p'))):
my_dataset.load_patches()
X_pos = df_to_mat(my_dataset.seen_patches_df)
X_neg = df_to_mat(my_dataset.unseen_patches_df)
X_train = np.concatenate((X_pos, X_neg))
y_pos = np.ones(df_to_mat(my_dataset.seen_patches_df).shape[0])
y_neg = np.zeros(df_to_mat(my_dataset.unseen_patches_df).shape[0])
y_train = np.concatenate((y_pos, y_neg))
clf = SVC(gamma=conf.gamma,
C=conf.C,
class_weight='balanced',
verbose=True)
logger.info('fitting')
clf.fit(X_train, y_train)
with open(os.path.join(conf.dataOutDir, 'clf.p'), 'wb') as f:
pk.dump(clf, f)
logger.info('Saved classifier.')
else:
with open(os.path.join(conf.dataOutDir, 'clf.p'), 'rb') as f:
clf = pk.load(f)
return conf, clf
def predict(conf, clf, out_dir=None, n_jobs=1):
"""
Parallel extraction and prediction of patches on every pixel
:imgs: list of images
:returns predictions
"""
# Set logger
if (out_dir is not None):
conf.dataOutDir = out_dir
save_path = os.path.join(conf.dataOutDir, 'vilar')
utls.setup_logging(conf.dataOutDir)
logger = logging.getLogger('vilar (predict)')
logger.info('---------------------------')
logger.info('Predicting on: ' + conf.ds_dir)
logger.info('type of sequence: ' + conf.seq_type)
logger.info('gaze filename: ' + conf.csvFileName_fg)
logger.info('Result dir:')
logger.info(conf.dataOutDir)
logger.info('---------------------------')
n_frames = conf.vilar_n_frames
if (n_frames == -1):
n_frames = len(conf.frameFileNames)
batch_size = n_frames
batch_idx = np.array_split(np.arange(0, n_frames), n_frames / n_jobs)
selem = morphology.square(5)
ims = []
for i in range(n_frames):
img = utls.imread(conf.frameFileNames[i])
img = (color.rgb2gray(img) * 255).astype(np.uint8)
#img = (color.rgb2gray(img)*255).astype(np.uint8)
#img = utls.imread(self.conf.frameFileNames[i])
img = median(img, selem=selem) # Uses square of size 3
ims.append(img) # make feat segment dictionary list
save_path = os.path.join(conf.dataOutDir, 'vilar')
if (not os.path.exists(save_path)):
os.mkdir(save_path)
for b in range(len(batch_idx)):
stack = list()
cnt = 0
for b_ in range(batch_idx[b].shape[0]):
i = batch_idx[b][b_]
stack.append(dict())
stack[cnt]['clf'] = clf
stack[cnt]['im'] = ims[i]
stack[cnt]['shape'] = ims[i].shape
stack[cnt]['f_num'] = i
stack[cnt]['ps'] = conf.patch_size
stack[cnt]['scale_factor'] = conf.scale_factor
stack[cnt]['save_path_patch'] = os.path.join(
save_path, 'all_patches_im_' + str(i) + '.npz')
stack[cnt]['save_path_pred'] = os.path.join(
save_path, 'pred_im_' + str(i) + '.npz')
cnt += 1
with Pool(processes=n_jobs) as pool:
pool.map(predict_job, stack)
return conf
def main(confs,
confunet,
out_dir=None,
train=True,
pred=True,
score=True,
resume_model=None):
n_points = 2000
if (out_dir is None):
now = datetime.datetime.now()
dateTime = now.strftime("%Y-%m-%d_%H-%M-%S")
out_dir = os.path.join(
confs[0].dataOutRoot, 'learning_exps',
'learning_' + confs[0].seq_type + '_' + dateTime)
dir_in = [c.dataOutDir for c in confs]
if (not os.path.exists(out_dir)):
os.mkdir(out_dir)
with open(os.path.join(out_dir, 'cfg.yml'), 'w') as outfile:
yaml.dump(confunet, outfile, default_flow_style=True)
datasets = []
utls.setup_logging(out_dir)
logger = logging.getLogger('learning_exp_unet')
logger.info('Starting learning experiment on:')
logger.info(dir_in)
logger.info('Gaze file: ' + str(confs[0].csvFileName_fg))
logger.info('')
if (not os.path.exists(os.path.join(out_dir, 'datasets.npz'))):
logger.info('Building target vectors')
for i in range(len(dir_in)):
with open(os.path.join(dir_in[i], 'cfg.yml'), 'r') as outfile:
conf = yaml.load(outfile)
logger.info('Dataset: ' + str(i + 1) + '/' + str(len(dir_in)))
dataset = learning_dataset.LearningDataset(conf)
npz_file = np.load(os.path.join(dir_in[i], 'results.npz'))
#seeds = np.asarray(utls.get_node_list_tracklets(npz_file['list_ksp'][-1]))
dataset.scores = npz_file['ksp_scores_mat'].astype(bool)
datasets.append(dataset)
if (not os.path.exists(out_dir)):
os.mkdir(out_dir)
logger.info('saving datasets to: ' + out_dir)
np.savez(os.path.join(out_dir, 'datasets.npz'),
**{'datasets': datasets})
dir_my_root = os.path.join(out_dir, 'my')
dir_true_root = os.path.join(out_dir, 'true')
n_folds = 4
if (train):
from nets import UNetBasic
logger.info('Loading datasets...')
datasets = np.load(os.path.join(out_dir, 'datasets.npz'))['datasets']
fold_ids = np.arange(0, 4)[::-1]
for i in range(n_folds):
logger.info('-----------------')
pred_fold = i
train_folds = np.asarray([
fold_ids[j] for j in range(n_folds)
if (fold_ids[j] != pred_fold)
])
logger.info('train_folds: ' + str(train_folds))
logger.info('pred_folds: ' + str(pred_fold))
logger.info('-----------------')
logger.info('Extracting X')
X = (resize_datasets([
datasets[train_folds[j]].X_all_images
for j in range(train_folds.shape[0])
], confunet.unet_im_size) * 255).astype(np.uint8)
logger.info('Extracting y')
y = (resize_datasets([
np.tile(
datasets[train_folds[j]].scores[:, :, np.newaxis, :].
astype(np.uint8) * 255, (1, 1, 3, 1))
for j in range(train_folds.shape[0])
], confunet.unet_im_size) * 255).astype(np.uint8)
# Set dirs
dir_my = os.path.join(dir_my_root, 'fold_' + str(pred_fold))
dir_my_train = os.path.join(dir_my, 'train')
logger.info('Writing _my_ train images/gts to disk...')
ims_my, gts_my = write_frames_train(X, y, dir_my_train, logger)
logger.info('Training U-Net on my segmentation...')
unet_my = UNetBasic.UNetBasic(confunet, dir_my_train, ims_my[0])
if (resume_model is not None):
model_path_my = get_model_path(dir_my_train,
take_min_loss=False)
initial_epoch_my = 0
if (len(model_path_my) == 0):
n_epochs_my = confunet.n_epochs
model_path_my = None
initial_epoch_my = 0
else:
n_epochs_my = confunet.n_epochs
initial_epoch_my = int(
os.path.split(model_path_my)[-1][6:8])
else:
model_path_my = None
n_epochs_my = confunet.n_epochs
initial_epoch_my = 0
unet_my.train(confunet,
ims_my,
gts_my,
dir_my_train,
n_epochs_my,
initial_epoch=initial_epoch_my,
dir_eval_clbk=dir_my,
resume_model=model_path_my)
logger.info('Extracting y')
y = (resize_datasets([
np.tile(
datasets[train_folds[j]].gt[:, :, np.newaxis, :].astype(
np.uint8) * 255, (1, 1, 3, 1))
for j in range(train_folds.shape[0])
], confunet.unet_im_size) * 255).astype(np.uint8)
dir_true = os.path.join(dir_true_root, 'fold_' + str(pred_fold))
dir_true_train = os.path.join(dir_true, 'train')
logger.info('Writing _true_ train images/gts to disk...')
ims_true, gts_true = write_frames_train(X, y, dir_true_train,
logger)
logger.info('Training U-Net on true segmentation...')
unet_true = UNetBasic.UNetBasic(confunet, dir_true_train,
ims_true[0])
if (resume_model is not None):
model_path_true = get_model_path(dir_true_train,
take_min_loss=False)
initial_epoch_true = 0
if (len(model_path_true) == 0):
n_epochs_true = confunet.n_epochs
model_path_true = None
initial_epoch_true = 0
else:
n_epochs_true = confunet.n_epochs
initial_epoch_true = int(
os.path.split(model_path_true)[-1][6:8])
else:
model_path_true = None
n_epochs_true = confunet.n_epochs
initial_epoch_true = 0
unet_true.train(confunet,
ims_true,
gts_true,
dir_true_train,
n_epochs_true,
initial_epoch=initial_epoch_true,
dir_eval_clbk=dir_true,
resume_model=model_path_true)
else:
logger.info('Results directory')
logger.info(dir_my_root)
logger.info(dir_true_root)
logger.info('Exist. Delete and re-compute')
logger.info('-----------------')
logger.info('Loading datasets...')
datasets = np.load(os.path.join(out_dir, 'datasets.npz'))['datasets']
if (pred):
from nets import UNetBasic
fold_ids = np.arange(0, 4)[::-1]
for i in range(n_folds):
pred_fold = i
logger.info('Predicting on fold_' + str(pred_fold))
dir_my = os.path.join(dir_my_root, 'fold_' + str(pred_fold))
dir_true = os.path.join(dir_true_root, 'fold_' + str(pred_fold))
#model_path_my = get_best_model_path(os.path.join(dir_my, 'train'))
model_path_my = get_model_path(os.path.join(dir_my, 'train'),
take_min_loss=False)
#model_path_true = get_best_model_path(os.path.join(dir_true,'train'))
model_path_true = get_model_path(os.path.join(dir_true, 'train'),
take_min_loss=False)
dir_true_pred = os.path.join(dir_true, 'pred')
dir_my_pred = os.path.join(dir_my, 'pred')
dir_true_pred_res = os.path.join(dir_true, 'pred_res')
dir_my_pred_res = os.path.join(dir_my, 'pred_res')
logger.info('Will use models:')
logger.info(model_path_my)
logger.info(model_path_true)
logger.info('Extracting y_my/y_true')
X = datasets[pred_fold].X_all_images
X = resize_stack(X, confunet.unet_im_size)
logger.info('Writing _my_ pred images to disk...')
ims_my = write_frames_pred(X, dir_my_pred, logger)
logger.info('Writing _true_ pred images to disk...')
ims_true = write_frames_pred(X, dir_true_pred, logger)
X = X.transpose((3, 0, 1, 2))
mean = np.mean(X.reshape(-1, 3), axis=0)
std = np.std(X.reshape(-1, 3), axis=0)
unet_true = UNetBasic.UNetBasic(confunet, dir_true_pred,
ims_true[0])
unet_my = UNetBasic.UNetBasic(confunet, dir_my_pred, ims_my[0])
# Get normalization parameters of training set
logger.info('Extracting X (train) normalization factors')
train_folds = np.asarray([
fold_ids[j] for j in range(n_folds)
if (fold_ids[j] != pred_fold)
])
im_list_train = [
datasets[train_folds[j]].conf.frameFileNames
for j in range(train_folds.shape[0])
]
im_list_train = [
item for sublist in im_list_train for item in sublist
]
input_y = unet_my.inputDimY
input_x = unet_my.inputDimX
n_chans = unet_my.nbrChannels
ims_train = UNetImpl.preprocess_imgs(unet_my, im_list_train,
input_y, input_x, n_chans)
#mean = np.mean(ims_train.reshape(-1,3), axis = 0)
#std = np.std(ims_train.reshape(-1,3), axis = 0)
#ims_train = utls.normalize_imgs(ims_train, mean, std)
logger.info('Predicting on my segmentation...')
preds_my = unet_my.eval(confunet, model_path_my, ims_my, mean, std)
logger.info('Predicting on true segmentation...')
preds_true = unet_true.eval(confunet, model_path_true, ims_true,
mean, std)
logger.info('Writing _my_ pred results images to disk...')
ims_my = write_frames_pred(preds_my,
dir_my_pred_res,
logger,
nchans=1)
logger.info('Writing _true_ pred results images to disk...')
ims_true = write_frames_pred(preds_true,
dir_true_pred_res,
logger,
nchans=1)
if (score):
for i in range(n_folds):
score_dict = dict()
pred_fold = i
logger.info('Scoring on fold_' + str(pred_fold))
dir_my = os.path.join(dir_my_root, 'fold_' + str(pred_fold))
dir_true = os.path.join(dir_true_root, 'fold_' + str(pred_fold))
dir_true_pred_res = os.path.join(dir_true, 'pred_res', 'img')
dir_my_pred_res = os.path.join(dir_my, 'pred_res', 'img')
fnames_true_pred_res = glob.glob(
os.path.join(dir_true_pred_res, '*.png'))
fnames_true_pred_res = sorted(fnames_true_pred_res)
fnames_my_pred_res = glob.glob(
os.path.join(dir_my_pred_res, '*.png'))
fnames_my_pred_res = sorted(fnames_my_pred_res)
my_preds = np.asarray([utls.imread(f) for f in fnames_my_pred_res
]).transpose(1, 2, 3, 0) / 255
true_preds = np.asarray(
[utls.imread(f)
for f in fnames_true_pred_res]).transpose(1, 2, 3, 0) / 255
gts = (resize_datasets([
np.tile(datasets[pred_fold].gt[:, :, np.newaxis, :],
(1, 1, 3, 1))
], confunet.unet_im_size) > 0).astype(np.uint8)
vals_gt = gts.ravel()
vals_my = my_preds.ravel()
vals_true = true_preds.ravel()
logger.info('Calculating metrics on my... ')
all_scores = get_all_scores(vals_gt, vals_my, n_points)
score_dict['conf'] = datasets[pred_fold].conf
score_dict['fold'] = pred_fold
score_dict['fpr_my'] = all_scores[0]
score_dict['tpr_my'] = all_scores[1]
score_dict['auc_my'] = all_scores[2]
score_dict['pr_my'] = all_scores[3]
score_dict['rc_my'] = all_scores[4]
score_dict['f1_my'] = all_scores[5]
score_dict['thr_my'] = all_scores[6]
logger.info('Calculating metrics on true... ')
all_scores = get_all_scores(vals_gt, vals_true, n_points)
score_dict['fpr_true'] = all_scores[0]
score_dict['tpr_true'] = all_scores[1]
score_dict['auc_true'] = all_scores[2]
score_dict['pr_true'] = all_scores[3]
score_dict['rc_true'] = all_scores[4]
score_dict['f1_true'] = all_scores[5]
score_dict['thr_true'] = all_scores[6]
score_dict['my_preds'] = my_preds
score_dict['true_preds'] = true_preds
logger.info('Saving results on fold: ' + str(pred_fold))
file_out = os.path.join(out_dir,
'scores_' + str(pred_fold) + '.npz')
np.savez(file_out, **score_dict)
def main(arg_cfg):
data = dict()
#Update config
cfg_dict = cfg.cfg()
arg_cfg['seq_type'] = cfg.datasetdir_to_type(arg_cfg['ds_dir'])
cfg_dict.update(arg_cfg)
conf = cfg.Bunch(cfg_dict)
#Write config to result dir
conf.dataOutDir = utls.getDataOutDir(conf.dataOutRoot, conf.ds_dir,
conf.resultDir, conf.out_dir_prefix,
conf.testing)
#Set logger
utls.setup_logging(conf.dataOutDir)
logger = logging.getLogger('feat_extr')
logger.info('---------------------------')
logger.info('starting feature extraction on: ' + conf.ds_dir)
logger.info('type of sequence: ' + conf.seq_type)
logger.info('gaze filename: ' + conf.csvFileName_fg)
logger.info('features type: ' + conf.feat_extr_algorithm)
logger.info('Result dir:')
logger.info(conf.dataOutDir)
logger.info('---------------------------')
#Make frame file names
gt_dir = os.path.join(conf.root_path, conf.ds_dir, conf.truth_dir)
gtFileNames = utls.makeFrameFileNames(conf.frame_prefix, conf.truth_dir,
conf.root_path, conf.ds_dir,
conf.frame_extension)
conf.frameFileNames = utls.makeFrameFileNames(conf.frame_prefix,
conf.frame_dir,
conf.root_path, conf.ds_dir,
conf.frame_extension)
conf.myGaze_fg = utls.readCsv(
os.path.join(conf.root_path, conf.ds_dir, conf.locs_dir,
conf.csvFileName_fg))
#conf.myGaze_bg = utls.readCsv(conf.csvName_bg)
gt_positives = utls.getPositives(gtFileNames)
if (conf.labelMatPath != ''):
conf.labelMatPath = os.path.join(conf.dataOutRoot, conf.ds_dir,
conf.frameDir, conf.labelMatPath)
conf.precomp_desc_path = os.path.join(conf.dataOutRoot, conf.ds_dir,
conf.feats_dir)
# ---------- Descriptors/superpixel costs
my_dataset = ds.Dataset(conf)
my_dataset.load_superpix_from_file()
my_dataset.calc_sp_feats_unet_gaze_rec(save=True)
with open(os.path.join(conf.dataOutDir, 'cfg.yml'), 'w') as outfile:
yaml.dump(conf, outfile, default_flow_style=True)
logger.info('Finished feature extraction: ' + conf.ds_dir)
return conf
def main(confs, out_dir=None):
alpha = 0.3
n_points = 2000
seq_type = confs[0].seq_type
if (out_dir is None):
now = datetime.datetime.now()
dateTime = now.strftime("%Y-%m-%d_%H-%M-%S")
out_dir = os.path.join(
confs[0].dataOutRoot, 'learning_exps',
'learning_' + confs[0].seq_type + '_' + dateTime)
dir_in = [c.dataOutDir for c in confs]
if (not os.path.exists(out_dir)):
os.mkdir(out_dir)
datasets = []
utls.setup_logging(out_dir)
logger = logging.getLogger('learning_exp')
logger.info('Starting learning experiment on:')
logger.info(dir_in)
logger.info('Gaze file: ' + str(confs[0].csvFileName_fg))
logger.info('')
if (not os.path.exists(os.path.join(out_dir, 'datasets.npz'))):
logger.info('Building target vectors')
for i in range(len(dir_in)):
with open(os.path.join(dir_in[i], 'cfg.yml'), 'r') as outfile:
conf = yaml.load(outfile)
logger.info('Dataset: ' + str(i + 1) + '/' + str(len(dir_in)))
files = sorted(
glob.glob(os.path.join(dir_in[i], 'pm_scores_iter*')))[-1]
#logger.info('Init. learner')
dataset = learning_dataset.LearningDataset(conf)
npz_file = np.load(os.path.join(dir_in[i], 'results.npz'))
seeds = np.asarray(
utls.get_node_list_tracklets(npz_file['list_ksp'][-1]))
if (confs[0].use_ss):
dataset.load_ss_from_file()
seeds = ss.thr_all_graphs(dataset.g_ss, seeds, conf.ss_thr)
dataset.set_seeds(seeds)
dataset.make_y_array(seeds)
dataset.make_y_array_true(dataset.gt)
datasets.append(dataset)
if (not os.path.exists(out_dir)):
os.mkdir(out_dir)
logger.info('saving datasets to: ' + out_dir)
np.savez(os.path.join(out_dir, 'datasets.npz'),
**{'datasets': datasets})
else:
logger.info('Loading datasets...')
datasets = np.load(os.path.join(out_dir, 'datasets.npz'))['datasets']
n_folds = 4
fold_ids = np.arange(0, 4)[::-1]
res_list = []
n_e = 150
if (not os.path.exists(os.path.join(out_dir, 'results.npz'))):
for i in range(n_folds):
logger.info('-----------------')
pred_fold = i
train_folds = np.asarray([
fold_ids[j] for j in range(n_folds)
if (fold_ids[j] != pred_fold)
])
logger.info('train_folds: ' + str(train_folds))
logger.info('pred_folds: ' + str(pred_fold))
logger.info('-----------------')
X_train = utls.concat_arr(
np.concatenate([
datasets[train_folds[j]].X
for j in range(train_folds.shape[0])
]))
y_train_my = np.concatenate([
datasets[train_folds[j]].y[:, 2]
for j in range(train_folds.shape[0])
])
y_train_true = np.concatenate([
datasets[train_folds[j]].y_true[:, 2]
for j in range(train_folds.shape[0])
])
logger.info('Extracting X_test')
X_test = utls.concat_arr(datasets[pred_fold].X)
logger.info('Extracting y_test')
y_test = datasets[pred_fold].y_true[:, 2]
logger.info('Fitting...')
bag_n_feats = confs[0].bag_n_feats_rf
bag_max_depth = confs[0].bag_max_depth_rf
logger.info('bag_n_feats: ' + str(bag_n_feats))
logger.info('bag_max_depth: ' + str(bag_max_depth))
n_trees = datasets[0].conf.T
clf_my = RandomForestClassifier(max_features=bag_n_feats,
class_weight='balanced',
n_estimators=n_trees)
clf_true = RandomForestClassifier(max_features=bag_n_feats,
class_weight='balanced',
n_estimators=n_trees)
clf_my.fit(X_train, y_train_my)
clf_true.fit(X_train, y_train_true)
logger.info('Predicting...')
probas_my = clf_my.predict_proba(X_test)[:, 1]
probas_true = clf_true.predict_proba(X_test)[:, 1]
#probas_my = rf.run(X_train,y_train_my,X_test,150)
#probas_true = rf.run(X_train,y_train_true,X_test,150)
logger.info('Computing ROC curves on true model')
fpr_true, tpr_true, thresholds_true = roc_curve(
y_test, probas_true)
auc_true = auc(fpr_true, tpr_true)
logger.info('auc_true: ' + str(auc_true))
logger.info('Computing ROC curves on my model')
fpr_my, tpr_my, thresholds_my = roc_curve(y_test,
probas_my,
pos_label=1)
auc_my = auc(fpr_my, tpr_my)
logger.info('auc_my: ' + str(auc_my))
logger.info('Computing prec-recall curves on true model')
precision_true, recall_true, _ = precision_recall_curve(
y_test, probas_true)
logger.info('Computing prec-recall curves on my model')
precision_my, recall_my, _ = precision_recall_curve(
y_test, probas_my)
dict_ = dict()
dict_['train_folds'] = train_folds
dict_['pred_fold'] = pred_fold
dict_['n_estimators'] = n_e
dict_['fpr_true'] = fpr_true
dict_['tpr_true'] = tpr_true
dict_['fpr_my'] = fpr_my
dict_['tpr_my'] = tpr_my
dict_['auc_true'] = auc_true
dict_['precision_true'] = precision_true
dict_['recall_true'] = recall_true
dict_['auc_true'] = auc_true
dict_['auc_my'] = auc_my
dict_['precision_my'] = precision_my
dict_['recall_my'] = recall_my
dict_['probas_my'] = probas_my
dict_['probas_true'] = probas_true
dict_['y_test'] = y_test
res_list.append(dict_)
file_out = os.path.join(out_dir, 'results.npz')
logger.info('Saving metrics to ')
np.savez(file_out, **{'res_list': res_list})
else:
logger.info('Loading results...')
res_list = np.load(os.path.join(out_dir, 'results.npz'))['res_list']
#Plot folds
colors = ['blue', 'darkorange', 'seagreen', 'yellow', 'blue']
lw = 1
plt.clf()
l_fpr_true = []
l_tpr_true = []
l_pr_true = []
l_rc_true = []
l_fpr_my = []
l_tpr_my = []
l_pr_my = []
l_rc_my = []
for i in range(len(res_list)):
fpr_true = res_list[i]['fpr_true']
tpr_true = res_list[i]['tpr_true']
#fpr_true, tpr_true = utls.my_interp(fpr_true, tpr_true, n_points)
l_fpr_true.append(fpr_true)
l_tpr_true.append(tpr_true)
fpr_my = res_list[i]['fpr_my']
tpr_my = res_list[i]['tpr_my']
#fpr_my, tpr_my = utls.my_interp(fpr_my, tpr_my, n_points)
l_fpr_my.append(fpr_my)
l_tpr_my.append(tpr_my)
pr_true = res_list[i]['precision_true']
rc_true = res_list[i]['recall_true']
#rc_true, pr_true = utls.my_interp(rc_true, pr_true, n_points)
l_rc_true.append(rc_true)
l_pr_true.append(pr_true)
pr_my = res_list[i]['precision_my']
rc_my = res_list[i]['recall_my']
#rc_my, pr_my = utls.my_interp(rc_my, pr_my, n_points)
l_rc_my.append(rc_my)
l_pr_my.append(pr_my)
rc_range_my = [
np.min([np.min(l_rc_my[i]) for i in range(len(l_rc_my))]),
np.max([np.max(l_rc_my[i]) for i in range(len(l_rc_my))])
]
rc_range_true = [
np.min([np.min(l_rc_true[i]) for i in range(len(l_rc_true))]),
np.max([np.max(l_rc_true[i]) for i in range(len(l_rc_true))])
]
rc_range = [
np.min((rc_range_my[0], rc_range_true[0])),
np.max((rc_range_my[1], rc_range_true[1]))
]
fpr_range_my = [
np.min([np.min(l_fpr_my[i]) for i in range(len(l_fpr_my))]),
np.max([np.max(l_fpr_my[i]) for i in range(len(l_fpr_my))])
]
fpr_range_true = [
np.min([np.min(l_fpr_true[i]) for i in range(len(l_fpr_true))]),
np.max([np.max(l_fpr_true[i]) for i in range(len(l_fpr_true))])
]
fpr_range = [
np.min((fpr_range_my[0], fpr_range_true[0])),
np.max((fpr_range_my[1], fpr_range_true[1]))
]
l_fpr_tpr_my_interp = np.asarray([
utls.my_interp(l_fpr_my[i], l_tpr_my[i], n_points, fpr_range)
for i in range(len(l_fpr_my))
]).transpose(1, 0, 2)
l_fpr_my = l_fpr_tpr_my_interp[0, ...]
l_tpr_my = l_fpr_tpr_my_interp[1, ...]
l_pr_rc_my_interp = np.asarray([
utls.my_interp(l_rc_my[i], l_pr_my[i], n_points, rc_range)
for i in range(len(l_rc_my))
]).transpose(1, 0, 2)
l_rc_my = l_pr_rc_my_interp[0, ...]
l_pr_my = l_pr_rc_my_interp[1, ...]
l_fpr_tpr_true_interp = np.asarray([
utls.my_interp(l_fpr_true[i], l_tpr_true[i], n_points, fpr_range)
for i in range(len(l_fpr_true))
]).transpose(1, 0, 2)
l_fpr_true = l_fpr_tpr_true_interp[0, ...]
l_tpr_true = l_fpr_tpr_true_interp[1, ...]
l_pr_rc_true_interp = np.asarray([
utls.my_interp(l_rc_true[i], l_pr_true[i], n_points, rc_range)
for i in range(len(l_rc_true))
]).transpose(1, 0, 2)
l_rc_true = l_pr_rc_true_interp[0, ...]
l_pr_true = l_pr_rc_true_interp[1, ...]
roc_xlim = [0, 1]
pr_rc_xlim = [0, 1]
logger.info('Concatenating results for scoring')
all_y_true = np.concatenate([r['y_test'] for r in res_list])
all_probas_my = np.concatenate([r['probas_my'] for r in res_list])
all_probas_true = np.concatenate([r['probas_true'] for r in res_list])
fpr_my_all, tpr_my_all, thresholds_my_all = roc_curve(all_y_true,
all_probas_my,
pos_label=1)
fpr_my_all, tpr_my_all = utls.my_interp(fpr_my_all, tpr_my_all, n_points)
fpr_true_all, tpr_true_all, thresholds_true_all = roc_curve(
all_y_true, all_probas_true, pos_label=1)
fpr_true_all, tpr_true_all = utls.my_interp(fpr_true_all, tpr_true_all,
n_points)
pr_my_all, rc_my_all, _ = precision_recall_curve(all_y_true, all_probas_my)
pr_my_all, rc_my_all = utls.my_interp(pr_my_all, rc_my_all, n_points)
pr_true_all, rc_true_all, _ = precision_recall_curve(
all_y_true, all_probas_true)
pr_true_all, rc_true_all = utls.my_interp(pr_true_all, rc_true_all,
n_points)
auc_my_all = auc(fpr_my_all, tpr_my_all)
probas_thr = np.linspace(0, 1, n_points)
f1_my = [f1_score(all_y_true, all_probas_my > p) for p in probas_thr]
probas_thr = np.linspace(0, 1, 200)
f1_true = [f1_score(all_y_true, all_probas_true > p) for p in probas_thr]
auc_true_all = auc(fpr_true_all, tpr_true_all)
# Plotting
lw = 3
plt.figure('tpr')
plt.plot(l_fpr_true.mean(axis=0),
l_tpr_true.mean(axis=0),
'-',
lw=lw,
color=colors[0],
label='all folds (true) (area = %0.4f, max_f1 = %0.4f)' %
(auc_true_all, np.max(f1_true)))
plt.fill_between(l_fpr_true.mean(axis=0),
l_tpr_true.mean(axis=0) + l_tpr_true.std(axis=0),
l_tpr_true.mean(axis=0) - l_tpr_true.std(axis=0),
facecolor=colors[0],
alpha=alpha)
plt.plot(l_fpr_my.mean(axis=0),
l_tpr_my.mean(axis=0),
'-',
lw=lw,
color=colors[1],
label='all folds (my) (area = %0.4f, max_f1 = %0.4f)' %
(auc_my_all, np.max(f1_my)))
plt.fill_between(l_fpr_my.mean(axis=0),
l_tpr_my.mean(axis=0) + l_tpr_my.std(axis=0),
l_tpr_my.mean(axis=0) - l_tpr_my.std(axis=0),
facecolor=colors[1],
alpha=alpha)
plt.legend()
plt.xlim(roc_xlim)
plt.xlabel('fpr')
plt.ylabel('tpr')
plt.suptitle('Sequence: ' + seq_type + '. Gaze: ' +
confs[0].csvFileName_fg)
plt.savefig(os.path.join(out_dir, 'folds_tpr_fpr.pdf'))
plt.figure('rc')
plt.plot(l_rc_true.mean(axis=0),
l_pr_true.mean(axis=0),
'-',
lw=lw,
color=colors[0],
label='all folds (true)')
plt.fill_between(l_rc_true.mean(axis=0),
l_pr_true.mean(axis=0) + l_pr_true.std(axis=0),
l_pr_true.mean(axis=0) - l_pr_true.std(axis=0),
facecolor=colors[0],
alpha=alpha)
plt.plot(l_rc_my.mean(axis=0),
l_pr_my.mean(axis=0),
'-',
lw=lw,
color=colors[1],
label='all folds (my)')
plt.fill_between(l_rc_my.mean(axis=0),
l_pr_my.mean(axis=0) + l_pr_my.std(axis=0),
l_pr_my.mean(axis=0) - l_pr_my.std(axis=0),
facecolor=colors[1],
alpha=alpha)
plt.legend()
plt.xlim(pr_rc_xlim)
plt.xlabel('recall')
plt.ylabel('precision')
plt.suptitle('Sequence: ' + seq_type + '. Gaze: ' +
confs[0].csvFileName_fg)
#plt.figure('rc').set_size_inches(18.5, 10.5)
plt.savefig(os.path.join(out_dir, 'folds_pr_rc.pdf'))
min_n_frames = np.min([len(d.conf.frameFileNames) for d in datasets])
dir_frames = os.path.join(out_dir, 'frames')
if (not os.path.exists(dir_frames)):
os.mkdir(dir_frames)
else:
logger.info('frames already exist, delete and re-run...')
#shutil.rmtree(dir_frames)
#os.mkdir(dir_frames)
logger.info('Generating prediction frames...')
#Plot by-frame predictions
for f in range(min_n_frames):
my = []
true = []
ims = []
for j in range(len(datasets)):
y_true = datasets[j].y
idx_y = np.where(y_true[:, 0] == f)[0]
y_true = y_true[idx_y]
probas_true = res_list[j]['probas_true'][idx_y]
probas_my = res_list[j]['probas_my'][idx_y]
scores_my = utls.get_scores_from_sps(y_true[:, 0:2],
datasets[j].get_labels(),
probas_my)[..., f]
my.append(scores_my)
scores_true = utls.get_scores_from_sps(y_true[:, 0:2],
datasets[j].get_labels(),
probas_true)[..., f]
true.append(scores_true)
cont_gt = segmentation.find_boundaries(datasets[j].gt[..., f],
mode='thick')
idx_cont_gt = np.where(cont_gt)
im = utls.imread(datasets[j].conf.frameFileNames[f])
im[idx_cont_gt[0], idx_cont_gt[1], :] = (255, 0, 0)
im = gaze.drawGazePoint(datasets[j].conf.myGaze_fg,
f,
im,
radius=7)
ims.append(im)
gs = gridspec.GridSpec(3, 4)
for c in range(4):
ax = plt.subplot(gs[0, c])
ax.imshow(true[c])
plt.title('true')
ax = plt.subplot(gs[1, c])
ax.imshow(my[c])
plt.title('my')
ax = plt.subplot(gs[2, c])
ax.imshow(ims[c])
plt.title('image')
plt.suptitle('Sequence: ' + seq_type + '. Frame: ' + str(f))
fig = plt.gcf()
fig.set_size_inches(18.5, 10.5)
plt.savefig(os.path.join(dir_frames, 'frame_' + str(f) + '.png'))