def rename_old_rundir(rundir):
if len(list(rundir.iterdir())) > 0:
timestamp = datetime.fromtimestamp(
rundir.stat().st_mtime).strftime('%Y-%m-%d_%H:%M:%S')
str_rnd = ''.join(random.choices(string.ascii_uppercase, k=3))
new_foldname = f'old_{timestamp}_{str_rnd}'
log.info(f'Existing experiment moved to {new_foldname}')
shutil.move(rundir, rundir.parent/new_foldname)
small.mkdir(rundir)
def demo_run(cls, cf, out, dataset, split_vids, tubes_dwein, neth):
vf_connections_dwti = cls._define_boxes_to_evaluate(
cf, dataset, split_vids, tubes_dwein)
vf_connections_dwti = sample_dict(
vf_connections_dwti, N=5, NP_SEED=0)
vfold = small.mkdir(out/'demovis')
cls._demovis_apply(vfold, neth, dataset, vf_connections_dwti)
def _simple_gpu_compute(
cls, out, dataset, neth, vf_connections_dwti
) -> Dict[Vid_daly, Dict[int, np.ndarray]]:
"""Progress saved on video-level scale"""
def isaver_eval_func(vid):
f_connections_dwti = vf_connections_dwti[vid]
video_path = dataset.videos[vid]['path']
finds = list(f_connections_dwti)
with vt_cv.video_capture_open(video_path) as vcap:
frames_u8 = vt_cv.video_sample(
vcap, finds, debug_filename=video_path)
f_cls_probs = {}
for find, frame_BGR in zip(finds, frames_u8):
connections_dwti = f_connections_dwti[find]
boxes = connections_dwti['boxes']
cls_probs = neth.score_boxes(frame_BGR, boxes) # N, (bcg+10)
f_cls_probs[find] = cls_probs
return f_cls_probs
vids_to_eval = list(vf_connections_dwti.keys())
isaver = snippets.Isaver_simple(
small.mkdir(out/'isave_rcnn_vid_eval'),
vids_to_eval, isaver_eval_func,
'::10', 120)
isaver_items = isaver.run()
vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]]
vf_cls_probs = dict(zip(vids_to_eval, isaver_items))
return vf_cls_probs
def _train_routine(cf, cf_add_d2, out,
cls_names, TRAIN_DATASET_NAME,
datalist, add_args):
num_classes = len(cls_names)
d2_output_dir = str(small.mkdir(out/'d2_output'))
d_cfg = set_detectron_cfg_base(
d2_output_dir, num_classes, cf['seed'])
d_cfg = set_detectron_cfg_train(
d_cfg, TRAIN_DATASET_NAME, cf_add_d2)
num_gpus = cf['num_gpus']
nargs = argparse.Namespace()
nargs.datalist = datalist
nargs.TRAIN_DATASET_NAME = TRAIN_DATASET_NAME
nargs.cls_names = cls_names
nargs.resume = True
nargs.trainer = DalyVideoDatasetTrainer
dist_url = _figure_out_disturl(add_args)
launch_w_logging(_detectron_train_function,
num_gpus,
num_machines=1,
machine_rank=0,
dist_url=dist_url,
args=(d_cfg, cf, nargs))
def _eval_routine(cf, cf_add_d2, out,
cls_names, TEST_DATASET_NAME,
datalist: Datalist,
model_to_eval):
num_classes = len(cls_names)
d2_output_dir = str(small.mkdir(out/'d2_output'))
d_cfg = set_detectron_cfg_base(
d2_output_dir, num_classes, cf['seed'])
d_cfg = set_detectron_cfg_test(
d_cfg, TEST_DATASET_NAME,
model_to_eval, cf['conf_thresh'], cf_add_d2)
simple_d2_setup(d_cfg)
predictor = DefaultPredictor(d_cfg)
cpu_device = torch.device("cpu")
def eval_func(dl_item: Dl_record):
video_path: Path = dl_item['video_path']
frame_number: int = dl_item['video_frame_number']
frame_time: float = dl_item['video_frame_time']
frame_u8 = get_frame_without_crashing(
video_path, frame_number, frame_time)
predictions = predictor(frame_u8)
cpu_instances = predictions["instances"].to(cpu_device)
return cpu_instances
df_isaver = snippets.Isaver_simple(
small.mkdir(out/'isaver'), datalist, eval_func, '::50')
predicted_datalist = df_isaver.run()
if cf['nms.enable']:
nms_thresh = cf['nms.thresh']
nmsed_predicted_datalist = []
for pred_item in predicted_datalist:
if cf['nms.batched']:
keep = batched_nms(pred_item.pred_boxes.tensor,
pred_item.scores, pred_item.pred_classes, nms_thresh)
else:
keep = nms(pred_item.pred_boxes.tensor,
pred_item.scores, nms_thresh)
nmsed_item = pred_item[keep]
nmsed_predicted_datalist.append(nmsed_item)
predicted_datalist = nmsed_predicted_datalist
log.info('AP v2:')
computeprint_ap_for_video_datalist(cls_names, datalist, predicted_datalist)
def additional_logging(rundir, start_gstep):
# Also log to rundir
id_string = snippets.get_experiment_id_string()
logfilename = small.mkdir(rundir) / '{}_{}.log'.format(
start_gstep, id_string)
out_filehandler = logging.FileHandler(str(logfilename))
LOG_FORMATTER = logging.Formatter("%(asctime)s %(levelname)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
out_filehandler.setFormatter(LOG_FORMATTER)
out_filehandler.setLevel(logging.INFO)
logging.getLogger().addHandler(out_filehandler)
# Also print platform info
log.info(snippets.platform_info())
def _involved_gpu_compute(
cls, out, dataset, neth,
vf_connections_dwti,
size_video_chunk,
) -> Dict[Vid_daly, Dict[int, np.ndarray]]:
frame_chunks = []
vids_to_eval = list(vf_connections_dwti.keys())
for vid in vids_to_eval:
f_connections_dwti = vf_connections_dwti[vid]
finds = np.array(list(f_connections_dwti))
finds_split = snippets.leqn_split(
finds, size_video_chunk)
for subset_finds in finds_split:
frame_chunks.append((vid, subset_finds))
def isaver_eval_func(frame_chunk):
vid, finds = frame_chunk
f_connections_dwti = vf_connections_dwti[vid]
video_path = dataset.videos[vid]['path']
with vt_cv.video_capture_open(video_path) as vcap:
frames_u8 = vt_cv.video_sample(
vcap, finds, debug_filename=video_path)
f_cls_probs = {}
for find, frame_BGR in zip(finds, frames_u8):
connections_dwti = f_connections_dwti[find]
boxes = connections_dwti['boxes']
cls_probs = neth.score_boxes(
frame_BGR, boxes) # N, (bcg+10)
f_cls_probs[find] = cls_probs
return f_cls_probs
isaver = snippets.Isaver_simple(
small.mkdir(out/'isave_rcnn_vid_eval'),
frame_chunks, isaver_eval_func,
save_interval=60,
log_interval=300)
isaver_items = isaver.run()
vf_cls_probs: Dict[Vid_daly, Dict[int, np.ndarray]]
vf_cls_probs = {}
for (vid, subset_finds), f_cls_probs in zip(
frame_chunks, isaver_items):
vf_cls_probs.setdefault(vid, {}).update(f_cls_probs)
return vf_cls_probs
def __init__(self, cf, cf_add_d2, dataset, out):
num_classes = len(dataset.action_names)
TEST_DATASET_NAME = 'daly_objaction_test'
# / Define d2 conf
d2_output_dir = str(small.mkdir(out / 'd2_output'))
d_cfg = set_detectron_cfg_base(d2_output_dir, num_classes, cf['seed'])
d_cfg = set_detectron_cfg_test(d_cfg,
TEST_DATASET_NAME,
cf['d2_rcnn.model'],
cf['d2_rcnn.conf_thresh'],
cf_add_d2,
freeze=False)
d_cfg.MODEL.PROPOSAL_GENERATOR.NAME = "PrecomputedProposals"
d_cfg.freeze()
# / Start d2
simple_d2_setup(d_cfg)
# Predictor without proposal generator
model = build_model(d_cfg)
model.eval()
checkpointer = DetectionCheckpointer(model)
checkpointer.load(d_cfg.MODEL.WEIGHTS)
MIN_SIZE_TEST = d_cfg.INPUT.MIN_SIZE_TEST
MAX_SIZE_TEST = d_cfg.INPUT.MAX_SIZE_TEST
transform_gen = d2_transforms.ResizeShortestEdge(
[MIN_SIZE_TEST, MIN_SIZE_TEST], MAX_SIZE_TEST)
# Instance monkeypatching
# https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module/50600307#50600307
model.forward = MethodType(genrcnn_rcnn_roiscores_forward, model)
self.d_cfg = d_cfg
self.rcnn_roiscores_model = model
self.cpu_device = torch.device("cpu")
self.transform_gen = transform_gen
def _demovis_apply(cls,
vfold, neth, dataset: Dataset_daly_ocv, vf_connections_dwti):
nicolas_labels = ['background', ] + cast(List[str], dataset.action_names)
for vid, f_connections_dwti in tqdm(
vf_connections_dwti.items(), 'nicphil_demovis'):
video_path = dataset.videos[vid]['path']
finds = list(f_connections_dwti)
with vt_cv.video_capture_open(video_path) as vcap:
frames_u8 = vt_cv.video_sample(
vcap, finds, debug_filename=video_path)
video_fold = small.mkdir(vfold/f'vid{vid}')
for find, frame_BGR in zip(finds, frames_u8):
connections_dwti = f_connections_dwti[find]
boxes = connections_dwti['boxes']
box_cls_probs = neth.score_boxes(frame_BGR, boxes) # N, (bcg+10)
# Draw and print
txt_output = []
image = frame_BGR.copy()
for i, cls_probs in enumerate(box_cls_probs):
box = boxes[i]
best_score_id = np.argmax(cls_probs)
best_score = cls_probs[best_score_id]
best_nicolas_label = nicolas_labels[best_score_id]
snippets.cv_put_box_with_text(image, box,
text='{} {} {:.2f}'.format(
i, best_nicolas_label, best_score))
line = (' '.join([f'{y}: {x:.3f}'
for x, y in zip(cls_probs, nicolas_labels)])
+ str(box))
txt_output.append(line)
cv2.imwrite(str(
video_fold/'Fr{:05d}.png'.format(find)), image)
with (video_fold/f'Fr{find:05d}_scores.txt').open('w') as f:
f.write('\n'.join(txt_output))
def tubefeats_dist_train_mlp(workfolder, cfg_dict, add_args):
"""
Training of MLP trainfeats in the same way as we finetune stuff
"""
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig_v2(cfg_dict)
Ncfg_daly.set_defcfg_v2(cfg)
cfg.set_defaults_yaml("""
inputs:
keyframes:
fold: ~
featname: ~
""")
cfg.set_defaults_yaml("""
inputs:
tubes_dwein: ~
tubes_dwein_feats:
fold: ~
kind: !def ['roipooled', ['fullframe', 'roipooled']]
ckpt: ~
seed: 42
split_assignment: !def ['train/val',
['train/val', 'trainval/test']]
data_scaler: !def ['no', ['keyframes', 'no']]
net:
n_outputs: !def [~, [10, 11]]
train:
lr: 1.0e-05
weight_decay: 5.0e-2
start_epoch: 0
n_epochs: 120
batch_size: 32
tubes:
top_n_matches: 1
stride: 4
frame_dist: 16
add_keyframes: True
period:
i_batch:
loss_log: '::'
i_epoch:
loss_log: '0::1'
q_eval: '::'
full_eval: '0,1,2,3,4::5'
eval:
full_tubes:
enabled: True
detect_mode: !def ['roipooled', ['fullframe', 'roipooled']]
nms: 0.3
field_nms: 'box_det_score' # hscore
field_det: 'box_det_score' # hscore*frame_cls_score
""")
cf = cfg.parse()
# Seeds
initial_seed = cf['seed']
torch.manual_seed(initial_seed)
# Data
# General DALY level preparation
dataset = Ncfg_daly.get_dataset(cf)
vgroup = Ncfg_daly.get_vids(cf, dataset)
sset_train, sset_eval = cf['split_assignment'].split('/')
vids_train, vids_eval = vgroup[sset_train], vgroup[sset_eval]
man_feats_kf = Manager_feats_keyframes(
cf['inputs.keyframes.fold'], cf['inputs.keyframes.featname'])
# wein tubes
tubes_dwein_d, tubes_dgt_d = load_gt_and_wein_tubes(
cf['inputs.tubes_dwein'], dataset, vgroup)
tubes_dwein_prov: Dict[I_dwein, Tube_daly_wein_as_provided] = \
small.load_pkl(cf['inputs.tubes_dwein'])
# synthetic tube labels
_, dwti_to_label_eval = qload_synthetic_tube_labels(
tubes_dgt_d[sset_eval], tubes_dwein_d[sset_eval], dataset)
# Ssset
tkfeats_train = man_feats_kf.split_off(vids_train, True)
tkfeats_eval = man_feats_kf.split_off(vids_eval, True)
tubes_dwein_train = tubes_dwein_d[sset_train]
tubes_dwein_eval = tubes_dwein_d[sset_eval]
tubes_dgt_train = tubes_dgt_d[sset_train]
tubes_dgt_eval = tubes_dgt_d[sset_eval]
# Interacting with big
assert cf['eval.full_tubes.enabled'], 'We train on them anyway'
top_n_matches = cf['train.tubes.top_n_matches']
stride = cf['train.tubes.stride']
detect_mode = cf['inputs.tubes_dwein_feats.kind']
man_feats_dwt = create_preextracted_feats_manager(
cf, None, detect_mode)
max_distance = cf['train.tubes.frame_dist']
output_dims = cf['net.n_outputs']
if detect_mode == 'fullframe':
# fullframes
labeled_frames: List[Frame_labeled] = \
prepare_label_fullframes_for_training(
tubes_dgt_train, dataset, stride, max_distance)
# associate to extracted feats
labeled_linked_frames: List[Frame_labeled] = \
_link_lframes_to_exfeats(labeled_frames, man_feats_dwt)
tdataset = TDataset_over_labeled_linked_frames(
labeled_linked_frames, man_feats_dwt)
assert output_dims == 10
D_in = man_feats_dwt.fullframe_feats.shape[-1]
elif detect_mode == 'roipooled':
# roitubes
tkfeats_train_numpy = man_feats_kf.split_off(vids_train, False)
keyframes_train = tkfeats_train_numpy['kf']
keyframe_feats_train = tkfeats_train_numpy['X']
labeled_boxes: List[Box_labeled] = \
prepare_label_roiboxes_for_training(
tubes_dgt_train, dataset, stride, max_distance,
tubes_dwein_train, keyframes_train, top_n_matches,
cf['train.tubes.add_keyframes'])
# associate roiboxes to extracted feats
labeled_linked_boxes: List[Box_labeled_linked] = \
_link_lboxes_to_exfeats(labeled_boxes, man_feats_dwt)
tdataset = TDataset_over_labeled_linked_boxes(
labeled_linked_boxes, man_feats_dwt, keyframe_feats_train)
assert output_dims == 11
D_in = man_feats_dwt.BIG.shape[-1]
else:
raise RuntimeError()
# Model
model = Net_mlp_onelayer(
D_in, output_dims, 32, 0.5)
loss_fn = torch.nn.CrossEntropyLoss(reduction='mean')
optimizer = torch.optim.AdamW(model.parameters(),
lr=cf['train.lr'], weight_decay=cf['train.weight_decay'])
ckpt = Checkpointer(model, optimizer)
# Restore previous run
rundir = small.mkdir(out/'rundir')
checkpoint_path = (Manager_checkpoint_name.find_last_checkpoint(rundir))
if '--new' in add_args:
Manager_checkpoint_name.rename_old_rundir(rundir)
checkpoint_path = None
start_epoch = (ckpt.restore_model_magic(checkpoint_path,
cf['inputs.ckpt'], cf['train.start_epoch']))
# Training
n_epochs = cf['train.n_epochs']
for i_epoch in range(start_epoch, n_epochs):
log.info(f'Started epoch {i_epoch=}')
model.train()
l_avg = snippets.misc.Averager()
avg_bs = snippets.misc.Averager()
# Loader reproducible even if we restore
rgen = np.random.default_rng(initial_seed+i_epoch)
loader = _get_trainloader_rnd_sampler(tdataset,
cf['train.batch_size'], rgen)
for i_batch, (data_input) in enumerate(loader):
# inputs converter
(meta,) = data_input
flat_labels = np.hstack([m['labels'] for m in meta])
flat_feats = np.vstack([m['feats'] for m in meta])
flat_labels_t = torch.from_numpy(flat_labels)
flat_feats_t = torch.from_numpy(flat_feats)
result = model(flat_feats_t)
pred_train = result['x_final']
loss = loss_fn(pred_train, flat_labels_t)
optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_bs.update(len(flat_labels))
l_avg.update(loss.item())
if check_step(i_batch, cf['period.i_batch.loss_log']):
Nb = len(loader)
loss_str = (f'loss(all/last):{l_avg.avg:.4f}/{l_avg.last:.4f}')
log.info(f'{i_epoch=}, {i_batch=}/{Nb}; {loss_str}')
log.info('Epoch stats: avg_batchsize {}, loader_size {} '.format(
avg_bs.avg, len(loader)))
ckpt.save_epoch(rundir, i_epoch)
if check_step(i_epoch, cf['period.i_epoch.loss_log']):
log.info(f'Loss at {i_epoch=}: {l_avg.avg}')
if check_step(i_epoch, cf['period.i_epoch.q_eval']):
model.eval()
kacc_train = quick_accuracy_over_kfeat(
tkfeats_train, model, True)*100
kacc_eval = quick_accuracy_over_kfeat(
tkfeats_eval, model, True)*100
model.train()
log.info(f'Qperf at {i_epoch=}: '
f'{kacc_train=:.2f} {kacc_eval=:.2f}')
if check_step(i_epoch, cf['period.i_epoch.full_eval']):
model.eval()
result = mlp_perf_kf_evaluate(
model, tkfeats_train, tkfeats_eval,
tubes_dwein_eval, tubes_dgt_eval,
dataset, output_dims)
result_fulltube = mlp_perf_fulltube_evaluate(
model, man_feats_dwt,
tubes_dwein_eval, tubes_dwein_prov,
tubes_dgt_eval, dwti_to_label_eval,
dataset, output_dims,
cf['eval.full_tubes.detect_mode'],
cf['eval.full_tubes.nms'],
cf['eval.full_tubes.field_nms'],
cf['eval.full_tubes.field_det'])
result.update(result_fulltube)
model.train()
log.info(f'Evalset perf at {i_epoch=}')
mlp_perf_display(result, sset_eval)
def get_subfolders(folder, subfolder_names=['out', 'temp']):
return [small.mkdir(folder / name) for name in subfolder_names]
def merge_scores_avstubes(workfolder, cfg_dict, add_args):
out, = snippets.get_subfolders(workfolder, ['out'])
cfg = snippets.YConfig(cfg_dict)
cfg.set_defaults_handling(raise_without_defaults=False)
Ncfg_dataset.set_dataset_seed(cfg)
Ncfg_tube_eval.set_defcfg(cfg)
cfg.set_defaults("""
tube_dict: ~
combinations:
enabled: False
sizes: ~
""")
cf = cfg.parse()
dataset, split_vids, av_gt_tubes = \
Ncfg_dataset.resolve_dataset_tubes(cf)
ts = {k: small.load_pkl(v) for k, v in cfg_dict['tube_dict'].items()}
if not cf['combinations.enabled']:
av_stubes = _meanpool_avstubes(list(ts.values()))
small.save_pkl(out / 'merged_av_stubes.pkl', av_stubes)
log.info('All combined score:')
Ncfg_tube_eval.evalprint_if(cf, av_stubes, av_gt_tubes)
return
sizes = cf['combinations.sizes']
combinations = [list(itertools.combinations(ts.keys(), r)) for r in sizes]
combinations = list(itertools.chain(*combinations))
log.info('Combinations: {}'.format(combinations))
comb_dfdicts = {}
for comb in combinations:
comb_name = '+'.join(comb)
comb_fold = small.mkdir(out / comb_name)
def compute():
to_merge = [ts[k] for k in comb]
av_stubes = _meanpool_avstubes(to_merge)
small.save_pkl(comb_fold / 'av_stubes.pkl', av_stubes)
dfdict = Ncfg_tube_eval.eval_as_df(cf, av_stubes, av_gt_tubes)
return dfdict
dfdict = small.stash2(comb_fold / 'stashed_dfdict.pkl')(compute)
comb_dfdicts[comb_name] = dfdict
log.info('Individual results:')
for comb_name, dfdict in comb_dfdicts.items():
log.info(f'Results for {comb_name}:')
_print_quick_evaluation_stats(dfdict)
log.info('Combined tables:')
big_ = {comb: pd.concat(dfdict) for comb, dfdict in comb_dfdicts.items()}
big = pd.concat(big_, axis=1)
for stat in big.index.levels[0]:
log.info(f'=== {stat} ===')
for thresh in big.columns.levels[1]:
X = (big.loc['ap'].loc[:, pd.IndexSlice[:,
thresh]].droplevel(1,
axis=1))
table = snippets.df_to_table_v2((X * 100).round(2))
log.info(f'{stat} for IOU {thresh}:\n{table}')
log.info('\n')