def iter_benchmark(iterator,
num_iter: int,
warmup: int = 5,
max_time_seconds: float = 60) -> Tuple[float, List[float]]:
"""
Benchmark an iterator/iterable for `num_iter` iterations with an extra
`warmup` iterations of warmup.
End early if `max_time_seconds` time is spent on iterations.
Returns:
float: average time (seconds) per iteration
list[float]: time spent on each iteration. Sometimes useful for further analysis.
"""
num_iter, warmup = int(num_iter), int(warmup)
iterator = iter(iterator)
for _ in range(warmup):
next(iterator)
timer = Timer()
all_times = []
for curr_iter in tqdm.trange(num_iter):
start = timer.seconds()
if start > max_time_seconds:
num_iter = curr_iter
break
next(iterator)
all_times.append(timer.seconds() - start)
avg = timer.seconds() / num_iter
return avg, all_times
def __init__(self, overall_iters, cfg, mode):
"""
overall_iters (int): the overall number of iterations of one epoch.
cfg (CfgNode): configs.
mode (str): `train`, `val`, or `test` mode.
"""
self.cfg = cfg
self.lr = None
self.loss = ScalarMeter(cfg.LOG_PERIOD)
self.full_ava_test = cfg.AVA.FULL_TEST_ON_VAL
self.mode = mode
self.iter_timer = Timer()
self.all_preds = []
self.all_ori_boxes = []
self.all_metadata = []
self.overall_iters = overall_iters
self.excluded_keys = read_exclusions(
os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.EXCLUSION_FILE))
self.categories, self.class_whitelist = read_labelmap(
os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.LABEL_MAP_FILE))
gt_filename = os.path.join(cfg.AVA.ANNOTATION_DIR,
cfg.AVA.GROUNDTRUTH_FILE)
self.full_groundtruth = read_csv(gt_filename, self.class_whitelist)
self.mini_groundtruth = get_ava_mini_groundtruth(self.full_groundtruth)
_, self.video_idx_to_name = ava_helper.load_image_lists(
cfg, mode == "train")
def benchmark_distributed(self, num_iter, warmup=10):
"""
Benchmark the dataloader in each distributed worker, and log results of
all workers. This helps understand the final performance as well as
the variances among workers.
It also prints startup time (first iter) of the dataloader.
"""
gpu = comm.get_world_size()
dataset = MapDataset(self.dataset, self.mapper)
n = self.num_workers
loader = build_batch_data_loader(dataset,
self.sampler,
self.total_batch_size,
num_workers=n)
timer = Timer()
loader = iter(loader)
next(loader)
startup_time = timer.seconds()
logger.info(
"Dataloader startup time: {:.2f} seconds".format(startup_time))
comm.synchronize()
avg, all_times = self._benchmark(loader, num_iter * max(n, 1),
warmup * max(n, 1))
del loader
self._log_time(
f"DataLoader ({gpu} GPUs x {n} workers, total bs={self.total_batch_size})",
avg,
all_times,
True,
)
class TestMeter(object):
def __init__(self, cfg):
self.cfg = cfg
self.forward_timer = Timer()
self.total_time = 0
self.cnt = 0
self.score = dict()
self.output_dir = Join(cfg.TEST.OUTPUT_DIR, cfg.TEST.DATASET)
self.save_img = cfg.TEST.SAVE_IMG
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
self.score_csv = open(Join(self.output_dir, "score.csv"), 'w')
self.score_csv.write("vid, image_id, psnr, ssim\n")
def forward_tic(self):
"""
Start to record time.
"""
self.forward_timer.reset()
def forward_toc(self):
"""
Stop to record time.
"""
self.forward_timer.pause()
self.total_time += self.forward_timer.seconds()
self.cnt += 1
def log_img_result(self, img_out, vid, img_id, psnr, ssim):
if vid not in self.score.keys():
self.score[vid] = {}
# log score
self.score[vid][img_id] = (psnr, ssim)
self.score_csv.write("{},{},{},{}\n".format(vid, img_id, psnr, ssim))
# save img
if self.save_img:
# if not os.path.exists(Join(self.output_dir, vid)):
# os.makedirs(Join(self.output_dir, vid))
img_out = cv2.cvtColor(img_out, cv2.COLOR_RGB2BGR)
cv2.imwrite(Join(self.output_dir, img_id), img_out)
def log_average_score(self):
score_per_vid = {}
for vid in self.score.keys():
psnrs = [x[0] for x in self.score[vid].values()]
ssims = [x[1] for x in self.score[vid].values()]
score_per_vid[vid] = (np.mean(psnrs), np.mean(ssims))
with open(Join(self.output_dir, 'videos_scores.csv'), 'w') as f:
f.write('video_id, psnr, ssim\n')
for vid in self.score.keys():
f.write("{},{},{}\n".format(vid, score_per_vid[vid][0],
score_per_vid[vid][1]))
return score_per_vid
def speed(self):
return self.total_time, self.total_time / self.cnt
def __init__(self, warmup_iter=3):
"""
Args:
warmup_iter (int): the number of iterations at the beginning to exclude
from timing.
"""
self._warmup_iter = warmup_iter
self._step_timer = Timer()
def load_lvis_json(json_file, image_root, dataset_name=None):
from lvis import LVIS
json_file = PathManager.get_local_path(json_file)
timer = Timer()
lvis_api = LVIS(json_file)
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds()))
if dataset_name is not None:
meta = get_lvis_instances_meta(dataset_name)
MetadataCatalog.get(dataset_name).set(**meta)
img_ids = sorted(lvis_api.imgs.keys())
imgs = lvis_api.load_imgs(img_ids)
anns = [lvis_api.img_ann_map[img_id] for img_id in img_ids]
ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image]
assert len(set(ann_ids)) == len(ann_ids), \
f"Annotation ids in '{json_file}' are not unique"
imgs_anns = list(zip(imgs, anns))
logger.info(f"Loaded {len(imgs_anns)} images in the LVIS format from {json_file}")
dataset_dicts = []
for (img_dict, anno_dict_list) in imgs_anns:
record = {}
file_name = img_dict["file_name"]
if img_dict["file_name"].startswith("COCO"):
file_name = file_name[-16:]
record["file_name"] = os.path.join(image_root, file_name)
record["height"] = img_dict["height"]
record["width"] = img_dict["width"]
record["not_exhaustive_category_ids"] = img_dict.get("not_exhaustive_category_ids", [])
record["neg_category_ids"] = img_dict.get("neg_category_ids", [])
image_id = record["image_id"] = img_dict["id"]
objs = []
for anno in anno_dict_list:
assert anno["image_id"] == image_id
obj = {"bbox": anno["bbox"], "bbox_mode": BoxMode.XYWH_ABS}
obj["category_id"] = anno["category_id"] - 1
segm = anno["segmentation"]
valid_segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6]
assert len(segm) == len(valid_segm), \
"Annotation contains an invalid polygon with < 3 points"
assert len(segm) > 0
obj["segmentation"] = segm
objs.append(obj)
record["annotations"] = objs
dataset_dicts.append(record)
return dataset_dicts
def __init__(self, warmup_iter=3):
"""
Args:
warmup_iter (int): the number of iterations at the beginning to exclude
from timing.
"""
self._warmup_iter = warmup_iter
self._step_timer = Timer() # 记录每个step 的时间
self._start_time = time.perf_counter() #记录当前时间 时间单位是s
self._total_timer = Timer() # 记录总的训练时间
def benchmark_data(args):
cfg = setup(args)
dataloader = build_detection_train_loader(cfg)
timer = Timer()
itr = iter(dataloader)
for i in range(10): # warmup
next(itr)
if i == 0:
startup_time = timer.seconds()
timer = Timer()
max_iter = 1000
for _ in tqdm.trange(max_iter):
next(itr)
logger.info("{} iters ({} images) in {} seconds.".format(
max_iter, max_iter * cfg.SOLVER.IMS_PER_BATCH, timer.seconds()))
logger.info("Startup time: {} seconds".format(startup_time))
vram = psutil.virtual_memory()
logger.info("RAM Usage: {:.2f}/{:.2f} GB".format(
(vram.total - vram.available) / 1024**3, vram.total / 1024**3))
# test for a few more rounds
for _ in range(10):
timer = Timer()
max_iter = 1000
for _ in tqdm.trange(max_iter):
next(itr)
logger.info("{} iters ({} images) in {} seconds.".format(
max_iter, max_iter * cfg.SOLVER.IMS_PER_BATCH, timer.seconds()))
def __call__(self):
timer = Timer()
json_file = PathManager.get_local_path(self.json_file)
with open(json_file, 'r') as file:
# imgs_anns = json.load(file)
imgs_anns = file.readlines()
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(
json_file, timer.seconds()))
logger.info("Loaded {} images in CrowdHuman format from {}".format(
len(imgs_anns), json_file))
dataset_dicts = []
# aspect_ratios = []
for idx, ann in enumerate(imgs_anns):
v = json.loads(ann)
record = {}
filename = v["filename"]
# NOTE when filename starts with '/', it is an absolute filename thus os.path.join doesn't work
if filename.startswith('/'):
filename = os.path.normpath(self.image_root + filename)
else:
filename = os.path.join(self.image_root, filename)
height, width = v["image_height"], v["image_width"]
record["file_name"] = filename
record["image_id"] = idx
record["height"] = height
record["width"] = width
objs = []
for anno in v.get('instances', []):
x1, y1, x2, y2 = anno['bbox']
w = x2 - x1
h = y2 - y1
obj = {
"category_id": anno['label'],
"bbox": anno['bbox'],
"vbbox": anno['vbbox'],
"is_ignored": anno.get('is_ignored', False),
'area': w * h,
# 'bbox_mode': BoxMode.XYXY_ABS
}
objs.append(obj)
# ratio = 1.0 * (height + 1) / (width + 1) # do something with ratio ?
record["annotations"] = objs
# dataset_dicts.append(record) # to print class histogram
dataset_dicts.append(ImageMeta.encode(
record)) #this saves up to x2 memory when serializing the data
# aspect_ratios.append(ratio)
return dataset_dicts
def update_meta(json_file, dataset_name=None):
from pyherbtools.herb import HERB
if dataset_name is not None and "test" not in dataset_name:
logger.info("Update Metadat of {} dataset".format(dataset_name))
timer = Timer()
json_file = PathManager.get_local_path(json_file)
with contextlib.redirect_stdout(io.StringIO()):
herb_api = HERB(json_file)
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(
json_file, timer.seconds()))
meta = MetadataCatalog.get(dataset_name)
cat_ids = sorted(herb_api.getCatIds())
cats = herb_api.loadCats(cat_ids)
# The categories in a custom json file may not be sorted.
thing_classes = [
c["name"] for c in sorted(cats, key=lambda x: x["id"])
]
meta.thing_classes = thing_classes
logger.info("Creating hierarchy target from given annotation")
order_family_hierarchy = torch.zeros(len(meta.family_map),
len(meta.order_map))
family_species_hierarchy = torch.zeros(len(meta.species_map),
len(meta.family_map))
for cat in cats:
order_id = meta.order_map[cat["order"]]
family_id = meta.family_map[cat["family"]]
species_id = meta.species_map[cat["name"]]
order_family_hierarchy[family_id][order_id] = 1
family_species_hierarchy[species_id][family_id] = 1
from torch import nn
order_family_hierarchy = nn.Softmax(dim=1)(order_family_hierarchy)
family_species_hierarchy = nn.Softmax(dim=1)(family_species_hierarchy)
meta.hierarchy_prior = {
"order|family": order_family_hierarchy,
"family|species": family_species_hierarchy
}
meta.cats = cats
meta.num_classes = {
"family": len(meta.family_map),
"order": len(meta.order_map),
"species": len(meta.species_map),
}
def __init__(self, max_iter, cfg):
"""
Args:
max_iter (int): the max number of iteration of the current epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.max_iter = max_iter
self.iter_timer = Timer()
self.num_samples = 0
self.stats = {}
def __init__(self, cfg):
self.cfg = cfg
self.forward_timer = Timer()
self.total_time = 0
self.cnt = 0
self.score = dict()
self.output_dir = Join(cfg.TEST.OUTPUT_DIR, cfg.TEST.DATASET)
self.save_img = cfg.TEST.SAVE_IMG
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
self.score_csv = open(Join(self.output_dir, "score.csv"), 'w')
self.score_csv.write("vid, image_id, psnr, ssim\n")
def __init__(self, epoch_iters, cfg):
"""
Args:
epoch_iters (int): the overall number of iterations of one epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.epoch_iters = epoch_iters
self.MAX_EPOCH = cfg.SOLVER.MAX_EPOCH * epoch_iters
self.iter_timer = Timer()
self.log_period = cfg.LOG_PERIOD
self.infos = None
self.num_samples = 0
def benchmark_data(args):
cfg = setup(args)
dataloader = build_detection_train_loader(cfg)
itr = iter(dataloader)
for _ in range(10): # warmup
next(itr)
timer = Timer()
max_iter = 1000
for _ in tqdm.trange(max_iter):
next(itr)
logger.info("{} iters ({} images) in {} seconds.".format(
max_iter, max_iter * cfg.SOLVER.IMS_PER_BATCH, timer.seconds()))
class EpochTimer:
"""
A timer which computes the epoch time.
"""
def __init__(self) -> None:
self.timer = Timer()
self.timer.reset()
self.epoch_times = []
def reset(self) -> None:
"""
Reset the epoch timer.
"""
self.timer.reset()
self.epoch_times = []
def epoch_tic(self):
"""
Start to record time.
"""
self.timer.reset()
def epoch_toc(self):
"""
Stop to record time.
"""
self.timer.pause()
self.epoch_times.append(self.timer.seconds())
def last_epoch_time(self):
"""
Get the time for the last epoch.
"""
assert len(self.epoch_times) > 0, "No epoch time has been recorded!"
return self.epoch_times[-1]
def avg_epoch_time(self):
"""
Calculate the average epoch time among the recorded epochs.
"""
assert len(self.epoch_times) > 0, "No epoch time has been recorded!"
return np.mean(self.epoch_times)
def median_epoch_time(self):
"""
Calculate the median epoch time among the recorded epochs.
"""
assert len(self.epoch_times) > 0, "No epoch time has been recorded!"
return np.median(self.epoch_times)
def __init__(self, epoch_iters, cfg):
"""
:param epoch_iters: iters in one epoch
:param cfg:
"""
self._cfg = cfg
self.epoch_iters = epoch_iters
# self.loss=ScalarMeter(cfg.LOG_PERIOD)
self.mse_loss = ScalarMeter(cfg.LOG_PERIOD)
self.entropy_loss = ScalarMeter(cfg.LOG_PERIOD)
self.combine_loss = ScalarMeter(cfg.LOG_PERIOD)
self.iter_timer = Timer()
self.lr = None
# self.loss_total=0.0
self.MAX_EPOCH = cfg.SOLVER.MAX_EPOCH * epoch_iters
def load_cub_json(ann_files, image_root, dataset_name=None):
images_txt, classes_txt, image_class_txt, train_test_split_txt = ann_files
split = 0
if 'test' in dataset_name:
split = 1
images_txt = open(images_txt, 'r').readlines()
image_class_txt = open(image_class_txt, 'r').readlines()
classes_txt = open(classes_txt, 'r').readlines()
train_test_split_txt = open(train_test_split_txt, 'r').readlines()
imgs = []
anns = []
classes = {}
for i in range(len(train_test_split_txt)):
image_id, curr_split = train_test_split_txt[i].split()
if int(curr_split) == split:
_, image_path = images_txt[i].split()
_, class_id = image_class_txt[i].split()
_, class_name = classes_txt[int(class_id) - 1].split()
curr_image = {"id": int(image_id), "file_name": image_path}
curr_ann = {"id": i, "category_id":int(class_id) - 1, "image_id":int(image_id)}
imgs.append(curr_image)
anns.append(curr_ann)
classes[int(class_id) - 1] = class_name
imgs_anns = list(zip(imgs, anns))
logger.info("Loaded {} images in CUB format from CUB-200".format(len(imgs_anns)))
dataset_dicts = []
ann_keys = ["category_id"]
logger.info("Convert CUB format into herbarium format")
timer = Timer()
meta = MetadataCatalog.get(dataset_name)
dataset_dicts = [process_per_record(anns, image_root, ann_keys, meta) for anns in imgs_anns]
logger.info("Processing Record takes {:.2f} seconds.".format(timer.seconds()))
return dataset_dicts
def __init__(self, overall_iters, cfg, mode):
"""
overall_iters (int): the overall number of iterations of one epoch.
cfg (CfgNode): configs.
mode (str): `train`, `val`, or `test` mode.
"""
is_custom_dataset = cfg.TRAIN.DATASET == "custom" and cfg.TEST.DATASET == "custom"
if is_custom_dataset:
logger.info("Creating AVA Meter for custom dataset in %s mode" %
mode)
else:
logger.info("Creating AVA Meter for AVA dataset in %s mode" % mode)
self.cfg = cfg
self.lr = None
self.loss = ScalarMeter(cfg.LOG_PERIOD)
self.full_ava_test = cfg.AVA.FULL_TEST_ON_VAL if not is_custom_dataset else cfg.CUSTOM_DATASET.FULL_TEST_ON_VAL
self.mode = mode
self.iter_timer = Timer()
self.all_preds = []
self.all_ori_boxes = []
self.all_metadata = []
self.overall_iters = overall_iters
self.excluded_keys = read_exclusions(
os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.EXCLUSION_FILE)
) if not is_custom_dataset else read_exclusions(
os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR,
cfg.CUSTOM_DATASET.EXCLUSION_FILE))
self.categories, self.class_whitelist = read_labelmap(
os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.LABEL_MAP_FILE)
) if not is_custom_dataset else read_labelmap(
os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR,
cfg.CUSTOM_DATASET.LABEL_MAP_FILE))
gt_filename = os.path.join(
cfg.AVA.ANNOTATION_DIR, cfg.AVA.GROUNDTRUTH_FILE
) if not is_custom_dataset else os.path.join(
cfg.CUSTOM_DATASET.ANNOTATION_DIR,
cfg.CUSTOM_DATASET.GROUNDTRUTH_FILE)
self.full_groundtruth = read_csv(gt_filename, self.class_whitelist)
self.mini_groundtruth = get_ava_mini_groundtruth(self.full_groundtruth)
_, self.video_idx_to_name = ava_helper.load_image_lists(
cfg, mode == "train"
) if not is_custom_dataset else custom_helper.load_image_lists(
cfg, mode == "train")
def test_avg_second(self) -> None:
"""
Test avg_seconds that counts the average time.
"""
for pause_second in (0.1, 0.15):
timer = Timer()
for t in (pause_second,) * 10:
if timer.is_paused():
timer.resume()
time.sleep(t)
timer.pause()
self.assertTrue(
math.isclose(pause_second, timer.avg_seconds(), rel_tol=1e-1),
msg="{}: {}".format(pause_second, timer.avg_seconds()),
)
def __init__(self, max_iter, cfg):
"""
Args:
max_iter (int): the max number of iteration of the current epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.max_iter = max_iter
self.iter_timer = Timer()
# Current minibatch errors (smoothed over a window).
self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
# Min errors (over the full val set).
self.min_top1_err = 100.0
self.min_top5_err = 100.0
# Number of misclassified examples.
self.num_top1_mis = 0
self.num_top5_mis = 0
self.num_samples = 0
def __init__(
self,
num_videos,
num_clips,
num_cls,
overall_iters,
multi_label=False,
ensemble_method="sum",
log_period=1,
):
"""
Construct tensors to store the predictions and labels. Expect to get
num_clips predictions from each video, and calculate the metrics on
num_videos videos.
Args:
num_videos (int): number of videos to test.
num_clips (int): number of clips sampled from each video for
aggregating the final prediction for the video.
num_cls (int): number of classes for each prediction.
overall_iters (int): overall iterations for testing.
multi_label (bool): if True, use map as the metric.
ensemble_method (str): method to perform the ensemble, options
include "sum", and "max".
"""
self.iter_timer = Timer()
self.num_clips = num_clips
self.overall_iters = overall_iters
self.multi_label = multi_label
self.ensemble_method = ensemble_method
self.log_period = log_period
# Initialize tensors.
self.video_preds = torch.zeros((num_videos, num_cls))
if multi_label:
self.video_preds -= 1e10
self.video_labels = (torch.zeros(
(num_videos, num_cls)) if multi_label else torch.zeros(
(num_videos)).long())
self.clip_count = torch.zeros((num_videos)).long()
self.view_count = torch.zeros((num_videos, num_clips)).long()
# Reset metric.
self.reset()
def __init__(self, epoch_iters, cfg):
"""
Args:
epoch_iters (int): the overall number of iterations of one epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.epoch_iters = epoch_iters
self.MAX_EPOCH = cfg.SOLVER.MAX_EPOCH * epoch_iters
self.iter_timer = Timer()
self.loss = ScalarMeter(cfg.LOG_PERIOD)
self.loss_total = 0.0
self.lr = None
# Current minibatch errors (smoothed over a window).
self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
# Number of misclassified examples.
self.num_top1_mis = 0
self.num_top5_mis = 0
self.num_samples = 0
def _load_coco_annotations(json_file: str):
"""
Load COCO annotations from a JSON file
Args:
json_file: str
Path to the file to load annotations from
Returns:
Instance of `pycocotools.coco.COCO` that provides access to annotations
data
"""
from pycocotools.coco import COCO
logger = logging.getLogger(__name__)
timer = Timer()
with contextlib.redirect_stdout(io.StringIO()):
coco_api = COCO(json_file)
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds()))
return coco_api
def _load_lvis_annotations(json_file: str):
"""
Load COCO annotations from a JSON file
Args:
json_file: str
Path to the file to load annotations from
Returns:
Instance of `pycocotools.coco.COCO` that provides access to annotations
data
"""
from lvis import LVIS
json_file = PathManager.get_local_path(json_file)
logger = logging.getLogger(__name__)
timer = Timer()
lvis_api = LVIS(json_file)
if timer.seconds() > 1:
logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds()))
return lvis_api
def __init__(self, epoch_iters, cfg):
"""
Args:
epoch_iters (int): the overall number of iterations of one epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.epoch_iters = epoch_iters
self.MAX_EPOCH = cfg.SOLVER.MAX_EPOCH * epoch_iters
self.iter_timer = Timer()
self.data_timer = Timer()
self.net_timer = Timer()
self.loss = ScalarMeter(cfg.LOG_PERIOD)
self.loss_total = 0.0
self.loss_verb = ScalarMeter(cfg.LOG_PERIOD)
self.loss_verb_total = 0.0
self.loss_noun = ScalarMeter(cfg.LOG_PERIOD)
self.loss_noun_total = 0.0
self.lr = None
# Current minibatch accuracies (smoothed over a window).
self.mb_top1_acc = ScalarMeter(cfg.LOG_PERIOD)
self.mb_top5_acc = ScalarMeter(cfg.LOG_PERIOD)
self.mb_verb_top1_acc = ScalarMeter(cfg.LOG_PERIOD)
self.mb_verb_top5_acc = ScalarMeter(cfg.LOG_PERIOD)
self.mb_noun_top1_acc = ScalarMeter(cfg.LOG_PERIOD)
self.mb_noun_top5_acc = ScalarMeter(cfg.LOG_PERIOD)
# Number of correctly classified examples.
self.num_top1_cor = 0
self.num_top5_cor = 0
self.num_verb_top1_cor = 0
self.num_verb_top5_cor = 0
self.num_noun_top1_cor = 0
self.num_noun_top5_cor = 0
self.num_samples = 0
self.output_dir = cfg.OUTPUT_DIR
def __init__(self, max_iter, cfg):
"""
Args:
max_iter (int): the max number of iteration of the current epoch.
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.max_iter = max_iter
self.iter_timer = Timer()
self.data_timer = Timer()
self.net_timer = Timer()
# Current minibatch errors (smoothed over a window).
self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
# Min errors (over the full val set).
self.min_top1_err = 100.0
self.min_top5_err = 100.0
# Number of misclassified examples.
self.num_top1_mis = 0
self.num_top5_mis = 0
self.num_samples = 0
self.all_preds = []
self.all_labels = []
self.output_dir = cfg.OUTPUT_DIR
self.extra_stats = {}
self.extra_stats_total = {}
self.log_period = cfg.LOG_PERIOD
def __init__(self, num_videos, num_clips, num_cls, overall_iters):
"""
Construct tensors to store the predictions and labels. Expect to get
num_clips predictions from each video, and calculate the metrics on
num_videos videos.
Args:
num_videos (int): number of videos to test.
num_clips (int): number of clips sampled from each video for
aggregating the final prediction for the video.
num_cls (int): number of classes for each prediction.
overall_iters (int): overall iterations for testing.
"""
self.iter_timer = Timer()
self.num_clips = num_clips
self.overall_iters = overall_iters
# Initialize tensors.
self.video_preds = torch.zeros((num_videos, num_cls))
self.video_labels = torch.zeros((num_videos)).long()
self.clip_count = torch.zeros((num_videos)).long()
# Reset metric.
self.reset()
def benchmark_eval(args):
cfg = setup(args)
if args.config_file.endswith(".yaml"):
model = build_model(cfg)
DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS)
cfg.defrost()
cfg.DATALOADER.NUM_WORKERS = 0
data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0])
else:
model = instantiate(cfg.model)
model.to(cfg.train.device)
DetectionCheckpointer(model).load(cfg.train.init_checkpoint)
cfg.dataloader.num_workers = 0
data_loader = instantiate(cfg.dataloader.test)
model.eval()
logger.info("Model:\n{}".format(model))
dummy_data = DatasetFromList(list(itertools.islice(data_loader, 100)),
copy=False)
def f():
while True:
yield from dummy_data
for k in range(5): # warmup
model(dummy_data[k])
max_iter = 300
timer = Timer()
with tqdm.tqdm(total=max_iter) as pbar:
for idx, d in enumerate(f()):
if idx == max_iter:
break
model(d)
pbar.update()
logger.info("{} iters in {} seconds.".format(max_iter, timer.seconds()))
def __init__(self, max_iter, max_epoch, cfg):
"""
Args:
max_iter (int): the max number of iteration of the current epoch.
max_epoch (int): the maximum number of epochs of the training phase
cfg (CfgNode): configs.
"""
self._cfg = cfg
self.max_iter = max_iter
self.MAX_EPOCH = max_epoch
self.iter_timer = Timer()
# Current minibatch errors (smoothed over a window).
self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD)
self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD)
# Min errors (over the full val set).
self.min_top1_err = 100.0
self.min_top5_err = 100.0
# Number of misclassified examples.
self.num_top1_mis = 0
self.num_top5_mis = 0
self.num_samples = 0
self.all_preds = []
self.all_labels = []
self.max_map = 0.0
def __init__(self, cfg, args, iter_every_epoch, **kwargs):
super().__init__()
# fmt: off
self.cfg = cfg
self.args = args
self.iter_every_epoch = iter_every_epoch
# fmt: on
self.timer = Timer()
self.device = torch.device(f'cuda:{comm.get_rank()}')
self.logger = logging.getLogger('tl')
self.num_gpu = comm.get_world_size()
self.distributed = comm.get_world_size() > 1
# torch.cuda.set_device(self.device)
# self.build_models(cfg=cfg)
self.to(self.device)
