def read_csv(csv_file, class_whitelist=None, load_score=False):
"""Loads boxes and class labels from a CSV file in the AVA format.
CSV file format described at https://research.google.com/ava/download.html.
Args:
csv_file: A file object.
class_whitelist: If provided, boxes corresponding to (integer) class labels
not in this set are skipped.
Returns:
boxes: A dictionary mapping each unique image key (string) to a list of
boxes, given as coordinates [y1, x1, y2, x2].
labels: A dictionary mapping each unique image key (string) to a list of
integer class lables, matching the corresponding box in `boxes`.
scores: A dictionary mapping each unique image key (string) to a list of
score values lables, matching the corresponding label in `labels`. If
scores are not provided in the csv, then they will default to 1.0.
"""
boxes = defaultdict(list)
labels = defaultdict(list)
scores = defaultdict(list)
with pathmgr.open(csv_file, "r") as f:
reader = csv.reader(f)
for row in reader:
assert len(row) in [7, 8], "Wrong number of columns: " + row
image_key = make_image_key(row[0], row[1])
x1, y1, x2, y2 = [float(n) for n in row[2:6]]
action_id = int(row[6])
if class_whitelist and action_id not in class_whitelist:
continue
score = 1.0
if load_score:
score = float(row[7])
boxes[image_key].append([y1, x1, y2, x2])
labels[image_key].append(action_id)
scores[image_key].append(score)
return boxes, labels, scores
def _cached_log_stream(filename):
# Use 1K buffer if writing to cloud storage.
io = pathmgr.open(filename,
"a",
buffering=1024 if "://" in filename else -1)
atexit.register(io.close)
return io
def _construct_loader(self):
"""
Construct the video loader.
"""
path_to_file = os.path.join(self.cfg.DATA.PATH_TO_DATA_DIR,
"{}.csv".format(self.mode))
assert pathmgr.exists(path_to_file), "{} dir not found".format(
path_to_file)
self._path_to_videos = []
self._labels = []
self._spatial_temporal_idx = []
with pathmgr.open(path_to_file, "r") as f:
for clip_idx, path_label in enumerate(f.read().splitlines()):
assert (len(
path_label.split(self.cfg.DATA.PATH_LABEL_SEPARATOR)) == 2)
path, label = path_label.split(
self.cfg.DATA.PATH_LABEL_SEPARATOR)
for idx in range(self._num_clips):
self._path_to_videos.append(
os.path.join(self.cfg.DATA.PATH_PREFIX, path))
self._labels.append(int(label))
self._spatial_temporal_idx.append(idx)
self._video_meta[clip_idx * self._num_clips + idx] = {}
assert (len(self._path_to_videos) >
0), "Failed to load Kinetics split {} from {}".format(
self._split_idx, path_to_file)
logger.info(
"Constructing kinetics dataloader (size: {}) from {}".format(
len(self._path_to_videos), path_to_file))
def retry_load_images(image_paths, retry=10, backend="pytorch"):
"""
This function is to load images with support of retrying for failed load.
Args:
image_paths (list): paths of images needed to be loaded.
retry (int, optional): maximum time of loading retrying. Defaults to 10.
backend (str): `pytorch` or `cv2`.
Returns:
imgs (list): list of loaded images.
"""
for i in range(retry):
imgs = []
for image_path in image_paths:
with pathmgr.open(image_path, "rb") as f:
img_str = np.frombuffer(f.read(), np.uint8)
img = cv2.imdecode(img_str, flags=cv2.IMREAD_COLOR)
imgs.append(img)
if all(img is not None for img in imgs):
if backend == "pytorch":
imgs = torch.as_tensor(np.stack(imgs))
return imgs
else:
logger.warn("Reading failed. Will retry.")
time.sleep(1.0)
if i == retry - 1:
raise Exception("Failed to load images {}".format(image_paths))
def load_image(self, im_path):
"""Prepares the image for network input with format of CHW RGB float"""
with pathmgr.open(im_path, "rb") as f:
with Image.open(f) as im:
im = im.convert("RGB")
im = torch.from_numpy(np.array(im).astype(np.float32) / 255.0)
# H W C to C H W
im = im.permute([2, 0, 1])
return im
def parse_bboxes_file(ann_filenames,
ann_is_gt_box,
detect_thresh,
boxes_sample_rate=1):
"""
Parse AVA bounding boxes files.
Args:
ann_filenames (list of str(s)): a list of AVA bounding boxes annotation files.
ann_is_gt_box (list of bools): a list of boolean to indicate whether the corresponding
ann_file is ground-truth. `ann_is_gt_box[i]` correspond to `ann_filenames[i]`.
detect_thresh (float): threshold for accepting predicted boxes, range [0, 1].
boxes_sample_rate (int): sample rate for test bounding boxes. Get 1 every `boxes_sample_rate`.
"""
all_boxes = {}
count = 0
unique_box_count = 0
for filename, is_gt_box in zip(ann_filenames, ann_is_gt_box):
with pathmgr.open(filename, "r") as f:
for line in f:
row = line.strip().split(",")
# When we use predicted boxes to train/eval, we need to
# ignore the boxes whose scores are below the threshold.
if not is_gt_box:
score = float(row[7])
if score < detect_thresh:
continue
video_name, frame_sec = row[0], int(row[1])
if frame_sec % boxes_sample_rate != 0:
continue
# Box with format [x1, y1, x2, y2] with a range of [0, 1] as float.
box_key = ",".join(row[2:6])
box = list(map(float, row[2:6]))
label = -1 if row[6] == "" else int(row[6])
if video_name not in all_boxes:
all_boxes[video_name] = {}
for sec in AVA_VALID_FRAMES:
all_boxes[video_name][sec] = {}
if box_key not in all_boxes[video_name][frame_sec]:
all_boxes[video_name][frame_sec][box_key] = [box, []]
unique_box_count += 1
all_boxes[video_name][frame_sec][box_key][1].append(label)
if label != -1:
count += 1
for video_name in all_boxes.keys():
for frame_sec in all_boxes[video_name].keys():
# Save in format of a list of [box_i, box_i_labels].
all_boxes[video_name][frame_sec] = list(
all_boxes[video_name][frame_sec].values())
return all_boxes, count, unique_box_count
def write_results(detections, filename):
"""Write prediction results into official formats."""
start = time.time()
boxes, labels, scores = detections
with pathmgr.open(filename, "w") as f:
for key in boxes.keys():
for box, label, score in zip(boxes[key], labels[key], scores[key]):
f.write("%s,%.03f,%.03f,%.03f,%.03f,%d,%.04f\n" %
(key, box[1], box[0], box[3], box[2], label, score))
logger.info("AVA results wrote to %s" % filename)
logger.info("\ttook %d seconds." % (time.time() - start))
def read_labelmap(labelmap_file):
"""Read label map and class ids."""
labelmap = []
class_ids = set()
name = ""
class_id = ""
with pathmgr.open(labelmap_file, "r") as f:
for line in f:
if line.startswith(" name:"):
name = line.split('"')[1]
elif line.startswith(" id:") or line.startswith(" label_id:"):
class_id = int(line.strip().split(" ")[-1])
labelmap.append({"id": class_id, "name": name})
class_ids.add(class_id)
return labelmap, class_ids
def read_exclusions(exclusions_file):
"""Reads a CSV file of excluded timestamps.
Args:
exclusions_file: A file object containing a csv of video-id,timestamp.
Returns:
A set of strings containing excluded image keys, e.g. "aaaaaaaaaaa,0904",
or an empty set if exclusions file is None.
"""
excluded = set()
if exclusions_file:
with pathmgr.open(exclusions_file, "r") as f:
reader = csv.reader(f)
for row in reader:
assert len(row) == 2, "Expected only 2 columns, got: " + row
excluded.add(make_image_key(row[0], row[1]))
return excluded
def _construct_loader(self):
"""
Construct the video loader.
"""
path_to_file = os.path.join(self.cfg.DATA.PATH_TO_DATA_DIR,
"{}.csv".format(self.mode))
assert pathmgr.exists(path_to_file), "{} dir not found".format(
path_to_file)
self._path_to_videos = []
self._labels = []
self._spatial_temporal_idx = []
self.cur_iter = 0
self.chunk_epoch = 0
self.epoch = 0.0
self.skip_rows = self.cfg.DATA.SKIP_ROWS
with pathmgr.open(path_to_file, "r") as f:
if self.use_chunk_loading:
rows = self._get_chunk(f, self.cfg.DATA.LOADER_CHUNK_SIZE)
else:
rows = f.read().splitlines()
for clip_idx, path_label in enumerate(rows):
fetch_info = path_label.split(
self.cfg.DATA.PATH_LABEL_SEPARATOR)
if len(fetch_info) == 2:
path, label = fetch_info
elif len(fetch_info) == 3:
path, fn, label = fetch_info
elif len(fetch_info) == 1:
path, label = fetch_info[0], 0
else:
raise RuntimeError(
"Failed to parse video fetch {} info {} retries.".
format(path_to_file, fetch_info))
for idx in range(self._num_clips):
self._path_to_videos.append(
os.path.join(self.cfg.DATA.PATH_PREFIX, path))
self._labels.append(int(label))
self._spatial_temporal_idx.append(idx)
self._video_meta[clip_idx * self._num_clips + idx] = {}
assert (len(self._path_to_videos) >
0), "Failed to load Kinetics split {} from {}".format(
self._split_idx, path_to_file)
logger.info(
"Constructing kinetics dataloader (size: {} skip_rows {}) from {} "
.format(len(self._path_to_videos), self.skip_rows, path_to_file))
def load_image_lists(cfg, is_train):
"""
Loading image paths from corresponding files.
Args:
cfg (CfgNode): config.
is_train (bool): if it is training dataset or not.
Returns:
image_paths (list[list]): a list of items. Each item (also a list)
corresponds to one video and contains the paths of images for
this video.
video_idx_to_name (list): a list which stores video names.
"""
list_filenames = [
os.path.join(cfg.AVA.FRAME_LIST_DIR, filename) for filename in (
cfg.AVA.TRAIN_LISTS if is_train else cfg.AVA.TEST_LISTS)
]
image_paths = defaultdict(list)
video_name_to_idx = {}
video_idx_to_name = []
for list_filename in list_filenames:
with pathmgr.open(list_filename, "r") as f:
f.readline()
for line in f:
row = line.split()
# The format of each row should follow:
# original_vido_id video_id frame_id path labels.
assert len(row) == 5
video_name = row[0]
if video_name not in video_name_to_idx:
idx = len(video_name_to_idx)
video_name_to_idx[video_name] = idx
video_idx_to_name.append(video_name)
data_key = video_name_to_idx[video_name]
image_paths[data_key].append(
os.path.join(cfg.AVA.FRAME_DIR, row[3]))
image_paths = [image_paths[i] for i in range(len(image_paths))]
logger.info("Finished loading image paths from: %s" %
", ".join(list_filenames))
return image_paths, video_idx_to_name
def load_image_lists(frame_list_file, prefix="", return_list=False):
"""
Load image paths and labels from a "frame list".
Each line of the frame list contains:
`original_vido_id video_id frame_id path labels`
Args:
frame_list_file (string): path to the frame list.
prefix (str): the prefix for the path.
return_list (bool): if True, return a list. If False, return a dict.
Returns:
image_paths (list or dict): list of list containing path to each frame.
If return_list is False, then return in a dict form.
labels (list or dict): list of list containing label of each frame.
If return_list is False, then return in a dict form.
"""
image_paths = defaultdict(list)
labels = defaultdict(list)
with pathmgr.open(frame_list_file, "r") as f:
assert f.readline().startswith("original_vido_id")
for line in f:
row = line.split()
# original_vido_id video_id frame_id path labels
assert len(row) == 5
video_name = row[0]
if prefix == "":
path = row[3]
else:
path = os.path.join(prefix, row[3])
image_paths[video_name].append(path)
frame_labels = row[-1].replace('"', "")
if frame_labels != "":
labels[video_name].append(
[int(x) for x in frame_labels.split(",")]
)
else:
labels[video_name].append([])
if return_list:
keys = image_paths.keys()
image_paths = [image_paths[key] for key in keys]
labels = [labels[key] for key in keys]
return image_paths, labels
return dict(image_paths), dict(labels)
def log_json_stats(stats, output_dir=None):
"""
Logs json stats.
Args:
stats (dict): a dictionary of statistical information to log.
"""
stats = {
k: decimal.Decimal("{:.3f}".format(v)) if isinstance(v, float) else v
for k, v in stats.items()
}
json_stats = simplejson.dumps(stats, sort_keys=True, use_decimal=True)
logger = get_logger(__name__)
logger.info("json_stats: {:s}".format(json_stats))
if du.is_root_proc() and output_dir:
filename = os.path.join(output_dir, "json_stats.log")
try:
with pathmgr.open(
filename, "a",
buffering=1024 if "://" in filename else -1) as f:
f.write("json_stats: {:s}\n".format(json_stats))
except Exception:
logger.info(
"Failed to write to json_stats.log: {}".format(json_stats))
def _prepare_im_tf(self, im_path):
with pathmgr.open(im_path, "rb") as f:
with Image.open(f) as im:
im = im.convert("RGB")
# Convert HWC/BGR/int to HWC/RGB/float format for applying transforms
train_size, test_size = (
self.cfg.DATA.TRAIN_CROP_SIZE,
self.cfg.DATA.TEST_CROP_SIZE,
)
if self.mode == "train":
aug_transform = transforms_imagenet_train(
img_size=(train_size, train_size),
color_jitter=self.cfg.AUG.COLOR_JITTER,
auto_augment=self.cfg.AUG.AA_TYPE,
interpolation=self.cfg.AUG.INTERPOLATION,
re_prob=self.cfg.AUG.RE_PROB,
re_mode=self.cfg.AUG.RE_MODE,
re_count=self.cfg.AUG.RE_COUNT,
mean=self.cfg.DATA.MEAN,
std=self.cfg.DATA.STD,
)
else:
t = []
size = int((256 / 224) * test_size)
t.append(
transforms_tv.Resize(
size, interpolation=3
), # to maintain same ratio w.r.t. 224 images
)
t.append(transforms_tv.CenterCrop(test_size))
t.append(transforms_tv.ToTensor())
t.append(
transforms_tv.Normalize(self.cfg.DATA.MEAN, self.cfg.DATA.STD))
aug_transform = transforms_tv.Compose(t)
im = aug_transform(im)
return im
def get_class_names(path, parent_path=None, subset_path=None):
"""
Read json file with entries {classname: index} and return
an array of class names in order.
If parent_path is provided, load and map all children to their ids.
Args:
path (str): path to class ids json file.
File must be in the format {"class1": id1, "class2": id2, ...}
parent_path (Optional[str]): path to parent-child json file.
File must be in the format {"parent1": ["child1", "child2", ...], ...}
subset_path (Optional[str]): path to text file containing a subset
of class names, separated by newline characters.
Returns:
class_names (list of strs): list of class names.
class_parents (dict): a dictionary where key is the name of the parent class
and value is a list of ids of the children classes.
subset_ids (list of ints): list of ids of the classes provided in the
subset file.
"""
try:
with pathmgr.open(path, "r") as f:
class2idx = json.load(f)
except Exception as err:
print("Fail to load file from {} with error {}".format(path, err))
return
max_key = max(class2idx.values())
class_names = [None] * (max_key + 1)
for k, i in class2idx.items():
class_names[i] = k
class_parent = None
if parent_path is not None and parent_path != "":
try:
with pathmgr.open(parent_path, "r") as f:
d_parent = json.load(f)
except EnvironmentError as err:
print("Fail to load file from {} with error {}".format(
parent_path, err))
return
class_parent = {}
for parent, children in d_parent.items():
indices = [
class2idx[c] for c in children if class2idx.get(c) is not None
]
class_parent[parent] = indices
subset_ids = None
if subset_path is not None and subset_path != "":
try:
with pathmgr.open(subset_path, "r") as f:
subset = f.read().split("\n")
subset_ids = [
class2idx[name] for name in subset
if class2idx.get(name) is not None
]
except EnvironmentError as err:
print("Fail to load file from {} with error {}".format(
subset_path, err))
return
return class_names, class_parent, subset_ids
def _construct_loader(self):
"""
Construct the video loader.
"""
# Loading label names.
with pathmgr.open(
os.path.join(
self.cfg.DATA.PATH_TO_DATA_DIR,
"something-something-v2-labels.json",
),
"r",
) as f:
label_dict = json.load(f)
# Loading labels.
label_file = os.path.join(
self.cfg.DATA.PATH_TO_DATA_DIR,
"something-something-v2-{}.json".format("train" if self.mode ==
"train" else "validation"),
)
with pathmgr.open(label_file, "r") as f:
label_json = json.load(f)
self._video_names = []
self._labels = []
for video in label_json:
video_name = video["id"]
template = video["template"]
template = template.replace("[", "")
template = template.replace("]", "")
label = int(label_dict[template])
self._video_names.append(video_name)
self._labels.append(label)
path_to_file = os.path.join(
self.cfg.DATA.PATH_TO_DATA_DIR,
"{}.csv".format("train" if self.mode == "train" else "val"),
)
assert pathmgr.exists(path_to_file), "{} dir not found".format(
path_to_file)
self._path_to_videos, _ = utils.load_image_lists(
path_to_file, self.cfg.DATA.PATH_PREFIX)
assert len(self._path_to_videos) == len(self._video_names), (
len(self._path_to_videos),
len(self._video_names),
)
# From dict to list.
new_paths, new_labels = [], []
for index in range(len(self._video_names)):
if self._video_names[index] in self._path_to_videos:
new_paths.append(
self._path_to_videos[self._video_names[index]])
new_labels.append(self._labels[index])
self._labels = new_labels
self._path_to_videos = new_paths
# Extend self when self._num_clips > 1 (during testing).
self._path_to_videos = list(
chain.from_iterable([[x] * self._num_clips
for x in self._path_to_videos]))
self._labels = list(
chain.from_iterable([[x] * self._num_clips for x in self._labels]))
self._spatial_temporal_idx = list(
chain.from_iterable([
range(self._num_clips)
for _ in range(len(self._path_to_videos))
]))
logger.info("Something-Something V2 dataloader constructed "
" (size: {}) from {}".format(len(self._path_to_videos),
path_to_file))
def perform_test(test_loader, model, test_meter, cfg, writer=None):
"""
For classification:
Perform mutli-view testing that uniformly samples N clips from a video along
its temporal axis. For each clip, it takes 3 crops to cover the spatial
dimension, followed by averaging the softmax scores across all Nx3 views to
form a video-level prediction. All video predictions are compared to
ground-truth labels and the final testing performance is logged.
For detection:
Perform fully-convolutional testing on the full frames without crop.
Args:
test_loader (loader): video testing loader.
model (model): the pretrained video model to test.
test_meter (TestMeter): testing meters to log and ensemble the testing
results.
cfg (CfgNode): configs. Details can be found in
slowfast/config/defaults.py
writer (TensorboardWriter object, optional): TensorboardWriter object
to writer Tensorboard log.
"""
# Enable eval mode.
model.eval()
test_meter.iter_tic()
for cur_iter, (inputs, labels, video_idx, time,
meta) in enumerate(test_loader):
if cfg.NUM_GPUS:
# Transfer the data to the current GPU device.
if isinstance(inputs, (list, )):
for i in range(len(inputs)):
inputs[i] = inputs[i].cuda(non_blocking=True)
else:
inputs = inputs.cuda(non_blocking=True)
# Transfer the data to the current GPU device.
labels = labels.cuda()
video_idx = video_idx.cuda()
for key, val in meta.items():
if isinstance(val, (list, )):
for i in range(len(val)):
val[i] = val[i].cuda(non_blocking=True)
else:
meta[key] = val.cuda(non_blocking=True)
test_meter.data_toc()
if cfg.DETECTION.ENABLE:
# Compute the predictions.
preds = model(inputs, meta["boxes"])
ori_boxes = meta["ori_boxes"]
metadata = meta["metadata"]
preds = preds.detach().cpu() if cfg.NUM_GPUS else preds.detach()
ori_boxes = (ori_boxes.detach().cpu()
if cfg.NUM_GPUS else ori_boxes.detach())
metadata = (metadata.detach().cpu()
if cfg.NUM_GPUS else metadata.detach())
if cfg.NUM_GPUS > 1:
preds = torch.cat(du.all_gather_unaligned(preds), dim=0)
ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes),
dim=0)
metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0)
test_meter.iter_toc()
# Update and log stats.
test_meter.update_stats(preds, ori_boxes, metadata)
test_meter.log_iter_stats(None, cur_iter)
elif cfg.TASK == "ssl" and cfg.MODEL.MODEL_NAME == "ContrastiveModel":
if not cfg.CONTRASTIVE.KNN_ON:
test_meter.finalize_metrics()
return test_meter
# preds = model(inputs, video_idx, time)
train_labels = (model.module.train_labels if hasattr(
model, "module") else model.train_labels)
yd, yi = model(inputs, video_idx, time)
batchSize = yi.shape[0]
K = yi.shape[1]
C = cfg.CONTRASTIVE.NUM_CLASSES_DOWNSTREAM # eg 400 for Kinetics400
candidates = train_labels.view(1, -1).expand(batchSize, -1)
retrieval = torch.gather(candidates, 1, yi)
retrieval_one_hot = torch.zeros((batchSize * K, C)).cuda()
retrieval_one_hot.scatter_(1, retrieval.view(-1, 1), 1)
yd_transform = yd.clone().div_(cfg.CONTRASTIVE.T).exp_()
probs = torch.mul(
retrieval_one_hot.view(batchSize, -1, C),
yd_transform.view(batchSize, -1, 1),
)
preds = torch.sum(probs, 1)
else:
# Perform the forward pass.
preds = model(inputs)
# Gather all the predictions across all the devices to perform ensemble.
if cfg.NUM_GPUS > 1:
preds, labels, video_idx = du.all_gather(
[preds, labels, video_idx])
if cfg.NUM_GPUS:
preds = preds.cpu()
labels = labels.cpu()
video_idx = video_idx.cpu()
test_meter.iter_toc()
# Update and log stats.
test_meter.update_stats(preds.detach(), labels.detach(),
video_idx.detach())
test_meter.log_iter_stats(cur_iter)
test_meter.iter_tic()
# Log epoch stats and print the final testing results.
if not cfg.DETECTION.ENABLE:
all_preds = test_meter.video_preds.clone().detach()
all_labels = test_meter.video_labels
if cfg.NUM_GPUS:
all_preds = all_preds.cpu()
all_labels = all_labels.cpu()
if writer is not None:
writer.plot_eval(preds=all_preds, labels=all_labels)
if cfg.TEST.SAVE_RESULTS_PATH != "":
save_path = os.path.join(cfg.OUTPUT_DIR,
cfg.TEST.SAVE_RESULTS_PATH)
if du.is_root_proc():
with pathmgr.open(save_path, "wb") as f:
pickle.dump([all_preds, all_labels], f)
logger.info("Successfully saved prediction results to {}".format(
save_path))
test_meter.finalize_metrics()
return test_meter
def visualize(cfg):
"""
Perform layer weights and activations visualization on the model.
Args:
cfg (CfgNode): configs. Details can be found in
slowfast/config/defaults.py
"""
if cfg.TENSORBOARD.ENABLE and (cfg.TENSORBOARD.MODEL_VIS.ENABLE
or cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE):
# Set up environment.
du.init_distributed_training(cfg)
# Set random seed from configs.
np.random.seed(cfg.RNG_SEED)
torch.manual_seed(cfg.RNG_SEED)
# Setup logging format.
logging.setup_logging(cfg.OUTPUT_DIR)
# Print config.
logger.info("Model Visualization with config:")
logger.info(cfg)
# Build the video model and print model statistics.
model = build_model(cfg)
model.eval()
if du.is_master_proc() and cfg.LOG_MODEL_INFO:
misc.log_model_info(model, cfg, use_train_input=False)
cu.load_test_checkpoint(cfg, model)
# Create video testing loaders.
vis_loader = loader.construct_loader(cfg, "test")
if cfg.DETECTION.ENABLE:
assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0
# Set up writer for logging to Tensorboard format.
if du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS):
writer = tb.TensorboardWriter(cfg)
else:
writer = None
if cfg.TENSORBOARD.PREDICTIONS_PATH != "":
assert not cfg.DETECTION.ENABLE, "Detection is not supported."
logger.info(
"Visualizing class-level performance from saved results...")
if writer is not None:
with pathmgr.open(cfg.TENSORBOARD.PREDICTIONS_PATH, "rb") as f:
preds, labels = pickle.load(f, encoding="latin1")
writer.plot_eval(preds, labels)
if cfg.TENSORBOARD.MODEL_VIS.ENABLE:
if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE:
assert (
not cfg.DETECTION.ENABLE
), "Detection task is currently not supported for Grad-CAM visualization."
if cfg.MODEL.ARCH in cfg.MODEL.SINGLE_PATHWAY_ARCH:
assert (
len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 1
), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format(
len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST))
elif cfg.MODEL.ARCH in cfg.MODEL.MULTI_PATHWAY_ARCH:
assert (
len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 2
), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format(
len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST))
else:
raise NotImplementedError(
"Model arch {} is not in {}".format(
cfg.MODEL.ARCH,
cfg.MODEL.SINGLE_PATHWAY_ARCH +
cfg.MODEL.MULTI_PATHWAY_ARCH,
))
logger.info("Visualize model analysis for {} iterations".format(
len(vis_loader)))
# Run visualization on the model
run_visualization(vis_loader, model, cfg, writer)
if cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE:
logger.info("Visualize Wrong Predictions for {} iterations".format(
len(vis_loader)))
perform_wrong_prediction_vis(vis_loader, model, cfg)
if writer is not None:
writer.close()
def _load_imdb(self):
split_path = os.path.join(self.cfg.DATA.PATH_TO_PRELOAD_IMDB,
f"{self.mode}.json")
with pathmgr.open(split_path, "r") as f:
data = f.read()
self._imdb = json.loads(data)
