def process_dataset(self, dataset_name, required, cfg):
timer = CodeTimer("building dynamics for {} with key {}".format(
dataset_name, required))
dataset_dicts = DatasetCatalog.get(dataset_name)
filtered_idx, filtered_dicts = filter_dataset(dataset_dicts,
[required])
worker_args = defaultdict(lambda: {"vid_dicts": []})
for d in filtered_dicts:
vid_num = d["image_id"] // 500
worker_args[vid_num]["vid_dicts"].append(d)
worker_args[vid_num]["required"] = required
if cfg.DEBUG:
data = [self.process_video(w) for w in worker_args.values()]
else:
with Pool(int(cpu_count())) as p:
data = p.map(self.process_video, worker_args.values())
inputs, targets, possible_flags, original_videos = zip(*data)
inputs = [torch.tensor(el) for el in inputs]
targets = [{k: torch.tensor(v)
for k, v in el.items()} for el in targets]
timer.done()
return inputs, targets, possible_flags, original_videos
def ai2thor_intphys_to_detectron(cfg, split, outfile):
timer = CodeTimer(f'started processing videos for {split}')
if split == "_val":
base_folder = os.path.join(
cfg.DATA_LOCATION, 'intphys_scenes_validation_dumped_perception')
elif split == "_train":
base_folder = os.path.join(cfg.DATA_LOCATION,
'intphys_scenes_dumped_perception')
else:
raise NotImplementedError
video_folders = list(
map(lambda x: os.path.join(base_folder, x),
sorted(os.listdir(base_folder))))
if cfg.DEBUG_VIDEOS is not None and cfg.DEBUG:
video_folders = cfg.DEBUG_VIDEOS
worker_args = zip(video_folders, range(len(video_folders)),
repeat(cfg.MIN_AREA))
if cfg.DEBUG:
dicts = [process_video(*w) for w in worker_args]
else:
with Pool(int(cpu_count())) as p:
dicts = p.starmap(process_video, worker_args)
dicts = list(itertools.chain.from_iterable(dicts))
write_serialized(dicts, outfile)
timer.done()
def intphys_to_detectron(cfg, split, outfile):
timer = CodeTimer("started processing videos for {}".format(split))
base_folder = split.split("_")[1:] if split != "_val" else ["train"]
base_folder = os.path.join(cfg.DATA_LOCATION, *base_folder)
video_folders = build_recursive_case_paths(base_folder, [])
start_vid_num = 0
if split == "_val":
video_folders = video_folders[:cfg.VAL_VIDEOS]
elif split == "_train":
video_folders = video_folders[cfg.VAL_VIDEOS:]
start_vid_num = cfg.VAL_VIDEOS
worker_args = []
for i, video in enumerate(video_folders, start=start_vid_num):
worker_args.append((video, i, cfg.MIN_AREA))
if cfg.DEBUG:
dicts = [process_video(*w) for w in worker_args]
else:
with Pool(int(cpu_count())) as p:
dicts = p.starmap(process_video, worker_args)
dicts = list(itertools.chain.from_iterable(dicts))
# print_mean_std_inv_depth(dicts)
write_serialized(dicts, outfile)
timer.done()
def shapes_world_to_detectron(cfg, split, out_file):
timer = CodeTimer("started processing videos")
video_list = sorted(os.listdir(cfg.DATAFOLDER))
if split != "_train":
video_list = video_list[:cfg.VAL_VIDEOS]
else:
video_list = video_list[cfg.VAL_VIDEOS:]
worker_args = []
for i, video in enumerate(video_list):
if i > cfg.MAX_VIDEOS:
break
video_folder = os.path.join(cfg.DATAFOLDER, video)
worker_args.append((video_folder, i, cfg.MIN_AREA))
if not cfg.DEBUG:
with Pool(int(cpu_count() / 2)) as p:
dicts = p.starmap(process_video, worker_args)
else:
dicts = [process_video(*w) for w in worker_args]
dicts = [d for vid in dicts for d in vid]
# dicts = sample_from_dataset(cfg, dicts, split)
with open(out_file, 'w') as f:
json.dump(dicts, f, indent=4)
timer.done()
def adept_to_detectron(cfg, split, outfile):
timer = CodeTimer("started processing videos for {}".format(split))
videos = get_video_folders(cfg, split)
# if "_val" in split:
# # videos = random.choices(videos, k=cfg.VAL_VIDEOS)
# videos = videos[:cfg.VAL_VIDEOS]
if cfg.DEBUG:
if len(cfg.DEBUG_VIDEOS) > 0:
videos = [
v for v in videos if v["video_folder"] in cfg.DEBUG_VIDEOS
]
dicts = [process_video(v, cfg, i) for i, v in enumerate(videos)]
else:
with Pool(int(cpu_count())) as p:
dicts = p.starmap(process_video,
zip(videos, repeat(cfg), range(len(videos))))
dicts = list(chain.from_iterable(dicts))
if "_val" in split:
dicts = random.choices(dicts, k=cfg.VAL_FRAMES)
write_serialized(dicts, outfile)
timer.done()
def set_means_and_stds(self, cfg):
######after computing them they should be saved to cfg so the model get's properly loaded######
if hasattr(cfg.ATTRIBUTES, "MEANS") and hasattr(
cfg.ATTRIBUTES, "STDS"):
means, stds = dict(cfg.ATTRIBUTES.MEANS), dict(cfg.ATTRIBUTES.STDS)
mins, maxes = 0, 0
else:
timer = CodeTimer(
"computing means and standard deviations of dataset")
#######compute the means and stds only on the validation set#############
dataset_dicts = [
DatasetCatalog.get(dataset_name)
for dataset_name in cfg.DATASETS.TEST if "_val" in dataset_name
]
#Get dataset dict without detectrono
"""data_cfg = self.data_cfg
for dataset_name in cfg.DATASETS.TEST:
if "_val" in dataset_name:
num_frames = utils.get_num_frames(data_cfg, "_val")
dataset_name, standard_format_json_file = utils.get_dataset_name_and_json(data_cfg, "_val")
dataset_dicts = utils.get_data_dicts(standard_format_json_file, num_frames)
required_fields = ["pred_box"] if cfg.DATASETS.USE_PREDICTED_BOXES else ["bbox"]
required_fields += ["attributes"]
_, dataset_dicts = image_based_to_annotation_based(dataset_dicts, required_fields)"""
dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
attributes = self.collate_attributes(dataset_dicts)
attributes = self.mask_out_irrelevant_values(attributes)
means = {
k: v.mean().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
stds = {
k: v.std().item()
if v.std() > 0.01 and k in self.continuous_terms else 1.0
for k, v in attributes.items()
}
maxes = {
k: v.max().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
mins = {
k: v.min().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
means_cfg = CfgNode(means)
stds_cfg = CfgNode(stds)
cfg.ATTRIBUTES.MEANS = means_cfg
cfg.ATTRIBUTES.STDS = stds_cfg
timer.done()
self.means = means
self.std_deviations = stds
self.maxes = maxes
self.mins = mins
def write_coco_format_json(cfg, split):
timer = CodeTimer("writting to coco")
dataset_name, standard_format_json_file = get_dataset_name_and_json(cfg, split)
dataset_dicts = DatasetCatalog.get(dataset_name)
_,filtered_dicts = filter_dataset(dataset_dicts, required_fields=["bbox", "bbox_mode", "segmentation"])
register_dataset(cfg, split, getter= lambda: filtered_dicts, name=dataset_name+"_for_coco")
coco_dict = convert_to_coco_dict(dataset_name+"_for_coco")
json_format_file = standard_format_json_file.replace(".json", "_coco_format.json")
with open(json_format_file, "w") as f:
json.dump(coco_dict, f)
timer.done()
def write_with_inferred_boxes(cfg, split):
# TODO: now there are invisible objects the detection mapper ignores that, will have to debug tomorrow
timer = CodeTimer("adding inferred boxes")
module_cfg = os.path.join(cfg.TRAINED_DETECTOR.EXP_DIR, "config.yaml")
module_cfg = load_cfg_from_file(module_cfg)
module_cfg.MODEL.WEIGHTS = cfg.TRAINED_DETECTOR.WEIGHTS_FILE
if cfg.DEBUG:
module_cfg.DATALOADER.NUM_WORKERS = 0
predictor = DetectorPredictor(module_cfg)
dataset_name, standard_format_json_file = get_dataset_name_and_json(
cfg, split)
data_loader = inference_detection_loader(
module_cfg.clone(), dataset_name, DetectionMapper(module_cfg.clone()))
worker_args = []
with torch.no_grad():
for inputs in data_loader:
outputs = predictor(inputs)
for i in range(len(outputs)):
worker_args.append(parse_worker_args(inputs[i], outputs[i]))
if cfg.DEBUG:
new_dicts = [add_inferred_boxes(*w) for w in worker_args]
else:
with Pool(int(cpu_count() / 4)) as p:
new_dicts = p.starmap(add_inferred_boxes, worker_args)
if 'PRED_BOX_SCORE_THRESHOLD' not in cfg:
assert '_val' in split, "start with validation split to compute detection threshold"
cfg.PRED_BOX_SCORE_THRESHOLD = infer_score_threshold(new_dicts)
new_dicts = filter_predicted_boxes_threshold(new_dicts,
cfg.PRED_BOX_SCORE_THRESHOLD)
with open(standard_format_json_file, 'w') as f:
json.dump(new_dicts, f, indent=4)
timer.done()
def write_with_inferred_attributes(cfg, split, attributes_key):
timer = CodeTimer(
"adding inferred attributes split:{}, attributes_key:{}".format(
split, attributes_key))
module_cfg = os.path.join(cfg.TRAINED_DERENDER.EXP_DIR, "cfg.yaml")
module_cfg = load_cfg_from_file(module_cfg)
module_cfg.MODEL.WEIGHTS = cfg.TRAINED_DERENDER.ATTRIBUTES_WEIGHTS_MAP[
attributes_key]
module_cfg.DATALOADER.OBJECTS_PER_BATCH = 1000 if cfg.BASE_NAME == "intphys" else 450
module_cfg.DATALOADER.NUM_WORKERS = 8 if cfg.BASE_NAME == "adept" else module_cfg.DATALOADER.NUM_WORKERS
if cfg.DEBUG:
module_cfg.DATALOADER.NUM_WORKERS = 0
module_cfg.DEBUG = True
module_cfg.DATALOADER.OBJECTS_PER_BATCH = 50
predictor = DerenderPredictor(module_cfg)
# if not cfg.DEBUG:
# gpu_ids = [_ for _ in range(torch.cuda.device_count())]
# predictor.derenderer = torch.nn.parallel.DataParallel(predictor.derenderer, gpu_ids)
dataset_name, standard_format_json_file = get_dataset_name_and_json(
cfg, split)
dataset = DatasetCatalog.get(dataset_name)
required_fields = [
"pred_box"
] if cfg.TRAINED_DERENDER.USE_INFERRED_BOXES else ["bbox"]
filtered_idx, \
mapped_dataset = image_based_to_annotation_based(dataset, required_fields)
mapped_dataset = DatasetFromList(mapped_dataset, copy=False)
mapper = DerenderMapper(cfg.TRAINED_DERENDER.USE_INFERRED_BOXES,
predictor.attributes,
for_inference=True,
use_depth=cfg.TRAINED_DERENDER.USE_DEPTH)
mapped_dataset = MapDataset(mapped_dataset, mapper)
data_loader = DataLoader(
dataset=mapped_dataset,
batch_size=module_cfg.DATALOADER.OBJECTS_PER_BATCH,
num_workers=module_cfg.DATALOADER.NUM_WORKERS,
shuffle=False)
fil_pointer = 0
with torch.no_grad():
for inputs in data_loader:
inputs = to_cuda(inputs)
outputs = predictor(inputs)
batch_size = list(outputs.values())[0].shape[0]
for oix, (img_idx, an_idx) in zip(
range(batch_size),
filtered_idx[fil_pointer:fil_pointer + batch_size]):
dataset[img_idx]["annotations"][an_idx][attributes_key] = \
{k: v[oix].item() for k, v in outputs.items()}
# {k: v[oix].item() if v[oix].size == 1
# else [float(el) for el in v[oix]]
# for k,v in outputs.items()}
fil_pointer = fil_pointer + batch_size
dataset = [fix_for_serialization(d) for d in dataset]
with open(standard_format_json_file, "w") as f:
json.dump(dataset, f, indent=4)
timer.done()
class JsonGenerator:
def __init__(self, data_cfg, split, target_physics, attributes_key,
vel_data_assoc):
self.timer = CodeTimer(
"building jsons for {}_{} with target {}".format(
data_cfg.BASE_NAME, split, target_physics))
self.process_object = PROCESS_OBJECT_MAP[(data_cfg.BASE_NAME,
target_physics)]
self.process_camera = PROCESS_CAMERA_MAP[(data_cfg.BASE_NAME,
target_physics)]
self.get_match_cost = MATCH_COST_MAP[data_cfg.BASE_NAME]
self.requires_vel = vel_data_assoc != "None"
self.vel_data_assoc = vel_data_assoc
self.target_physics = target_physics
self.attributes_key = attributes_key
self.build_physics_jsons(data_cfg, split)
def process_frame(self, data_cfg, prev_video_dict, video_dict):
# process_object = PROCESS_OBJECT_MAP[data_cfg.BASE_NAME]
prev_video_anns = self.match_annotations(prev_video_dict, video_dict)
objects = [
self.process_object(attributes=an[self.attributes_key],
prev_attributes=prev_an[self.attributes_key]
if prev_an is not None else None,
object_id=an["object_id"],
segmentation=an["segmentation"])
for prev_an, an in zip(prev_video_anns, video_dict["annotations"])
]
objects = [obj for obj in objects if obj is not None]
camera = self.process_camera(video_dict["camera"])
state_scene = deepcopy(_DUMMY_FRAME)
state_scene["objects"] = objects
state_scene["suggested_view"]["camera"] = camera
return state_scene
def video2json(self, data_cfg, video_dict, out_dir, vid_num):
# init,end = data_cfg.SHAPESWORLD_JSON.FRAMES_RANGE_PER_VIDEO
frame_range = sorted(list(video_dict.keys()))
assert np.unique(frame_range).size == np.arange(
frame_range[0], frame_range[-1] + 1).size
if not self.requires_vel:
video_dict[frame_range[0] - 1] = None
else:
frame_range = frame_range[1:]
scene_states = [
self.process_frame(data_cfg, video_dict[f - 1], video_dict[f])
for f in frame_range
]
# video_name = 'video_' + str(vid_num).zfill(5) + ".json"
video_name = os.path.basename(
video_dict[frame_range[0]]["original_video"]) + ".json"
out_path = os.path.join(out_dir, video_name)
with open(out_path, "w") as f:
json.dump(
{
"scene_states": scene_states,
"debug": {
"is_possible":
video_dict[frame_range[0]]["is_possible"],
"original_video":
video_dict[frame_range[0]]["original_video"]
}
},
f,
indent=4)
def build_physics_jsons(self, data_cfg, split):
dataset_name, standard_format_json_file = get_dataset_name_and_json(
data_cfg, split)
dataset = DatasetCatalog.get(dataset_name)
required_fields_values = {self.attributes_key: {"visible": 1}}
_, dataset = filter_dataset(
dataset, required_fields_values=required_fields_values)
videos_dicts = frames2videos(dataset)
out_dir = get_jsons_directory(data_cfg, self.target_physics,
self.attributes_key, dataset_name)
os.makedirs(out_dir, exist_ok=True)
worker_args = [(data_cfg, vid_dict, out_dir, vid_num)
for vid_num, vid_dict in videos_dicts.items()]
if data_cfg.DEBUG:
[self.video2json(*w) for w in worker_args]
else:
with Pool(int(cpu_count())) as p:
p.starmap(self.video2json, worker_args)
self.timer.done()
def match_annotations(self, prev_video_dict, video_dict):
if not self.requires_vel:
return [None] * len(video_dict["annotations"])
prev_anns_matched = []
if self.vel_data_assoc == "ground_truth":
#ground truth should have proper object ids for data association
prev_map = {
prev_an["object_id"]: prev_an
for prev_an in prev_video_dict["annotations"]
}
elif self.vel_data_assoc == "heuristic":
prev_map = match_current_to_prev_anns(
video_dict["annotations"], prev_video_dict["annotations"],
self.attributes_key)
else:
raise NotImplementedError
for an in video_dict["annotations"]:
if an["object_id"] in prev_map:
prev_anns_matched.append(prev_map[an["object_id"]])
else:
prev_anns_matched.append(None)
return prev_anns_matched