Exemplo n.º 1
0
    def process_dataset(self, dataset_name, required, cfg):
        timer = CodeTimer("building dynamics for {} with key {}".format(
            dataset_name, required))
        dataset_dicts = DatasetCatalog.get(dataset_name)
        filtered_idx, filtered_dicts = filter_dataset(dataset_dicts,
                                                      [required])

        worker_args = defaultdict(lambda: {"vid_dicts": []})
        for d in filtered_dicts:
            vid_num = d["image_id"] // 500
            worker_args[vid_num]["vid_dicts"].append(d)
            worker_args[vid_num]["required"] = required

        if cfg.DEBUG:
            data = [self.process_video(w) for w in worker_args.values()]
        else:
            with Pool(int(cpu_count())) as p:
                data = p.map(self.process_video, worker_args.values())

        inputs, targets, possible_flags, original_videos = zip(*data)
        inputs = [torch.tensor(el) for el in inputs]
        targets = [{k: torch.tensor(v)
                    for k, v in el.items()} for el in targets]
        timer.done()
        return inputs, targets, possible_flags, original_videos
Exemplo n.º 2
0
def ai2thor_intphys_to_detectron(cfg, split, outfile):
    timer = CodeTimer(f'started processing videos for {split}')
    if split == "_val":
        base_folder = os.path.join(
            cfg.DATA_LOCATION, 'intphys_scenes_validation_dumped_perception')
    elif split == "_train":
        base_folder = os.path.join(cfg.DATA_LOCATION,
                                   'intphys_scenes_dumped_perception')
    else:
        raise NotImplementedError

    video_folders = list(
        map(lambda x: os.path.join(base_folder, x),
            sorted(os.listdir(base_folder))))

    if cfg.DEBUG_VIDEOS is not None and cfg.DEBUG:
        video_folders = cfg.DEBUG_VIDEOS

    worker_args = zip(video_folders, range(len(video_folders)),
                      repeat(cfg.MIN_AREA))

    if cfg.DEBUG:
        dicts = [process_video(*w) for w in worker_args]
    else:
        with Pool(int(cpu_count())) as p:
            dicts = p.starmap(process_video, worker_args)

    dicts = list(itertools.chain.from_iterable(dicts))

    write_serialized(dicts, outfile)
    timer.done()
Exemplo n.º 3
0
def intphys_to_detectron(cfg, split, outfile):
    timer = CodeTimer("started processing videos for {}".format(split))
    base_folder = split.split("_")[1:] if split != "_val" else ["train"]
    base_folder = os.path.join(cfg.DATA_LOCATION, *base_folder)
    video_folders = build_recursive_case_paths(base_folder, [])

    start_vid_num = 0
    if split == "_val":
        video_folders = video_folders[:cfg.VAL_VIDEOS]
    elif split == "_train":
        video_folders = video_folders[cfg.VAL_VIDEOS:]
        start_vid_num = cfg.VAL_VIDEOS

    worker_args = []
    for i, video in enumerate(video_folders, start=start_vid_num):
        worker_args.append((video, i, cfg.MIN_AREA))

    if cfg.DEBUG:
        dicts = [process_video(*w) for w in worker_args]
    else:
        with Pool(int(cpu_count())) as p:
            dicts = p.starmap(process_video, worker_args)

    dicts = list(itertools.chain.from_iterable(dicts))

    # print_mean_std_inv_depth(dicts)

    write_serialized(dicts, outfile)
    timer.done()
Exemplo n.º 4
0
def shapes_world_to_detectron(cfg, split, out_file):
    timer = CodeTimer("started processing videos")
    video_list = sorted(os.listdir(cfg.DATAFOLDER))

    if split != "_train":
        video_list = video_list[:cfg.VAL_VIDEOS]
    else:
        video_list = video_list[cfg.VAL_VIDEOS:]

    worker_args = []
    for i, video in enumerate(video_list):
        if i > cfg.MAX_VIDEOS:
            break
        video_folder = os.path.join(cfg.DATAFOLDER, video)
        worker_args.append((video_folder, i, cfg.MIN_AREA))

    if not cfg.DEBUG:
        with Pool(int(cpu_count() / 2)) as p:
            dicts = p.starmap(process_video, worker_args)
    else:
        dicts = [process_video(*w) for w in worker_args]

    dicts = [d for vid in dicts for d in vid]
    # dicts = sample_from_dataset(cfg, dicts, split)

    with open(out_file, 'w') as f:
        json.dump(dicts, f, indent=4)

    timer.done()
Exemplo n.º 5
0
def adept_to_detectron(cfg, split, outfile):
    timer = CodeTimer("started processing videos for {}".format(split))
    videos = get_video_folders(cfg, split)
    # if "_val" in split:
    #     # videos = random.choices(videos, k=cfg.VAL_VIDEOS)
    #     videos = videos[:cfg.VAL_VIDEOS]

    if cfg.DEBUG:
        if len(cfg.DEBUG_VIDEOS) > 0:
            videos = [
                v for v in videos if v["video_folder"] in cfg.DEBUG_VIDEOS
            ]
        dicts = [process_video(v, cfg, i) for i, v in enumerate(videos)]
    else:
        with Pool(int(cpu_count())) as p:
            dicts = p.starmap(process_video,
                              zip(videos, repeat(cfg), range(len(videos))))

    dicts = list(chain.from_iterable(dicts))

    if "_val" in split:
        dicts = random.choices(dicts, k=cfg.VAL_FRAMES)

    write_serialized(dicts, outfile)
    timer.done()
    def set_means_and_stds(self, cfg):
        ######after computing them they should be saved to cfg so the model get's properly loaded######
        if hasattr(cfg.ATTRIBUTES, "MEANS") and hasattr(
                cfg.ATTRIBUTES, "STDS"):
            means, stds = dict(cfg.ATTRIBUTES.MEANS), dict(cfg.ATTRIBUTES.STDS)
            mins, maxes = 0, 0
        else:
            timer = CodeTimer(
                "computing means and standard deviations of dataset")
            #######compute the means and stds only on the validation  set#############
            dataset_dicts = [
                DatasetCatalog.get(dataset_name)
                for dataset_name in cfg.DATASETS.TEST if "_val" in dataset_name
            ]
            #Get dataset dict without detectrono
            """data_cfg = self.data_cfg
            for dataset_name in cfg.DATASETS.TEST:
                if "_val" in dataset_name:
                    num_frames = utils.get_num_frames(data_cfg, "_val")
                    dataset_name, standard_format_json_file = utils.get_dataset_name_and_json(data_cfg, "_val")
                    dataset_dicts = utils.get_data_dicts(standard_format_json_file, num_frames)

            required_fields = ["pred_box"] if cfg.DATASETS.USE_PREDICTED_BOXES else ["bbox"]
            required_fields += ["attributes"]
            _, dataset_dicts = image_based_to_annotation_based(dataset_dicts, required_fields)"""

            dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
            attributes = self.collate_attributes(dataset_dicts)
            attributes = self.mask_out_irrelevant_values(attributes)
            means = {
                k: v.mean().item() if k in self.continuous_terms else 0.0
                for k, v in attributes.items()
            }
            stds = {
                k: v.std().item()
                if v.std() > 0.01 and k in self.continuous_terms else 1.0
                for k, v in attributes.items()
            }
            maxes = {
                k: v.max().item() if k in self.continuous_terms else 0.0
                for k, v in attributes.items()
            }
            mins = {
                k: v.min().item() if k in self.continuous_terms else 0.0
                for k, v in attributes.items()
            }
            means_cfg = CfgNode(means)
            stds_cfg = CfgNode(stds)
            cfg.ATTRIBUTES.MEANS = means_cfg
            cfg.ATTRIBUTES.STDS = stds_cfg
            timer.done()

        self.means = means
        self.std_deviations = stds
        self.maxes = maxes
        self.mins = mins
Exemplo n.º 7
0
def write_coco_format_json(cfg, split):
    timer = CodeTimer("writting to coco")
    dataset_name, standard_format_json_file = get_dataset_name_and_json(cfg, split)

    dataset_dicts = DatasetCatalog.get(dataset_name)
    _,filtered_dicts = filter_dataset(dataset_dicts, required_fields=["bbox", "bbox_mode", "segmentation"])
    register_dataset(cfg, split, getter= lambda: filtered_dicts, name=dataset_name+"_for_coco")

    coco_dict = convert_to_coco_dict(dataset_name+"_for_coco")

    json_format_file = standard_format_json_file.replace(".json", "_coco_format.json")
    with open(json_format_file, "w") as f:
        json.dump(coco_dict, f)
    timer.done()
def write_with_inferred_boxes(cfg, split):
    # TODO: now there are invisible objects the detection mapper  ignores that, will have to debug tomorrow
    timer = CodeTimer("adding inferred boxes")
    module_cfg = os.path.join(cfg.TRAINED_DETECTOR.EXP_DIR, "config.yaml")
    module_cfg = load_cfg_from_file(module_cfg)
    module_cfg.MODEL.WEIGHTS = cfg.TRAINED_DETECTOR.WEIGHTS_FILE
    if cfg.DEBUG:
        module_cfg.DATALOADER.NUM_WORKERS = 0

    predictor = DetectorPredictor(module_cfg)

    dataset_name, standard_format_json_file = get_dataset_name_and_json(
        cfg, split)
    data_loader = inference_detection_loader(
        module_cfg.clone(), dataset_name, DetectionMapper(module_cfg.clone()))

    worker_args = []
    with torch.no_grad():
        for inputs in data_loader:
            outputs = predictor(inputs)
            for i in range(len(outputs)):
                worker_args.append(parse_worker_args(inputs[i], outputs[i]))

    if cfg.DEBUG:
        new_dicts = [add_inferred_boxes(*w) for w in worker_args]
    else:
        with Pool(int(cpu_count() / 4)) as p:
            new_dicts = p.starmap(add_inferred_boxes, worker_args)

    if 'PRED_BOX_SCORE_THRESHOLD' not in cfg:
        assert '_val' in split, "start with validation split to compute detection threshold"
        cfg.PRED_BOX_SCORE_THRESHOLD = infer_score_threshold(new_dicts)

    new_dicts = filter_predicted_boxes_threshold(new_dicts,
                                                 cfg.PRED_BOX_SCORE_THRESHOLD)

    with open(standard_format_json_file, 'w') as f:
        json.dump(new_dicts, f, indent=4)

    timer.done()
def write_with_inferred_attributes(cfg, split, attributes_key):
    timer = CodeTimer(
        "adding inferred attributes split:{}, attributes_key:{}".format(
            split, attributes_key))
    module_cfg = os.path.join(cfg.TRAINED_DERENDER.EXP_DIR, "cfg.yaml")
    module_cfg = load_cfg_from_file(module_cfg)
    module_cfg.MODEL.WEIGHTS = cfg.TRAINED_DERENDER.ATTRIBUTES_WEIGHTS_MAP[
        attributes_key]

    module_cfg.DATALOADER.OBJECTS_PER_BATCH = 1000 if cfg.BASE_NAME == "intphys" else 450
    module_cfg.DATALOADER.NUM_WORKERS = 8 if cfg.BASE_NAME == "adept" else module_cfg.DATALOADER.NUM_WORKERS

    if cfg.DEBUG:
        module_cfg.DATALOADER.NUM_WORKERS = 0
        module_cfg.DEBUG = True
        module_cfg.DATALOADER.OBJECTS_PER_BATCH = 50

    predictor = DerenderPredictor(module_cfg)

    # if not cfg.DEBUG:
    #     gpu_ids = [_ for _ in range(torch.cuda.device_count())]
    #     predictor.derenderer = torch.nn.parallel.DataParallel(predictor.derenderer, gpu_ids)

    dataset_name, standard_format_json_file = get_dataset_name_and_json(
        cfg, split)
    dataset = DatasetCatalog.get(dataset_name)
    required_fields = [
        "pred_box"
    ] if cfg.TRAINED_DERENDER.USE_INFERRED_BOXES else ["bbox"]
    filtered_idx, \
    mapped_dataset = image_based_to_annotation_based(dataset, required_fields)
    mapped_dataset = DatasetFromList(mapped_dataset, copy=False)
    mapper = DerenderMapper(cfg.TRAINED_DERENDER.USE_INFERRED_BOXES,
                            predictor.attributes,
                            for_inference=True,
                            use_depth=cfg.TRAINED_DERENDER.USE_DEPTH)
    mapped_dataset = MapDataset(mapped_dataset, mapper)

    data_loader = DataLoader(
        dataset=mapped_dataset,
        batch_size=module_cfg.DATALOADER.OBJECTS_PER_BATCH,
        num_workers=module_cfg.DATALOADER.NUM_WORKERS,
        shuffle=False)

    fil_pointer = 0
    with torch.no_grad():
        for inputs in data_loader:
            inputs = to_cuda(inputs)
            outputs = predictor(inputs)
            batch_size = list(outputs.values())[0].shape[0]
            for oix, (img_idx, an_idx) in zip(
                    range(batch_size),
                    filtered_idx[fil_pointer:fil_pointer + batch_size]):

                dataset[img_idx]["annotations"][an_idx][attributes_key] = \
                    {k: v[oix].item() for k, v in outputs.items()}
                # {k: v[oix].item() if v[oix].size == 1
                #                   else [float(el) for el in v[oix]]
                # for k,v in outputs.items()}

            fil_pointer = fil_pointer + batch_size

    dataset = [fix_for_serialization(d) for d in dataset]

    with open(standard_format_json_file, "w") as f:
        json.dump(dataset, f, indent=4)

    timer.done()
Exemplo n.º 10
0
class JsonGenerator:
    def __init__(self, data_cfg, split, target_physics, attributes_key,
                 vel_data_assoc):
        self.timer = CodeTimer(
            "building jsons for {}_{} with target  {}".format(
                data_cfg.BASE_NAME, split, target_physics))
        self.process_object = PROCESS_OBJECT_MAP[(data_cfg.BASE_NAME,
                                                  target_physics)]
        self.process_camera = PROCESS_CAMERA_MAP[(data_cfg.BASE_NAME,
                                                  target_physics)]
        self.get_match_cost = MATCH_COST_MAP[data_cfg.BASE_NAME]

        self.requires_vel = vel_data_assoc != "None"
        self.vel_data_assoc = vel_data_assoc
        self.target_physics = target_physics
        self.attributes_key = attributes_key

        self.build_physics_jsons(data_cfg, split)

    def process_frame(self, data_cfg, prev_video_dict, video_dict):
        # process_object = PROCESS_OBJECT_MAP[data_cfg.BASE_NAME]
        prev_video_anns = self.match_annotations(prev_video_dict, video_dict)
        objects = [
            self.process_object(attributes=an[self.attributes_key],
                                prev_attributes=prev_an[self.attributes_key]
                                if prev_an is not None else None,
                                object_id=an["object_id"],
                                segmentation=an["segmentation"])
            for prev_an, an in zip(prev_video_anns, video_dict["annotations"])
        ]
        objects = [obj for obj in objects if obj is not None]

        camera = self.process_camera(video_dict["camera"])

        state_scene = deepcopy(_DUMMY_FRAME)
        state_scene["objects"] = objects
        state_scene["suggested_view"]["camera"] = camera
        return state_scene

    def video2json(self, data_cfg, video_dict, out_dir, vid_num):
        # init,end = data_cfg.SHAPESWORLD_JSON.FRAMES_RANGE_PER_VIDEO
        frame_range = sorted(list(video_dict.keys()))
        assert np.unique(frame_range).size == np.arange(
            frame_range[0], frame_range[-1] + 1).size
        if not self.requires_vel:
            video_dict[frame_range[0] - 1] = None
        else:
            frame_range = frame_range[1:]

        scene_states = [
            self.process_frame(data_cfg, video_dict[f - 1], video_dict[f])
            for f in frame_range
        ]

        # video_name = 'video_' + str(vid_num).zfill(5) + ".json"
        video_name = os.path.basename(
            video_dict[frame_range[0]]["original_video"]) + ".json"
        out_path = os.path.join(out_dir, video_name)

        with open(out_path, "w") as f:
            json.dump(
                {
                    "scene_states": scene_states,
                    "debug": {
                        "is_possible":
                        video_dict[frame_range[0]]["is_possible"],
                        "original_video":
                        video_dict[frame_range[0]]["original_video"]
                    }
                },
                f,
                indent=4)

    def build_physics_jsons(self, data_cfg, split):
        dataset_name, standard_format_json_file = get_dataset_name_and_json(
            data_cfg, split)
        dataset = DatasetCatalog.get(dataset_name)
        required_fields_values = {self.attributes_key: {"visible": 1}}

        _, dataset = filter_dataset(
            dataset, required_fields_values=required_fields_values)
        videos_dicts = frames2videos(dataset)
        out_dir = get_jsons_directory(data_cfg, self.target_physics,
                                      self.attributes_key, dataset_name)
        os.makedirs(out_dir, exist_ok=True)

        worker_args = [(data_cfg, vid_dict, out_dir, vid_num)
                       for vid_num, vid_dict in videos_dicts.items()]

        if data_cfg.DEBUG:
            [self.video2json(*w) for w in worker_args]
        else:
            with Pool(int(cpu_count())) as p:
                p.starmap(self.video2json, worker_args)
        self.timer.done()

    def match_annotations(self, prev_video_dict, video_dict):
        if not self.requires_vel:
            return [None] * len(video_dict["annotations"])
        prev_anns_matched = []
        if self.vel_data_assoc == "ground_truth":
            #ground truth should have proper object ids for  data association
            prev_map = {
                prev_an["object_id"]: prev_an
                for prev_an in prev_video_dict["annotations"]
            }
        elif self.vel_data_assoc == "heuristic":
            prev_map = match_current_to_prev_anns(
                video_dict["annotations"], prev_video_dict["annotations"],
                self.attributes_key)
        else:
            raise NotImplementedError

        for an in video_dict["annotations"]:
            if an["object_id"] in prev_map:
                prev_anns_matched.append(prev_map[an["object_id"]])
            else:
                prev_anns_matched.append(None)
        return prev_anns_matched