def add_panel_seg_config(cfg: CfgNode):
"""
Add config for the panel seg adapted retinanet.
Args:
cfg (CfgNode): The config node that will be extended.
"""
# Name of the validation data set
cfg.MODEL.RETINANET.PANEL_IN_FEATURES = []
cfg.MODEL.RETINANET.LABEL_IN_FEATURES = []
cfg.MODEL.RETINANET.NUM_LABEL_CLASSES = 50
cfg.MODEL.PANEL_FPN = CfgNode()
cfg.MODEL.PANEL_FPN.IN_FEATURES = []
cfg.MODEL.PANEL_FPN.OUT_CHANNELS = 0
cfg.MODEL.LABEL_FPN = CfgNode()
cfg.MODEL.LABEL_FPN.IN_FEATURES = []
cfg.MODEL.LABEL_FPN.OUT_CHANNELS = 0
cfg.MODEL.PANEL_ANCHOR_GENERATOR = CfgNode()
cfg.MODEL.PANEL_ANCHOR_GENERATOR.SIZES = []
cfg.MODEL.PANEL_ANCHOR_GENERATOR.ASPECT_RATIOS = []
cfg.MODEL.LABEL_ANCHOR_GENERATOR = CfgNode()
cfg.MODEL.LABEL_ANCHOR_GENERATOR.SIZES = []
cfg.MODEL.LABEL_ANCHOR_GENERATOR.ASPECT_RATIOS = []
def get_means_and_stds_from_dataset(cfg):
if hasattr(cfg.ATTRIBUTES, "MEANS") and hasattr(
cfg.ATTRIBUTES, "STDS"):
return dict(cfg.ATTRIBUTES.MEANS), dict(cfg.ATTRIBUTES.STDS)
print("computing means and standard deviations of dataset")
start = time.time()
dataset_dicts = [
DatasetCatalog.get(dataset_name)
for dataset_name in cfg.DATASETS.TEST
]
dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
attributes = collate_attributes(dataset_dicts)
attributes = mask_out_irrelevant_values(attributes)
means = {k: v.mean().item() for k, v in attributes.items()}
stds = {
k: v.std().item() if v.std() > 0.01 else 1.0
for k, v in attributes.items()
}
means_cfg = CfgNode(means)
stds_cfg = CfgNode(stds)
cfg.ATTRIBUTES.MEANS = means_cfg
cfg.ATTRIBUTES.STDS = stds_cfg
print("done after {}".format(time.time() - start))
return means, stds
def get_cfg(args):
cfg = CfgNode()
if "module" in args:
module_cfg = _MODULE_CONFIG_MAP[args.module](args.dataset)
cfg.MODULE_CFG = module_cfg
if "dataset" in args:
cfg.DATA_CFG = data_get_cfg(args.dataset)
return cfg
def add_efficientdet_config(cfg):
_C = cfg
_C.MODEL.EFFICIENTNET = CfgNode()
_C.MODEL.EFFICIENTNET.VERSION = "b0"
_C.MODEL.EFFICIENTNET.PRETRAINED = True
_C.MODEL.BIFPN = CfgNode()
_C.MODEL.BIFPN.F_CHANNELS = 64
_C.MODEL.BIFPN.NUM_LAYERS = 2
def set_means_and_stds(self, cfg):
######after computing them they should be saved to cfg so the model get's properly loaded######
if hasattr(cfg.ATTRIBUTES, "MEANS") and hasattr(
cfg.ATTRIBUTES, "STDS"):
means, stds = dict(cfg.ATTRIBUTES.MEANS), dict(cfg.ATTRIBUTES.STDS)
mins, maxes = 0, 0
else:
timer = CodeTimer(
"computing means and standard deviations of dataset")
#######compute the means and stds only on the validation set#############
dataset_dicts = [
DatasetCatalog.get(dataset_name)
for dataset_name in cfg.DATASETS.TEST if "_val" in dataset_name
]
#Get dataset dict without detectrono
"""data_cfg = self.data_cfg
for dataset_name in cfg.DATASETS.TEST:
if "_val" in dataset_name:
num_frames = utils.get_num_frames(data_cfg, "_val")
dataset_name, standard_format_json_file = utils.get_dataset_name_and_json(data_cfg, "_val")
dataset_dicts = utils.get_data_dicts(standard_format_json_file, num_frames)
required_fields = ["pred_box"] if cfg.DATASETS.USE_PREDICTED_BOXES else ["bbox"]
required_fields += ["attributes"]
_, dataset_dicts = image_based_to_annotation_based(dataset_dicts, required_fields)"""
dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts))
attributes = self.collate_attributes(dataset_dicts)
attributes = self.mask_out_irrelevant_values(attributes)
means = {
k: v.mean().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
stds = {
k: v.std().item()
if v.std() > 0.01 and k in self.continuous_terms else 1.0
for k, v in attributes.items()
}
maxes = {
k: v.max().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
mins = {
k: v.min().item() if k in self.continuous_terms else 0.0
for k, v in attributes.items()
}
means_cfg = CfgNode(means)
stds_cfg = CfgNode(stds)
cfg.ATTRIBUTES.MEANS = means_cfg
cfg.ATTRIBUTES.STDS = stds_cfg
timer.done()
self.means = means
self.std_deviations = stds
self.maxes = maxes
self.mins = mins
def test_from_config(self):
cfg = CfgNode_()
cfg.TRACKER_HEADS = CfgNode_()
cfg.TRACKER_HEADS.TRACKER_NAME = "IOUWeightedHungarianBBoxIOUTracker"
cfg.TRACKER_HEADS.VIDEO_HEIGHT = int(self._img_size[0])
cfg.TRACKER_HEADS.VIDEO_WIDTH = int(self._img_size[1])
cfg.TRACKER_HEADS.MAX_NUM_INSTANCES = self._max_num_instances
cfg.TRACKER_HEADS.MAX_LOST_FRAME_COUNT = self._max_lost_frame_count
cfg.TRACKER_HEADS.MIN_BOX_REL_DIM = self._min_box_rel_dim
cfg.TRACKER_HEADS.MIN_INSTANCE_PERIOD = self._min_instance_period
cfg.TRACKER_HEADS.TRACK_IOU_THRESHOLD = self._track_iou_threshold
tracker = build_tracker_head(cfg)
self.assertTrue(tracker._video_height == self._img_size[0])
def __init__(
self,
cfg: CfgNode,
n_iter=150,
gamma=0.5,
nms_thresh=0.9,
mapper: Callable = DatasetMapper,
):
"""Implements the DAG algorithm
Parameters
----------
cfg : CfgNode
Config object used to train the model
n_iter : int, optional
Number of iterations to run the algorithm on each image, by default 150
gamma : float, optional
Perturbation weight, by default 0.5
nms_thresh : float, optional
NMS threshold of RPN; higher it is, more dense set of proposals, by default 0.9
mapper : Callable, optional
Can specify own DatasetMapper logic, by default DatasetMapper
"""
self.n_iter = n_iter
self.gamma = gamma
# Modify config
self.cfg = cfg.clone() # cfg can be modified by model
# To generate more dense proposals
self.cfg.MODEL.RPN.NMS_THRESH = nms_thresh
self.cfg.MODEL.RPN.POST_NMS_TOPK_TEST = 3000
# Init model
self.model = build_model(self.cfg)
self.model.eval()
# Load weights
checkpointer = DetectionCheckpointer(self.model)
checkpointer.load(cfg.MODEL.WEIGHTS)
self.aug = T.ResizeShortestEdge(
[cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST],
cfg.INPUT.MAX_SIZE_TEST)
self.input_format = cfg.INPUT.FORMAT
assert self.input_format in ["RGB", "BGR"], self.input_format
# Init dataloader on training dataset
dataset_mapper = mapper(cfg, is_train=False)
# self.data_loader = build_detection_test_loader(
# cfg, cfg.DATASETS['TRAIN'][0], mapper=dataset_mapper
# )
self.data_loader = build_detection_test_loader(cfg,
"benign_train",
mapper=dataset_mapper)
self.device = self.model.device
self.n_classes = self.cfg.MODEL.ROI_HEADS.NUM_CLASSES
self.metadata = MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0])
# HACK Only specific for this dataset
self.metadata.thing_classes = ["box", "logo"]
def get_bootstrap_dataset_config() -> CN:
_C = CN()
_C.DATASET = ""
# ratio used to mix data loaders
_C.RATIO = 0.1
# image loader
_C.IMAGE_LOADER = CN(new_allowed=True)
_C.IMAGE_LOADER.TYPE = ""
_C.IMAGE_LOADER.BATCH_SIZE = 4
_C.IMAGE_LOADER.NUM_WORKERS = 4
_C.IMAGE_LOADER.CATEGORIES = []
_C.IMAGE_LOADER.MAX_COUNT_PER_CATEGORY = 1_000_000
_C.IMAGE_LOADER.CATEGORY_TO_CLASS_MAPPING = CN(new_allowed=True)
# inference
_C.INFERENCE = CN()
# batch size for model inputs
_C.INFERENCE.INPUT_BATCH_SIZE = 4
# batch size to group model outputs
_C.INFERENCE.OUTPUT_BATCH_SIZE = 2
# sampled data
_C.DATA_SAMPLER = CN(new_allowed=True)
_C.DATA_SAMPLER.TYPE = ""
_C.DATA_SAMPLER.USE_GROUND_TRUTH_CATEGORIES = False
# filter
_C.FILTER = CN(new_allowed=True)
_C.FILTER.TYPE = ""
return _C
def build_inference_based_loaders(
cfg: CfgNode, model: torch.nn.Module) -> List[InferenceBasedLoader]:
loaders = []
ratios = []
for dataset_spec in cfg.BOOTSTRAP_DATASETS:
dataset_cfg = get_bootstrap_dataset_config().clone()
dataset_cfg.merge_from_other_cfg(CfgNode(dataset_spec))
loader = build_inference_based_loader(cfg, dataset_cfg, model)
loaders.append(loader)
ratios.append(dataset_cfg.RATIO)
return loaders, ratios
def get_infer_topk_cfg():
cfg = get_d2_cfg()
cfg.TEST.TOPK_CAT = CfgNode()
cfg.TEST.TOPK_CAT.ENABLED = True
cfg.TEST.TOPK_CAT.K = 10000
cfg.TEST.TOPK_CAT.MIN_SCORE = 1.0e-7
# Images used to estimate initial score threshold, with mask branch off.
# Set to be greater than LVIS validation set, so we do a full pass with the
# mask branch off.
cfg.TEST.TOPK_CAT.NUM_ESTIMATE = 30000
return cfg
def build_inference_based_loaders(
cfg: CfgNode, model: torch.nn.Module
) -> Tuple[List[InferenceBasedLoader], List[float]]:
loaders = []
ratios = []
embedder = build_densepose_embedder(cfg).to(device=model.device) # pyre-ignore[16]
for dataset_spec in cfg.BOOTSTRAP_DATASETS:
dataset_cfg = get_bootstrap_dataset_config().clone()
dataset_cfg.merge_from_other_cfg(CfgNode(dataset_spec))
loader = build_inference_based_loader(cfg, dataset_cfg, model, embedder)
loaders.append(loader)
ratios.append(dataset_cfg.RATIO)
return loaders, ratios
def add_custom_param(cfg):
"""
In order to add custom config parameter in the .yaml those parameter must
be initialised
"""
# Model
cfg.MODEL_CUSTOM = CfgNode()
cfg.MODEL_CUSTOM.BACKBONE = CfgNode()
cfg.MODEL_CUSTOM.BACKBONE.EFFICIENTNET_ID = 5
cfg.MODEL_CUSTOM.BACKBONE.LOAD_PRETRAIN = False
# DATASET
cfg.NUM_CLASS = 19
cfg.DATASET_PATH = "/home/ubuntu/Elix/cityscapes"
cfg.TRAIN_JSON = "gtFine/cityscapes_panoptic_train.json"
cfg.VALID_JSON = "gtFine/cityscapes_panoptic_val.json"
cfg.PRED_DIR = "preds"
cfg.PRED_JSON = "cityscapes_panoptic_preds.json"
# Transfom
cfg.TRANSFORM = CfgNode()
cfg.TRANSFORM.NORMALIZE = CfgNode()
cfg.TRANSFORM.NORMALIZE.MEAN = (106.433, 116.617, 119.559)
cfg.TRANSFORM.NORMALIZE.STD = (65.496, 67.6, 74.123)
cfg.TRANSFORM.RESIZE = CfgNode()
cfg.TRANSFORM.RESIZE.HEIGHT = 512
cfg.TRANSFORM.RESIZE.WIDTH = 1024
cfg.TRANSFORM.RANDOMCROP = CfgNode()
cfg.TRANSFORM.RANDOMCROP.HEIGHT = 512
cfg.TRANSFORM.RANDOMCROP.WIDTH = 1024
cfg.TRANSFORM.HFLIP = CfgNode()
cfg.TRANSFORM.HFLIP.PROB = 0.5
# Solver
cfg.SOLVER.NAME = "SGD"
cfg.SOLVER.ACCUMULATE_GRAD = 1
# Runner
cfg.BATCH_SIZE = 1
cfg.CHECKPOINT_PATH = ""
cfg.PRECISION = 32
# Callbacks
cfg.CALLBACKS = CfgNode()
cfg.CALLBACKS.CHECKPOINT_DIR = None
# Inference
cfg.INFERENCE = CfgNode()
cfg.INFERENCE.AREA_TRESH = 0
def __init__(self, cfg: CfgNode, val_augmentation: Sequence[Augmentation],
period: int):
super().__init__()
self.cfg = cfg.clone()
self.cfg.DATASETS.TRAIN = cfg.DATASETS.TEST
self._loader = iter(
build_detection_train_loader(
self.cfg,
mapper=DatasetMapper(self.cfg,
is_train=True,
augmentations=val_augmentation),
))
self._period = period
self.num_steps = 0
def get_config(device_name):
cfg = get_cfg()
if device_name == 'SERVER':
cfg.merge_from_file('configs/Teacher_dod.yaml')
cfg.MODEL.WEIGHTS = \
model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
elif device_name == 'CLIENT':
cfg.DOD = CfgNode()
cfg.DOD.MIN_STRIDE = 8
cfg.DOD.MAX_STRIDE = 64
cfg.DOD.NUM_UPDATES = 8
cfg.DATASETS.NUM_CLASSES = -1 # will be overrided by config file
cfg.merge_from_file('configs/Student_dod.yaml')
return cfg
def load_bootstrap_config(cfg: CN):
"""
Bootstrap datasets are given as a list of `dict` that are not automatically
converted into CfgNode. This method processes all bootstrap dataset entries
and ensures that they are in CfgNode format and comply with the specification
"""
if not cfg.BOOTSTRAP_DATASETS:
return
bootstrap_datasets_cfgnodes = []
for dataset_cfg in cfg.BOOTSTRAP_DATASETS:
_C = get_bootstrap_dataset_config().clone()
_C.merge_from_other_cfg(CN(dataset_cfg))
bootstrap_datasets_cfgnodes.append(_C)
cfg.BOOTSTRAP_DATASETS = bootstrap_datasets_cfgnodes
def _convert_dict_2_CfgNode(config):
ret_cfg = CfgNode()
for k in config:
if isinstance(config[k], dict):
setattr(ret_cfg, k, _convert_dict_2_CfgNode(config[k]))
pass
else:
val = config[k]
if isinstance(config[k], list):
if len(config[k]) > 0 and isinstance(config[k][0], dict):
temp = list(map(dict, config[k]))
val = temp
pass
setattr(ret_cfg, k, val)
return ret_cfg
def add_export_config(cfg):
"""
Args:
cfg (CfgNode): a detectron2 config
Returns:
CfgNode:
an updated config with new options that will be used by :class:`Caffe2Tracer`.
"""
is_frozen = cfg.is_frozen()
cfg.defrost()
cfg.EXPORT_CAFFE2 = CfgNode()
cfg.EXPORT_CAFFE2.USE_HEATMAP_MAX_KEYPOINT = False
if is_frozen:
cfg.freeze()
return cfg
def _create_gradient_clipper(cfg: CfgNode) -> _GradientClipper:
"""
Creates gradient clipping closure to clip by value or by norm,
according to the provided config.
"""
cfg = cfg.clone()
def clip_grad_norm(p: _GradientClipperInput):
torch.nn.utils.clip_grad_norm_(p, cfg.CLIP_VALUE, cfg.NORM_TYPE)
def clip_grad_value(p: _GradientClipperInput):
torch.nn.utils.clip_grad_value_(p, cfg.CLIP_VALUE)
_GRADIENT_CLIP_TYPE_TO_CLIPPER = {
GradientClipType.VALUE: clip_grad_value,
GradientClipType.NORM: clip_grad_norm,
}
return _GRADIENT_CLIP_TYPE_TO_CLIPPER[GradientClipType(cfg.CLIP_TYPE)]
def _test_end_value(cfg_dict):
cfg = get_cfg()
cfg.merge_from_other_cfg(CfgNode(cfg_dict))
p = nn.Parameter(torch.zeros(0))
opt = torch.optim.SGD([p], lr=cfg.SOLVER.BASE_LR)
scheduler = build_lr_scheduler(cfg, opt)
p.sum().backward()
opt.step()
self.assertEqual(opt.param_groups[0]["lr"],
cfg.SOLVER.BASE_LR * cfg.SOLVER.WARMUP_FACTOR)
lrs = []
for _ in range(cfg.SOLVER.MAX_ITER):
scheduler.step()
lrs.append(opt.param_groups[0]["lr"])
self.assertAlmostEqual(lrs[-1], cfg.SOLVER.BASE_LR_END)
def model_fn(model_dir: str) -> DefaultPredictor:
r"""Load trained model
Parameters
----------
model_dir : str
S3 location of the model directory
Returns
-------
DefaultPredictor
PyTorch model created by using Detectron2 API
"""
path_cfg, path_model = None, None
for p_file in Path(model_dir).iterdir():
if p_file.suffix == ".json":
path_cfg = p_file
if p_file.suffix == ".pth":
path_model = p_file
LOGGER.info(f"Using configuration specified in {path_cfg}")
LOGGER.info(f"Using model saved at {path_model}")
if path_model is None:
err_msg = "Missing model PTH file"
LOGGER.error(err_msg)
raise RuntimeError(err_msg)
if path_cfg is None:
err_msg = "Missing configuration JSON file"
LOGGER.error(err_msg)
raise RuntimeError(err_msg)
with open(str(path_cfg)) as fid:
cfg = CfgNode(json.load(fid))
cfg.MODEL.WEIGHTS = str(path_model)
cfg.MODEL.DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
return DefaultPredictor(cfg)
def add_retinanet_deploy_config(cfg: CfgNode):
_C = cfg
_C.MODEL.DEPLOY = CfgNode()
# Input shape for the deployed model.
_C.MODEL.DEPLOY.INPUT_SHAPE = (1, 3, 540, 960)
# ONNX opset version.
_C.MODEL.DEPLOY.OPSET_VERSION = 9
# Path to the exported ONNX model file.
_C.MODEL.DEPLOY.ONNX_PATH = "models/retinanet.onnx"
# Path to the serialized TensorRT engine file.
_C.MODEL.DEPLOY.ENGINE_PATH = "models/retinanet.engine"
# Path to the exported anchors.
_C.MODEL.DEPLOY.ANCHORS_PATH = "models/retinanet_anchors.pth"
# Maximum batch size for the TensorRT engine.
_C.MODEL.DEPLOY.MAX_BATCH_SIZE = 1
# Maximum workspace size for the TensorRT engine.
_C.MODEL.DEPLOY.MAX_WORKSPACE_SIZE = 1 << 25
# Whether to use FP16 precision.
_C.MODEL.DEPLOY.FP16_MODE = True
# Whether to force rebuilding the TensorRT engine.
_C.MODEL.DEPLOY.FORCE_REBUILD = False
def get_bootstrap_dataset_config() -> CN:
_C = CN()
_C.DATASET = ""
# ratio used to mix data loaders
_C.RATIO = 0.1
# image loader
_C.IMAGE_LOADER = CN(new_allowed=True)
_C.IMAGE_LOADER.TYPE = ""
_C.IMAGE_LOADER.BATCH_SIZE = 4
_C.IMAGE_LOADER.NUM_WORKERS = 4
# inference
_C.INFERENCE = CN()
# batch size for model inputs
_C.INFERENCE.INPUT_BATCH_SIZE = 4
# batch size to group model outputs
_C.INFERENCE.OUTPUT_BATCH_SIZE = 2
# sampled data
_C.DATA_SAMPLER = CN(new_allowed=True)
_C.DATA_SAMPLER.TYPE = ""
# filter
_C.FILTER = CN(new_allowed=True)
_C.FILTER.TYPE = ""
return _C
def run(self):
# Core function of your process
input = self.getInput(0)
# Get parameters :
param = self.getParam()
if len(input.data["images"]) > 0:
param.cfg["epochs"] = int(param.cfg["maxIter"] * param.cfg["batchSize"] / len(input.data["images"]))
# complete class names if input dataset has no background class
if not (input.has_bckgnd_class):
tmp_dict = {0: "background"}
for k, name in input.data["metadata"]["category_names"].items():
tmp_dict[k + 1] = name
input.data["metadata"]["category_names"] = tmp_dict
input.has_bckgnd_class = True
param.cfg["classes"] = len(input.data["metadata"]["category_names"])
# Call beginTaskRun for initialization
self.beginTaskRun()
if param.cfg["expertModeCfg"] == "":
# Get default config
cfg = get_cfg()
# Add specific deeplab config
add_deeplab_config(cfg)
cfg.merge_from_file(os.path.dirname(os.path.realpath(__file__)) + "/model/configs/deeplab_v3_plus_R_103_os16_mg124_poly_90k_bs16.yaml")
# Generic dataset names that will be used
cfg.DATASETS.TRAIN = ("datasetTrain",)
cfg.DATASETS.TEST = ("datasetTest",)
cfg.SOLVER.MAX_ITER = param.cfg["maxIter"]
cfg.SOLVER.WARMUP_FACTOR = 0.001
cfg.SOLVER.WARMUP_ITERS = param.cfg["maxIter"] // 5
cfg.SOLVER.POLY_LR_FACTOR = 0.9
cfg.SOLVER.POLY_LR_CONSTANT_FACTOR = 0.0
cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES = param.cfg["classes"]
cfg.SOLVER.BASE_LR = param.cfg["learningRate"]
cfg.MODEL.SEM_SEG_HEAD.ASPP_CHANNELS = 256
cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE = 4
cfg.SOLVER.IMS_PER_BATCH = param.cfg["batchSize"]
cfg.DATALOADER.NUM_WORKERS = 0
cfg.INPUT_SIZE = (param.cfg["inputWidth"], param.cfg["inputHeight"])
cfg.TEST.EVAL_PERIOD = param.cfg["evalPeriod"]
cfg.SPLIT_TRAIN_TEST = param.cfg["splitTrainTest"]
cfg.SPLIT_TRAIN_TEST_SEED = -1
cfg.MODEL.BACKBONE.FREEZE_AT = 5
cfg.CLASS_NAMES = [name for k, name in input.data["metadata"]["category_names"].items()]
if param.cfg["earlyStopping"]:
cfg.PATIENCE = param.cfg["patience"]
else:
cfg.PATIENCE = -1
if param.cfg["outputFolder"] == "":
cfg.OUTPUT_DIR = os.path.dirname(os.path.realpath(__file__)) + "/output"
elif os.path.isdir(param.cfg["outputFolder"]):
cfg.OUTPUT_DIR = param.cfg["outputFolder"]
else:
print("Incorrect output folder path")
else:
cfg = None
with open(param.cfg["expertModeCfg"], 'r') as file:
cfg_data = file.read()
cfg = CfgNode.load_cfg(cfg_data)
if cfg is not None:
deeplabutils.register_train_test(input.data["images"], input.data["metadata"],
train_ratio=cfg.SPLIT_TRAIN_TEST / 100,
seed=cfg.SPLIT_TRAIN_TEST_SEED)
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
str_datetime = datetime.now().strftime("%d-%m-%YT%Hh%Mm%Ss")
model_folder = cfg.OUTPUT_DIR + os.path.sep + str_datetime
cfg.OUTPUT_DIR = model_folder
if not os.path.isdir(model_folder):
os.mkdir(model_folder)
cfg.OUTPUT_DIR = model_folder
self.trainer = deeplabutils.MyTrainer(cfg, self)
self.trainer.resume_or_load(resume=False)
print("Starting training job...")
launch(self.trainer.train, num_gpus_per_machine=1)
print("Training job finished.")
self.trainer = None
gc.collect()
torch.cuda.empty_cache()
with open(cfg.OUTPUT_DIR+"/Detectron2_DeepLabV3Plus_Train_Config.yaml", 'w') as file:
file.write(cfg.dump())
else:
print("Error : can't load config file "+param.cfg["expertModeCfg"])
# Call endTaskRun to finalize process
self.endTaskRun()
# ----------------------------------------------------------------------------- # Convention about Training / Test specific parameters # ----------------------------------------------------------------------------- # Whenever an argument can be either used for training or for testing, the # corresponding name will be post-fixed by a _TRAIN for a training parameter, # or _TEST for a test-specific parameter. # For example, the number of images during training will be # IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be # IMAGES_PER_BATCH_TEST # ----------------------------------------------------------------------------- # Config definition # ----------------------------------------------------------------------------- _C = CN() # The version number, to upgrade from old configs to new ones if any # changes happen. It's recommended to keep a VERSION in your config file. _C.VERSION = 2 _C.MODEL = CN() _C.MODEL.LOAD_PROPOSALS = False _C.MODEL.MASK_ON = False _C.MODEL.KEYPOINT_ON = False _C.MODEL.DEVICE = "cuda" _C.MODEL.META_ARCHITECTURE = "GeneralizedRCNN" # Path (possibly with schema like catalog:// or detectron2://) to a checkpoint file # to be loaded to the model. You can find available models in the model zoo. _C.MODEL.WEIGHTS = ""
def load_yaml_with_base(filename: str, *args, **kwargs):
with reroute_load_yaml_with_base():
return _CfgNode.load_yaml_with_base(filename, *args, **kwargs)
def main(config):
root = expanduser(config["base"]["root"])
imgs_root = expanduser(config["base"]["imgs_root"])
jsons_dir = join(root, "jsons")
model_dir = join(root, "outputs")
scale = float(config["test_model"]["scale"])
do_show = config["test_model"]["do_show"]
register_data(jsons_dir, imgs_root)
# Need this datasets line, in order for metadata to have .thing_classes attribute
datasets = DatasetCatalog.get("test_data")
metadata = MetadataCatalog.get("test_data")
# Read the cfg back in:
with open(join(model_dir, "cfg.txt"), "r") as f:
cfg = f.read()
# Turn into CfgNode obj:
cfg = CfgNode.load_cfg(cfg)
# Use the weights from the model trained on our custom dataset:
cfg.MODEL.WEIGHTS = join(model_dir, "model_final.pth") # TODO: have option to use snapshot instead
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.01 # make small so I can make PR curve for broad range of scores
# cfg.DATASETS.TEST = ("val_data", ) # should already be saved from train_model.py
print("Generating predictor ...")
predictor = DefaultPredictor(cfg)
# For saving images with predicted labels:
output_imgs_dir = join(model_dir, "test_pred_imgs")
makedirs(output_imgs_dir, exist_ok=True)
# For saving detection predictions as csv:
output_csv = join(model_dir, "all_test_preds.csv")
csv_file_handle = open(output_csv, "w", newline="")
atexit.register(csv_file_handle.close)
col_names = ["img", "x1", "y1", "x2", "y2", "score", "thing","dummy_id"]
csv_writer = csv.DictWriter(csv_file_handle, fieldnames=col_names)
csv_writer.writeheader()
# Select 5 random images to visualize,
# but save the prediction results for all imgs:
rando_idxs = np.random.choice(range(len(datasets)), 5, replace=False).tolist()
for i,d in enumerate(datasets):
print(f"Predicting on image {i+1} of {len(datasets)} ...", end="\r")
id = d["image_id"]
img = cv2.imread(d["file_name"])
detected = predictor(img)
# Visualize:
visualizer = Visualizer(img[:, :, ::-1],
metadata=metadata,
scale=scale,
instance_mode=ColorMode)
visualizer = visualizer.draw_instance_predictions(detected["instances"].to("cpu"))
# Save the first 5 images from the random draw:
if i in rando_idxs:
pred_img = visualizer.get_image()[:, :, ::-1]
cv2.imwrite(join(output_imgs_dir, ("predicted_" + basename(d["file_name"]))), pred_img)
if do_show:
cv2.imshow(f"prediction on image {id}", pred_img)
print(f"Press any key to go to the next image ({i+1}/5) ...")
key = cv2.waitKey(0) & 0xFF
if key == ord("q"):
print("Quitting ...")
break
cv2.destroyAllWindows()
# Stream the predicted box coords and scores to a csv:
preds = detected['instances'].to('cpu')
boxes = preds.pred_boxes
thing_ids = preds.pred_classes.tolist()
scores = preds.scores
num_boxes = np.array(scores.size())[0]
for i in range(0, num_boxes):
coords = boxes[i].tensor.numpy()
score = float(scores[i].numpy())
thing_id = thing_ids[i] # is int
thing_class = metadata.thing_classes[thing_id]
csv_writer.writerow({col_names[0]: basename(d["file_name"]),
col_names[1]: int(coords[0][0]), # x1
col_names[2]: int(coords[0][1]), # y1
col_names[3]: int(coords[0][2]), # x2
col_names[4]: int(coords[0][3]), # y2
col_names[5]: score, # score
col_names[6]: thing_class, # thing
col_names[7]: i}) # dummy id
print(f"Finished predicting on all {len(datasets)} images from the test data fraction.")
print(f"Results are stored in {output_csv}")
print(f"5 sample test images are stored in {output_imgs_dir}\n"
"Note that the 5 sample test images show all detections with a score greater than 0.01. "
"This low score cutoff is for test purposes and is intentional. "
"You should expect to see many false positive labels.\n")
# Clear GPU memory
torch.cuda.empty_cache()
def get_adept_cfg(dataset_basename):
_C = CfgNode()
_C.TYPE = "adept"
_C.DEBUG = False
_C.VERSION = "paper-adept" #options: "paper-adept", "intphys-adept"
_C.RESUME = False
_C.RESUME_DIR = ""
_C.SEED = -1
_C.CUDNN_BENCHMARK = False
_C.DEBUG_VIDEOS = [
] #["output/intphys/.data_tmp/adept_jsons/pred_attr_82649/intphys_dev_O1/video_00054.json"]
_C.ANALYZE_RESULTS_FOLDER = "None"
#"output/intphys/adept/bk_distributed_exp"
#"/home/aldo/cora-derender/output/adept/adept/bk_distributed_exp"
#"/all/home/aldo/cora-derenderer/output/adept/adept/distributed_exp/"
#'/all/home/aldo/cora-derenderer/output/intphys/adept/distributed_exp_complete_bk' #"None" #"output/intphys/adept/exp_00025"
# -----------------------------------------------------------------------------
# Datasets
# -----------------------------------------------------------------------------
_C.DATASETS = CfgNode()
_C.DATASETS.BASE_NAME = dataset_basename
if _C.DATASETS.BASE_NAME == "intphys":
####### INTPHYS ###########
_C.ATTRIBUTES_KEYS = (
"pred_attr_82649",
# "pred_attr_03227",
# "pred_attr_06472",
# "pred_attr_12664",
# "pred_attr_22318",
# "pred_attr_41337",
)
_C.DATASETS.TEST = (
"intphys_dev_O1",
"intphys_dev_O2",
"intphys_dev_O3",
# "intphys_dev-meta_O1",
# "intphys_dev-meta_O2",
# "intphys_dev-meta_O3"
)
elif _C.DATASETS.BASE_NAME == "adept":
_C.ATTRIBUTES_KEYS = (
# "attributes",
"pred_attr_43044",
# "pred_attr_00650",
# 'pred_attr_10580',
# "pred_attr_18377",
# "pred_attr_34216"
)
_C.DATASETS.TEST = (
# "adept_val",
# "adept_train", #TODO get back to only test
"adept_human_create",
"adept_human_vanish",
"adept_human_short-overturn",
"adept_human_long-overturn",
"adept_human_visible-discontinuous",
"adept_human_invisible-discontinuous",
"adept_human_delay",
"adept_human_block",
)
# -----------------------------------------------------------------------------
# Model
# -----------------------------------------------------------------------------
_C.MODEL = CfgNode()
_C.MODEL.META_ARCHITECTURE = "PARTICLE_FILTER"
# Particles to be used in the particle filter
_C.MODEL.N_PARTICLES = 128
# Threshold for minimal area for an objects to be considered visible
_C.MODEL.AREA_THRESHOLD = 200.
# -----------------------------------------------------------------------------
# Dynamics Model
# -----------------------------------------------------------------------------
_C.MODEL.STEP = CfgNode()
_C.MODEL.STEP.PERTURBATION = CfgNode()
# Whether to perturb the objects
_C.MODEL.STEP.PERTURBATION.TO_PERTURB = True
# Sigma in the velocity term
_C.MODEL.STEP.PERTURBATION.VELOCITY_SIGMA = [.01, .06]
_C.MODEL.STEP.PERTURBATION.SCALE_SIGMA = .0005
# Sigma in the location term
_C.MODEL.STEP.PERTURBATION.LOCATION_SIGMA = [.01, .06]
# Sigma in the velocity term, multiplicative
_C.MODEL.STEP.PERTURBATION.VELOCITY_LAMBDA = [.01, .06]
# -----------------------------------------------------------------------------
# Magic in the dynamics model
# -----------------------------------------------------------------------------
_C.MODEL.STEP.MAGIC = CfgNode()
# Whether to use magic
_C.MODEL.STEP.MAGIC.USE_MAGIC = True
# The probability to disappear
_C.MODEL.STEP.MAGIC.DISAPPEAR_PROBABILITY = .02
# The penalty for magically disappearing
_C.MODEL.STEP.MAGIC.DISAPPEAR_PENALTY = 10.
# The probability for magically stopping
_C.MODEL.STEP.MAGIC.STOP_PROBABILITY = .02
# The penalty for magically stopping
_C.MODEL.STEP.MAGIC.STOP_PENALTY = 1.
# The probability for magically accelerating
_C.MODEL.STEP.MAGIC.ACCELERATE_PROBABILITY = .04
# The penalty for magically accelerating
_C.MODEL.STEP.MAGIC.ACCELERATE_PENALTY = 1.
# The magnitude for magically accelerating
_C.MODEL.STEP.MAGIC.ACCELERATE_LAMBDA = 1.5
# -----------------------------------------------------------------------------
# Particle filter
# -----------------------------------------------------------------------------
# The period for particle filter to resample
_C.MODEL.RESAMPLE = CfgNode()
# Resample every period
_C.MODEL.RESAMPLE.PERIOD = 1
# Scaling on nll
_C.MODEL.RESAMPLE.FACTOR = 1.
# -----------------------------------------------------------------------------
# Mass sampler
# -----------------------------------------------------------------------------
_C.MODEL.MASS = CfgNode()
# Whether to sample mass
_C.MODEL.MASS.TO_SAMPLE_MASS = False
# The log mean of mass
_C.MODEL.MASS.LOG_MASS_MU = 0
# The log stdev of mass
_C.MODEL.MASS.LOG_MASS_SIGMA = 1
# -----------------------------------------------------------------------------
# Observation Model
# -----------------------------------------------------------------------------
_C.MODEL.UPDATING = CfgNode()
_C.MODEL.UPDATING.MATCHED = CfgNode()
# Loss for matched object updating
_C.MODEL.UPDATING.MATCHED.LOSS = "Smoothed_L_Half"
# Sigma in the location term
_C.MODEL.UPDATING.MATCHED.LOCATION_SIGMA = .2
# Sigma in the velocity term
_C.MODEL.UPDATING.MATCHED.VELOCITY_SIGMA = .2
_C.MODEL.UPDATING.MATCHED.SCALE_SIGMA = .05
_C.MODEL.UPDATING.UNMATCHED_BELIEF = CfgNode()
# Base Penalty coefficient for unseen object
_C.MODEL.UPDATING.UNMATCHED_BELIEF.BASE_PENALTY = 1.
# Penalty coefficient for unseen object w.r.t. mask area shown
_C.MODEL.UPDATING.UNMATCHED_BELIEF.MASK_PENALTY = .0001
_C.MODEL.UPDATING.UNMATCHED_OBSERVATION = CfgNode()
# Penalty for object appearing
_C.MODEL.UPDATING.UNMATCHED_OBSERVATION.PENALTY = .02
_C.MODEL.UPDATING.UNMATCHED_OBSERVATION.MAX_PENALTY = 12.
_C.MODEL.MATCHER = CfgNode()
# PENALTY FOR MISMATCHED OBJECT TYPES, ONLY BETWEEN OCCLUDER AND OTHER
_C.MODEL.MATCHER.TYPE_PENALTY = 10.
# PENALTY FOR MISMATCHED OBJECT COLOR
_C.MODEL.MATCHER.COLOR_PENALTY = 12.
# PENALTY FOR MISMATCHED OBJECT WHEN THEY ARE AFAR
_C.MODEL.MATCHER.DISTANCE_PENALTY = 14. if _C.VERSION == "intphys-adept" else 20.
# THE THRESHOLD FOR OBJECT BEING AFAR
_C.MODEL.MATCHER.DISTANCE_THRESHOLD = 2.
# THE BASE PENALTY BETWEEN PLACEHOLDER AND OBJECTS
_C.MODEL.MATCHER.BASE_PENALTY = 8.
# when more than 5 objects creating more objects should not happen
_C.MODEL.MATCHER.BASE_PENALTY_HIGH = 16
return _C
def setup(args, config):
"""
Create configs and perform basic setups.
"""
from detectron2.config import CfgNode
# detectron2 default cfg
# cfg = get_cfg()
cfg = CfgNode()
cfg.SEED = -1
cfg.CUDNN_BENCHMARK = False
cfg.DATASETS = CfgNode()
cfg.SOLVER = CfgNode()
cfg.DATALOADER = CfgNode()
cfg.DATALOADER.ASPECT_RATIO_GROUPING = False
cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS = True
cfg.DATALOADER.SAMPLER_TRAIN = "TrainingSampler"
cfg.MODEL = CfgNode()
cfg.MODEL.KEYPOINT_ON = False
cfg.MODEL.LOAD_PROPOSALS = False
cfg.MODEL.WEIGHTS = ""
if args.config_file:
cfg.merge_from_file(args.config_file)
cfg.OUTPUT_DIR = f'{args.tl_outdir}/detectron2'
cfg.merge_from_list(args.opts)
cfg = D2Utils.cfg_merge_from_easydict(cfg, config)
cfg.freeze()
default_setup(
cfg, args
) # if you don't like any of the default setup, write your own setup code
return cfg
def get_dynamics_cfg(dataset_base_name):
_C = CfgNode()
_C.DEBUG = False
_C.TYPE = "dynamics"
_C.DATASETS = CfgNode()
# _C.DATASETS.USE_PREDICTED_ATTRIBUTES = True
_C.DATASETS.BASE_NAME = dataset_base_name
_C.DATALOADER = CfgNode()
_C.DATALOADER.NUM_WORKERS = 4
_C.DATALOADER.BATCH_SIZE = 15
_C.MODEL = CfgNode()
_C.MODEL.ARCHITECTURE = "interaction"
_C.MODEL.RNN_NUM_LAYERS = 6
_C.MODEL.DROP_OUT = 0.2
_C.MODEL.HIDDEN_SIZE = 300
_C.SOLVER = CfgNode()
_C.SOLVER.BASE_LR = 0.001
_C.SOLVER.BIAS_LR_FACTOR = 2
_C.SOLVER.OPT_TYPE = "Adam"
_C.SOLVER.MOMENTUM = 0.996
_C.SOLVER.ADAM_BETA = 0.9999427846237621
_C.SOLVER.WEIGHT_DECAY = 0.0005
_C.SOLVER.WEIGHT_DECAY_BIAS = 0
_C.SOLVER.GAMMA = 0.5
_C.SOLVER.STEPS = (999999, 999999)
_C.SOLVER.WARMUP_FACTOR = 1.0 / 3
_C.SOLVER.WARMUP_ITERS = 500
_C.SOLVER.WARMUP_METHOD = "linear"
_C.SOLVER.MAX_TIME_SECS = 999999999
# Print metrics every _ seconds
_C.SOLVER.PRINT_METRICS_TIME = 180 #TODO: set back to 180
# write to tensorboard summary every_ secconds
_C.SOLVER.TENSORBOARD_SECS = 1
# Checkpoint every _ seconds
_C.SOLVER.CHECKPOINT_SECS = 1200
# Run validation every _ seconds
_C.SOLVER.VALIDATION_SECS = 60 #TODO: change back to 300
_C.SOLVER.VALIDATION_MAX_SECS = 9999
if _C.DATASETS.BASE_NAME == "intphys":
_C.DATASETS.TRAIN = ("intphys_val", )
_C.DATASETS.TEST = (
"intphys_dev_O1",
"intphys_dev_O2",
"intphys_dev_O3",
# "intphys_dev-meta_O1",
# "intphys_dev-meta_O2",
# "intphys_dev-meta_O3"
)
_C.ATTRIBUTES_KEYS = ("pred_attr_401469", "pred_attr_003227")
elif _C.DATASETS.BASE_NAME == "adept":
_C.ATTRIBUTES_KEYS = (
"attributes",
# "pred_attr_00650",
"pred_attr_43044",
# 'pred_attr_10580',
# "pred_attr_18377",
# "pred_attr_34216"
)
_C.DATASETS.TRAIN = ("adept_train", )
_C.DATASETS.TEST = (
"adept_val",
# "adept_train",
# "adept_human_create",
# "adept_human_vanish",
# "adept_human_short-overturn",
# "adept_human_long-overturn",
# "adept_human_visible-discontinuous",
# "adept_human_invisible-discontinuous",
# "adept_human_delay",
# "adept_human_block",
)
return _C
def get_derender_config(dataset_name):
_C = CfgNode()
_C.DEBUG = False
_C.TYPE = "derender"
_C.ATTRIBUTES = CfgNode()
_C.RESUME = False
_C.RESUME_DIR = ""
_C.SEED = -1
_C.CUDNN_BENCHMARK = False
# -----------------------------------------------------------------------------
# Model
# -----------------------------------------------------------------------------
_C.MODEL = CfgNode()
# Number of derender visual feature channels
_C.MODEL.FEATURE_CHANNELS = 512
# number of hidden layers after backbone
_C.MODEL.NUM_MID_LAYERS = 2
# Number of intermediate layer channels
_C.MODEL.MID_CHANNELS = 256
_C.INPUT = CfgNode()
_C.DATASETS = CfgNode()
_C.DATASETS.USE_PREDICTED_BOXES = False
_C.DATASETS.BASE_NAME = dataset_name
# -----------------------------------------------------------------------------
# DataLoader
# -----------------------------------------------------------------------------
_C.DATALOADER = CfgNode()
# Number of data loading threads
if _C.DEBUG:
_C.DATALOADER.NUM_WORKERS = 0
else:
_C.DATALOADER.NUM_WORKERS = 6
if _C.DEBUG:
_C.DATALOADER.VAL_BATCH_SIZE = 80
else:
_C.DATALOADER.VAL_BATCH_SIZE = 160
# ---------------------------------------------------------------------------- #
# Solver
# ---------------------------------------------------------------------------- #
_C.SOLVER = CfgNode()
#maximum number of seconds for training
_C.SOLVER.MAX_TIME_SECS = 43200
# Print metrics every _ seconds
_C.SOLVER.PRINT_METRICS_TIME = 180 #TODO: set back to 180
# write to tensorboard summary every_ secconds
_C.SOLVER.TENSORBOARD_SECS = 1
# Checkpoint every _ seconds
_C.SOLVER.CHECKPOINT_SECS = 600
# Run validation every _ seconds
_C.SOLVER.VALIDATION_SECS = 600 #TODO: change back to 300
_C.SOLVER.VALIDATION_MAX_SECS = 60
_C.SOLVER.BASE_LR = 6.658777172739463e-5
_C.SOLVER.BIAS_LR_FACTOR = 2
_C.SOLVER.OPT_TYPE = "Adam" #options "Adam" "SGD"
_C.SOLVER.MOMENTUM = 0.9960477666835778 #found via Bayesian Optimization
_C.SOLVER.ADAM_BETA = 0.9999427846237621
_C.SOLVER.WEIGHT_DECAY = 0.0005
_C.SOLVER.WEIGHT_DECAY_BIAS = 0
#Factor of reduction at iteration == el for el in SOLVER.STEPS
_C.SOLVER.GAMMA = 0.3
_C.SOLVER.STEPS = (80000, 100000)
# _C.SOLVER.STEPS = (3000,5000)
_C.SOLVER.WARMUP_FACTOR = 1.0 / 3
_C.SOLVER.WARMUP_ITERS = 500
_C.SOLVER.WARMUP_METHOD = "linear"
######### INTPHYS ###############
if _C.DATASETS.BASE_NAME == "intphys":
_C.DATASETS.TRAIN = ("intphys_train", )
# _C.DATASETS.TRAIN = ("intphys_val",)
_C.DATASETS.TEST = ("intphys_val", )
_C.DATASETS.USE_DEPTH = True
_C.DATALOADER.OBJECTS_PER_BATCH = 160 # TODO: get back to 160
_C.ATTRIBUTES.NAME = "intphys"
_C.MODEL.ADD_CAMERA = True
# Input channels for the model (segmented depth map, depth map)
_C.MODEL.IN_CHANNELS = 2
# The size of pooling kernel in the last layer of the resnet34
_C.MODEL.POOLING_KERNEL_SIZE = (8, 8)
####### ADEPT #################
elif _C.DATASETS.BASE_NAME == "adept":
_C.DATASETS.TRAIN = ("adept_train", )
# _C.DATASETS.TRAIN = ("adept_val",)
_C.DATASETS.TEST = ("adept_val", )
_C.DATASETS.USE_DEPTH = False
_C.ATTRIBUTES.NAME = "adept"
_C.MODEL.ADD_CAMERA = False
if _C.DEBUG:
_C.DATALOADER.OBJECTS_PER_BATCH = 20 # TODO: get back to 40
else:
_C.DATALOADER.OBJECTS_PER_BATCH = 120 # TODO: get back to 40
# Input channels for the model (segmented depth map, depth map)
_C.MODEL.IN_CHANNELS = 12
# The size of pooling kernel in the last layer of the resnet34
_C.MODEL.POOLING_KERNEL_SIZE = (10, 15)
elif _C.DATASETS.BASE_NAME == "ai2thor-intphys":
_C.DATASETS.TRAIN = ("ai2thor-intphys_train", )
# _C.DATASETS.TRAIN = ("ai2thor-intphys_val",)
_C.DATASETS.TEST = ("ai2thor-intphys_val", )
_C.DATASETS.USE_DEPTH = True
_C.ATTRIBUTES.NAME = "ai2thor"
_C.MODEL.ADD_CAMERA = False
if _C.DEBUG:
_C.DATALOADER.OBJECTS_PER_BATCH = 20
else:
_C.DATALOADER.OBJECTS_PER_BATCH = 120
# Input channels for the model (segmented depth map, depth map)
_C.MODEL.IN_CHANNELS = 2
# The size of pooling kernel in the last layer of the resnet34
_C.MODEL.POOLING_KERNEL_SIZE = (8, 16)
else:
raise NotImplementedError
return _C
