def setup_custom_environment(custom_module_path):
"""Load custom environment setup from a Python source file and run the setup
function.
"""
module = import_file("wetectron.utils.env.custom_module",
custom_module_path)
assert hasattr(module, "setup_environment") and callable(
module.setup_environment), (
"Custom environment module defined in {} does not have the "
"required callable attribute 'setup_environment'."
).format(custom_module_path)
module.setup_environment()
def _load_file(self, f):
# catalog lookup
if f.startswith("catalog://"):
paths_catalog = import_file("wetectron.config.paths_catalog",
self.cfg.PATHS_CATALOG, True)
catalog_f = paths_catalog.ModelCatalog.get(f[len("catalog://"):])
self.logger.info("{} points to {}".format(f, catalog_f))
f = catalog_f
# download url files
if f.startswith("http"):
# if the file is a url path, download it and cache it
cached_f = cache_url(f)
self.logger.info("url {} cached in {}".format(f, cached_f))
f = cached_f
# convert Caffe2 checkpoint from pkl
if f.endswith(".pkl"):
return load_c2_format(self.cfg, f)
# load native detectron.pytorch checkpoint
loaded = super(DetectronCheckpointer, self)._load_file(f)
if "model" not in loaded:
loaded = dict(model=loaded)
return loaded
def make_data_loader(cfg, is_train=True, is_distributed=False, start_iter=0):
# seed = cfg.SEED
# def _init_fn(worker_id):
# np.random.seed(int(seed))
num_gpus = get_world_size()
if is_train:
images_per_batch = cfg.SOLVER.IMS_PER_BATCH
assert (
images_per_batch % num_gpus == 0
), "SOLVER.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.".format(
images_per_batch, num_gpus)
images_per_gpu = images_per_batch // num_gpus
shuffle = True
num_iters = cfg.SOLVER.MAX_ITER
else:
images_per_batch = cfg.TEST.IMS_PER_BATCH
assert (
images_per_batch % num_gpus == 0
), "TEST.IMS_PER_BATCH ({}) must be divisible by the number of GPUs ({}) used.".format(
images_per_batch, num_gpus)
images_per_gpu = images_per_batch // num_gpus
shuffle = False if not is_distributed else True
num_iters = None
start_iter = 0
if images_per_gpu > 1:
logger = logging.getLogger(__name__)
logger.warning(
"When using more than one image per GPU you may encounter "
"an out-of-memory (OOM) error if your GPU does not have "
"sufficient memory. If this happens, you can reduce "
"SOLVER.IMS_PER_BATCH (for training) or "
"TEST.IMS_PER_BATCH (for inference). For training, you must "
"also adjust the learning rate and schedule length according "
"to the linear scaling rule. See for example: "
"https://github.com/facebookresearch/Detectron/blob/master/configs/getting_started/tutorial_1gpu_e2e_faster_rcnn_R-50-FPN.yaml#L14"
)
# group images which have similar aspect ratio. In this case, we only
# group in two cases: those with width / height > 1, and the other way around,
# but the code supports more general grouping strategy
aspect_grouping = [1] if cfg.DATALOADER.ASPECT_RATIO_GROUPING else []
paths_catalog = import_file("wetectron.config.paths_catalog",
cfg.PATHS_CATALOG, True)
DatasetCatalog = paths_catalog.DatasetCatalog
dataset_list = cfg.DATASETS.TRAIN if is_train else cfg.DATASETS.TEST
proposal_files = cfg.PROPOSAL_FILES.TRAIN if is_train else cfg.PROPOSAL_FILES.TEST
# If bbox aug is enabled in testing, simply set transforms to None and we will apply transforms later
transforms = None if not is_train and cfg.TEST.BBOX_AUG.ENABLED else build_transforms(
cfg, is_train)
datasets, data_args = build_dataset(dataset_list, transforms,
DatasetCatalog, is_train,
proposal_files)
if is_train:
# save category_id to label name mapping
save_labels(datasets, cfg.OUTPUT_DIR)
data_loaders = []
for dataset in datasets:
#shuffle = False if cfg.SOLVER.CLASS_BATCH else True
sampler = make_data_sampler(dataset, shuffle,
is_distributed) ##Randomsampler
batch_sampler = make_batch_data_sampler(
dataset, sampler, aspect_grouping, images_per_gpu, data_args,
is_train, cfg.SOLVER.CLASS_BATCH, num_iters, start_iter)
collator = BBoxAugCollator() if not is_train and cfg.TEST.BBOX_AUG.ENABLED else \
BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY)
num_workers = cfg.DATALOADER.NUM_WORKERS
data_loader = torch.utils.data.DataLoader(
dataset,
num_workers=num_workers,
batch_sampler=batch_sampler,
collate_fn=collator,
worker_init_fn=worker_init_reset_seed)
data_loaders.append(data_loader)
'''co_mat = np.zeros([20,20])
if is_train:
for data in data_loaders[0]:
for d in data[1]:
im_labels = d.get_field('labels').unique().subtract(1).tolist()
for l in im_labels:
co_mat[l,l] += 1
if len(im_labels) >= 2:
comb = list(itertools.combinations(im_labels,2))
for c in comb:
co_mat[c[0], c[1]] += 1
import IPython; IPython.embed()
'''
'''for a in range(20):
for b in range(20):
co_mat[b,a] = co_mat[a,b]
for i in range(20):
co_mat2[i] = co_mat[i]/co_mat[i][i]
for x in range(20):
for y in range(20):
if co_mat2[x,y] == 0:
co_mat2[x,y] = 1e-9
co_occur = torch.Tensor(co_mat2).to('cuda')
'''
if is_train:
# during training, a single (possibly concatenated) data_loader is returned
assert len(data_loaders) == 1
return data_loaders[0]
return data_loaders