def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train_wsl.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
print('reprint snapshot name for the result: ', checkpoints['final'])
if 'voc_' in cfg.TRAIN.DATASETS[0]:
TEST_DATASETS = cfg.TEST.DATASETS
TEST_PROPOSAL_FILES = cfg.TEST.PROPOSAL_FILES
cfg.immutable(False)
cfg.TEST.DATASETS = cfg.TRAIN.DATASETS
cfg.TEST.PROPOSAL_FILES = cfg.TRAIN.PROPOSAL_FILES
cfg.immutable(True)
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
print('reprint snapshot name for the result: ',
checkpoints['final'])
cfg.immutable(False)
cfg.TEST.DATASETS = TEST_DATASETS
cfg.TEST.PROPOSAL_FILES = TEST_PROPOSAL_FILES
cfg.immutable(True)
cfg.immutable(False)
cfg.TEST.BBOX_AUG.ENABLED = False
cfg.VIS = False
cfg.immutable(True)
_ = checkpoints.pop('final', None)
for snapshot in sorted(checkpoints.keys(), reverse=True):
test_model(checkpoints[snapshot], args.multi_gpu_testing,
args.opts)
print('reprint snapshot name for the result: ', snapshot,
checkpoints[snapshot])
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']
)
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def __init__(self,
config,
im_list,
model=None,
gpu_id=0): # im_list passed from cityscapes dataset
self.nb_features = config['nb_features']
self.split = config['split']
self.im_list = im_list
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
if not cfg.is_immutable(
): # just in case feature extractor has not been set up already
dset = b'cityscapes_fine_instanceonly_seg_' + self.split
args = Namespace(
cfg_file=MASK_RCNN_CONFIG,
wait=True,
multi_gpu_testing=False,
range=None, #[0, 3],
opts=['OUTPUT_DIR', config['save']])
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
if model is None or model == False:
self.model = initialize_model_from_cfg(instanciate_head_also=True)
else:
self.model = model
gpu_dev = caffe2_core.DeviceOption(caffe2_pb2.CUDA, gpu_id)
name_scope = 'gpu_{}'.format(gpu_id)
# Subsampler - originally inside the FPN network. But we don't want to predict the subsampled features.
# Instead, we want to predict the features, and then use the same subsampling operator to obtain the subsampled features
with caffe2_core.NameScope(name_scope):
with caffe2_core.DeviceScope(gpu_dev):
self.subsampler = caffe2_core.CreateOperator(
"MaxPool", # operator
["predicted_fpn_res5_2_sum"], #input blobs
["predicted_fpn_res5_2_sum_subsampled_2x"], #output blobs
kernel=1,
pad=0,
stride=2,
deterministic=1)
self.timers = {
k: Timer()
for k in [
'im_detect_bbox', 'im_detect_mask', 'misc_bbox', 'misc_mask',
'im_forward_backbone'
]
}
# For evaluation with respect to the dataset's gt, we save the prediction of the annotated frame for each sequence
self.num_classes = cfg.MODEL.NUM_CLASSES
self.num_images = len(self.im_list)
self.all_boxes_ann_frame, self.all_segms_ann_frame, _ = empty_results(
self.num_classes, self.num_images)
self.id_sequences = []
self.gpu_id = gpu_id
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def run(self):
split_l = self.jsondata["sample"].split(",")
samples = [sample.strip() for sample in split_l if sample]
root_dir = "/".join(samples[0].rstrip("/").split("/")[:-1])
if not all([sample.startswith(root_dir) for sample in samples]):
print("Please make sure all the samples in the same parent path.")
sys.exit()
self.parseJsonToCFG(cfg)
flag = self.checkPath(root_dir, samples, self.retrain)
anno_path = convert(root_dir, samples, flag)
print("Dataset Create Success!")
categories = readCategoryFromJson(
osp.join(
anno_path, "%s_%s_train.json" %
(self.datasets_name.lower(), self.datasets_year)))
# Remember to undo the gpu number setting
parameter_l = [
"MODEL.NUM_CLASSES",
len(categories) + 1, "NUM_GPUS", 4, "TRAIN.IMS_PER_BATCH", 1
]
merge_cfg_from_list(parameter_l)
assert_and_infer_cfg()
if not osp.exists(self.jsondata["model_Path"]):
os.mkdir(self.jsondata["model_Path"])
with open(osp.join(self.jsondata["model_Path"], "model.yaml"),
"w") as src:
src.write(yaml.dump(cfg))
with open(osp.join(self.jsondata["model_Path"], "classes.txt"),
"w") as src:
for category in categories:
src.write(category)
self.checkSymLink(root_dir)
losses, mAP = self.beginTrain()
return losses, mAP
def main():
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
cfg.NUM_GPUS = 1
assert_and_infer_cfg()
logger.info('Conerting model with config:')
logger.info(pprint.pformat(cfg))
assert not cfg.MODEL.KEYPOINTS_ON, "Keypoint model not supported."
assert not cfg.MODEL.MASK_ON, "Mask model not supported."
assert not cfg.FPN.FPN_ON, "FPN not supported."
assert not cfg.RETINANET.RETINANET_ON, "RetinaNet model not supported."
# load model from cfg
model, blobs = load_model(args)
net = core.Net('')
net.Proto().op.extend(copy.deepcopy(model.net.Proto().op))
net.Proto().external_input.extend(
copy.deepcopy(model.net.Proto().external_input))
net.Proto().external_output.extend(
copy.deepcopy(model.net.Proto().external_output))
net.Proto().type = args.net_execution_type
net.Proto().num_workers = 1 if args.net_execution_type == 'simple' else 4
# Reset the device_option, change to unscope name and replace python operators
convert_net(args, net.Proto(), blobs)
# add operators for bbox
add_bbox_ops(args, net, blobs)
if args.fuse_af:
print('Fusing affine channel...')
net, blobs = mutils.fuse_net_affine(
net, blobs)
if args.use_nnpack:
mutils.update_mobile_engines(net.Proto())
# generate init net
empty_blobs = ['data', 'im_info']
init_net = gen_init_net(net, blobs, empty_blobs)
if args.device == 'gpu':
[net, init_net] = convert_model_gpu(args, net, init_net)
net.Proto().name = args.net_name
init_net.Proto().name = args.net_name + "_init"
if args.test_img is not None:
verify_model(args, [net, init_net], args.test_img)
_save_models(net, init_net, args)
def main():
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
cfg.NUM_GPUS = 1
assert_and_infer_cfg()
logger.info('Conerting model with config:')
logger.info(pprint.pformat(cfg))
assert not cfg.MODEL.KEYPOINTS_ON, "Keypoint model not supported."
assert not cfg.MODEL.MASK_ON, "Mask model not supported."
assert not cfg.FPN.FPN_ON, "FPN not supported."
assert not cfg.RETINANET.RETINANET_ON, "RetinaNet model not supported."
# load model from cfg
model, blobs = load_model(args)
net = core.Net('')
net.Proto().op.extend(copy.deepcopy(model.net.Proto().op))
net.Proto().external_input.extend(
copy.deepcopy(model.net.Proto().external_input))
net.Proto().external_output.extend(
copy.deepcopy(model.net.Proto().external_output))
net.Proto().type = args.net_execution_type
net.Proto().num_workers = 1 if args.net_execution_type == 'simple' else 4
# Reset the device_option, change to unscope name and replace python operators
convert_net(args, net.Proto(), blobs)
# add operators for bbox
add_bbox_ops(args, net, blobs)
if args.fuse_af:
print('Fusing affine channel...')
net, blobs = mutils.fuse_net_affine(
net, blobs)
if args.use_nnpack:
mutils.update_mobile_engines(net.Proto())
# generate init net
empty_blobs = ['data', 'im_info']
init_net = gen_init_net(net, blobs, empty_blobs)
if args.device == 'gpu':
[net, init_net] = convert_model_gpu(args, net, init_net)
net.Proto().name = args.net_name
init_net.Proto().name = args.net_name + "_init"
if args.test_img is not None:
verify_model(args, [net, init_net], args.test_img)
_save_models(net, init_net, args)
def test_renamed_key_from_list(self):
# You should see logger messages like:
# "Key EXAMPLE.RENAMED.KEY was renamed to EXAMPLE.KEY;
# please update your config"
opts = ['EXAMPLE.RENAMED.KEY', 'foobar']
with self.assertRaises(AttributeError):
_ = cfg.EXAMPLE.RENAMED.KEY # noqa
with self.assertRaises(KeyError):
core_config.merge_cfg_from_list(opts)
def test_renamed_key_from_list(self):
# You should see logger messages like:
# "Key EXAMPLE.RENAMED.KEY was renamed to EXAMPLE.KEY;
# please update your config"
opts = ['EXAMPLE.RENAMED.KEY', 'foobar']
with self.assertRaises(AttributeError):
_ = cfg.EXAMPLE.RENAMED.KEY # noqa
with self.assertRaises(KeyError):
core_config.merge_cfg_from_list(opts)
def test_deprecated_key_from_list(self):
# You should see logger messages like:
# "Deprecated config key (ignoring): MODEL.DILATION"
opts = ['FINAL_MSG', 'foobar', 'MODEL.DILATION', 2]
with self.assertRaises(AttributeError):
_ = cfg.FINAL_MSG # noqa
with self.assertRaises(AttributeError):
_ = cfg.MODEL.DILATION # noqa
core_config.merge_cfg_from_list(opts)
with self.assertRaises(AttributeError):
_ = cfg.FINAL_MSG # noqa
with self.assertRaises(AttributeError):
_ = cfg.MODEL.DILATION # noqa
def test_deprecated_key_from_list(self):
# You should see logger messages like:
# "Deprecated config key (ignoring): MODEL.DILATION"
opts = ['FINAL_MSG', 'foobar', 'MODEL.DILATION', 2]
with self.assertRaises(AttributeError):
_ = cfg.FINAL_MSG # noqa
with self.assertRaises(AttributeError):
_ = cfg.MODEL.DILATION # noqa
core_config.merge_cfg_from_list(opts)
with self.assertRaises(AttributeError):
_ = cfg.FINAL_MSG # noqa
with self.assertRaises(AttributeError):
_ = cfg.MODEL.DILATION # noqa
def test_merge_cfg_from_list(self):
opts = [
'TRAIN.SCALES', '(100, )', 'MODEL.TYPE', 'foobar', 'NUM_GPUS', 2
]
assert len(cfg.TRAIN.SCALES) > 0
assert cfg.TRAIN.SCALES[0] != 100
assert cfg.MODEL.TYPE != 'foobar'
assert cfg.NUM_GPUS != 2
core_config.merge_cfg_from_list(opts)
assert type(cfg.TRAIN.SCALES) is tuple
assert len(cfg.TRAIN.SCALES) == 1
assert cfg.TRAIN.SCALES[0] == 100
assert cfg.MODEL.TYPE == 'foobar'
assert cfg.NUM_GPUS == 2
def test_merge_cfg_from_list(self):
opts = [
'TRAIN.SCALES', '(100, )', 'MODEL.TYPE', u'foobar', 'NUM_GPUS', 2
]
assert len(cfg.TRAIN.SCALES) > 0
assert cfg.TRAIN.SCALES[0] != 100
assert cfg.MODEL.TYPE != 'foobar'
assert cfg.NUM_GPUS != 2
core_config.merge_cfg_from_list(opts)
assert type(cfg.TRAIN.SCALES) is tuple
assert len(cfg.TRAIN.SCALES) == 1
assert cfg.TRAIN.SCALES[0] == 100
assert cfg.MODEL.TYPE == 'foobar'
assert cfg.NUM_GPUS == 2
def main():
args = parse_args()
merge_cfg_from_file(args.cfg)
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
model, blobs = convert_tools.load_model(args)
convert_main_net(args, model.net.Proto(), blobs)
if args.mask_dir:
convert_mask_net(args, model.mask_net.Proto())
if args.corresp:
classes = getattr(dummy_datasets, 'get_{}_dataset'.format(args.corresp))().classes
corresp = '\n'.join('{} {}'.format(i, classes[i]) for i, _ in enumerate(classes))
with open(args.out_dir + '/corresp.txt', 'w') as f:
f.write(corresp)
return 0
def main():
args = parse_args()
merge_cfg_from_file(args.cfg)
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
model, blobs = convert_tools.load_model(args)
convert_main_net(args, model.net, blobs)
if args.mask_dir:
convert_mask_net(args, model.mask_net)
if args.coco:
classes = dummy_datasets.get_coco_dataset().classes
corresp = '\n'.join('{} {}'.format(i, classes[i])
for i, _ in enumerate(classes))
with open(args.out_dir + '/corresp.txt', 'w') as f:
f.write(corresp)
return 0
def parseJsonToCFG(self, cfg):
merge_cfg_from_file(self.jsondata["parameterPath"])
para_dict = self.jsondata
model_dict = para_dict.pop("modelParameter")
solver_l = ["base_lr", "gamma", "max_iter", "steps"]
train_l = ["scales", "max_size"]
test_l = ["scale", "max_size", "nms"]
parameter_l = [
"TRAIN.DATASETS", ("voc_2007_train", ), "TEST.DATASETS",
("voc_2007_val", )
]
for key in model_dict:
if key in solver_l:
parameter_l.extend(
["SOLVER.%s" % key.upper(), model_dict[key]])
if key in train_l:
parameter_l.extend(["TRAIN.%s" % key.upper(), model_dict[key]])
if key in test_l:
parameter_l.extend(["TEST.%s" % key.upper(), model_dict[key]])
parameter_l.extend(["OUTPUT_DIR", para_dict["model_Path"]])
merge_cfg_from_list(parameter_l)
resume_weights_file = f
weights_file = os.path.join(cfg.TEST.WEIGHTS, resume_weights_file)
logger.info(
'========> Resuming from checkpoint {} at iter {}'.
format(weights_file, checkpoint_iter)
)
run_inference(
weights_file,
ind_range=args.range,
multi_gpu_testing=args.multi_gpu_testing,
check_expected_results=True,
)
if __name__ == '__main__':
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
logger = setup_logging(__name__)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
checkNewCheckpoint(args, cfg, logger)
roi_data_loader._minibatch_queue.qsize(),
cfg.DATA_LOADER.MINIBATCH_QUEUE_SIZE
)
)
# Sleep to simulate the time taken by running a little network
time.sleep(opts.sleep_time)
# To inspect:
# blobs = workspace.FetchBlobs(all_blobs)
# from IPython import embed; embed()
logger.info('Shutting down data loader...')
roi_data_loader.shutdown()
if __name__ == '__main__':
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
logger = setup_logging(__name__)
logger.setLevel(logging.DEBUG)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
np.random.seed(cfg.RNG_SEED)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Running with config:')
logger.info(pprint.pformat(cfg))
main(args)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# test model
logger.info("creat test model ...")
test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS,
gpu_id=0)
logger.info("created test model ...")
train_data = DataLoader(root,
"train_id.txt",
cfg,
test_model,
is_train=True)
# creat mode
model, weights_file, start_iter, checkpoints = create_model(
True, cfg, output_dir)
# test blob
print(workspace.Blobs())
# create input blob
blob_names = ['data_stage2', 'gt_label_stage2']
for gpu_id in range(cfg.NUM_GPUS):
with c2_utils.NamedCudaScope(gpu_id):
for blob_name in blob_names:
workspace.CreateBlob(core.ScopedName(blob_name))
# Override random weight initialization with weights from a saved model
if weights_file:
nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0)
# Even if we're randomly initializing we still need to synchronize
# parameters across GPUs
nu.broadcast_parameters(model)
workspace.CreateNet(model.net)
logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir)))
dump_proto_files(model, output_dir)
writer = SummaryWriter(log_dir=output_dir)
training_stats = TrainingStats(model, writer)
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
logger.info("start train ...")
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
# feed data
# print("{} iter starting feed data...".format(cur_iter))
data_stage2, gt_label = train_data.next_batch()
with c2_utils.NamedCudaScope(gpu_id):
workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2)
workspace.FeedBlob(core.ScopedName('gt_label_stage2'), gt_label)
# print("workspace.RunNet(model.net.Proto().name)")
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
workspace.RunNet(model.net.Proto().name)
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats(cur_iter)
training_stats.LogIterStats(cur_iter, lr)
writer.add_scalar('learning_rate', lr, cur_iter)
# print("end of RunNet")
if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter:
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if cur_iter == start_iter + training_stats.LOG_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
handle_critical_error(model, 'Loss is NaN')
# Save the final model
checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# save train loss and metric
state_file = os.path.join(output_dir, 'training_state.json')
training_stats.SaveTrainingStates(state_file)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def main():
workspace.GlobalInit(["caffe2", "--caffe2_log_level=0"])
args = parse_args()
logger.info("Called with args:")
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
cfg.NUM_GPUS = 1
assert_and_infer_cfg()
logger.info("Converting model with config:")
logger.info(pprint.pformat(cfg))
# script will stop when it can't find an operator rather
# than stopping based on these flags
#
# assert not cfg.MODEL.KEYPOINTS_ON, "Keypoint model not supported."
# assert not cfg.MODEL.MASK_ON, "Mask model not supported."
# assert not cfg.FPN.FPN_ON, "FPN not supported."
# assert not cfg.RETINANET.RETINANET_ON, "RetinaNet model not supported."
# load model from cfg
model, blobs = load_model(args)
net = core.Net("")
net.Proto().op.extend(copy.deepcopy(model.net.Proto().op))
net.Proto().external_input.extend(
copy.deepcopy(model.net.Proto().external_input))
net.Proto().external_output.extend(
copy.deepcopy(model.net.Proto().external_output))
net.Proto().type = args.net_execution_type
net.Proto().num_workers = 1 if args.net_execution_type == "simple" else 4
# Reset the device_option, change to unscope name and replace python operators
convert_net(args, net.Proto(), blobs)
# add operators for bbox
add_bbox_ops(args, net, blobs)
if args.fuse_af:
print("Fusing affine channel...")
net, blobs = mutils.fuse_net_affine(net, blobs)
if args.use_nnpack:
mutils.update_mobile_engines(net.Proto())
# generate init net
empty_blobs = ["data", "im_info"]
init_net = gen_init_net(net, blobs, empty_blobs)
if args.device == "gpu":
[net, init_net] = convert_model_gpu(args, net, init_net)
net.Proto().name = args.net_name
init_net.Proto().name = args.net_name + "_init"
if args.test_img is not None:
verify_model(args, [net, init_net], args.test_img)
if args.logdb == 1:
output_file = os.path.join(args.out_dir, "model.logfiledb")
_export_to_logfiledb(args, net, init_net, empty_blobs, output_file)
else:
_save_models(net, init_net, args)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
if os.path.exists('./detectron/datasets/data/coco'):
shutil.rmtree('./detectron/datasets/data/coco')
os.makedirs('./detectron/datasets/data/coco')
if 'dior_2nd' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_2nd ./detectron/datasets/data/coco/annotations'
)
elif 'dior_3rd' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_3rd ./detectron/datasets/data/coco/annotations'
)
elif 'dior_4th' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_4th ./detectron/datasets/data/coco/annotations'
)
elif 'dior_5th' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_5th ./detectron/datasets/data/coco/annotations'
)
elif '2020.10.6' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN ./detectron/datasets/data/coco/annotations'
)
else:
raise Exception
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def __init__(self,
split,
frame_ss,
nSeq,
features,
savedir,
size=None,
loaded_model=None):
super(CityscapesDatasetAndFeatures, self).__init__()
self.split = split
self.frame_ss = frame_ss
self.nSeq = nSeq or 1
self.features = features
self.FPNfeatures = u'fpn_res5_2_sum' in self.features \
or u'fpn_res4_5_sum' in self.features \
or u'fpn_res3_3_sum' in self.features \
or u'fpn_res2_2_sum' in self.features
self.limitSize = size
# Check which features have been precomputed and load them if they have been found
logger.info('Searching for precomputed features...')
self.precompute_features_dir = PRECOMPUTED_FEATURES_DIR
self.potential_precomputed_feature_types = [
u'fpn_res5_2_sum', u'fpn_res4_5_sum', u'fpn_res3_3_sum',
u'fpn_res2_2_sum'
]
self.load_precomputed = self.check_requested_features_that_could_be_precomputed_were(
)
if self.load_precomputed:
self.precomputed_features_index, self.precomputed_features = \
self.load_requested_precomputed_features()
self.requested_fpn_features = [] + \
([u'fpn_res5_2_sum'] if 'fpn_res5_2_sum' in self.features else []) + \
([u'fpn_res4_5_sum'] if 'fpn_res4_5_sum' in self.features else []) + \
([u'fpn_res3_3_sum'] if 'fpn_res3_3_sum' in self.features else []) + \
([u'fpn_res2_2_sum'] if 'fpn_res2_2_sum' in self.features else [])
import detectron.utils.c2 as c2_utils
c2_utils.import_detectron_ops()
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
dset = b'cityscapes_fine_instanceonly_seg_sequences_' + self.split
if not cfg.is_immutable(
): # just in case feature extractor has not been set up already
# Preparing where to load data from and how using coco api
args = Namespace(cfg_file=MASK_RCNN_CONFIG,
wait=True,
multi_gpu_testing=False,
opts=['OUTPUT_DIR', savedir])
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
assert os.path.exists(cfg.TEST.WEIGHTS), \
'need path to pretrained instance segmentation model'
assert not cfg.MODEL.RPN_ONLY, 'end to end model required'
dataset = JsonDataset(dset)
self.dataset = dataset
self.im_list = self.dataset.get_roidb()
# Preparing the model from which we obtain the features
if not self.load_precomputed:
if loaded_model is None:
model = initialize_model_from_cfg()
self.model = model
else:
self.model = loaded_model
else:
self.model = False
# Store config for further use in running the Mask RCNN head
logger.info('Cityscapes dataset, size : %d' % len(self))
import sys
import yaml
import json
import pprint
from detectron.core.config import assert_and_infer_cfg
from detectron.core.config import cfg
from detectron.core.config import merge_cfg_from_file
from detectron.core.config import merge_cfg_from_list
if __name__ == "__main__":
para = sys.stdin.read()
para_dict = json.loads(para)
model_dict = para_dict.pop("modelParameter")
solver_l = ["base_lr", "gamma", "max_iter", "steps"]
train_l = ["scales", "max_size"]
test_l = ["scale", "max_size", "nms"]
parameter_l = list()
for key in model_dict:
if key in solver_l:
parameter_l.extend(["SOLVER.%s" % key.upper(), model_dict[key]])
if key in train_l:
parameter_l.extend(["TRAIN.%s" % key.upper(), model_dict[key]])
if key in test_l:
parameter_l.extend(["TEST.%s" % key.upper(), model_dict[key]])
parameter_l.extend(["OUTPUT_DIR", para_dict["model_Path"]])
merge_cfg_from_list(parameter_l)
assert_and_infer_cfg()
with open("%s/model.yaml" % para_dict["model_Path"], "w") as src:
src.write(yaml.dump(cfg))
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
fs = open('imgnames.pkl', 'rb')
roidbnames = pickle.load(fs)
fs.close()
logger.info('Loading dataset: {}'.format(cfg.TRAIN.DATASETS))
dataset_names = cfg.TRAIN.DATASETS
proposal_files = cfg.TRAIN.PROPOSAL_FILES
roidb = get_training_roidb(dataset_names, proposal_files)
logger.info('{:d} roidb entries'.format(len(roidb)))
total_num = len(roidb)
# bitmap idx indicated for training
bitmapRoidb = BitMap(total_num)
# initial samples
# initial_num = int(total_num*0.2)
# for i in range(initial_num):
# bitmapRoidb.set(i)
#
# train_roidb = [roidb[i] for i in range(initial_num)]
initialidx = []
train_roidb = []
for i, x in enumerate(roidb):
if x['image'].split('/')[-1] in roidbnames:
initialidx.append(i)
train_roidb.append(x)
for i in initialidx:
bitmapRoidb.set(i)
logger.info('{:d} the number initial roidb entries'.format(
len(train_roidb)))
# append flipped images
train_roidb = flipped_roidb_for_training(train_roidb)
logger.info('{:d} the number initial roidb entries'.format(
len(train_roidb)))
alamount = 0
ssamount = 0
gamma = 0.95
# control al proportion
al_proportion_checkpoint = [
int(x * total_num * 0.4) for x in np.linspace(0.2, 1, 10)
]
# control ss proportion
ss_proportion_checkpoint = [
int(x * total_num) for x in np.linspace(0.2, 2, 10)
]
next_iters = 90000
sum_iters = next_iters
'''load the lasted checkpoints'''
checkpoints = detectron.utils.train.train_model(sum_iters, train_roidb,
cfg.TRAIN.WEIGHTS)
while True:
# to do a test on the test dataset
test_model(checkpoints[(sum_iters - 1)], args.multi_gpu_testing,
args.opts)
if sum_iters > cfg.SOLVER.MAX_ITER:
break
# next detect unlabeled samples
unlabeledidx = list(set(range(total_num)) - set(bitmapRoidb.nonzero()))
# labeled samples
labeledidx = list(set(bitmapRoidb.nonzero()))
# detect unlabeled samples
BBoxes, YClass, Scores, al_candidate_idx, ALScore = detect_im(
checkpoints[(sum_iters - 1)],
roidb,
gamma,
idxs=unlabeledidx,
gpu_id=0)
al_avg_idx = np.argsort(np.array(ALScore))
al_candidate_idx = [al_candidate_idx[i] for i in al_avg_idx]
gamma = max(gamma - 0.05, 0.7)
# the ss candidate idx
ss_candidate_idx = [
i for i in unlabeledidx if i not in al_candidate_idx
]
# update roidb for next training
train_roidb = replace_roidb(roidb, BBoxes, YClass, ss_candidate_idx)
# control the proportion
if alamount + len(al_candidate_idx) >= al_proportion_checkpoint[0]:
al_candidate_idx = al_candidate_idx[:int(
al_proportion_checkpoint[0] - alamount)]
tmp = al_proportion_checkpoint.pop(0)
al_proportion_checkpoint.append(al_proportion_checkpoint[-1])
if ssamount + len(ss_candidate_idx) >= ss_proportion_checkpoint[0]:
ss_candidate_idx = ss_candidate_idx[:int(
ss_proportion_checkpoint[0] - ssamount)]
tmp = ss_proportion_checkpoint.pop(0)
ss_proportion_checkpoint.append(ss_proportion_checkpoint[-1])
# record ss and al factor
alamount += len(al_candidate_idx)
ssamount += len(ss_candidate_idx)
logger.info('alfactor:{},ssfactor:{}'.format(alamount / total_num,
ssamount / total_num))
# for idx in al_candidate_idx:
# bitmapRoidb.set(idx)
next_train_idx = bitmapRoidb.nonzero()
next_train_idx.extend(ss_candidate_idx)
train_roidb = blur_image(train_roidb, ss_candidate_idx)
# the next training roidb
train_roidb = [train_roidb[i] for i in next_train_idx]
# flipped the roidb
train_roidb = flipped_roidb_for_training(train_roidb)
# the next training iters
next_iters = 30000
sum_iters += next_iters
checkpoints = detectron.utils.train.train_model(
sum_iters, train_roidb, checkpoints[(sum_iters - next_iters - 1)])
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']
)
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# test model
logger.info("creat test model ...")
test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS, gpu_id=0)
logger.info("created test model ...")
#cfg.TRAIN.IMS_PER_BATCH = 1
train_data = DataLoader(root, "val_id.txt", cfg, test_model, is_train=False)
# creat mode
model, weights_file, start_iter, checkpoints = create_model(False, cfg, output_dir)
# test blob
print(workspace.Blobs())
# create input blob
blob_names = ['data_stage2']
for gpu_id in range(cfg.NUM_GPUS):
with c2_utils.NamedCudaScope(gpu_id):
for blob_name in blob_names:
workspace.CreateBlob(core.ScopedName(blob_name))
# Override random weight initialization with weights from a saved model
if weights_file:
nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0)
# Even if we're randomly initializing we still need to synchronize
# parameters across GPUs
nu.broadcast_parameters(model)
workspace.CreateNet(model.net)
logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir)))
logger.info("start test ...")
save_root = os.path.join(output_dir, 'fusion')
if not os.path.exists(save_root):
os.makedirs(save_root)
for cur_iter in range(10000):
# feed data
# print("{} iter starting feed data...".format(cur_iter))
data_stage2, gt_label, meta = train_data.next_batch()
'''#
print('input0-20 sungalsses max score:', np.max(data_stage2[0, 4, :, :]))
print('input20-40 sungalsses max score:', np.max(data_stage2[0, 24, :, :]))
print('input0-20 glovess max score:', np.max(data_stage2[0, 3, :, :]))
print('input20-40 glovess max score:', np.max(data_stage2[0, 23, :, :]))
#'''
with c2_utils.NamedCudaScope(gpu_id):
workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2)
# print("workspace.RunNet(model.net.Proto().name)")
with c2_utils.NamedCudaScope(gpu_id):
workspace.RunNet(model.net.Proto().name)
batch_probs = workspace.FetchBlob(core.ScopedName('probs_human_NCHW_stage2'))
batch_probs = batch_probs.transpose((0, 2, 3, 1))
assert len(meta) == batch_probs.shape[0]
#print('batch_probs shape:', batch_probs.shape)
for i in range(len(meta)):
probs = cv2.resize(batch_probs[i], (meta[i]['width'], meta[i]['height']), interpolation=cv2.INTER_LINEAR)
probs = probs.transpose((2,0,1))
print('sungalsses max score:', np.max(probs[4, :, :]))
print('glovess max score:', np.max(probs[3, :, :]))
#print('probs shape:', probs.shape)
cv2.imwrite(os.path.join(save_root, meta[i]['id']+'.png'), probs.argmax(0))
print("prossed ", cur_iter)
