def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train_wsl.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
print('reprint snapshot name for the result: ', checkpoints['final'])
if 'voc_' in cfg.TRAIN.DATASETS[0]:
TEST_DATASETS = cfg.TEST.DATASETS
TEST_PROPOSAL_FILES = cfg.TEST.PROPOSAL_FILES
cfg.immutable(False)
cfg.TEST.DATASETS = cfg.TRAIN.DATASETS
cfg.TEST.PROPOSAL_FILES = cfg.TRAIN.PROPOSAL_FILES
cfg.immutable(True)
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
print('reprint snapshot name for the result: ',
checkpoints['final'])
cfg.immutable(False)
cfg.TEST.DATASETS = TEST_DATASETS
cfg.TEST.PROPOSAL_FILES = TEST_PROPOSAL_FILES
cfg.immutable(True)
cfg.immutable(False)
cfg.TEST.BBOX_AUG.ENABLED = False
cfg.VIS = False
cfg.immutable(True)
_ = checkpoints.pop('final', None)
for snapshot in sorted(checkpoints.keys(), reverse=True):
test_model(checkpoints[snapshot], args.multi_gpu_testing,
args.opts)
print('reprint snapshot name for the result: ', snapshot,
checkpoints[snapshot])
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']
)
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def beginTrain(self):
workspace.GlobalInit([
'caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'
])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints, losses = train_model()
# Test the trained model
self.test_model(checkpoints["final"])
dataset_name, _ = get_inference_dataset(0)
output_dir = get_output_dir(dataset_name, training=False)
with open(osp.join(output_dir, "res.pkl"), "rb") as src:
mAP = pickle.load(src)
return losses, mAP
def main():
c2_utils.import_contrib_ops()
c2_utils.import_detectron_ops()
cv2.ocl.setUseOpenCL(False)
args = parse_args()
input_file = args.input_file
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
merge_cfg_from_file(
'/home/LAB/wusj/exp/KL-Loss/configs/e2e_faster_rcnn_R-50-FPN_2x_entropy.yaml'
)
assert_and_infer_cfg(cache_urls=False)
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
workspace.ResetWorkspace()
np.random.seed(cfg.RNG_SEED)
with open(input_file, 'r') as f:
config = json.load(f)
parameter = config['parameter']
inference_id = parameter['inferenceId']
dataset = parameter['dataSetName']
model_file = parameter['modelPkl']
task_id = inference_id
image_id_list = parameter['imageIdList']
load_data(dataset, image_id_list)
roidb, result = test_net(model_file, dataset)
tmp = []
for i, entry in enumerate(roidb):
tmp.append(entry)
tmp.sort(cmp=compare)
selectNum = parameter['selectNum']
output = {}
# time.sleep(3)
output['annotationList'] = []
output['selectImageIdList'] = []
output['remainImageIdList'] = []
for i in range(selectNum):
output['annotationList'].append(result[str(tmp[i]['id'])])
output['selectImageIdList'].append(str(tmp[i]['id']))
for i in range(parameter['selectNum'], len(parameter['imageIdList'])):
output['remainImageIdList'].append(str(tmp[i]['id']))
output['remainImageNum'] = len(output['remainImageIdList'])
output['inferenceId'] = inference_id
output['dataSetName'] = dataset
result_output_dir = '/home/LAB/wusj/fastwash_tmp/inference/'
with open(result_output_dir + 'result_' + task_id, 'wt') as f2:
json.dump(output, f2)
def main():
c2_utils.import_contrib_ops()
c2_utils.import_detectron_ops()
cv2.ocl.setUseOpenCL(False)
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
merge_cfg_from_file(
'/home/LAB/wusj/exp/KL-Loss/configs/e2e_faster_rcnn_R-50-FPN_2x_entropy.yaml'
)
assert_and_infer_cfg(cache_urls=False)
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
workspace.ResetWorkspace()
np.random.seed(cfg.RNG_SEED)
args = parse_args()
input_file = args.input_file
result_output_dir = args.output_dir
with open(input_file, 'r') as f:
config = json.load(f)
dataset = config['dataSetName']
model_file = args.model_file
task_id = config['id']
image_id_list = config['imageIdList']
load_data(dataset, image_id_list)
roidb, result = test_net(model_file, dataset)
config['inferenceResult'] = result
with open(result_output_dir + 'result_' + task_id, 'wt') as f2:
json.dump(config, f2)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# test model
logger.info("creat test model ...")
test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS,
gpu_id=0)
logger.info("created test model ...")
train_data = DataLoader(root,
"train_id.txt",
cfg,
test_model,
is_train=True)
# creat mode
model, weights_file, start_iter, checkpoints = create_model(
True, cfg, output_dir)
# test blob
print(workspace.Blobs())
# create input blob
blob_names = ['data_stage2', 'gt_label_stage2']
for gpu_id in range(cfg.NUM_GPUS):
with c2_utils.NamedCudaScope(gpu_id):
for blob_name in blob_names:
workspace.CreateBlob(core.ScopedName(blob_name))
# Override random weight initialization with weights from a saved model
if weights_file:
nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0)
# Even if we're randomly initializing we still need to synchronize
# parameters across GPUs
nu.broadcast_parameters(model)
workspace.CreateNet(model.net)
logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir)))
dump_proto_files(model, output_dir)
writer = SummaryWriter(log_dir=output_dir)
training_stats = TrainingStats(model, writer)
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
logger.info("start train ...")
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
# feed data
# print("{} iter starting feed data...".format(cur_iter))
data_stage2, gt_label = train_data.next_batch()
with c2_utils.NamedCudaScope(gpu_id):
workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2)
workspace.FeedBlob(core.ScopedName('gt_label_stage2'), gt_label)
# print("workspace.RunNet(model.net.Proto().name)")
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
workspace.RunNet(model.net.Proto().name)
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats(cur_iter)
training_stats.LogIterStats(cur_iter, lr)
writer.add_scalar('learning_rate', lr, cur_iter)
# print("end of RunNet")
if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter:
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if cur_iter == start_iter + training_stats.LOG_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
handle_critical_error(model, 'Loss is NaN')
# Save the final model
checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# save train loss and metric
state_file = os.path.join(output_dir, 'training_state.json')
training_stats.SaveTrainingStates(state_file)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']
)
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# test model
logger.info("creat test model ...")
test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS, gpu_id=0)
logger.info("created test model ...")
#cfg.TRAIN.IMS_PER_BATCH = 1
train_data = DataLoader(root, "val_id.txt", cfg, test_model, is_train=False)
# creat mode
model, weights_file, start_iter, checkpoints = create_model(False, cfg, output_dir)
# test blob
print(workspace.Blobs())
# create input blob
blob_names = ['data_stage2']
for gpu_id in range(cfg.NUM_GPUS):
with c2_utils.NamedCudaScope(gpu_id):
for blob_name in blob_names:
workspace.CreateBlob(core.ScopedName(blob_name))
# Override random weight initialization with weights from a saved model
if weights_file:
nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0)
# Even if we're randomly initializing we still need to synchronize
# parameters across GPUs
nu.broadcast_parameters(model)
workspace.CreateNet(model.net)
logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir)))
logger.info("start test ...")
save_root = os.path.join(output_dir, 'fusion')
if not os.path.exists(save_root):
os.makedirs(save_root)
for cur_iter in range(10000):
# feed data
# print("{} iter starting feed data...".format(cur_iter))
data_stage2, gt_label, meta = train_data.next_batch()
'''#
print('input0-20 sungalsses max score:', np.max(data_stage2[0, 4, :, :]))
print('input20-40 sungalsses max score:', np.max(data_stage2[0, 24, :, :]))
print('input0-20 glovess max score:', np.max(data_stage2[0, 3, :, :]))
print('input20-40 glovess max score:', np.max(data_stage2[0, 23, :, :]))
#'''
with c2_utils.NamedCudaScope(gpu_id):
workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2)
# print("workspace.RunNet(model.net.Proto().name)")
with c2_utils.NamedCudaScope(gpu_id):
workspace.RunNet(model.net.Proto().name)
batch_probs = workspace.FetchBlob(core.ScopedName('probs_human_NCHW_stage2'))
batch_probs = batch_probs.transpose((0, 2, 3, 1))
assert len(meta) == batch_probs.shape[0]
#print('batch_probs shape:', batch_probs.shape)
for i in range(len(meta)):
probs = cv2.resize(batch_probs[i], (meta[i]['width'], meta[i]['height']), interpolation=cv2.INTER_LINEAR)
probs = probs.transpose((2,0,1))
print('sungalsses max score:', np.max(probs[4, :, :]))
print('glovess max score:', np.max(probs[3, :, :]))
#print('probs shape:', probs.shape)
cv2.imwrite(os.path.join(save_root, meta[i]['id']+'.png'), probs.argmax(0))
print("prossed ", cur_iter)
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
if os.path.exists('./detectron/datasets/data/coco'):
shutil.rmtree('./detectron/datasets/data/coco')
os.makedirs('./detectron/datasets/data/coco')
if 'dior_2nd' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_2nd ./detectron/datasets/data/coco/annotations'
)
elif 'dior_3rd' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_3rd ./detectron/datasets/data/coco/annotations'
)
elif 'dior_4th' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_4th ./detectron/datasets/data/coco/annotations'
)
elif 'dior_5th' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN_5th ./detectron/datasets/data/coco/annotations'
)
elif '2020.10.6' in cfg.OUTPUT_DIR:
os.system(
'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014'
)
os.system(
'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 '
)
os.system(
'ln -s /home/wsh/dior/coco/annotationsN ./detectron/datasets/data/coco/annotations'
)
else:
raise Exception
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)