def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run checkpoints = detectron.utils.train.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run checkpoints = detectron.utils.train_wsl.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts) print('reprint snapshot name for the result: ', checkpoints['final']) if 'voc_' in cfg.TRAIN.DATASETS[0]: TEST_DATASETS = cfg.TEST.DATASETS TEST_PROPOSAL_FILES = cfg.TEST.PROPOSAL_FILES cfg.immutable(False) cfg.TEST.DATASETS = cfg.TRAIN.DATASETS cfg.TEST.PROPOSAL_FILES = cfg.TRAIN.PROPOSAL_FILES cfg.immutable(True) test_model(checkpoints['final'], args.multi_gpu_testing, args.opts) print('reprint snapshot name for the result: ', checkpoints['final']) cfg.immutable(False) cfg.TEST.DATASETS = TEST_DATASETS cfg.TEST.PROPOSAL_FILES = TEST_PROPOSAL_FILES cfg.immutable(True) cfg.immutable(False) cfg.TEST.BBOX_AUG.ENABLED = False cfg.VIS = False cfg.immutable(True) _ = checkpoints.pop('final', None) for snapshot in sorted(checkpoints.keys(), reverse=True): test_model(checkpoints[snapshot], args.multi_gpu_testing, args.opts) print('reprint snapshot name for the result: ', snapshot, checkpoints[snapshot])
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'] ) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run checkpoints = detectron.utils.train.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def beginTrain(self): workspace.GlobalInit([ 'caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1' ]) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run checkpoints, losses = train_model() # Test the trained model self.test_model(checkpoints["final"]) dataset_name, _ = get_inference_dataset(0) output_dir = get_output_dir(dataset_name, training=False) with open(osp.join(output_dir, "res.pkl"), "rb") as src: mAP = pickle.load(src) return losses, mAP
def main(): c2_utils.import_contrib_ops() c2_utils.import_detectron_ops() cv2.ocl.setUseOpenCL(False) args = parse_args() input_file = args.input_file workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) merge_cfg_from_file( '/home/LAB/wusj/exp/KL-Loss/configs/e2e_faster_rcnn_R-50-FPN_2x_entropy.yaml' ) assert_and_infer_cfg(cache_urls=False) smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) workspace.ResetWorkspace() np.random.seed(cfg.RNG_SEED) with open(input_file, 'r') as f: config = json.load(f) parameter = config['parameter'] inference_id = parameter['inferenceId'] dataset = parameter['dataSetName'] model_file = parameter['modelPkl'] task_id = inference_id image_id_list = parameter['imageIdList'] load_data(dataset, image_id_list) roidb, result = test_net(model_file, dataset) tmp = [] for i, entry in enumerate(roidb): tmp.append(entry) tmp.sort(cmp=compare) selectNum = parameter['selectNum'] output = {} # time.sleep(3) output['annotationList'] = [] output['selectImageIdList'] = [] output['remainImageIdList'] = [] for i in range(selectNum): output['annotationList'].append(result[str(tmp[i]['id'])]) output['selectImageIdList'].append(str(tmp[i]['id'])) for i in range(parameter['selectNum'], len(parameter['imageIdList'])): output['remainImageIdList'].append(str(tmp[i]['id'])) output['remainImageNum'] = len(output['remainImageIdList']) output['inferenceId'] = inference_id output['dataSetName'] = dataset result_output_dir = '/home/LAB/wusj/fastwash_tmp/inference/' with open(result_output_dir + 'result_' + task_id, 'wt') as f2: json.dump(output, f2)
def main(): c2_utils.import_contrib_ops() c2_utils.import_detectron_ops() cv2.ocl.setUseOpenCL(False) workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) merge_cfg_from_file( '/home/LAB/wusj/exp/KL-Loss/configs/e2e_faster_rcnn_R-50-FPN_2x_entropy.yaml' ) assert_and_infer_cfg(cache_urls=False) smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) workspace.ResetWorkspace() np.random.seed(cfg.RNG_SEED) args = parse_args() input_file = args.input_file result_output_dir = args.output_dir with open(input_file, 'r') as f: config = json.load(f) dataset = config['dataSetName'] model_file = args.model_file task_id = config['id'] image_id_list = config['imageIdList'] load_data(dataset, image_id_list) roidb, result = test_net(model_file, dataset) config['inferenceResult'] = result with open(result_output_dir + 'result_' + task_id, 'wt') as f2: json.dump(config, f2)
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # test model logger.info("creat test model ...") test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS, gpu_id=0) logger.info("created test model ...") train_data = DataLoader(root, "train_id.txt", cfg, test_model, is_train=True) # creat mode model, weights_file, start_iter, checkpoints = create_model( True, cfg, output_dir) # test blob print(workspace.Blobs()) # create input blob blob_names = ['data_stage2', 'gt_label_stage2'] for gpu_id in range(cfg.NUM_GPUS): with c2_utils.NamedCudaScope(gpu_id): for blob_name in blob_names: workspace.CreateBlob(core.ScopedName(blob_name)) # Override random weight initialization with weights from a saved model if weights_file: nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0) # Even if we're randomly initializing we still need to synchronize # parameters across GPUs nu.broadcast_parameters(model) workspace.CreateNet(model.net) logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir))) dump_proto_files(model, output_dir) writer = SummaryWriter(log_dir=output_dir) training_stats = TrainingStats(model, writer) CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS) logger.info("start train ...") for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER): # feed data # print("{} iter starting feed data...".format(cur_iter)) data_stage2, gt_label = train_data.next_batch() with c2_utils.NamedCudaScope(gpu_id): workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2) workspace.FeedBlob(core.ScopedName('gt_label_stage2'), gt_label) # print("workspace.RunNet(model.net.Proto().name)") training_stats.IterTic() lr = model.UpdateWorkspaceLr(cur_iter, lr_policy.get_lr_at_iter(cur_iter)) workspace.RunNet(model.net.Proto().name) if cur_iter == start_iter: nu.print_net(model) training_stats.IterToc() training_stats.UpdateIterStats(cur_iter) training_stats.LogIterStats(cur_iter, lr) writer.add_scalar('learning_rate', lr, cur_iter) # print("end of RunNet") if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter: checkpoints[cur_iter] = os.path.join( output_dir, 'model_iter{}.pkl'.format(cur_iter)) nu.save_model_to_weights_file(checkpoints[cur_iter], model) if cur_iter == start_iter + training_stats.LOG_PERIOD: # Reset the iteration timer to remove outliers from the first few # SGD iterations training_stats.ResetIterTimer() if np.isnan(training_stats.iter_total_loss): handle_critical_error(model, 'Loss is NaN') # Save the final model checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl') nu.save_model_to_weights_file(checkpoints['final'], model) # save train loss and metric state_file = os.path.join(output_dir, 'training_state.json') training_stats.SaveTrainingStates(state_file) # Execute the training run checkpoints = detectron.utils.train.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'] ) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # test model logger.info("creat test model ...") test_model = test_engine.initialize_model_from_cfg(cfg.TEST.WEIGHTS, gpu_id=0) logger.info("created test model ...") #cfg.TRAIN.IMS_PER_BATCH = 1 train_data = DataLoader(root, "val_id.txt", cfg, test_model, is_train=False) # creat mode model, weights_file, start_iter, checkpoints = create_model(False, cfg, output_dir) # test blob print(workspace.Blobs()) # create input blob blob_names = ['data_stage2'] for gpu_id in range(cfg.NUM_GPUS): with c2_utils.NamedCudaScope(gpu_id): for blob_name in blob_names: workspace.CreateBlob(core.ScopedName(blob_name)) # Override random weight initialization with weights from a saved model if weights_file: nu.initialize_gpu_from_weights_file(model, weights_file, gpu_id=0) # Even if we're randomly initializing we still need to synchronize # parameters across GPUs nu.broadcast_parameters(model) workspace.CreateNet(model.net) logger.info('Outputs saved to: {:s}'.format(os.path.abspath(output_dir))) logger.info("start test ...") save_root = os.path.join(output_dir, 'fusion') if not os.path.exists(save_root): os.makedirs(save_root) for cur_iter in range(10000): # feed data # print("{} iter starting feed data...".format(cur_iter)) data_stage2, gt_label, meta = train_data.next_batch() '''# print('input0-20 sungalsses max score:', np.max(data_stage2[0, 4, :, :])) print('input20-40 sungalsses max score:', np.max(data_stage2[0, 24, :, :])) print('input0-20 glovess max score:', np.max(data_stage2[0, 3, :, :])) print('input20-40 glovess max score:', np.max(data_stage2[0, 23, :, :])) #''' with c2_utils.NamedCudaScope(gpu_id): workspace.FeedBlob(core.ScopedName('data_stage2'), data_stage2) # print("workspace.RunNet(model.net.Proto().name)") with c2_utils.NamedCudaScope(gpu_id): workspace.RunNet(model.net.Proto().name) batch_probs = workspace.FetchBlob(core.ScopedName('probs_human_NCHW_stage2')) batch_probs = batch_probs.transpose((0, 2, 3, 1)) assert len(meta) == batch_probs.shape[0] #print('batch_probs shape:', batch_probs.shape) for i in range(len(meta)): probs = cv2.resize(batch_probs[i], (meta[i]['width'], meta[i]['height']), interpolation=cv2.INTER_LINEAR) probs = probs.transpose((2,0,1)) print('sungalsses max score:', np.max(probs[4, :, :])) print('glovess max score:', np.max(probs[3, :, :])) #print('probs shape:', probs.shape) cv2.imwrite(os.path.join(save_root, meta[i]['id']+'.png'), probs.argmax(0)) print("prossed ", cur_iter)
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run if os.path.exists('./detectron/datasets/data/coco'): shutil.rmtree('./detectron/datasets/data/coco') os.makedirs('./detectron/datasets/data/coco') if 'dior_2nd' in cfg.OUTPUT_DIR: os.system( 'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014' ) os.system( 'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 ' ) os.system( 'ln -s /home/wsh/dior/coco/annotationsN_2nd ./detectron/datasets/data/coco/annotations' ) elif 'dior_3rd' in cfg.OUTPUT_DIR: os.system( 'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014' ) os.system( 'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 ' ) os.system( 'ln -s /home/wsh/dior/coco/annotationsN_3rd ./detectron/datasets/data/coco/annotations' ) elif 'dior_4th' in cfg.OUTPUT_DIR: os.system( 'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014' ) os.system( 'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 ' ) os.system( 'ln -s /home/wsh/dior/coco/annotationsN_4th ./detectron/datasets/data/coco/annotations' ) elif 'dior_5th' in cfg.OUTPUT_DIR: os.system( 'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014' ) os.system( 'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 ' ) os.system( 'ln -s /home/wsh/dior/coco/annotationsN_5th ./detectron/datasets/data/coco/annotations' ) elif '2020.10.6' in cfg.OUTPUT_DIR: os.system( 'ln -s /home/wsh/dior/coco/coco_train2014 ./detectron/datasets/data/coco/coco_train2014' ) os.system( 'ln -s /home/wsh/dior/coco/coco_val2014 ./detectron/datasets/data/coco/coco_val2014 ' ) os.system( 'ln -s /home/wsh/dior/coco/annotationsN ./detectron/datasets/data/coco/annotations' ) else: raise Exception checkpoints = detectron.utils.train.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)