def result_getter(): if is_parent: # Parent case: # In this case we're either running inference on the entire dataset in a # single process or (if multi_gpu_testing is True) using this process to # launch subprocesses that each run inference on a range of the dataset all_results = {} for i in range(len(cfg.TEST.DATASETS)): dataset_name, proposal_file = get_inference_dataset(i) output_dir = get_output_dir(dataset_name, training=False) results = parent_func( weights_file, dataset_name, proposal_file, output_dir, multi_gpu=multi_gpu_testing ) all_results.update(results) return all_results else: # Subprocess child case: # In this case test_net was called via subprocess.Popen to execute on a # range of inputs on a single dataset dataset_name, proposal_file = get_inference_dataset(0, is_parent=False) output_dir = get_output_dir(dataset_name, training=False) return child_func( weights_file, dataset_name, proposal_file, output_dir, ind_range=ind_range, gpu_id=gpu_id )
def load_detections(detection_file, dataset, thresholds): detections = json.load(open(detection_file)) roidb = dataset.get_roidb() classes = dataset.classes #apply class thresholds thres_dets = [det for det in detections if det['score'] > thresholds[classes[det['category_id']]]] #save thresholded detections to file output_dir = os.path.abspath(get_output_dir(dataset.name, training=False)) res_file = os.path.join( output_dir, 'bbox_' + dataset.name + '_results_thresh.json' ) with open(res_file,'w') as out_file: json.dump(thres_dets, out_file) #get per-image detections detections_per_image = [] for entry in roidb: #get all detections for image im_dets = [det for det in thres_dets if det['image_id'] == entry['id']] #convert bbox from xywh to xyxy format for det in im_dets: det["bbox"] = box_utils.xywh_to_xyxy(det["bbox"]) detections_per_image.append(im_dets) return detections_per_image, res_file
def test_restore_checkpoint(): # Create Model model = model_builder.create(cfg.MODEL.TYPE, train=True) add_momentum_init_ops(model) init_weights(model) # Fill input blobs roidb = combined_roidb_for_training(cfg.TRAIN.DATASETS, cfg.TRAIN.PROPOSAL_FILES) model_builder.add_training_inputs(model, roidb=roidb) workspace.CreateNet(model.net) # Bookkeeping for checkpoint creation iter_num = 0 checkpoints = {} output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) chk_file_path = os.path.join(output_dir, 'model_iter{}.pkl'.format(iter_num)) checkpoints[iter_num] = chk_file_path # Save model weights nu.save_model_to_weights_file(checkpoints[iter_num], model) orig_gpu_0_params, orig_all_params = get_params(model) # Change the model weights init_weights(model) # Reload the weights in the model nu.initialize_gpu_from_weights_file(model, chk_file_path, gpu_id=0) nu.broadcast_parameters(model) shutil.rmtree(cfg.OUTPUT_DIR) _, restored_all_params = get_params(model) # Check if all params are loaded correctly for scoped_name, blob in orig_all_params.items(): np.testing.assert_array_equal(blob, restored_all_params[scoped_name]) # Check if broadcast_parameters works for scoped_name, blob in restored_all_params.items(): unscoped_name = c2_utils.UnscopeName(scoped_name) np.testing.assert_array_equal(blob, orig_gpu_0_params[unscoped_name])
def __init__(self, save=True, sort_keys=True): self.save = save if save: self.output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) self.log_path = os.path.join(self.output_dir, 'log.json') open(self.log_path, "w").close() self.sort_keys = sort_keys
def res_top_result(model, entry, save_png=True): im_info = workspace.FetchBlob(core.ScopedName('im_info')) blob_h, blob_w, scale = im_info[0] img = cv2.imread(entry['image']) img_h, img_w, _ = img.shape probs_human = workspace.FetchBlob(core.ScopedName('probs_human_NCHW'))[0, :, :, :] probs_crop = np.zeros((probs_human.shape[0], int(img_h*scale), int(img_w*scale)), dtype=np.float32) for i in range(probs_crop.shape[0]): probs_up4 = cv2.resize(probs_human[i], None, None, fx=4, fy=4, interpolation=cv2.INTER_LINEAR) probs_crop[i, :, :] = probs_up4[0:int(img_h*scale), 0:int(img_w*scale)] probs_resize = np.zeros((probs_crop.shape[0], int(img_h), int(img_w)), dtype=np.float32) for i in range(probs_crop.shape[0]): probs_resize[i, :, :] = cv2.resize(probs_crop[i], (img_w, img_h), interpolation=cv2.INTER_LINEAR) dataset_name = cfg.TEST.DATASETS[0] output_dir = os.path.join(get_output_dir(dataset_name, training=False), 'vis_res_top') if not os.path.exists(output_dir): os.makedirs(output_dir) # print(output_dir) if save_png: cv2.imwrite(os.path.join(output_dir, entry['id']+'.png'), probs_resize.argmax(0)) return probs_resize
def beginTrain(self): workspace.GlobalInit([ 'caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1' ]) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info() logger.info("cuda version : {}".format(cuda_ver)) logger.info("cudnn version: {}".format(cudnn_ver)) logger.info("nvidia-smi output:\n{}".format(smi_output)) logger.info('Training with config:') logger.info(pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) # Execute the training run checkpoints, losses = train_model() # Test the trained model self.test_model(checkpoints["final"]) dataset_name, _ = get_inference_dataset(0) output_dir = get_output_dir(dataset_name, training=False) with open(osp.join(output_dir, "res.pkl"), "rb") as src: mAP = pickle.load(src) return losses, mAP
def validate_tracking_params(weights_file, dataset_name): dataset = JsonDataset(dataset_name) output_dir = os.path.abspath(get_output_dir(dataset.name, training=False)) #tracking parameter files class_thresh_file = os.path.join(output_dir, 'AP_thresholds.txt') obs_model_file = os.path.join(output_dir, 'observation_model.txt') meas_cov_file = os.path.join(output_dir, 'meas_cov.txt') #generate param files if they do not exist if (not os.path.exists(class_thresh_file) or not os.path.exists(obs_model_file) or not os.path.exists(meas_cov_file)): logger.info('validation files for validation dataset %s do not ' 'exist. Generating them now' % (dataset.name)) #this performs inference on all images and calls the evaluation #script, generating the validation files test_net_on_dataset(weights_file,dataset.name,None,output_dir) #read parameters from files cla_thresh = load_class_thresholds(class_thresh_file, dataset) observation_model = np.loadtxt(obs_model_file, delimiter=',') ekf_sensor_noise = np.loadtxt(meas_cov_file, delimiter=',') return cla_thresh, observation_model, ekf_sensor_noise
def test_restore_checkpoint(): # Create Model model = model_builder.create(cfg.MODEL.TYPE, train=True) add_momentum_init_ops(model) init_weights(model) # Fill input blobs roidb = combined_roidb_for_training( cfg.TRAIN.DATASETS, cfg.TRAIN.PROPOSAL_FILES ) model_builder.add_training_inputs(model, roidb=roidb) workspace.CreateNet(model.net) # Bookkeeping for checkpoint creation iter_num = 0 checkpoints = {} output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) chk_file_path = os.path.join(output_dir, 'model_iter{}.pkl'.format(iter_num)) checkpoints[iter_num] = chk_file_path # Save model weights nu.save_model_to_weights_file(checkpoints[iter_num], model) orig_gpu_0_params, orig_all_params = get_params(model) # Change the model weights init_weights(model) # Reload the weights in the model nu.initialize_gpu_from_weights_file(model, chk_file_path, gpu_id=0) nu.broadcast_parameters(model) shutil.rmtree(cfg.OUTPUT_DIR) _, restored_all_params = get_params(model) # Check if all params are loaded correctly for scoped_name, blob in orig_all_params.items(): np.testing.assert_array_equal(blob, restored_all_params[scoped_name]) # Check if broadcast_parameters works for scoped_name, blob in restored_all_params.items(): unscoped_name = c2_utils.UnscopeName(scoped_name) np.testing.assert_array_equal(blob, orig_gpu_0_params[unscoped_name])
def create_model(): """Build the model and look for saved model checkpoints in case we can resume from one. """ logger = logging.getLogger(__name__) start_iter = 0 checkpoints = {} output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) weights_file = cfg.TRAIN.WEIGHTS if cfg.TRAIN.AUTO_RESUME: # Check for the final model (indicates training already finished) final_path = os.path.join(output_dir, 'model_final.pkl') if os.path.exists(final_path): logger.info('model_final.pkl exists; no need to train!') # return None, None, None, {'final': final_path}, output_dir files = os.listdir(output_dir) for f in files: iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f) if len(iter_string) > 0: checkpoint_iter = int(iter_string[0]) checkpoints[checkpoint_iter] = os.path.join(output_dir, f) checkpoints['final'] = final_path return None, None, None, checkpoints, output_dir if cfg.TRAIN.COPY_WEIGHTS: copyfile( weights_file, os.path.join(output_dir, os.path.basename(weights_file))) logger.info('Copy {} to {}'.format(weights_file, output_dir)) # Find the most recent checkpoint (highest iteration number) files = os.listdir(output_dir) for f in files: iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f) if len(iter_string) > 0: checkpoint_iter = int(iter_string[0]) if checkpoint_iter > start_iter: # Start one iteration immediately after the checkpoint iter start_iter = checkpoint_iter + 1 resume_weights_file = f if start_iter > 0: # Override the initialization weights with the found checkpoint weights_file = os.path.join(output_dir, resume_weights_file) logger.info( '========> Resuming from checkpoint {} at start iter {}'. format(weights_file, start_iter) ) logger.info('Building model: {}'.format(cfg.MODEL.TYPE)) model = model_builder_wsl.create(cfg.MODEL.TYPE, train=True) if cfg.MEMONGER: optimize_memory(model) # Performs random weight initialization as defined by the model workspace.RunNetOnce(model.param_init_net) return model, weights_file, start_iter, checkpoints, output_dir
def train_model(): """Model training loop.""" model, weights_file, start_iter, checkpoints, output_dir = create_model() if 'final' in checkpoints: # The final model was found in the output directory, so nothing to do return checkpoints with SummaryWriter(log_dir=get_output_dir(cfg.TRAIN.DATASETS) + "/events") as writer: writer.write_graph([model.net]) logger = logging.getLogger(__name__) setup_model_for_training(model, weights_file, output_dir) training_stats = TrainingStats(model) CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS) error_count = 0 for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER): if model.roi_data_loader.has_stopped(): handle_critical_error(model, 'roi_data_loader failed') training_stats.IterTic() lr = model.UpdateWorkspaceLr(cur_iter, lr_policy.get_lr_at_iter(cur_iter)) try: workspace.RunNet(model.net.Proto().name) except: error_count += 1 logger.warn("Error in iter {}, error count: {}".format( cur_iter, error_count)) if not cfg.CONTINUE_ON_ERROR: raise if cur_iter == start_iter: nu.print_net(model) training_stats.IterToc() training_stats.UpdateIterStats() training_stats.LogIterStats(cur_iter, lr) if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter: checkpoints[cur_iter] = os.path.join( output_dir, 'model_iter{}.pkl'.format(cur_iter)) nu.save_model_to_weights_file(checkpoints[cur_iter], model) if cur_iter == start_iter + cfg.TRAIN.EPOCH_PERIOD: # Reset the iteration timer to remove outliers from the first few # SGD iterations training_stats.ResetIterTimer() if np.isnan(training_stats.iter_total_loss): handle_critical_error(model, 'Loss is NaN') # Save the final model checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl') nu.save_model_to_weights_file(checkpoints['final'], model) # Shutdown data loading threads model.roi_data_loader.shutdown() return checkpoints
def result_getter(): if is_parent: # Parent case: # In this case we're either running inference on the entire dataset in a # single process or (if multi_gpu_testing is True) using this process to # launch subprocesses that each run inference on a range of the dataset all_results = {} subset_pointer = None if cfg.VOC_SUBSET != '': subset_pointer = result_getter #any dummy object could be used that is more advanced than 'object()' or similar builtins. subset_pointer.subset = np.load(cfg.VOC_SUBSET) print('loading subset') for i in range(len(cfg.TEST.DATASETS)): dataset_name, proposal_file = get_inference_dataset(i) output_dir = get_output_dir(dataset_name, training=False) print('len before', len(subset_pointer.subset)) results = parent_func(weights_file, dataset_name, proposal_file, output_dir, multi_gpu=multi_gpu_testing, subset_pointer=subset_pointer) all_results.update(results) return all_results else: # Subprocess child case: # In this case test_net was called via subprocess.Popen to execute on a # range of inputs on a single dataset dataset_name, proposal_file = get_inference_dataset( 0, is_parent=False) output_dir = get_output_dir(dataset_name, training=False) return child_func(weights_file, dataset_name, proposal_file, output_dir, ind_range=ind_range, gpu_id=gpu_id)
def vis_training(cur_iter): prefix = '' if cfg.WEBLY.MINING: prefix = 'mining_' if not (cfg.WSL.DEBUG or (cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)): return output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) sample_dir = os.path.join(output_dir, 'webly_sample') if not os.path.exists(sample_dir): os.makedirs(sample_dir) for gpu_id in range(cfg.NUM_GPUS): data_ids = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data_ids')) ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data')) labels_oh = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, 'labels_oh')) im_score = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cls_prob')) roi_score = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, prefix + 'rois_pred')) # roi_score_softmax = workspace.FetchBlob('gpu_{}/{}'.format( # gpu_id, prefix + 'rois_pred_softmax')) rois = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, prefix + 'rois')) # anchor_argmax = workspace.FetchBlob('gpu_{}/{}'.format( # gpu_id, 'anchor_argmax')) preffix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id) save_im(labels_oh, im_score, ims, cfg.PIXEL_MEANS, preffix, sample_dir) save_rois(labels_oh, im_score, roi_score, ims, rois, cfg.PIXEL_MEANS, preffix, '', sample_dir) # continue if cfg.WEBLY.ENTROPY: pass else: continue class_weight = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, prefix + 'rois_class_weight')) rois_pred_hatE = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, prefix + 'rois_pred_hatE')) rois_pred_E = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, prefix + 'rois_pred_E')) y_logN__logy = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, prefix + 'rois_pred_y_logN__logy')) save_entropy(labels_oh, im_score, class_weight, roi_score, ims, rois, cfg.PIXEL_MEANS, preffix, '', sample_dir, rois_pred_hatE, rois_pred_E, y_logN__logy)
def result_getter(): if is_parent: # Parent case: # In this case we're either running inference on the entire dataset in a # single process or (if multi_gpu_testing is True) using this process to # launch subprocesses that each run inference on a range of the dataset all_results = {} dataset_info, proposal_file = get_inference_dataset() output_dir = osp.join(get_output_dir(dataset_info, training=False), weights_file.strip().split('/')[-1].split('.')[0]) if not osp.exists(output_dir): os.makedirs(output_dir) results = parent_func( weights_file, dataset_info, proposal_file, output_dir, multi_gpu=multi_gpu_testing ) all_results.update(results) return all_results else: # Subprocess child case: # In this case test_net was called via subprocess.Popen to execute on a # range of inputs on a single dataset dataset_info, proposal_file = get_inference_dataset(is_parent=False) output_dir = osp.join(get_output_dir(dataset_info, training=False), weights_file.strip().split('/')[-1].split('.')[0]) if not osp.exists(output_dir): os.makedirs(output_dir) return child_func( weights_file, dataset_info, proposal_file, output_dir, ind_range=ind_range, gpu_id=gpu_id )
def write_filtered_detections(dataset, filtered_boxes, filtered_depths, use_hmm): output_dir = os.path.abspath(get_output_dir(dataset.name, training=False)) if use_hmm: res_file = os.path.join( output_dir, 'bbox_' + dataset.name + '_results_EKF_HMM.json' ) else: res_file = os.path.join( output_dir, 'bbox_' + dataset.name + '_results_EKF.json' ) _write_coco_bbox_results_file(dataset, filtered_boxes, filtered_depths, res_file) return res_file
def load_data(dataset_name, image_id_list=None): imgs = [] annotation_file = dataset_catalog.get_ann_fn(dataset_name) dataset = json.load(open(annotation_file, 'r')) for img in dataset['images']: imgs.append(img) output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) file_name = os.path.join(output_dir, 'train_init.json') select = [] image_id_set = set() for i in image_id_list: image_id_set.add(int(i)) for i in range(0, len(imgs)): if (imgs[i]['id'] in image_id_set): select.append(imgs[i]) dataset['images'] = select with open(file_name, 'wt') as f: json.dump(dataset, f) return dataset
def create_model(): """Build the model and look for saved model checkpoints in case we can resume from one. """ logger = logging.getLogger(__name__) start_iter = 0 checkpoints = {} output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) weights_file = cfg.TRAIN.WEIGHTS if cfg.TRAIN.AUTO_RESUME: # Check for the final model (indicates training already finished) final_path = os.path.join(output_dir, 'model_final.pkl') if os.path.exists(final_path): logger.info('model_final.pkl exists; no need to train!') return None, None, None, {'final': final_path}, output_dir # Find the most recent checkpoint (highest iteration number) files = os.listdir(output_dir) for f in files: iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f) if len(iter_string) > 0: checkpoint_iter = int(iter_string[0]) if checkpoint_iter > start_iter: # Start one iteration immediately after the checkpoint iter start_iter = checkpoint_iter + 1 resume_weights_file = f if start_iter > 0: # Override the initialization weights with the found checkpoint weights_file = os.path.join(output_dir, resume_weights_file) logger.info( '========> Resuming from checkpoint {} at start iter {}'. format(weights_file, start_iter) ) logger.info('Building model: {}'.format(cfg.MODEL.TYPE)) model = model_builder.create(cfg.MODEL.TYPE, train=True) if cfg.MEMONGER: optimize_memory(model) # Performs random weight initialization as defined by the model workspace.RunNetOnce(model.param_init_net) return model, weights_file, start_iter, checkpoints, output_dir
def run_tracking(validation_dataset, tracking_datasets, weights_file, timestep, use_hmm=True, no_filtering=False, visualize=False, step=False): class_thresh, obs_model, meas_cov = validate_tracking_params(weights_file, validation_dataset) res_files = [] json_datasets = [] for i in range(len(tracking_datasets)): tracking_set_name = tracking_datasets[i] tracking_set_json = JsonDataset(tracking_set_name) detections, res_file = get_detections(weights_file, tracking_set_json, class_thresh) if not no_filtering: filtered_boxes, filtered_depths = do_kalman_filtering(detections, tracking_set_json, timestep, meas_cov, obs_model, use_hmm=use_hmm, viz=visualize, step=step) res_file = write_filtered_detections(tracking_set_json, filtered_boxes, filtered_depths, use_hmm) res_files.append(res_file) json_datasets.append(tracking_set_json) #perform the evaluation for all datasets tracking_dataset_name = '-'.join(tracking_datasets) tracking_output_dir = os.path.abspath(get_output_dir(tracking_dataset_name, training=False)) evaluate_tracking(res_files, json_datasets, tracking_output_dir, use_matlab=True)
def get_detections(weights_file, dataset, class_thresh): output_dir = os.path.abspath(get_output_dir(dataset.name, training=False)) res_file = os.path.join( output_dir, 'bbox_' + dataset.name + '_results.json' ) #generate detections if they do not exist if not os.path.exists(res_file): logger.info('results file %s does not exist. Generating results ' 'for dataset: %s' % (res_file, dataset.name)) all_boxes, all_depths, all_segms, all_keyps = test_net( weights_file, dataset.name, None, output_dir) _write_coco_bbox_results_file(dataset, all_boxes, all_depths, res_file) detections, thresh_det_file = load_detections(res_file, dataset, class_thresh) return detections, thresh_det_file
def main(): # Initialize C2 workspace.GlobalInit( ['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1']) # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO) args = parse_args() if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) if cfg.LOGFILE: output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) print("OUTPUT_DIR=", output_dir) root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) fh = logging.FileHandler(os.path.join(output_dir, cfg.LOGFILE)) fh.setLevel(logging.INFO) root_logger.addHandler(fh) logger.info('Called with args:') logger.info(args) assert_and_infer_cfg() logger.info('Training with config:') logger.info(pprint.pformat(cfg)) # Note that while we set the numpy random seed network training will not be # deterministic in general. There are sources of non-determinism that cannot # be removed with a reasonble execution-speed tradeoff (such as certain # non-deterministic cudnn functions). np.random.seed(cfg.RNG_SEED) random.seed(cfg.RNG_SEED) # Execute the training run checkpoints = detectron.utils.train.train_model() # Test the trained model if not args.skip_test: test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def log_json(self, stats_main, stats_extra): stats_main.update(stats_extra) json_stats = json.dumps(stats_main) print("-" * 100) print("iter: {}, loss: {:.4}, eta: {}, time: {:.0f} ms, lr: {:.3}". format( stats_main['iter'], stats_main['loss'], stats_main['eta'], stats_main['time'] * 1000, stats_main['lr'], )) stats_str_list = [ "{}: {:.3}".format(k, v) for k, v in stats_extra.items() ] stats_str_list.sort() print(", ".join(stats_str_list)) if self.save: with open(self.log_path, "a") as f: f.write(json_stats + "\n") # tensorboard stats_main.pop('eta', None) i = stats_main.pop('iter', None) for key in stats_main.keys(): key_split = key.split("_") if key_split[0] == "loss": if len(key_split) > 1 and key_split[1] == "rpn": tag = "loss_rpn" else: tag = "loss" else: tag = "meta" stats_main["{}/{}".format(tag, key)] = stats_main[key] stats_main.pop(key, None) with SummaryWriter(log_dir=get_output_dir(cfg.TRAIN.DATASETS) + "/events") as writer: writer.write_scalars(stats_main, i)
def vis_training(cur_iter): # if not (cfg.WSL.DEBUG or # (cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)): # return if cur_iter % 100 != 0: return output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) sample_dir = os.path.join(output_dir, 'DB_sample') if not os.path.exists(sample_dir): os.makedirs(sample_dir) for gpu_id in range(cfg.NUM_GPUS): db2_add = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'db2_add')) conv5_3 = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'conv5_3')) ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data')) prefix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id) if False: save_im(ims, cfg.PIXEL_MEANS, prefix, sample_dir) save_conv(ims, db2_add, cfg.PIXEL_MEANS, prefix, "_db", sample_dir) save_conv(ims, conv5_3, cfg.PIXEL_MEANS, prefix, "_con53", sample_dir)
def main(args): logger = logging.getLogger(__name__) merge_cfg_from_file(args.cfg) cfg.NUM_GPUS = 1 if "mot-classes" in args.opts: dummy_dataset = dummy_datasets.get_mot_dataset() cfg.NUM_CLASSES = 14 else: dummy_dataset = dummy_datasets.get_coco_dataset() cfg.NUM_CLASSES = 81 for i, weights_file in enumerate(args.weights_pre_list): args.weights_pre_list[i] = cache_url(weights_file, cfg.DOWNLOAD_CACHE) for i, weights_file in enumerate(args.weights_post_list): args.weights_post_list[i] = cache_url(weights_file, cfg.DOWNLOAD_CACHE) assert_and_infer_cfg(cache_urls=False) logger.info('Testing with config:') logger.info(pprint.pformat(cfg)) # If True: Evaluation with respect to specified parameters (model from # specifig training iterations or inference hyper-parameters) # If False: Infer test sequence for evaluation on the MOT benchmark server EVAL = "eval" in args.opts train_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) train_dir_split = train_dir.split("/") train_dir = os.path.join("/".join(train_dir_split[:-1]) + \ args.model_suffix, train_dir_split[-1]) model_list = [] files = os.listdir(train_dir) if EVAL: test_dir = "/".join(get_output_dir(cfg.TEST.DATASETS, training=False).split("/")[:-1]) + args.model_suffix # Evaluation with respect to inference hyper-parameters if HYPER_PARAM is not None: test_dir = os.path.join(test_dir, HYPER_PARAM.lower()) model_param = ((args.model, param) for param in PARAM_RANGE) # Evaluation with respect to weights from specific training iterations else: model_param = [] for f in files: if f.startswith("model_") and f.endswith(".pkl"): iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f) if len(iter_string) > 0: model_param.append((f, int(iter_string[0]))) model_param.sort(key=lambda tup: tup[1]) if "model_final.pkl" in files: model_param.append(("model_final.pkl", "final")) # Tracking evaluation by interface to matlab engine seq_map_path = os.path.join(test_dir, "seq_map.txt") if not os.path.exists(test_dir): os.makedirs(test_dir) with open(seq_map_path, "w+") as seq_map: seq_map.write("name\n") for dataset in cfg.TEST.DATASETS: seq_map.write(get_im_dir(dataset).split("/")[-2] + '\n') seq_map_path = os.path.relpath(os.path.abspath(seq_map_path), os.path.expanduser(os.path.join(args.devkit_path, "seqmaps"))) matlab_eng = get_matlab_engine(args.devkit_path) eval_datections = lambda res_dir, gt_dir: eval_detections_matlab( matlab_eng, seq_map_path, res_dir, gt_dir, 'MOT17') else: if args.model is not None: model_param = ((args.model, None),) else: model_param = (("model_final.pkl", "final"),) # Iterate through (model, parameter) tuples for i, (model, param) in enumerate(model_param): if EVAL and (i + 1 + args.offset) % (args.skip + 1) != 0: logger.info("Skipping {}".format(model)) continue # Hyper parameter inference elif HYPER_PARAM is not None: cfg.immutable(False) setattr(cfg.TRCNN, HYPER_PARAM, param) assert_and_infer_cfg(cache_urls=False) print(cfg.TRCNN) if not EVAL or param >= args.start_at: weights_list = args.weights_pre_list + [os.path.join(train_dir, model)] + \ args.weights_post_list preffix_list = args.preffix_list if len(args.preffix_list) \ else [""] * (len(args.weights_pre_list) + len(args.weights_post_list) + 1) workspace.ResetWorkspace() model = infer_engine.initialize_mixed_model_from_cfg(weights_list, preffix_list=preffix_list) logger.info("Processing {}".format(param)) timing = [] # iterate through test sequences for dataset in cfg.TEST.DATASETS: tracking = Tracking(args.thresh, cfg.TRCNN.MAX_BACK_TRACK) logger.info("Processing dataset {}".format(dataset)) im_dir = get_im_dir(dataset) vis = None if EVAL: output_file = os.path.join(test_dir, str(param), im_dir.split("/")[-2] + ".txt") # Visualize detections along with tracking detection file creation else: output_dir = os.path.join("outputs/MOT17", im_dir.split("/")[-2]) if "vis" in args.opts: vis = { "output-dir": output_dir, "dummy-dataset": dummy_dataset, "show-class": "show-class" in args.opts, "show-track": "show-track" in args.opts, "thresh": args.thresh, "track-thresh": cfg.TRCNN.DETECTION_THRESH, "n-colors": 15, } output_file = os.path.join("outputs/MOT17", im_dir.split("/")[-2] + ".txt") # Use custom proposals if provided if "proposals" in args.opts: proposals = pickle.load(open(os.path.join('/', *(im_dir.split("/")[:-1] + \ ["det/proposals.pkl"])), 'r')) else: proposals = None head, tail = os.path.split(output_file) if not os.path.exists(head): os.makedirs(head) start = time.time() # Run inference infer_track_sequence(model, im_dir, tracking, proposals=proposals, vis=vis, det_file=output_file) delta = time.time() - start freq = float(len(os.listdir(im_dir))) / delta timing.append(freq) # Save evaluation results if EVAL: val_directory = os.path.abspath(head) + "/" eval_datections(val_directory, os.path.abspath(os.path.join(*im_dir.split("/")[:-2])) + "/") with open(val_directory + "eval.txt", "r") as f: temp = f.readline().strip() with open(val_directory + "eval.txt", "w+") as f: f.write("{},{}".format(temp, np.average(timing)))
def visualize(distmat, json_dataset, query_ids=None, gallery_ids=None, query_cams=None, gallery_cams=None, query_paths=None, gallery_paths=None): print(distmat.shape) output_dir = os.path.join( get_output_dir(json_dataset.name, training=False), 'vis') if not os.path.exists(output_dir): os.makedirs(output_dir) m, n = distmat.shape # Sort and find correct matches indices = np.argsort(distmat, axis=1) matches = (gallery_ids[indices] == query_ids[:, np.newaxis]) for i in range(m): # Filter out the same id and same camera valid = ((gallery_ids[indices[i]] != query_ids[i]) | (gallery_cams[indices[i]] != query_cams[i])) y_true = matches[i, valid] y_score = -distmat[i][indices[i]][valid] if not np.any(y_true): continue query_path = query_paths[i] query_name = os.path.basename(query_path) # copyfile(query_path, os.path.join(output_dir, query_name)) im_query = cv2.imread(query_path, cv2.IMREAD_COLOR) bs = 4 ms = 10 im_list = np.zeros((im_query.shape[0] + bs * 2, im_query.shape[1] * 11 + ms * 2 + ms * 2 * 10, 3), np.uint8) im_list[:, :, :] = 255 im_list[bs:-bs, 0:im_query.shape[1], :] = im_query st = im_query.shape[1] + ms * 2 gallery_paths_this = gallery_paths[indices[i]][valid] for j in range(10): gallery_path = gallery_paths_this[j] gallery_name = os.path.basename(gallery_path) # copyfile( # gallery_path, # os.path.join( # output_dir, '{}_{}_{}_{}_{}'.format( # query_name, j, y_true[j], y_score[j], gallery_name))) im_gallery = cv2.imread(gallery_path, cv2.IMREAD_COLOR) im_gallery = cv2.resize( im_gallery, (im_query.shape[1], im_query.shape[0]), interpolation=cv2.INTER_CUBIC) if y_true[j]: im_list[:, st + ms - bs:st + ms + im_gallery.shape[1] + bs, :] = [0, 255, 0] else: im_list[:, st + ms - bs:st + ms + im_gallery.shape[1] + bs, :] = [0, 0, 255] im_list[bs:-bs, st + ms:st + ms + im_gallery.shape[1], :] = im_gallery st += im_gallery.shape[1] + 2 * ms cv2.imwrite(os.path.join(output_dir, query_name), im_list)
if __name__ == '__main__': workspace.GlobalInit(['caffe2', '--caffe2_log_level=0']) logger = setup_logging(__name__) args = parse_args() logger.info('Called with args:') logger.info(args) if args.cfg_file is not None: merge_cfg_from_file(args.cfg_file) if args.opts is not None: merge_cfg_from_list(args.opts) assert_and_infer_cfg() logger.info('Testing with config:') logger.info(pprint.pformat(cfg)) output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) # Check for the final model (indicates training already finished) final_path = os.path.join(output_dir, 'model_final.pkl') while not os.path.exists(final_path) and args.wait: logger.info('Waiting for \'{}\' to exist...'.format(final_path)) time.sleep(10) # cfg.immutable(False) # cfg.TEST.NMS = 1.1 # cfg.TEST.SCORE_THRESH = -9999999999.0 # cfg.TEST.DETECTIONS_PER_IM = 10000000000 # cfg.immutable(True) # run_inference( # final_path, # ind_range=args.range,
def vis_training(cur_iter): if not (cfg.WSL.DEBUG or (cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)): return output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) sample_dir = os.path.join(output_dir, 'sample') if not os.path.exists(sample_dir): os.makedirs(sample_dir) for gpu_id in range(cfg.NUM_GPUS): data_ids = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data_ids')) ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data')) cpg = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cpg')) labels_oh = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, 'labels_oh')) if cfg.WSL.DEBUG: print('gpu_id: ', gpu_id, 'cpg: ', cpg.shape, cpg.max(), cpg.min(), cpg.mean()) prefix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id) save_im(ims, cfg.PIXEL_MEANS, prefix, sample_dir) save_cpg(cpg, labels_oh, prefix, sample_dir) im_scores = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cls_prob')) roi_scores = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, 'rois_pred')) rois = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'rois')) if cfg.WSL.CSC and (not cfg.MODEL.MASK_ON or True): csc = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'csc')) if cfg.WSL.DEBUG: print('gpu_id: ', gpu_id, 'csc: ', csc.shape, csc.max(), csc.min(), csc.mean()) save_csc(csc, labels_oh, im_scores, roi_scores, ims, rois, cfg.PIXEL_MEANS, prefix, '', sample_dir) if cfg.WSL.CENTER_LOSS and False: center_S = workspace.FetchBlob('gpu_{}/S'.format(gpu_id)) save_center(center_S, labels_oh, roi_scores, ims, rois, cfg.PIXEL_MEANS, prefix, '', sample_dir) if not cfg.MODEL.MASK_ON: continue if cfg.WSL.CSC: mask_csc = workspace.FetchBlob('gpu_{}/mask_{}'.format( gpu_id, 'csc')) if cfg.WSL.DEBUG: print('gpu_id: ', gpu_id, 'mask_csc: ', mask_csc.shape, mask_csc.max(), mask_csc.min(), mask_csc.mean()) save_csc(mask_csc, labels_oh, im_scores, roi_scores, ims, rois, cfg.PIXEL_MEANS, prefix, 'mask', sample_dir) if 'deeplab' in cfg.MRCNN.ROI_MASK_HEAD: blobs_name = [ 'mask_labels_oh', 'mask_fc8', 'mask_fc8_up', 'mask_fc8_crf_fg', 'mask_fc8_bg' ] sigmoids = [0, 1, 1, 0, 0] for blob_name, sigmoid in zip(blobs_name, sigmoids): data = workspace.FetchBlob('gpu_{}/{}'.format( gpu_id, blob_name)) if cfg.WSL.DEBUG: print('gpu_id: ', gpu_id, ' ', blob_name, ': ', data.shape, data.max(), data.min(), data.mean()) if sigmoid: save_sigmoid(data, labels_oh, prefix, blob_name, sample_dir) else: save_common(data, labels_oh, prefix, blob_name, sample_dir) continue mask_crf = workspace.FetchBlob( 'gpu_{}/mask_fc8_crf_fg_up'.format(gpu_id)) save_common(mask_crf, labels_oh, prefix, 'mask_fc8_crf_fg_up', sample_dir) mask = workspace.FetchBlob('gpu_{}/mask_fc8_up'.format(gpu_id)) save_pixels_pkl(data_ids, cpg, mask, mask_crf, labels_oh, sample_dir)
def grid_search(): dataset_name, proposal_file = get_inference_dataset(0, is_parent=False) roidb, dataset, start_ind, end_ind, total_num_images = get_roidb_and_dataset( dataset_name, proposal_file, None ) num_images = len(roidb) num_classes = cfg.MODEL.NUM_CLASSES subinds = np.array_split(range(num_images), cfg.NUM_GPUS) tag = 'detection' output_dir = get_output_dir(cfg.TEST.DATASETS, training=False) det_file = os.path.join(output_dir, 'detections.pkl') outputs = load_object(det_file) print(len(outputs)) all_dets_cache = outputs['all_boxes'] print(len(all_dets_cache)) all_boxes_cache = [] all_scores_cache = [] for i, entry in enumerate(roidb): print(i) max_det = all_dets_cache[1][i].shape[0] print(max_det, num_classes) boxes = np.zeros((max_det, 4), dtype=np.float32) scores = np.zeros((max_det, num_classes), dtype=np.float32) boxes[:] = -1 scores[:] = -1 for j in range(num_classes): if len(all_dets_cache[j]) > 0: pass else: continue scores[:, j] = all_dets_cache[j][i][:, 4] boxes[:, 0:4] = all_dets_cache[1][i][:, :4] boxes = np.tile(boxes, (1, scores.shape[1])) print(scores.shape, boxes.shape) all_boxes_cache.append(boxes) all_scores_cache.append(scores) timers = defaultdict(Timer) resultss = [] nmses = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.0] threshs = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1] max_per_images = [10000, 1000, 100, 10, 1] for nms in nmses: for thresh in threshs: for max_per_image in max_per_images: print("----------------------------------------------------") print('NUM: ', nms, ' Thresh: ', thresh, ' MAX_PER_IM: ', max_per_image) cfg.immutable(False) cfg.TEST.NMS = nms cfg.TEST.SCORE_THRESH = thresh cfg.TEST.DETECTIONS_PER_IM = max_per_image cfg.immutable(True) all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images) for i, entry in enumerate(roidb): # print(i) timers['im_detect_bbox'].tic() scores = all_scores_cache[i] boxes = all_boxes_cache[i] # print(scores.shape, boxes.shape) timers['im_detect_bbox'].toc() timers['misc_bbox'].tic() scores, boxes, cls_boxes_i = box_results_with_nms_and_limit(scores, boxes) timers['misc_bbox'].toc() extend_results(i, all_boxes, cls_boxes_i) results = task_evaluation.evaluate_all( dataset, all_boxes, all_segms, all_keyps, output_dir ) print(results) print(resultss) f = open('grid_search.csv', 'wb') wr = csv.writer(f, dialect='excel') wr.writerows(resultss)
def create_cpg_net(train=True): logger = logging.getLogger(__name__) FREEZE_CONV_BODY = cfg.TRAIN.FREEZE_CONV_BODY FREEZE_AT = cfg.TRAIN.FREEZE_AT WSL_CSC = cfg.WSL.CSC CENTER_LOSS = cfg.WSL.CENTER_LOSS MIN_ENTROPY_LOSS = cfg.WSL.MIN_ENTROPY_LOSS MASK_ON = cfg.MODEL.MASK_ON EXECUTION_TYPE = cfg.MODEL.EXECUTION_TYPE cfg.immutable(False) cfg.TRAIN.FREEZE_CONV_BODY = False cfg.TRAIN.FREEZE_AT = 0 cfg.WSL.CSC = False cfg.WSL.CPG = False cfg.WSL.CENTER_LOSS = False cfg.WSL.MIN_ENTROPY_LOSS = False cfg.MODEL.MASK_ON = False cfg.MODEL.EXECUTION_TYPE = b'simple' cfg.immutable(True) output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True) for gpu_id in range(cfg.NUM_GPUS): logger.info('Building model: {}'.format('gpu_' + str(gpu_id) + '_' + cfg.MODEL.TYPE + '_cpg')) model = model_builder_wsl.create('gpu_' + str(gpu_id) + '_' + cfg.MODEL.TYPE + '_cpg', train=train) # workspace.RunNetOnce(model.param_init_net) #----------------------------------------------------------------------------------- # logger.info( # 'Outputs saved to: {:s}'.format(os.path.abspath(output_dir))) # dump_proto_files(model, output_dir) if cfg.MEMONGER and False: start_op, end_op = OP_surgery_head(model, gpu_id) optimize_memory_cpg(model, gpu_id) OP_surgery_back(model, gpu_id, start_op, end_op) namescope = 'gpu_{}/'.format(gpu_id) model.net._net.op[0].input[ 0] = namescope + cfg.WSL.CPG_PRE_BLOB + '_grad' model.net._net.op[-1].output[ -1] = namescope + cfg.WSL.CPG_DATA_BLOB + '_grad' # share_surgery(model, gpu_id) else: OP_surgery(model, gpu_id) Input_surgery(model, gpu_id) workspace.CreateBlob('gpu_' + str(gpu_id) + '/' + cfg.WSL.CPG_PRE_BLOB + '_grad') optimize_memory_cpg(model, gpu_id) #----------------------------------------------------------------------------------- # nu.broadcast_parameters(model) workspace.CreateNet(model.net) logger.info('Outputs saved to: {:s}'.format( os.path.abspath(output_dir))) dump_proto_files(model, output_dir) cfg.immutable(False) cfg.TRAIN.FREEZE_CONV_BODY = FREEZE_CONV_BODY cfg.TRAIN.FREEZE_AT = FREEZE_AT cfg.WSL.CSC = WSL_CSC cfg.WSL.CENTER_LOSS = CENTER_LOSS cfg.WSL.MIN_ENTROPY_LOSS = MIN_ENTROPY_LOSS cfg.MODEL.MASK_ON = MASK_ON cfg.MODEL.EXECUTION_TYPE = EXECUTION_TYPE cfg.immutable(True)