def result_getter():
if is_parent:
# Parent case:
# In this case we're either running inference on the entire dataset in a
# single process or (if multi_gpu_testing is True) using this process to
# launch subprocesses that each run inference on a range of the dataset
all_results = {}
for i in range(len(cfg.TEST.DATASETS)):
dataset_name, proposal_file = get_inference_dataset(i)
output_dir = get_output_dir(dataset_name, training=False)
results = parent_func(
weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=multi_gpu_testing
)
all_results.update(results)
return all_results
else:
# Subprocess child case:
# In this case test_net was called via subprocess.Popen to execute on a
# range of inputs on a single dataset
dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)
output_dir = get_output_dir(dataset_name, training=False)
return child_func(
weights_file,
dataset_name,
proposal_file,
output_dir,
ind_range=ind_range,
gpu_id=gpu_id
)
def result_getter():
if is_parent:
# Parent case:
# In this case we're either running inference on the entire dataset in a
# single process or (if multi_gpu_testing is True) using this process to
# launch subprocesses that each run inference on a range of the dataset
all_results = {}
for i in range(len(cfg.TEST.DATASETS)):
dataset_name, proposal_file = get_inference_dataset(i)
output_dir = get_output_dir(dataset_name, training=False)
results = parent_func(
weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=multi_gpu_testing
)
all_results.update(results)
return all_results
else:
# Subprocess child case:
# In this case test_net was called via subprocess.Popen to execute on a
# range of inputs on a single dataset
dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)
output_dir = get_output_dir(dataset_name, training=False)
return child_func(
weights_file,
dataset_name,
proposal_file,
output_dir,
ind_range=ind_range,
gpu_id=gpu_id
)
def load_detections(detection_file, dataset, thresholds):
detections = json.load(open(detection_file))
roidb = dataset.get_roidb()
classes = dataset.classes
#apply class thresholds
thres_dets = [det for det in detections if det['score'] > thresholds[classes[det['category_id']]]]
#save thresholded detections to file
output_dir = os.path.abspath(get_output_dir(dataset.name, training=False))
res_file = os.path.join(
output_dir, 'bbox_' + dataset.name + '_results_thresh.json'
)
with open(res_file,'w') as out_file:
json.dump(thres_dets, out_file)
#get per-image detections
detections_per_image = []
for entry in roidb:
#get all detections for image
im_dets = [det for det in thres_dets if det['image_id'] == entry['id']]
#convert bbox from xywh to xyxy format
for det in im_dets:
det["bbox"] = box_utils.xywh_to_xyxy(det["bbox"])
detections_per_image.append(im_dets)
return detections_per_image, res_file
def test_restore_checkpoint():
# Create Model
model = model_builder.create(cfg.MODEL.TYPE, train=True)
add_momentum_init_ops(model)
init_weights(model)
# Fill input blobs
roidb = combined_roidb_for_training(cfg.TRAIN.DATASETS,
cfg.TRAIN.PROPOSAL_FILES)
model_builder.add_training_inputs(model, roidb=roidb)
workspace.CreateNet(model.net)
# Bookkeeping for checkpoint creation
iter_num = 0
checkpoints = {}
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
chk_file_path = os.path.join(output_dir,
'model_iter{}.pkl'.format(iter_num))
checkpoints[iter_num] = chk_file_path
# Save model weights
nu.save_model_to_weights_file(checkpoints[iter_num], model)
orig_gpu_0_params, orig_all_params = get_params(model)
# Change the model weights
init_weights(model)
# Reload the weights in the model
nu.initialize_gpu_from_weights_file(model, chk_file_path, gpu_id=0)
nu.broadcast_parameters(model)
shutil.rmtree(cfg.OUTPUT_DIR)
_, restored_all_params = get_params(model)
# Check if all params are loaded correctly
for scoped_name, blob in orig_all_params.items():
np.testing.assert_array_equal(blob, restored_all_params[scoped_name])
# Check if broadcast_parameters works
for scoped_name, blob in restored_all_params.items():
unscoped_name = c2_utils.UnscopeName(scoped_name)
np.testing.assert_array_equal(blob, orig_gpu_0_params[unscoped_name])
def __init__(self, save=True, sort_keys=True):
self.save = save
if save:
self.output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
self.log_path = os.path.join(self.output_dir, 'log.json')
open(self.log_path, "w").close()
self.sort_keys = sort_keys
def res_top_result(model, entry, save_png=True):
im_info = workspace.FetchBlob(core.ScopedName('im_info'))
blob_h, blob_w, scale = im_info[0]
img = cv2.imread(entry['image'])
img_h, img_w, _ = img.shape
probs_human = workspace.FetchBlob(core.ScopedName('probs_human_NCHW'))[0, :, :, :]
probs_crop = np.zeros((probs_human.shape[0], int(img_h*scale), int(img_w*scale)), dtype=np.float32)
for i in range(probs_crop.shape[0]):
probs_up4 = cv2.resize(probs_human[i], None, None, fx=4, fy=4, interpolation=cv2.INTER_LINEAR)
probs_crop[i, :, :] = probs_up4[0:int(img_h*scale), 0:int(img_w*scale)]
probs_resize = np.zeros((probs_crop.shape[0], int(img_h), int(img_w)), dtype=np.float32)
for i in range(probs_crop.shape[0]):
probs_resize[i, :, :] = cv2.resize(probs_crop[i], (img_w, img_h), interpolation=cv2.INTER_LINEAR)
dataset_name = cfg.TEST.DATASETS[0]
output_dir = os.path.join(get_output_dir(dataset_name, training=False), 'vis_res_top')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# print(output_dir)
if save_png:
cv2.imwrite(os.path.join(output_dir, entry['id']+'.png'), probs_resize.argmax(0))
return probs_resize
def beginTrain(self):
workspace.GlobalInit([
'caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'
])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
smi_output, cuda_ver, cudnn_ver = c2_utils.get_nvidia_info()
logger.info("cuda version : {}".format(cuda_ver))
logger.info("cudnn version: {}".format(cudnn_ver))
logger.info("nvidia-smi output:\n{}".format(smi_output))
logger.info('Training with config:')
logger.info(pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints, losses = train_model()
# Test the trained model
self.test_model(checkpoints["final"])
dataset_name, _ = get_inference_dataset(0)
output_dir = get_output_dir(dataset_name, training=False)
with open(osp.join(output_dir, "res.pkl"), "rb") as src:
mAP = pickle.load(src)
return losses, mAP
def validate_tracking_params(weights_file,
dataset_name):
dataset = JsonDataset(dataset_name)
output_dir = os.path.abspath(get_output_dir(dataset.name, training=False))
#tracking parameter files
class_thresh_file = os.path.join(output_dir, 'AP_thresholds.txt')
obs_model_file = os.path.join(output_dir, 'observation_model.txt')
meas_cov_file = os.path.join(output_dir, 'meas_cov.txt')
#generate param files if they do not exist
if (not os.path.exists(class_thresh_file) or
not os.path.exists(obs_model_file) or
not os.path.exists(meas_cov_file)):
logger.info('validation files for validation dataset %s do not '
'exist. Generating them now' % (dataset.name))
#this performs inference on all images and calls the evaluation
#script, generating the validation files
test_net_on_dataset(weights_file,dataset.name,None,output_dir)
#read parameters from files
cla_thresh = load_class_thresholds(class_thresh_file, dataset)
observation_model = np.loadtxt(obs_model_file, delimiter=',')
ekf_sensor_noise = np.loadtxt(meas_cov_file, delimiter=',')
return cla_thresh, observation_model, ekf_sensor_noise
def test_restore_checkpoint():
# Create Model
model = model_builder.create(cfg.MODEL.TYPE, train=True)
add_momentum_init_ops(model)
init_weights(model)
# Fill input blobs
roidb = combined_roidb_for_training(
cfg.TRAIN.DATASETS, cfg.TRAIN.PROPOSAL_FILES
)
model_builder.add_training_inputs(model, roidb=roidb)
workspace.CreateNet(model.net)
# Bookkeeping for checkpoint creation
iter_num = 0
checkpoints = {}
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
chk_file_path = os.path.join(output_dir, 'model_iter{}.pkl'.format(iter_num))
checkpoints[iter_num] = chk_file_path
# Save model weights
nu.save_model_to_weights_file(checkpoints[iter_num], model)
orig_gpu_0_params, orig_all_params = get_params(model)
# Change the model weights
init_weights(model)
# Reload the weights in the model
nu.initialize_gpu_from_weights_file(model, chk_file_path, gpu_id=0)
nu.broadcast_parameters(model)
shutil.rmtree(cfg.OUTPUT_DIR)
_, restored_all_params = get_params(model)
# Check if all params are loaded correctly
for scoped_name, blob in orig_all_params.items():
np.testing.assert_array_equal(blob, restored_all_params[scoped_name])
# Check if broadcast_parameters works
for scoped_name, blob in restored_all_params.items():
unscoped_name = c2_utils.UnscopeName(scoped_name)
np.testing.assert_array_equal(blob, orig_gpu_0_params[unscoped_name])
def create_model():
"""Build the model and look for saved model checkpoints in case we can
resume from one.
"""
logger = logging.getLogger(__name__)
start_iter = 0
checkpoints = {}
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
weights_file = cfg.TRAIN.WEIGHTS
if cfg.TRAIN.AUTO_RESUME:
# Check for the final model (indicates training already finished)
final_path = os.path.join(output_dir, 'model_final.pkl')
if os.path.exists(final_path):
logger.info('model_final.pkl exists; no need to train!')
# return None, None, None, {'final': final_path}, output_dir
files = os.listdir(output_dir)
for f in files:
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
if len(iter_string) > 0:
checkpoint_iter = int(iter_string[0])
checkpoints[checkpoint_iter] = os.path.join(output_dir, f)
checkpoints['final'] = final_path
return None, None, None, checkpoints, output_dir
if cfg.TRAIN.COPY_WEIGHTS:
copyfile(
weights_file,
os.path.join(output_dir, os.path.basename(weights_file)))
logger.info('Copy {} to {}'.format(weights_file, output_dir))
# Find the most recent checkpoint (highest iteration number)
files = os.listdir(output_dir)
for f in files:
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
if len(iter_string) > 0:
checkpoint_iter = int(iter_string[0])
if checkpoint_iter > start_iter:
# Start one iteration immediately after the checkpoint iter
start_iter = checkpoint_iter + 1
resume_weights_file = f
if start_iter > 0:
# Override the initialization weights with the found checkpoint
weights_file = os.path.join(output_dir, resume_weights_file)
logger.info(
'========> Resuming from checkpoint {} at start iter {}'.
format(weights_file, start_iter)
)
logger.info('Building model: {}'.format(cfg.MODEL.TYPE))
model = model_builder_wsl.create(cfg.MODEL.TYPE, train=True)
if cfg.MEMONGER:
optimize_memory(model)
# Performs random weight initialization as defined by the model
workspace.RunNetOnce(model.param_init_net)
return model, weights_file, start_iter, checkpoints, output_dir
def train_model():
"""Model training loop."""
model, weights_file, start_iter, checkpoints, output_dir = create_model()
if 'final' in checkpoints:
# The final model was found in the output directory, so nothing to do
return checkpoints
with SummaryWriter(log_dir=get_output_dir(cfg.TRAIN.DATASETS) +
"/events") as writer:
writer.write_graph([model.net])
logger = logging.getLogger(__name__)
setup_model_for_training(model, weights_file, output_dir)
training_stats = TrainingStats(model)
CHECKPOINT_PERIOD = int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)
error_count = 0
for cur_iter in range(start_iter, cfg.SOLVER.MAX_ITER):
if model.roi_data_loader.has_stopped():
handle_critical_error(model, 'roi_data_loader failed')
training_stats.IterTic()
lr = model.UpdateWorkspaceLr(cur_iter,
lr_policy.get_lr_at_iter(cur_iter))
try:
workspace.RunNet(model.net.Proto().name)
except:
error_count += 1
logger.warn("Error in iter {}, error count: {}".format(
cur_iter, error_count))
if not cfg.CONTINUE_ON_ERROR:
raise
if cur_iter == start_iter:
nu.print_net(model)
training_stats.IterToc()
training_stats.UpdateIterStats()
training_stats.LogIterStats(cur_iter, lr)
if (cur_iter + 1) % CHECKPOINT_PERIOD == 0 and cur_iter > start_iter:
checkpoints[cur_iter] = os.path.join(
output_dir, 'model_iter{}.pkl'.format(cur_iter))
nu.save_model_to_weights_file(checkpoints[cur_iter], model)
if cur_iter == start_iter + cfg.TRAIN.EPOCH_PERIOD:
# Reset the iteration timer to remove outliers from the first few
# SGD iterations
training_stats.ResetIterTimer()
if np.isnan(training_stats.iter_total_loss):
handle_critical_error(model, 'Loss is NaN')
# Save the final model
checkpoints['final'] = os.path.join(output_dir, 'model_final.pkl')
nu.save_model_to_weights_file(checkpoints['final'], model)
# Shutdown data loading threads
model.roi_data_loader.shutdown()
return checkpoints
def result_getter():
if is_parent:
# Parent case:
# In this case we're either running inference on the entire dataset in a
# single process or (if multi_gpu_testing is True) using this process to
# launch subprocesses that each run inference on a range of the dataset
all_results = {}
subset_pointer = None
if cfg.VOC_SUBSET != '':
subset_pointer = result_getter #any dummy object could be used that is more advanced than 'object()' or similar builtins.
subset_pointer.subset = np.load(cfg.VOC_SUBSET)
print('loading subset')
for i in range(len(cfg.TEST.DATASETS)):
dataset_name, proposal_file = get_inference_dataset(i)
output_dir = get_output_dir(dataset_name, training=False)
print('len before', len(subset_pointer.subset))
results = parent_func(weights_file,
dataset_name,
proposal_file,
output_dir,
multi_gpu=multi_gpu_testing,
subset_pointer=subset_pointer)
all_results.update(results)
return all_results
else:
# Subprocess child case:
# In this case test_net was called via subprocess.Popen to execute on a
# range of inputs on a single dataset
dataset_name, proposal_file = get_inference_dataset(
0, is_parent=False)
output_dir = get_output_dir(dataset_name, training=False)
return child_func(weights_file,
dataset_name,
proposal_file,
output_dir,
ind_range=ind_range,
gpu_id=gpu_id)
def vis_training(cur_iter):
prefix = ''
if cfg.WEBLY.MINING:
prefix = 'mining_'
if not (cfg.WSL.DEBUG or
(cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)):
return
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
sample_dir = os.path.join(output_dir, 'webly_sample')
if not os.path.exists(sample_dir):
os.makedirs(sample_dir)
for gpu_id in range(cfg.NUM_GPUS):
data_ids = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data_ids'))
ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data'))
labels_oh = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, 'labels_oh'))
im_score = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cls_prob'))
roi_score = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, prefix + 'rois_pred'))
# roi_score_softmax = workspace.FetchBlob('gpu_{}/{}'.format(
# gpu_id, prefix + 'rois_pred_softmax'))
rois = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, prefix + 'rois'))
# anchor_argmax = workspace.FetchBlob('gpu_{}/{}'.format(
# gpu_id, 'anchor_argmax'))
preffix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id)
save_im(labels_oh, im_score, ims, cfg.PIXEL_MEANS, preffix, sample_dir)
save_rois(labels_oh, im_score, roi_score, ims, rois, cfg.PIXEL_MEANS,
preffix, '', sample_dir)
# continue
if cfg.WEBLY.ENTROPY:
pass
else:
continue
class_weight = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, prefix + 'rois_class_weight'))
rois_pred_hatE = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, prefix + 'rois_pred_hatE'))
rois_pred_E = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, prefix + 'rois_pred_E'))
y_logN__logy = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, prefix + 'rois_pred_y_logN__logy'))
save_entropy(labels_oh, im_score, class_weight, roi_score, ims, rois,
cfg.PIXEL_MEANS, preffix, '', sample_dir, rois_pred_hatE,
rois_pred_E, y_logN__logy)
def result_getter():
if is_parent:
# Parent case:
# In this case we're either running inference on the entire dataset in a
# single process or (if multi_gpu_testing is True) using this process to
# launch subprocesses that each run inference on a range of the dataset
all_results = {}
dataset_info, proposal_file = get_inference_dataset()
output_dir = osp.join(get_output_dir(dataset_info, training=False), weights_file.strip().split('/')[-1].split('.')[0])
if not osp.exists(output_dir):
os.makedirs(output_dir)
results = parent_func(
weights_file,
dataset_info,
proposal_file,
output_dir,
multi_gpu=multi_gpu_testing
)
all_results.update(results)
return all_results
else:
# Subprocess child case:
# In this case test_net was called via subprocess.Popen to execute on a
# range of inputs on a single dataset
dataset_info, proposal_file = get_inference_dataset(is_parent=False)
output_dir = osp.join(get_output_dir(dataset_info, training=False), weights_file.strip().split('/')[-1].split('.')[0])
if not osp.exists(output_dir):
os.makedirs(output_dir)
return child_func(
weights_file,
dataset_info,
proposal_file,
output_dir,
ind_range=ind_range,
gpu_id=gpu_id
)
def write_filtered_detections(dataset, filtered_boxes, filtered_depths, use_hmm):
output_dir = os.path.abspath(get_output_dir(dataset.name, training=False))
if use_hmm:
res_file = os.path.join(
output_dir, 'bbox_' + dataset.name + '_results_EKF_HMM.json'
)
else:
res_file = os.path.join(
output_dir, 'bbox_' + dataset.name + '_results_EKF.json'
)
_write_coco_bbox_results_file(dataset, filtered_boxes, filtered_depths, res_file)
return res_file
def load_data(dataset_name, image_id_list=None):
imgs = []
annotation_file = dataset_catalog.get_ann_fn(dataset_name)
dataset = json.load(open(annotation_file, 'r'))
for img in dataset['images']:
imgs.append(img)
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
file_name = os.path.join(output_dir, 'train_init.json')
select = []
image_id_set = set()
for i in image_id_list:
image_id_set.add(int(i))
for i in range(0, len(imgs)):
if (imgs[i]['id'] in image_id_set):
select.append(imgs[i])
dataset['images'] = select
with open(file_name, 'wt') as f:
json.dump(dataset, f)
return dataset
def create_model():
"""Build the model and look for saved model checkpoints in case we can
resume from one.
"""
logger = logging.getLogger(__name__)
start_iter = 0
checkpoints = {}
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
weights_file = cfg.TRAIN.WEIGHTS
if cfg.TRAIN.AUTO_RESUME:
# Check for the final model (indicates training already finished)
final_path = os.path.join(output_dir, 'model_final.pkl')
if os.path.exists(final_path):
logger.info('model_final.pkl exists; no need to train!')
return None, None, None, {'final': final_path}, output_dir
# Find the most recent checkpoint (highest iteration number)
files = os.listdir(output_dir)
for f in files:
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
if len(iter_string) > 0:
checkpoint_iter = int(iter_string[0])
if checkpoint_iter > start_iter:
# Start one iteration immediately after the checkpoint iter
start_iter = checkpoint_iter + 1
resume_weights_file = f
if start_iter > 0:
# Override the initialization weights with the found checkpoint
weights_file = os.path.join(output_dir, resume_weights_file)
logger.info(
'========> Resuming from checkpoint {} at start iter {}'.
format(weights_file, start_iter)
)
logger.info('Building model: {}'.format(cfg.MODEL.TYPE))
model = model_builder.create(cfg.MODEL.TYPE, train=True)
if cfg.MEMONGER:
optimize_memory(model)
# Performs random weight initialization as defined by the model
workspace.RunNetOnce(model.param_init_net)
return model, weights_file, start_iter, checkpoints, output_dir
def run_tracking(validation_dataset, tracking_datasets, weights_file, timestep,
use_hmm=True, no_filtering=False, visualize=False, step=False):
class_thresh, obs_model, meas_cov = validate_tracking_params(weights_file,
validation_dataset)
res_files = []
json_datasets = []
for i in range(len(tracking_datasets)):
tracking_set_name = tracking_datasets[i]
tracking_set_json = JsonDataset(tracking_set_name)
detections, res_file = get_detections(weights_file,
tracking_set_json,
class_thresh)
if not no_filtering:
filtered_boxes, filtered_depths = do_kalman_filtering(detections,
tracking_set_json,
timestep,
meas_cov,
obs_model,
use_hmm=use_hmm,
viz=visualize,
step=step)
res_file = write_filtered_detections(tracking_set_json,
filtered_boxes, filtered_depths,
use_hmm)
res_files.append(res_file)
json_datasets.append(tracking_set_json)
#perform the evaluation for all datasets
tracking_dataset_name = '-'.join(tracking_datasets)
tracking_output_dir = os.path.abspath(get_output_dir(tracking_dataset_name, training=False))
evaluate_tracking(res_files, json_datasets, tracking_output_dir, use_matlab=True)
def get_detections(weights_file, dataset, class_thresh):
output_dir = os.path.abspath(get_output_dir(dataset.name, training=False))
res_file = os.path.join(
output_dir, 'bbox_' + dataset.name + '_results.json'
)
#generate detections if they do not exist
if not os.path.exists(res_file):
logger.info('results file %s does not exist. Generating results '
'for dataset: %s' % (res_file, dataset.name))
all_boxes, all_depths, all_segms, all_keyps = test_net(
weights_file, dataset.name, None, output_dir)
_write_coco_bbox_results_file(dataset, all_boxes, all_depths, res_file)
detections, thresh_det_file = load_detections(res_file, dataset, class_thresh)
return detections, thresh_det_file
def main():
# Initialize C2
workspace.GlobalInit(
['caffe2', '--caffe2_log_level=0', '--caffe2_gpu_memory_tracking=1'])
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('detectron.roi_data.loader').setLevel(logging.INFO)
args = parse_args()
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
if cfg.LOGFILE:
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
print("OUTPUT_DIR=", output_dir)
root_logger = logging.getLogger()
root_logger.setLevel(logging.INFO)
fh = logging.FileHandler(os.path.join(output_dir, cfg.LOGFILE))
fh.setLevel(logging.INFO)
root_logger.addHandler(fh)
logger.info('Called with args:')
logger.info(args)
assert_and_infer_cfg()
logger.info('Training with config:')
logger.info(pprint.pformat(cfg))
# Note that while we set the numpy random seed network training will not be
# deterministic in general. There are sources of non-determinism that cannot
# be removed with a reasonble execution-speed tradeoff (such as certain
# non-deterministic cudnn functions).
np.random.seed(cfg.RNG_SEED)
random.seed(cfg.RNG_SEED)
# Execute the training run
checkpoints = detectron.utils.train.train_model()
# Test the trained model
if not args.skip_test:
test_model(checkpoints['final'], args.multi_gpu_testing, args.opts)
def log_json(self, stats_main, stats_extra):
stats_main.update(stats_extra)
json_stats = json.dumps(stats_main)
print("-" * 100)
print("iter: {}, loss: {:.4}, eta: {}, time: {:.0f} ms, lr: {:.3}".
format(
stats_main['iter'],
stats_main['loss'],
stats_main['eta'],
stats_main['time'] * 1000,
stats_main['lr'],
))
stats_str_list = [
"{}: {:.3}".format(k, v) for k, v in stats_extra.items()
]
stats_str_list.sort()
print(", ".join(stats_str_list))
if self.save:
with open(self.log_path, "a") as f:
f.write(json_stats + "\n")
# tensorboard
stats_main.pop('eta', None)
i = stats_main.pop('iter', None)
for key in stats_main.keys():
key_split = key.split("_")
if key_split[0] == "loss":
if len(key_split) > 1 and key_split[1] == "rpn":
tag = "loss_rpn"
else:
tag = "loss"
else:
tag = "meta"
stats_main["{}/{}".format(tag, key)] = stats_main[key]
stats_main.pop(key, None)
with SummaryWriter(log_dir=get_output_dir(cfg.TRAIN.DATASETS) +
"/events") as writer:
writer.write_scalars(stats_main, i)
def vis_training(cur_iter):
# if not (cfg.WSL.DEBUG or
# (cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)):
# return
if cur_iter % 100 != 0:
return
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
sample_dir = os.path.join(output_dir, 'DB_sample')
if not os.path.exists(sample_dir):
os.makedirs(sample_dir)
for gpu_id in range(cfg.NUM_GPUS):
db2_add = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'db2_add'))
conv5_3 = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'conv5_3'))
ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data'))
prefix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id)
if False:
save_im(ims, cfg.PIXEL_MEANS, prefix, sample_dir)
save_conv(ims, db2_add, cfg.PIXEL_MEANS, prefix, "_db", sample_dir)
save_conv(ims, conv5_3, cfg.PIXEL_MEANS, prefix, "_con53", sample_dir)
def main(args):
logger = logging.getLogger(__name__)
merge_cfg_from_file(args.cfg)
cfg.NUM_GPUS = 1
if "mot-classes" in args.opts:
dummy_dataset = dummy_datasets.get_mot_dataset()
cfg.NUM_CLASSES = 14
else:
dummy_dataset = dummy_datasets.get_coco_dataset()
cfg.NUM_CLASSES = 81
for i, weights_file in enumerate(args.weights_pre_list):
args.weights_pre_list[i] = cache_url(weights_file, cfg.DOWNLOAD_CACHE)
for i, weights_file in enumerate(args.weights_post_list):
args.weights_post_list[i] = cache_url(weights_file, cfg.DOWNLOAD_CACHE)
assert_and_infer_cfg(cache_urls=False)
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
# If True: Evaluation with respect to specified parameters (model from
# specifig training iterations or inference hyper-parameters)
# If False: Infer test sequence for evaluation on the MOT benchmark server
EVAL = "eval" in args.opts
train_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
train_dir_split = train_dir.split("/")
train_dir = os.path.join("/".join(train_dir_split[:-1]) + \
args.model_suffix, train_dir_split[-1])
model_list = []
files = os.listdir(train_dir)
if EVAL:
test_dir = "/".join(get_output_dir(cfg.TEST.DATASETS,
training=False).split("/")[:-1]) + args.model_suffix
# Evaluation with respect to inference hyper-parameters
if HYPER_PARAM is not None:
test_dir = os.path.join(test_dir, HYPER_PARAM.lower())
model_param = ((args.model, param) for param in PARAM_RANGE)
# Evaluation with respect to weights from specific training iterations
else:
model_param = []
for f in files:
if f.startswith("model_") and f.endswith(".pkl"):
iter_string = re.findall(r'(?<=model_iter)\d+(?=\.pkl)', f)
if len(iter_string) > 0:
model_param.append((f, int(iter_string[0])))
model_param.sort(key=lambda tup: tup[1])
if "model_final.pkl" in files:
model_param.append(("model_final.pkl", "final"))
# Tracking evaluation by interface to matlab engine
seq_map_path = os.path.join(test_dir, "seq_map.txt")
if not os.path.exists(test_dir):
os.makedirs(test_dir)
with open(seq_map_path, "w+") as seq_map:
seq_map.write("name\n")
for dataset in cfg.TEST.DATASETS:
seq_map.write(get_im_dir(dataset).split("/")[-2] + '\n')
seq_map_path = os.path.relpath(os.path.abspath(seq_map_path),
os.path.expanduser(os.path.join(args.devkit_path, "seqmaps")))
matlab_eng = get_matlab_engine(args.devkit_path)
eval_datections = lambda res_dir, gt_dir: eval_detections_matlab(
matlab_eng, seq_map_path, res_dir, gt_dir, 'MOT17')
else:
if args.model is not None:
model_param = ((args.model, None),)
else:
model_param = (("model_final.pkl", "final"),)
# Iterate through (model, parameter) tuples
for i, (model, param) in enumerate(model_param):
if EVAL and (i + 1 + args.offset) % (args.skip + 1) != 0:
logger.info("Skipping {}".format(model))
continue
# Hyper parameter inference
elif HYPER_PARAM is not None:
cfg.immutable(False)
setattr(cfg.TRCNN, HYPER_PARAM, param)
assert_and_infer_cfg(cache_urls=False)
print(cfg.TRCNN)
if not EVAL or param >= args.start_at:
weights_list = args.weights_pre_list + [os.path.join(train_dir, model)] + \
args.weights_post_list
preffix_list = args.preffix_list if len(args.preffix_list) \
else [""] * (len(args.weights_pre_list) + len(args.weights_post_list) + 1)
workspace.ResetWorkspace()
model = infer_engine.initialize_mixed_model_from_cfg(weights_list,
preffix_list=preffix_list)
logger.info("Processing {}".format(param))
timing = []
# iterate through test sequences
for dataset in cfg.TEST.DATASETS:
tracking = Tracking(args.thresh, cfg.TRCNN.MAX_BACK_TRACK)
logger.info("Processing dataset {}".format(dataset))
im_dir = get_im_dir(dataset)
vis = None
if EVAL:
output_file = os.path.join(test_dir, str(param),
im_dir.split("/")[-2] + ".txt")
# Visualize detections along with tracking detection file creation
else:
output_dir = os.path.join("outputs/MOT17", im_dir.split("/")[-2])
if "vis" in args.opts:
vis = {
"output-dir": output_dir,
"dummy-dataset": dummy_dataset,
"show-class": "show-class" in args.opts,
"show-track": "show-track" in args.opts,
"thresh": args.thresh,
"track-thresh": cfg.TRCNN.DETECTION_THRESH,
"n-colors": 15,
}
output_file = os.path.join("outputs/MOT17",
im_dir.split("/")[-2] + ".txt")
# Use custom proposals if provided
if "proposals" in args.opts:
proposals = pickle.load(open(os.path.join('/', *(im_dir.split("/")[:-1] + \
["det/proposals.pkl"])), 'r'))
else:
proposals = None
head, tail = os.path.split(output_file)
if not os.path.exists(head):
os.makedirs(head)
start = time.time()
# Run inference
infer_track_sequence(model, im_dir, tracking, proposals=proposals,
vis=vis, det_file=output_file)
delta = time.time() - start
freq = float(len(os.listdir(im_dir))) / delta
timing.append(freq)
# Save evaluation results
if EVAL:
val_directory = os.path.abspath(head) + "/"
eval_datections(val_directory,
os.path.abspath(os.path.join(*im_dir.split("/")[:-2])) + "/")
with open(val_directory + "eval.txt", "r") as f:
temp = f.readline().strip()
with open(val_directory + "eval.txt", "w+") as f:
f.write("{},{}".format(temp, np.average(timing)))
def visualize(distmat,
json_dataset,
query_ids=None,
gallery_ids=None,
query_cams=None,
gallery_cams=None,
query_paths=None,
gallery_paths=None):
print(distmat.shape)
output_dir = os.path.join(
get_output_dir(json_dataset.name, training=False), 'vis')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
m, n = distmat.shape
# Sort and find correct matches
indices = np.argsort(distmat, axis=1)
matches = (gallery_ids[indices] == query_ids[:, np.newaxis])
for i in range(m):
# Filter out the same id and same camera
valid = ((gallery_ids[indices[i]] != query_ids[i]) |
(gallery_cams[indices[i]] != query_cams[i]))
y_true = matches[i, valid]
y_score = -distmat[i][indices[i]][valid]
if not np.any(y_true): continue
query_path = query_paths[i]
query_name = os.path.basename(query_path)
# copyfile(query_path, os.path.join(output_dir, query_name))
im_query = cv2.imread(query_path, cv2.IMREAD_COLOR)
bs = 4
ms = 10
im_list = np.zeros((im_query.shape[0] + bs * 2,
im_query.shape[1] * 11 + ms * 2 + ms * 2 * 10, 3),
np.uint8)
im_list[:, :, :] = 255
im_list[bs:-bs, 0:im_query.shape[1], :] = im_query
st = im_query.shape[1] + ms * 2
gallery_paths_this = gallery_paths[indices[i]][valid]
for j in range(10):
gallery_path = gallery_paths_this[j]
gallery_name = os.path.basename(gallery_path)
# copyfile(
# gallery_path,
# os.path.join(
# output_dir, '{}_{}_{}_{}_{}'.format(
# query_name, j, y_true[j], y_score[j], gallery_name)))
im_gallery = cv2.imread(gallery_path, cv2.IMREAD_COLOR)
im_gallery = cv2.resize(
im_gallery, (im_query.shape[1], im_query.shape[0]),
interpolation=cv2.INTER_CUBIC)
if y_true[j]:
im_list[:, st + ms - bs:st + ms + im_gallery.shape[1] +
bs, :] = [0, 255, 0]
else:
im_list[:, st + ms - bs:st + ms + im_gallery.shape[1] +
bs, :] = [0, 0, 255]
im_list[bs:-bs, st + ms:st + ms +
im_gallery.shape[1], :] = im_gallery
st += im_gallery.shape[1] + 2 * ms
cv2.imwrite(os.path.join(output_dir, query_name), im_list)
if __name__ == '__main__':
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
logger = setup_logging(__name__)
args = parse_args()
logger.info('Called with args:')
logger.info(args)
if args.cfg_file is not None:
merge_cfg_from_file(args.cfg_file)
if args.opts is not None:
merge_cfg_from_list(args.opts)
assert_and_infer_cfg()
logger.info('Testing with config:')
logger.info(pprint.pformat(cfg))
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
# Check for the final model (indicates training already finished)
final_path = os.path.join(output_dir, 'model_final.pkl')
while not os.path.exists(final_path) and args.wait:
logger.info('Waiting for \'{}\' to exist...'.format(final_path))
time.sleep(10)
# cfg.immutable(False)
# cfg.TEST.NMS = 1.1
# cfg.TEST.SCORE_THRESH = -9999999999.0
# cfg.TEST.DETECTIONS_PER_IM = 10000000000
# cfg.immutable(True)
# run_inference(
# final_path,
# ind_range=args.range,
def vis_training(cur_iter):
if not (cfg.WSL.DEBUG or
(cfg.WSL.SAMPLE and cur_iter % cfg.WSL.SAMPLE_ITER == 0)):
return
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
sample_dir = os.path.join(output_dir, 'sample')
if not os.path.exists(sample_dir):
os.makedirs(sample_dir)
for gpu_id in range(cfg.NUM_GPUS):
data_ids = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data_ids'))
ims = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'data'))
cpg = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cpg'))
labels_oh = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, 'labels_oh'))
if cfg.WSL.DEBUG:
print('gpu_id: ', gpu_id, 'cpg: ', cpg.shape, cpg.max(), cpg.min(),
cpg.mean())
prefix = 'iter_' + str(cur_iter) + '_gpu_' + str(gpu_id)
save_im(ims, cfg.PIXEL_MEANS, prefix, sample_dir)
save_cpg(cpg, labels_oh, prefix, sample_dir)
im_scores = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'cls_prob'))
roi_scores = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, 'rois_pred'))
rois = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'rois'))
if cfg.WSL.CSC and (not cfg.MODEL.MASK_ON or True):
csc = workspace.FetchBlob('gpu_{}/{}'.format(gpu_id, 'csc'))
if cfg.WSL.DEBUG:
print('gpu_id: ', gpu_id, 'csc: ', csc.shape, csc.max(),
csc.min(), csc.mean())
save_csc(csc, labels_oh, im_scores, roi_scores, ims, rois,
cfg.PIXEL_MEANS, prefix, '', sample_dir)
if cfg.WSL.CENTER_LOSS and False:
center_S = workspace.FetchBlob('gpu_{}/S'.format(gpu_id))
save_center(center_S, labels_oh, roi_scores, ims, rois,
cfg.PIXEL_MEANS, prefix, '', sample_dir)
if not cfg.MODEL.MASK_ON:
continue
if cfg.WSL.CSC:
mask_csc = workspace.FetchBlob('gpu_{}/mask_{}'.format(
gpu_id, 'csc'))
if cfg.WSL.DEBUG:
print('gpu_id: ', gpu_id, 'mask_csc: ', mask_csc.shape,
mask_csc.max(), mask_csc.min(), mask_csc.mean())
save_csc(mask_csc, labels_oh, im_scores, roi_scores, ims, rois,
cfg.PIXEL_MEANS, prefix, 'mask', sample_dir)
if 'deeplab' in cfg.MRCNN.ROI_MASK_HEAD:
blobs_name = [
'mask_labels_oh', 'mask_fc8', 'mask_fc8_up', 'mask_fc8_crf_fg',
'mask_fc8_bg'
]
sigmoids = [0, 1, 1, 0, 0]
for blob_name, sigmoid in zip(blobs_name, sigmoids):
data = workspace.FetchBlob('gpu_{}/{}'.format(
gpu_id, blob_name))
if cfg.WSL.DEBUG:
print('gpu_id: ', gpu_id, ' ', blob_name, ': ', data.shape,
data.max(), data.min(), data.mean())
if sigmoid:
save_sigmoid(data, labels_oh, prefix, blob_name,
sample_dir)
else:
save_common(data, labels_oh, prefix, blob_name, sample_dir)
continue
mask_crf = workspace.FetchBlob(
'gpu_{}/mask_fc8_crf_fg_up'.format(gpu_id))
save_common(mask_crf, labels_oh, prefix, 'mask_fc8_crf_fg_up',
sample_dir)
mask = workspace.FetchBlob('gpu_{}/mask_fc8_up'.format(gpu_id))
save_pixels_pkl(data_ids, cpg, mask, mask_crf, labels_oh,
sample_dir)
def grid_search():
dataset_name, proposal_file = get_inference_dataset(0, is_parent=False)
roidb, dataset, start_ind, end_ind, total_num_images = get_roidb_and_dataset(
dataset_name, proposal_file, None
)
num_images = len(roidb)
num_classes = cfg.MODEL.NUM_CLASSES
subinds = np.array_split(range(num_images), cfg.NUM_GPUS)
tag = 'detection'
output_dir = get_output_dir(cfg.TEST.DATASETS, training=False)
det_file = os.path.join(output_dir, 'detections.pkl')
outputs = load_object(det_file)
print(len(outputs))
all_dets_cache = outputs['all_boxes']
print(len(all_dets_cache))
all_boxes_cache = []
all_scores_cache = []
for i, entry in enumerate(roidb):
print(i)
max_det = all_dets_cache[1][i].shape[0]
print(max_det, num_classes)
boxes = np.zeros((max_det, 4), dtype=np.float32)
scores = np.zeros((max_det, num_classes), dtype=np.float32)
boxes[:] = -1
scores[:] = -1
for j in range(num_classes):
if len(all_dets_cache[j]) > 0:
pass
else:
continue
scores[:, j] = all_dets_cache[j][i][:, 4]
boxes[:, 0:4] = all_dets_cache[1][i][:, :4]
boxes = np.tile(boxes, (1, scores.shape[1]))
print(scores.shape, boxes.shape)
all_boxes_cache.append(boxes)
all_scores_cache.append(scores)
timers = defaultdict(Timer)
resultss = []
nmses = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.0]
threshs = [1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
max_per_images = [10000, 1000, 100, 10, 1]
for nms in nmses:
for thresh in threshs:
for max_per_image in max_per_images:
print("----------------------------------------------------")
print('NUM: ', nms, ' Thresh: ', thresh, ' MAX_PER_IM: ', max_per_image)
cfg.immutable(False)
cfg.TEST.NMS = nms
cfg.TEST.SCORE_THRESH = thresh
cfg.TEST.DETECTIONS_PER_IM = max_per_image
cfg.immutable(True)
all_boxes, all_segms, all_keyps = empty_results(num_classes, num_images)
for i, entry in enumerate(roidb):
# print(i)
timers['im_detect_bbox'].tic()
scores = all_scores_cache[i]
boxes = all_boxes_cache[i]
# print(scores.shape, boxes.shape)
timers['im_detect_bbox'].toc()
timers['misc_bbox'].tic()
scores, boxes, cls_boxes_i = box_results_with_nms_and_limit(scores, boxes)
timers['misc_bbox'].toc()
extend_results(i, all_boxes, cls_boxes_i)
results = task_evaluation.evaluate_all(
dataset, all_boxes, all_segms, all_keyps, output_dir
)
print(results)
print(resultss)
f = open('grid_search.csv', 'wb')
wr = csv.writer(f, dialect='excel')
wr.writerows(resultss)
def create_cpg_net(train=True):
logger = logging.getLogger(__name__)
FREEZE_CONV_BODY = cfg.TRAIN.FREEZE_CONV_BODY
FREEZE_AT = cfg.TRAIN.FREEZE_AT
WSL_CSC = cfg.WSL.CSC
CENTER_LOSS = cfg.WSL.CENTER_LOSS
MIN_ENTROPY_LOSS = cfg.WSL.MIN_ENTROPY_LOSS
MASK_ON = cfg.MODEL.MASK_ON
EXECUTION_TYPE = cfg.MODEL.EXECUTION_TYPE
cfg.immutable(False)
cfg.TRAIN.FREEZE_CONV_BODY = False
cfg.TRAIN.FREEZE_AT = 0
cfg.WSL.CSC = False
cfg.WSL.CPG = False
cfg.WSL.CENTER_LOSS = False
cfg.WSL.MIN_ENTROPY_LOSS = False
cfg.MODEL.MASK_ON = False
cfg.MODEL.EXECUTION_TYPE = b'simple'
cfg.immutable(True)
output_dir = get_output_dir(cfg.TRAIN.DATASETS, training=True)
for gpu_id in range(cfg.NUM_GPUS):
logger.info('Building model: {}'.format('gpu_' + str(gpu_id) + '_' +
cfg.MODEL.TYPE + '_cpg'))
model = model_builder_wsl.create('gpu_' + str(gpu_id) + '_' +
cfg.MODEL.TYPE + '_cpg',
train=train)
# workspace.RunNetOnce(model.param_init_net)
#-----------------------------------------------------------------------------------
# logger.info(
# 'Outputs saved to: {:s}'.format(os.path.abspath(output_dir)))
# dump_proto_files(model, output_dir)
if cfg.MEMONGER and False:
start_op, end_op = OP_surgery_head(model, gpu_id)
optimize_memory_cpg(model, gpu_id)
OP_surgery_back(model, gpu_id, start_op, end_op)
namescope = 'gpu_{}/'.format(gpu_id)
model.net._net.op[0].input[
0] = namescope + cfg.WSL.CPG_PRE_BLOB + '_grad'
model.net._net.op[-1].output[
-1] = namescope + cfg.WSL.CPG_DATA_BLOB + '_grad'
# share_surgery(model, gpu_id)
else:
OP_surgery(model, gpu_id)
Input_surgery(model, gpu_id)
workspace.CreateBlob('gpu_' + str(gpu_id) + '/' +
cfg.WSL.CPG_PRE_BLOB + '_grad')
optimize_memory_cpg(model, gpu_id)
#-----------------------------------------------------------------------------------
# nu.broadcast_parameters(model)
workspace.CreateNet(model.net)
logger.info('Outputs saved to: {:s}'.format(
os.path.abspath(output_dir)))
dump_proto_files(model, output_dir)
cfg.immutable(False)
cfg.TRAIN.FREEZE_CONV_BODY = FREEZE_CONV_BODY
cfg.TRAIN.FREEZE_AT = FREEZE_AT
cfg.WSL.CSC = WSL_CSC
cfg.WSL.CENTER_LOSS = CENTER_LOSS
cfg.WSL.MIN_ENTROPY_LOSS = MIN_ENTROPY_LOSS
cfg.MODEL.MASK_ON = MASK_ON
cfg.MODEL.EXECUTION_TYPE = EXECUTION_TYPE
cfg.immutable(True)
