def run_ResNet50_accuracy(engine_file, batch_size, num_images, verbose=False):
if verbose:
logging.info("Running ResNet50 accuracy test with:")
logging.info(" engine_file: {:}".format(engine_file))
logging.info(" batch_size: {:}".format(batch_size))
logging.info(" num_images: {:}".format(num_images))
runner = EngineRunner(engine_file, verbose=verbose)
input_dtype, input_format = get_input_format(runner.engine)
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.INT8:
if input_format == trt.TensorFormat.LINEAR:
format_string = "int8_linear"
elif input_format == trt.TensorFormat.CHW4:
format_string = "int8_chw4"
image_dir = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"imagenet/ResNet50", format_string)
accuracy_runner = ImageNetAccuracyRunner(runner,
batch_size,
image_dir,
num_images,
verbose=verbose)
return accuracy_runner.run()
def run_dlrm_accuracy(engine_file,
batch_size,
num_pairs=10000000,
verbose=False):
if verbose:
logging.info("Running DLRM accuracy test with:")
logging.info(" engine_file: {:}".format(engine_file))
logging.info(" batch_size: {:}".format(batch_size))
logging.info(" num_pairs: {:}".format(num_pairs))
runner = EngineRunner(engine_file, verbose=verbose)
pair_dir = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"criteo", "full_recalib")
input_dtype, input_format = get_input_format(runner.engine)
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.HALF:
format_string = "fp16"
elif input_dtype == trt.DataType.INT8:
format_string = "int8"
if input_format == trt.TensorFormat.CHW4:
format_string += "_chw4"
else:
raise NotImplementedError(
"Unsupported DataType {:}".format(input_dtype))
numerical_inputs = np.load(
os.path.join(pair_dir, "numeric_{:}.npy".format(format_string)))
categ_inputs = np.load(os.path.join(pair_dir, "categorical_int32.npy"))
predictions = []
refs = []
batch_idx = 0
for pair_idx in range(0, int(num_pairs), batch_size):
actual_batch_size = batch_size if pair_idx + batch_size <= num_pairs else num_pairs - pair_idx
numerical_input = np.ascontiguousarray(
numerical_inputs[pair_idx:pair_idx + actual_batch_size])
categ_input = np.ascontiguousarray(categ_inputs[pair_idx:pair_idx +
actual_batch_size])
start_time = time.time()
outputs = runner([numerical_input, categ_input], actual_batch_size)
if verbose:
logging.info(
"Batch {:d} (Size {:}) >> Inference time: {:f}".format(
batch_idx, actual_batch_size,
time.time() - start_time))
predictions.extend(outputs[0][:actual_batch_size])
batch_idx += 1
ground_truths = np.load(os.path.join(
pair_dir, "ground_truth.npy"))[:num_pairs].tolist()
return evaluate(ground_truths, predictions)
def run_3dunet_accuracy(engine_file, batch_size, num_images, verbose=False):
runner = EngineRunner(engine_file, verbose=verbose)
input_dtype, input_format = get_input_format(runner.engine)
if verbose:
logging.info("Running UNET accuracy test with:")
logging.info(" engine_file: {:}".format(engine_file))
logging.info(" batch_size: {:}".format(batch_size))
logging.info(" num_images: {:}".format(num_images))
logging.info(" input_dtype: {:}".format(input_dtype))
logging.info(" input_format: {:}".format(input_format))
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.INT8:
if input_format == trt.TensorFormat.LINEAR:
format_string = "int8_linear"
elif input_dtype == trt.DataType.HALF:
if input_format == trt.TensorFormat.LINEAR:
format_string = "fp16_linear"
elif input_format == trt.TensorFormat.DHWC8:
format_string = "fp16_dhwc8"
image_dir = os.path.join(os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"brats", "brats_npy", format_string)
if num_images is None:
num_images = 67
image_list = []
with open("data_maps/brats/val_map.txt") as f:
for line in f:
image_list.append(line.split()[0])
predictions = []
batch_idx = 0
for image_idx in range(0, num_images, batch_size):
actual_batch_size = batch_size if image_idx + batch_size <= num_images else num_images - image_idx
batch_images = np.ascontiguousarray(np.stack([np.load(os.path.join(image_dir, name + ".npy")) for name in image_list[image_idx:image_idx + actual_batch_size]]))
start_time = time.time()
outputs = runner([batch_images], actual_batch_size)
print(np.mean(batch_images[0].astype(np.float32)))
print(np.std(batch_images[0].astype(np.float32)))
print(np.mean(outputs[0].astype(np.float32)))
print(np.std(outputs[0].astype(np.float32)))
if verbose:
logging.info("Batch {:d} (Size {:}) >> Inference time: {:f}".format(batch_idx, actual_batch_size, time.time() - start_time))
predictions.extend(outputs[0][:actual_batch_size])
batch_idx += 1
logging.warning("3D-Unet standalone accuracy checker does not have accuracy checking yet! Always return 1.0")
return 1.0
def run_SSDResNet34_accuracy(engine_file,
batch_size,
num_images,
verbose=False,
output_file="build/out/SSDResNet34/dump.json"):
threshold = 0.20
runner = EngineRunner(engine_file, verbose=verbose)
input_dtype, input_format = get_input_format(runner.engine)
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.INT8:
if input_format == trt.TensorFormat.LINEAR:
format_string = "int8_linear"
elif input_format == trt.TensorFormat.CHW4:
format_string = "int8_chw4"
image_dir = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/val2017/SSDResNet34", format_string)
val_annotate = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/annotations/instances_val2017.json")
coco = COCO(annotation_file=val_annotate)
image_ids = coco.getImgIds()
cat_ids = coco.getCatIds()
# Class 0 is background
cat_ids.insert(0, 0)
num_images = min(num_images, len(image_ids))
logging.info(
"Running validation on {:} images. Please wait...".format(num_images))
coco_detections = []
batch_idx = 0
for image_idx in range(0, num_images, batch_size):
end_idx = min(image_idx + batch_size, num_images)
img = []
img_sizes = []
for idx in range(image_idx, end_idx):
image_id = image_ids[idx]
img.append(
np.load(
os.path.join(image_dir,
coco.imgs[image_id]["file_name"] + ".npy")))
img_sizes.append(
[coco.imgs[image_id]["height"], coco.imgs[image_id]["width"]])
img = np.stack(img)
start_time = time.time()
outputs = runner([img], batch_size=batch_size)
trt_detections = outputs[0]
if verbose:
logging.info("Batch {:d} >> Inference time: {:f}".format(
batch_idx,
time.time() - start_time))
for idx in range(0, end_idx - image_idx):
keep_count = trt_detections[idx * (200 * 7 + 1) +
200 * 7].view('int32')
trt_detections_batch = trt_detections[idx * (200 * 7 + 1):idx *
(200 * 7 + 1) +
keep_count * 7].reshape(
keep_count, 7)
image_height = img_sizes[idx][0]
image_width = img_sizes[idx][1]
for prediction_idx in range(0, keep_count):
loc = trt_detections_batch[prediction_idx, [2, 1, 4, 3]]
label = trt_detections_batch[prediction_idx, 6]
score = float(trt_detections_batch[prediction_idx, 5])
bbox_coco_fmt = [
loc[0] * image_width,
loc[1] * image_height,
(loc[2] - loc[0]) * image_width,
(loc[3] - loc[1]) * image_height,
]
coco_detection = {
"image_id": image_ids[image_idx + idx],
"category_id": cat_ids[int(label)],
"bbox": bbox_coco_fmt,
"score": score,
}
coco_detections.append(coco_detection)
batch_idx += 1
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output_file, "w") as f:
json.dump(coco_detections, f)
cocoDt = coco.loadRes(output_file)
eval = COCOeval(coco, cocoDt, 'bbox')
eval.params.imgIds = image_ids[:num_images]
eval.evaluate()
eval.accumulate()
eval.summarize()
map_score = eval.stats[0]
logging.info("Get mAP score = {:f} Target = {:f}".format(
map_score, threshold))
return (map_score >= threshold * 0.99)
def run_SSDMobileNet_accuracy(engine_file,
batch_size,
num_images,
verbose=False,
output_file="build/out/SSDMobileNet/dump.json"):
logging.info(
"Running SSDMobileNet functionality test for engine [ {:} ] with batch size {:}"
.format(engine_file, batch_size))
runner = EngineRunner(engine_file, verbose=verbose)
input_dtype, input_format = get_input_format(runner.engine)
if input_dtype == trt.DataType.FLOAT:
format_string = "fp32"
elif input_dtype == trt.DataType.INT8:
if input_format == trt.TensorFormat.LINEAR:
format_string = "int8_linear"
elif input_format == trt.TensorFormat.CHW4:
format_string = "int8_chw4"
image_dir = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/val2017/SSDMobileNet", format_string)
annotations_path = os.path.join(
os.getenv("PREPROCESSED_DATA_DIR", "build/preprocessed_data"),
"coco/annotations/instances_val2017.json")
val_map = "data_maps/coco/val_map.txt"
if len(glob(image_dir)) == 0:
logging.warn("Cannot find data directory in ({:})".format(image_dir))
pytest.skip("Cannot find data directory ({:})".format(image_dir))
coco = COCO(annotation_file=annotations_path)
coco_detections = []
image_ids = coco.getImgIds()
num_images = min(num_images, len(image_ids))
logging.info(
"Running validation on {:} images. Please wait...".format(num_images))
batch_idx = 0
for image_idx in range(0, num_images, batch_size):
batch_image_ids = image_ids[image_idx:image_idx + batch_size]
actual_batch_size = len(batch_image_ids)
batch_images = np.ascontiguousarray(
np.stack([
np.load(
os.path.join(image_dir,
coco.imgs[id]["file_name"] + ".npy"))
for id in batch_image_ids
]))
start_time = time.time()
[outputs] = runner([batch_images], actual_batch_size)
if verbose:
logging.info("Batch {:d} >> Inference time: {:f}".format(
batch_idx,
time.time() - start_time))
batch_detections = outputs.reshape(batch_size,
100 * 7 + 1)[:actual_batch_size]
for detections, image_id in zip(batch_detections, batch_image_ids):
keep_count = detections[100 * 7].view('int32')
image_width = coco.imgs[image_id]["width"]
image_height = coco.imgs[image_id]["height"]
for detection in detections[:keep_count * 7].reshape(
keep_count, 7):
score = float(detection[PredictionLayout.CONFIDENCE])
bbox_coco_fmt = [
detection[PredictionLayout.XMIN] * image_width,
detection[PredictionLayout.YMIN] * image_height,
(detection[PredictionLayout.XMAX] -
detection[PredictionLayout.XMIN]) * image_width,
(detection[PredictionLayout.YMAX] -
detection[PredictionLayout.YMIN]) * image_height,
]
coco_detection = {
"image_id": image_id,
"category_id": int(detection[PredictionLayout.LABEL]),
"bbox": bbox_coco_fmt,
"score": score,
}
coco_detections.append(coco_detection)
batch_idx += 1
output_dir = os.path.dirname(output_file)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(output_file, "w") as f:
json.dump(coco_detections, f)
cocoDt = coco.loadRes(output_file)
eval = COCOeval(coco, cocoDt, 'bbox')
eval.params.imgIds = image_ids[:num_images]
eval.evaluate()
eval.accumulate()
eval.summarize()
map_score = eval.stats[0]
logging.info("Get mAP score = {:f} Target = {:f}".format(
map_score, 0.22386))
return map_score