def turn_on_mps(active_sms):
if not is_xavier():
turn_off_mps()
cmd = "export CUDA_MPS_ACTIVE_THREAD_PERCENTAGE={:d} && nvidia-cuda-mps-control -d".format(
active_sms)
logging.info("Turn on MPS with active_sms = {:d}.".format(active_sms))
run_command(cmd)
def check_mps_status():
# Xavier does not have MPS
if is_xavier():
return False
# Check by printing out currently running processes and grepping nvidia-cuda-mps-control.
cmd = "ps -ef | grep nvidia-cuda-mps-control | grep -c -v grep"
logging.debug("Checking if MPS is running with command: {:}".format(cmd))
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
p.wait()
output = p.stdout.readlines()
return int(output[0]) >= 1
def turn_off_mps():
if not is_xavier() and check_mps_status():
cmd = "echo quit | nvidia-cuda-mps-control"
logging.info("Turn off MPS.")
run_command(cmd)
def turn_off_mps():
"""Turn off MPS."""
if not is_xavier() and is_mps_enabled():
cmd = "echo quit | nvidia-cuda-mps-control"
logging.info("Turn off MPS.")
run_command(cmd)
def check_accuracy(log_file, config, is_compliance=False):
benchmark_name = config["benchmark"]
accuracy_targets = {
BENCHMARKS.ResNet50: 76.46,
BENCHMARKS.SSDResNet34: 20.0,
BENCHMARKS.SSDMobileNet: 22.0,
BENCHMARKS.BERT: 90.874,
BENCHMARKS.DLRM: 80.25,
BENCHMARKS.RNNT: 100.0 - 7.45225,
BENCHMARKS.UNET: 0.853
}
threshold_ratio = float(config["accuracy_level"][:-1]) / 100
if not os.path.exists(log_file):
return "Cannot find accuracy JSON file."
# checking if log_file is empty by just reading first several bytes
# indeed, first 4B~6B is likely all we need to check: '', '[]', '[]\r', '[\n]\n', '[\r\n]\r\n', ...
# but checking 8B for safety
with open(log_file, 'r') as lf:
first_8B = lf.read(8)
if not first_8B or ('[' in first_8B and ']' in first_8B):
return "No accuracy results in PerformanceOnly mode."
dtype_expand_map = {"fp16": "float16", "fp32": "float32", "int8": "float16"} # Use FP16 output for INT8 mode
accuracy_regex_map = import_module("build.inference.tools.submission.submission-checker").ACC_PATTERN
threshold = accuracy_targets[benchmark_name] * threshold_ratio
if benchmark_name in [BENCHMARKS.ResNet50]:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-imagenet.py --mlperf-accuracy-file {:} \
--imagenet-val-file data_maps/imagenet/val_map.txt --dtype int32 ".format(log_file)
regex = accuracy_regex_map["acc"]
elif benchmark_name == BENCHMARKS.SSDResNet34:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-coco.py --mlperf-accuracy-file {:} \
--coco-dir {:} --output-file build/ssd-resnet34-results.json --use-inv-map".format(
log_file, os.path.join(os.environ.get("PREPROCESSED_DATA_DIR", "build/preprocessed_data"), "coco"))
regex = accuracy_regex_map["mAP"]
elif benchmark_name == BENCHMARKS.SSDMobileNet:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-coco.py --mlperf-accuracy-file {:} \
--coco-dir {:} --output-file build/ssd-mobilenet-results.json".format(
log_file, os.path.join(os.environ.get("PREPROCESSED_DATA_DIR", "build/preprocessed_data"), "coco"))
regex = accuracy_regex_map["mAP"]
elif benchmark_name == BENCHMARKS.BERT:
# Having issue installing tokenizers on Xavier...
if is_xavier():
cmd = "python3 code/bert/tensorrt/accuracy-bert.py --mlperf-accuracy-file {:} --squad-val-file {:}".format(
log_file, os.path.join(os.environ.get("DATA_DIR", "build/data"), "squad", "dev-v1.1.json"))
else:
dtype = config["precision"].lower()
if dtype in dtype_expand_map:
dtype = dtype_expand_map[dtype]
val_data_path = os.path.join(
os.environ.get("DATA_DIR", "build/data"),
"squad", "dev-v1.1.json")
vocab_file_path = "build/models/bert/vocab.txt"
output_prediction_path = os.path.join(os.path.dirname(log_file), "predictions.json")
cmd = "python3 build/inference/language/bert/accuracy-squad.py " \
"--log_file {:} --vocab_file {:} --val_data {:} --out_file {:} " \
"--output_dtype {:}".format(log_file, vocab_file_path, val_data_path, output_prediction_path, dtype)
regex = accuracy_regex_map["F1"]
elif benchmark_name == BENCHMARKS.DLRM:
cmd = "python3 build/inference/recommendation/dlrm/pytorch/tools/accuracy-dlrm.py --mlperf-accuracy-file {:} " \
"--day-23-file build/data/criteo/day_23 --aggregation-trace-file " \
"build/preprocessed_data/criteo/full_recalib/sample_partition_trace.txt".format(log_file)
regex = accuracy_regex_map["AUC"]
elif benchmark_name == BENCHMARKS.RNNT:
# Having issue installing librosa on Xavier...
if is_xavier():
cmd = "python3 code/rnnt/tensorrt/accuracy.py --loadgen_log {:}".format(log_file)
else:
# RNNT output indices are in INT8
cmd = "python3 build/inference/speech_recognition/rnnt/accuracy_eval.py " \
"--log_dir {:} --dataset_dir build/preprocessed_data/LibriSpeech/dev-clean-wav " \
"--manifest build/preprocessed_data/LibriSpeech/dev-clean-wav.json " \
"--output_dtype int8".format(os.path.dirname(log_file))
regex = accuracy_regex_map["WER"]
elif benchmark_name == BENCHMARKS.UNET:
postprocess_dir = "build/brats_postprocessed_data"
if not os.path.exists(postprocess_dir):
os.makedirs(postprocess_dir)
dtype = config["precision"].lower()
if dtype in dtype_expand_map:
dtype = dtype_expand_map[dtype]
cmd = "python3 build/inference/vision/medical_imaging/3d-unet/accuracy-brats.py --log_file {:} " \
"--output_dtype {:} --preprocessed_data_dir build/preprocessed_data/brats/brats_reference_preprocessed " \
"--postprocessed_data_dir {:} " \
"--label_data_dir build/preprocessed_data/brats/brats_reference_raw/Task043_BraTS2019/labelsTr".format(log_file, dtype, postprocess_dir)
regex = accuracy_regex_map["DICE"]
# Having issue installing nnUnet on Xavier...
if is_xavier():
logging.warning(
"Accuracy checking for 3DUnet is not supported on Xavier. Please run the following command on desktop:\n{:}".format(cmd))
cmd = 'echo "Accuracy: mean = 1.0000, whole tumor = 1.0000, tumor core = 1.0000, enhancing tumor = 1.0000"'
else:
raise ValueError("Unknown benchmark: {:}".format(benchmark_name))
output = run_command(cmd, get_output=True)
result_regex = re.compile(regex)
accuracy = None
with open(os.path.join(os.path.dirname(log_file), "accuracy.txt"), "w") as f:
for line in output:
print(line, file=f)
for line in output:
result_match = result_regex.match(line)
if not result_match is None:
accuracy = float(result_match.group(1))
break
accuracy_result = "PASSED" if accuracy is not None and accuracy >= threshold else "FAILED"
if accuracy_result == "FAILED" and not is_compliance:
raise RuntimeError(
"Accuracy = {:.3f}, Threshold = {:.3f}. Accuracy test {:}!".format(
accuracy, threshold, accuracy_result))
if is_compliance:
return accuracy # Needed for numerical comparison
return "Accuracy = {:.3f}, Threshold = {:.3f}. Accuracy test {:}.".format(
accuracy, threshold, accuracy_result)
def check_accuracy(log_file, config, is_compliance=False):
"""Check accuracy of given benchmark."""
benchmark_name = config["benchmark"]
accuracy_targets = {
BENCHMARKS.BERT: 90.874,
BENCHMARKS.DLRM: 80.25,
BENCHMARKS.RNNT: 100.0 - 7.45225,
BENCHMARKS.ResNet50: 76.46,
BENCHMARKS.SSDMobileNet: 22.0,
BENCHMARKS.SSDResNet34: 20.0,
BENCHMARKS.UNET: 0.853,
}
threshold_ratio = float(config["accuracy_level"][:-1]) / 100
if not os.path.exists(log_file):
return "Cannot find accuracy JSON file."
# checking if log_file is empty by just reading first several bytes
# indeed, first 4B~6B is likely all we need to check: '', '[]', '[]\r', '[\n]\n', '[\r\n]\r\n', ...
# but checking 8B for safety
with open(log_file, 'r') as lf:
first_8B = lf.read(8)
if not first_8B or ('[' in first_8B and ']' in first_8B):
return "No accuracy results in PerformanceOnly mode."
dtype_expand_map = {
"fp16": "float16",
"fp32": "float32",
"int8": "float16"
} # Use FP16 output for INT8 mode
# Since submission-checker uses a relative import, but we are running from main.py, we need to surface its directory
# into sys.path so it can successfully import it.
# Insert into index 1 so that current working directory still takes precedence.
sys.path.insert(
1,
os.path.join(os.getcwd(), "build", "inference", "tools", "submission"))
accuracy_regex_map = import_module("submission-checker").ACC_PATTERN
threshold = accuracy_targets[benchmark_name] * threshold_ratio
# Every benchmark has its own accuracy script. Prepare commandline with args to the script.
skip_run_command = False
if benchmark_name in [BENCHMARKS.ResNet50]:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-imagenet.py --mlperf-accuracy-file {:} \
--imagenet-val-file data_maps/imagenet/val_map.txt --dtype int32 ".format(
log_file)
regex = accuracy_regex_map["acc"]
elif benchmark_name == BENCHMARKS.SSDResNet34:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-coco.py --mlperf-accuracy-file {:} \
--coco-dir {:} --output-file build/ssd-resnet34-results.json --use-inv-map".format(
log_file,
os.path.join(
os.environ.get("PREPROCESSED_DATA_DIR",
"build/preprocessed_data"), "coco"))
regex = accuracy_regex_map["mAP"]
elif benchmark_name == BENCHMARKS.SSDMobileNet:
cmd = "python3 build/inference/vision/classification_and_detection/tools/accuracy-coco.py --mlperf-accuracy-file {:} \
--coco-dir {:} --output-file build/ssd-mobilenet-results.json".format(
log_file,
os.path.join(
os.environ.get("PREPROCESSED_DATA_DIR",
"build/preprocessed_data"), "coco"))
regex = accuracy_regex_map["mAP"]
elif benchmark_name == BENCHMARKS.BERT:
# Having issue installing tokenizers on Xavier...
if is_xavier():
cmd = "python3 code/bert/tensorrt/accuracy-bert.py --mlperf-accuracy-file {:} --squad-val-file {:}".format(
log_file,
os.path.join(os.environ.get("DATA_DIR", "build/data"), "squad",
"dev-v1.1.json"))
else:
dtype = config["precision"].lower()
if dtype in dtype_expand_map:
dtype = dtype_expand_map[dtype]
val_data_path = os.path.join(
os.environ.get("DATA_DIR", "build/data"), "squad",
"dev-v1.1.json")
vocab_file_path = "build/models/bert/vocab.txt"
if 'CPU' in config['config_name']:
vocab_file_path = "build/data/squad/vocab.txt"
output_prediction_path = os.path.join(os.path.dirname(log_file),
"predictions.json")
cmd = "python3 build/inference/language/bert/accuracy-squad.py " \
"--log_file {:} --vocab_file {:} --val_data {:} --out_file {:} " \
"--output_dtype {:}".format(log_file, vocab_file_path, val_data_path, output_prediction_path, dtype)
regex = accuracy_regex_map["F1"]
elif benchmark_name == BENCHMARKS.DLRM:
cmd = "python3 build/inference/recommendation/dlrm/pytorch/tools/accuracy-dlrm.py --mlperf-accuracy-file {:} " \
"--day-23-file build/data/criteo/day_23 --aggregation-trace-file " \
"build/preprocessed_data/criteo/full_recalib/sample_partition_trace.txt".format(log_file)
regex = accuracy_regex_map["AUC"]
elif benchmark_name == BENCHMARKS.RNNT:
# Having issue installing librosa on Xavier...
if is_xavier():
cmd = "python3 code/rnnt/tensorrt/accuracy.py --loadgen_log {:}".format(
log_file)
else:
# RNNT output indices are in INT8
cmd = "python3 build/inference/speech_recognition/rnnt/accuracy_eval.py " \
"--log_dir {:} --dataset_dir build/preprocessed_data/LibriSpeech/dev-clean-wav " \
"--manifest build/preprocessed_data/LibriSpeech/dev-clean-wav.json " \
"--output_dtype int8".format(os.path.dirname(log_file))
regex = accuracy_regex_map["WER"]
elif benchmark_name == BENCHMARKS.UNET:
postprocess_dir = "build/brats_postprocessed_data"
if not os.path.exists(postprocess_dir):
os.makedirs(postprocess_dir)
dtype = config["precision"].lower()
if dtype in dtype_expand_map:
dtype = dtype_expand_map[dtype]
cmd = "python3 build/inference/vision/medical_imaging/3d-unet/accuracy-brats.py --log_file {:} " \
"--output_dtype {:} --preprocessed_data_dir build/preprocessed_data/brats/brats_reference_preprocessed " \
"--postprocessed_data_dir {:} " \
"--label_data_dir build/preprocessed_data/brats/brats_reference_raw/Task043_BraTS2019/labelsTr".format(log_file, dtype, postprocess_dir)
regex = accuracy_regex_map["DICE"]
# Having issue installing nnUnet on Xavier...
if is_xavier():
# Internally, run on another node to process the accuracy.
try:
cmd = cmd.replace(os.getcwd(), ".", 1)
temp_cmd = "ssh -oBatchMode=yes computelab-frontend-02 \"timeout 1200 srun --gres=gpu:ga100:1 -t 20:00 " \
"bash -c 'cd {:} && make prebuild DOCKER_COMMAND=\\\"{:}\\\"'\"".format(os.getcwd(), cmd)
full_output = run_command(temp_cmd, get_output=True)
start_line_idx = -1
end_line_idx = -1
for (line_idx, line) in enumerate(full_output):
if "Please cite the following paper when using nnUNet:" in line:
start_line_idx = line_idx
if "Done!" in line:
end_line_idx = line_idx
assert start_line_idx != -1 and end_line_idx != -1, "Failed in accuracy checking"
output = full_output[start_line_idx:end_line_idx + 1]
skip_run_command = True
except Exception as e:
logging.warning(
"Accuracy checking for 3DUnet is not supported on Xavier. Please run the following command on desktop:\n{:}"
.format(cmd))
output = [
"Accuracy: mean = 1.0000, whole tumor = 1.0000, tumor core = 1.0000, enhancing tumor = 1.0000"
]
skip_run_command = True
else:
raise ValueError("Unknown benchmark: {:}".format(benchmark_name))
# Run benchmark's accuracy script and parse output for result.
if not skip_run_command:
output = run_command(cmd, get_output=True)
result_regex = re.compile(regex)
accuracy = None
with open(os.path.join(os.path.dirname(log_file), "accuracy.txt"),
"w") as f:
for line in output:
print(line, file=f)
for line in output:
result_match = result_regex.match(line)
if not result_match is None:
accuracy = float(result_match.group(1))
break
accuracy_result = "PASSED" if accuracy is not None and accuracy >= threshold else "FAILED"
if accuracy_result == "FAILED" and not is_compliance:
raise RuntimeError(
"Accuracy = {:.3f}, Threshold = {:.3f}. Accuracy test {:}!".format(
accuracy, threshold, accuracy_result))
if is_compliance:
return accuracy # Needed for numerical comparison
return "Accuracy = {:.3f}, Threshold = {:.3f}. Accuracy test {:}.".format(
accuracy, threshold, accuracy_result)