def run_tests(loader=None):
"""Runs tests of the variant specified by CLI arguments. If "cpu" is specified,
CPU-only tests will be run and similarly for "gpu." Both are run if "both" is
passed in. If "static" is specified, the tests are run per their order in the
given static json file. Otherwise, the test order is automatically determined.
"""
parser.add_argument(
"--type", help="Type of tests to run (one of: cpu, gpu, both)", required=True
)
parser.add_argument(
"--static",
help="Static json w/ list of tests (use ONLY if NetworkX unavailable)",
)
args = parser.parse_args()
translator_path = os.path.join(
Path(os.path.abspath(__file__)).parents[2], "res", "test", "translator.json"
)
with open(translator_path) as f:
tests_setup = json.load(f)
if load_static or args.static:
with open(args.static, "r") as f:
ordered_json = json.load(f)
ordered_tests = []
if (args.type == "both" or args.type == "cpu") and "cpu" in ordered_json:
ordered_tests += ordered_json["cpu"]
if (args.type == "both" or args.type == "gpu") and "gpu" in ordered_json:
ordered_tests += ordered_json["gpu"]
else:
ordered_tests = get_ordered_tests(tests_setup, args.type)
test_classes = []
for test in ordered_tests:
test_classes.append(getattr(sys.modules[__name__], test))
generic_main(test_classes, loader)
with open(info_path, "r") as f:
lines = f.readlines()
# Relevant_results here refers to lines specifically in the format:
# <timestamp> RigAligner.h:<line_number> Ceres Solver Report: Iterations: <iters>, Initial cost: <initial_cost>, Final cost: <final_cost>, Termination: CONVERGENCE
record = {}
metric_values = lines[-4].split("Final cost: ")[-1].strip()
metric, _ = metric_values.split(", ", 1)
record["final cost"] = float(metric)
return record
def test_run(self):
"""Run test for RigAligner.
Raises:
AssertionError: If incorrect results are produced.
"""
aligned_rig_fn = "rig_aligned.json"
self.io_args.rig_reference = os.path.join(self.io_args.input_root,
"rig_calibrated.json")
self.io_args.rig_out = os.path.join(self.io_args.output_root,
aligned_rig_fn)
self.run_app("RigAligner")
record = self.parse_align_results(self.io_args.log_dir)
self.check_metrics(record)
if __name__ == "__main__":
generic_main([RigAlignerTest])
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for RigAnalyzer.
Raises:
AssertionError: If incorrect results are produced.
"""
rig_analysis_dir = "rig_analysis"
analysis_root = os.path.join(self.io_args.output_root,
rig_analysis_dir)
os.makedirs(analysis_root, exist_ok=True)
self.io_args.output_obj = os.path.join(analysis_root, "final.obj")
self.io_args.output_equirect = os.path.join(analysis_root,
"equirect.ppm")
self.io_args.output_camera = os.path.join(analysis_root, "camera.ppm")
self.io_args.output_camera_id = "0"
self.io_args.output_cross_section = os.path.join(
analysis_root, "cross.ppm")
self.run_app("RigAnalyzer")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, rig_analysis_dir),
output=analysis_root,
)
if __name__ == "__main__":
generic_main([RigAnalyzerTest])
Raises:
AssertionError: If incorrect results are produced.
"""
CalibrationTest.setup_flags(self)
log_file = os.path.join(self.io_args.log_dir,
"CalibrationLibMain.INFO")
# CalibrationLibMain assumes the operating frame to be 000000
lib_main_input = self.io_args.color_full + "_000000"
if not os.path.exists(lib_main_input):
shutil.copytree(self.io_args.color_full, lib_main_input)
for cam in os.listdir(lib_main_input):
cur_img = os.path.join(lib_main_input, cam,
f"{self.io_args.first}.png")
new_img = os.path.join(lib_main_input, cam, "000000.png")
os.rename(cur_img, new_img)
self.run_app(
"CalibrationLibMain",
args=
f"{self.io_args.rig_out} {self.io_args.matches} {self.io_args.rig_in} {lib_main_input}",
log_file=log_file,
)
record = parse_calibration_results(self.io_args.log_dir,
bin_name="CalibrationLibMain")
self.check_metrics(record)
if __name__ == "__main__":
generic_main([CalibrationLibMainTest])
"""Run test for AlignColors.
Raises:
AssertionError: If incorrect results are produced.
"""
aligned_dir = "color_full_alt_aligned"
self.io_args.output = os.path.join(self.io_args.output_root,
aligned_dir)
self.io_args.color = os.path.join(self.io_args.input_root,
"color_full_alt")
self.io_args.first = self.io_args.last = "000010"
self.io_args.calibrated_rig = os.path.join(self.io_args.input_root,
"rig_calibrated_alt.json")
self.io_args.rig_red = os.path.join(self.io_args.input_root,
"red.json")
self.io_args.rig_green = os.path.join(self.io_args.input_root,
"green.json")
self.io_args.rig_blue = os.path.join(self.io_args.input_root,
"blue.json")
self.run_app("AlignColors")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, aligned_dir),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([AlignColorsTest])
modes_str = (
"mono_eqr,stereo_eqr,ftheta_ring,dodecahedron,icosahedron,rig_from_json"
)
modes = modes_str.split(",")
rig_simulations_dir = "rig_simulations"
simulations_root = os.path.join(self.io_args.output_root, rig_simulations_dir)
self.io_args.rig_in = self.io_args.rig
self.io_args.skybox_path = os.path.join(self.io_args.testing_dir, "skybox.png")
self.io_args.dest_mono = os.path.join(simulations_root, "mono.png")
self.io_args.dest_mono_depth = os.path.join(simulations_root, "mono_depth.png")
self.io_args.dest_left = os.path.join(simulations_root, "left.png")
self.io_args.dest_right = os.path.join(simulations_root, "right.png")
self.io_args.dest_stereo = os.path.join(simulations_root, "stereo.png")
for mode in modes:
self.io_args.mode = mode
self.io_args.rig_out = os.path.join(simulations_root, f"{mode}.json")
self.io_args.dest_cam_images = os.path.join(simulations_root, mode)
os.makedirs(self.io_args.dest_cam_images, exist_ok=True)
self.run_app("RigSimulator")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, rig_simulations_dir),
output=simulations_root,
)
if __name__ == "__main__":
generic_main([RigSimulatorTest])
Raises:
AssertionError: If incorrect results are produced.
"""
derp_command = self.run_app("DerpCLI")
# Rephotography error is computed on a particular level (e.g. finest level)
self.io_args.disparity = os.path.join(self.io_args.disparity_levels,
config.TEST_LEVEL)
self.io_args.color = os.path.join(self.io_args.color,
config.TEST_LEVEL)
rephoto_command = self.run_app("ComputeRephotographyErrors")
derp_records = self.parse_rephoto_errors(self.io_args.log_dir)
total_rephoto_error = (derp_records["error_r"] +
derp_records["error_g"] +
derp_records["error_b"]) / 3
record = {
"test_name": self.__class__.__name__,
"total_rephoto_error": total_rephoto_error,
"r_rephoto_error": derp_records["error_r"],
"g_rephoto_error": derp_records["error_g"],
"b_rephoto_error": derp_records["error_b"],
"derpcli_invocation": derp_command,
"computerephotography_invocation": rephoto_command,
}
self.check_metrics(record)
if __name__ == "__main__":
generic_main([DerpCLITest])
info_path = os.path.join(log_dir, f"{bin_name}.INFO")
with open(info_path, "r") as f:
lines = f.readlines()
for line in lines:
if "Warning" in line:
lines.remove(line)
records = {}
traces_str = "nonempty traces"
traces_line = _get_line_with_str(lines, traces_str, 0)
traces_half = traces_line.split("found ")[1].strip()
records["calibration_trace_count"] = int(traces_half.split(" ")[0])
error_str = "median"
error_line = _get_line_with_str(lines, error_str, -1)
error_half = error_line.split(error_str)[1].strip()
records["calibration_median_error"] = float(error_half.split(" ")[0])
if parse_timing:
timing_str = "Aggregate timing"
timing_line = _get_line_with_str(lines, timing_str, 0)
times = _get_time_split(timing_line)
records["calibration_cpu_time"] = (times["cpu"], )
records["calibration_wall_time"] = times["wall"]
return records
if __name__ == "__main__":
generic_main([CalibrationTest])
class SimpleMeshRendererTest(DepTest):
"""Unit test class for SimpleMeshRenderer.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for SimpleMeshRenderer.
Raises:
AssertionError: If incorrect results are produced.
"""
meshes_dir = "meshes"
formats_str = "cubecolor,cubedisp,eqrcolor,eqrdisp,lr180,snapcolor,snapdisp,tb3dof,tbstereo"
formats = formats_str.split(",")
self.io_args.color = os.path.join(self.io_args.color,
config.TEST_LEVEL)
self.io_args.disparity = os.path.join(self.io_args.disparity_levels,
config.TEST_LEVEL)
for format in formats:
self.io_args.output = os.path.join(self.io_args.output_root,
meshes_dir, format)
self.io_args.format = format
self.run_app("SimpleMeshRenderer")
if __name__ == "__main__":
generic_main([SimpleMeshRendererTest])
import os
import test_config as config
from test_master_class import DepTest, generic_main
class ExportPointCloudTest(DepTest):
"""Unit test class for ExportPointCloud.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
point_cloud_fn = "point_cloud.xyz"
self.io_args.color = os.path.join(self.io_args.color,
config.TEST_LEVEL)
self.io_args.disparity = os.path.join(self.io_args.disparity_levels,
config.TEST_LEVEL)
self.io_args.output = os.path.join(self.io_args.output_root,
point_cloud_fn)
self.run_app("ExportPointCloud")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, point_cloud_fn),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([ExportPointCloudTest])
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for ConvertToBinary.
Raises:
AssertionError: If incorrect results are produced.
"""
fused_dir = "fused"
binary_dir = os.path.join(self.io_args.output_root, fused_dir)
os.makedirs(binary_dir, exist_ok=True)
self.io_args.color = self.io_args.color_full
self.io_args.disparity = os.path.join(
self.io_args.disparity_levels, config.TEST_LEVEL
)
self.io_args.bin = os.path.join(self.io_args.testing_dir, "bin")
self.io_args.fused = binary_dir
self.io_args.cameras = config.TEST_CAM
self.run_app("ConvertToBinary")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, fused_dir), output=binary_dir
)
if __name__ == "__main__":
generic_main([ConvertToBinaryTest])
class GenerateForegroundMasksTest(DepTest):
"""Unit test class for GenerateForegroundMasks.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for GenerateForegroundMasks.
Raises:
AssertionError: If incorrect results are produced.
"""
self.io_args.color = os.path.join(self.io_args.color,
config.TEST_LEVEL)
self.io_args.background_color = os.path.join(
self.io_args.background_color, config.TEST_LEVEL)
self.io_args.foreground_masks = os.path.join(
self.io_args.foreground_masks, config.TEST_LEVEL)
self.run_app("GenerateForegroundMasks")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir,
"foreground_masks_levels", config.TEST_LEVEL),
output=self.io_args.foreground_masks,
)
if __name__ == "__main__":
generic_main([GenerateForegroundMasksTest])
"""Unit test class for ProjectEquirectsToCameras.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for ProjectEquirectsToCameras.
Raises:
AssertionError: If incorrect results are produced.
"""
projected_dir = "equirect_projections"
eqr_masks = "equirect_foreground_masks"
self.io_args.eqr_masks = os.path.join(self.io_args.testing_dir,
eqr_masks)
self.io_args.first, self.io_args.last = min_max_frame_from_data_dir(
self.io_args.eqr_masks)
self.io_args.output = os.path.join(self.io_args.output_root,
projected_dir)
self.run_app("ProjectEquirectsToCameras")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, projected_dir),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([ProjectEquirectsToCamerasTest])
"""Unit test class for LayerDisparities.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for LayerDisparities.
Raises:
AssertionError: If incorrect results are produced.
"""
merged_disparities = "merged_disparities"
self.io_args.background_disp = os.path.join(
self.io_args.background_disp, config.TEST_LEVEL)
self.io_args.foreground_disp = os.path.join(
self.io_args.disparity_levels, config.TEST_LEVEL)
self.io_args.output = os.path.join(self.io_args.output_root,
merged_disparities)
self.io_args.first, self.io_args.last = min_max_frame_from_data_dir(
self.io_args.foreground_disp)
self.run_app("LayerDisparities")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, merged_disparities),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([LayerDisparitiesTest])
import test_config as config
from test_master_class import DepTest, generic_main
class GenerateCameraOverlapsTest(DepTest):
"""Unit test class for GenerateCameraOverlaps.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for GenerateCameraOverlaps.
Raises:
AssertionError: If incorrect results are produced.
"""
overlap_dir = "overlaps"
self.io_args.color = os.path.join(self.io_args.color,
config.TEST_LEVEL)
self.io_args.output = overlap_dir
self.run_app("GenerateCameraOverlaps")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir, overlap_dir),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([GenerateCameraOverlapsTest])
"foreground_masks_in": "",
"foreground_masks_out": foreground_masks_out,
},
}
disparity_upsample_dir = "disparity_upsample"
self.io_args.color = os.path.join(self.io_args.color, output_level)
self.io_args.background_disp = os.path.join(
self.io_args.background_disp_levels, output_level)
self.io_args.disparity = os.path.join(self.io_args.disparity_levels,
input_level)
for test_type, mask_paths in test_type_to_masks.items():
self.io_args.foreground_masks_in = mask_paths[
"foreground_masks_in"]
self.io_args.foreground_masks_out = mask_paths[
"foreground_masks_out"]
self.io_args.output = os.path.join(self.io_args.output_root,
disparity_upsample_dir,
test_type)
self.run_app("UpsampleDisparity")
self.check_against_truth(
truth=os.path.join(self.io_args.truth_dir,
disparity_upsample_dir, test_type),
output=self.io_args.output,
)
if __name__ == "__main__":
generic_main([UpsampleDisparityTest])
dict[str, float]: Map of relevant metric names to their values.
"""
info_path = os.path.join(log_dir, "RigCompare.INFO")
with open(info_path, "r") as f:
lines = f.readlines()
avg_results = lines[-5:]
record = {}
for line in avg_results:
metric_value = line.split("-")[-1].strip()
metric, value = metric_value.split(": ")
record[metric.strip()] = float(value)
return record
def test_run(self):
"""Run test for RigCompare.
Raises:
AssertionError: If incorrect results are produced.
"""
self.io_args.reference = os.path.join(self.io_args.input_root,
"rig_calibrated.json")
self.run_app("RigCompare")
record = self.parse_diffs(self.io_args.log_dir)
self.check_metrics(record)
if __name__ == "__main__":
generic_main([RigCompareTest])
class RawToRgbTest(DepTest):
"""Unit test class for RawToRgb.
Attributes:
name (str): String representation of the class name.
"""
def test_run(self):
"""Run test for RawToRgb.
Raises:
AssertionError: If incorrect results are produced.
"""
processed_image = "000000.png"
self.io_args.input_image_path = os.path.join(self.io_args.input_root,
"000000.raw")
self.io_args.output_image_path = os.path.join(self.io_args.testing_dir,
processed_image)
self.io_args.isp_config_path = os.path.join(self.io_args.input_root,
"isp.json")
self.run_app("RawToRgb")
self.check_against_truth(
truth=os.path.join(self.io_args.testing_dir, processed_image),
output=self.io_args.output_image_path,
)
if __name__ == "__main__":
generic_main([RawToRgbTest])