def run_tests(loader=None): """Runs tests of the variant specified by CLI arguments. If "cpu" is specified, CPU-only tests will be run and similarly for "gpu." Both are run if "both" is passed in. If "static" is specified, the tests are run per their order in the given static json file. Otherwise, the test order is automatically determined. """ parser.add_argument( "--type", help="Type of tests to run (one of: cpu, gpu, both)", required=True ) parser.add_argument( "--static", help="Static json w/ list of tests (use ONLY if NetworkX unavailable)", ) args = parser.parse_args() translator_path = os.path.join( Path(os.path.abspath(__file__)).parents[2], "res", "test", "translator.json" ) with open(translator_path) as f: tests_setup = json.load(f) if load_static or args.static: with open(args.static, "r") as f: ordered_json = json.load(f) ordered_tests = [] if (args.type == "both" or args.type == "cpu") and "cpu" in ordered_json: ordered_tests += ordered_json["cpu"] if (args.type == "both" or args.type == "gpu") and "gpu" in ordered_json: ordered_tests += ordered_json["gpu"] else: ordered_tests = get_ordered_tests(tests_setup, args.type) test_classes = [] for test in ordered_tests: test_classes.append(getattr(sys.modules[__name__], test)) generic_main(test_classes, loader)
with open(info_path, "r") as f: lines = f.readlines() # Relevant_results here refers to lines specifically in the format: # <timestamp> RigAligner.h:<line_number> Ceres Solver Report: Iterations: <iters>, Initial cost: <initial_cost>, Final cost: <final_cost>, Termination: CONVERGENCE record = {} metric_values = lines[-4].split("Final cost: ")[-1].strip() metric, _ = metric_values.split(", ", 1) record["final cost"] = float(metric) return record def test_run(self): """Run test for RigAligner. Raises: AssertionError: If incorrect results are produced. """ aligned_rig_fn = "rig_aligned.json" self.io_args.rig_reference = os.path.join(self.io_args.input_root, "rig_calibrated.json") self.io_args.rig_out = os.path.join(self.io_args.output_root, aligned_rig_fn) self.run_app("RigAligner") record = self.parse_align_results(self.io_args.log_dir) self.check_metrics(record) if __name__ == "__main__": generic_main([RigAlignerTest])
name (str): String representation of the class name. """ def test_run(self): """Run test for RigAnalyzer. Raises: AssertionError: If incorrect results are produced. """ rig_analysis_dir = "rig_analysis" analysis_root = os.path.join(self.io_args.output_root, rig_analysis_dir) os.makedirs(analysis_root, exist_ok=True) self.io_args.output_obj = os.path.join(analysis_root, "final.obj") self.io_args.output_equirect = os.path.join(analysis_root, "equirect.ppm") self.io_args.output_camera = os.path.join(analysis_root, "camera.ppm") self.io_args.output_camera_id = "0" self.io_args.output_cross_section = os.path.join( analysis_root, "cross.ppm") self.run_app("RigAnalyzer") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, rig_analysis_dir), output=analysis_root, ) if __name__ == "__main__": generic_main([RigAnalyzerTest])
Raises: AssertionError: If incorrect results are produced. """ CalibrationTest.setup_flags(self) log_file = os.path.join(self.io_args.log_dir, "CalibrationLibMain.INFO") # CalibrationLibMain assumes the operating frame to be 000000 lib_main_input = self.io_args.color_full + "_000000" if not os.path.exists(lib_main_input): shutil.copytree(self.io_args.color_full, lib_main_input) for cam in os.listdir(lib_main_input): cur_img = os.path.join(lib_main_input, cam, f"{self.io_args.first}.png") new_img = os.path.join(lib_main_input, cam, "000000.png") os.rename(cur_img, new_img) self.run_app( "CalibrationLibMain", args= f"{self.io_args.rig_out} {self.io_args.matches} {self.io_args.rig_in} {lib_main_input}", log_file=log_file, ) record = parse_calibration_results(self.io_args.log_dir, bin_name="CalibrationLibMain") self.check_metrics(record) if __name__ == "__main__": generic_main([CalibrationLibMainTest])
"""Run test for AlignColors. Raises: AssertionError: If incorrect results are produced. """ aligned_dir = "color_full_alt_aligned" self.io_args.output = os.path.join(self.io_args.output_root, aligned_dir) self.io_args.color = os.path.join(self.io_args.input_root, "color_full_alt") self.io_args.first = self.io_args.last = "000010" self.io_args.calibrated_rig = os.path.join(self.io_args.input_root, "rig_calibrated_alt.json") self.io_args.rig_red = os.path.join(self.io_args.input_root, "red.json") self.io_args.rig_green = os.path.join(self.io_args.input_root, "green.json") self.io_args.rig_blue = os.path.join(self.io_args.input_root, "blue.json") self.run_app("AlignColors") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, aligned_dir), output=self.io_args.output, ) if __name__ == "__main__": generic_main([AlignColorsTest])
modes_str = ( "mono_eqr,stereo_eqr,ftheta_ring,dodecahedron,icosahedron,rig_from_json" ) modes = modes_str.split(",") rig_simulations_dir = "rig_simulations" simulations_root = os.path.join(self.io_args.output_root, rig_simulations_dir) self.io_args.rig_in = self.io_args.rig self.io_args.skybox_path = os.path.join(self.io_args.testing_dir, "skybox.png") self.io_args.dest_mono = os.path.join(simulations_root, "mono.png") self.io_args.dest_mono_depth = os.path.join(simulations_root, "mono_depth.png") self.io_args.dest_left = os.path.join(simulations_root, "left.png") self.io_args.dest_right = os.path.join(simulations_root, "right.png") self.io_args.dest_stereo = os.path.join(simulations_root, "stereo.png") for mode in modes: self.io_args.mode = mode self.io_args.rig_out = os.path.join(simulations_root, f"{mode}.json") self.io_args.dest_cam_images = os.path.join(simulations_root, mode) os.makedirs(self.io_args.dest_cam_images, exist_ok=True) self.run_app("RigSimulator") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, rig_simulations_dir), output=simulations_root, ) if __name__ == "__main__": generic_main([RigSimulatorTest])
Raises: AssertionError: If incorrect results are produced. """ derp_command = self.run_app("DerpCLI") # Rephotography error is computed on a particular level (e.g. finest level) self.io_args.disparity = os.path.join(self.io_args.disparity_levels, config.TEST_LEVEL) self.io_args.color = os.path.join(self.io_args.color, config.TEST_LEVEL) rephoto_command = self.run_app("ComputeRephotographyErrors") derp_records = self.parse_rephoto_errors(self.io_args.log_dir) total_rephoto_error = (derp_records["error_r"] + derp_records["error_g"] + derp_records["error_b"]) / 3 record = { "test_name": self.__class__.__name__, "total_rephoto_error": total_rephoto_error, "r_rephoto_error": derp_records["error_r"], "g_rephoto_error": derp_records["error_g"], "b_rephoto_error": derp_records["error_b"], "derpcli_invocation": derp_command, "computerephotography_invocation": rephoto_command, } self.check_metrics(record) if __name__ == "__main__": generic_main([DerpCLITest])
info_path = os.path.join(log_dir, f"{bin_name}.INFO") with open(info_path, "r") as f: lines = f.readlines() for line in lines: if "Warning" in line: lines.remove(line) records = {} traces_str = "nonempty traces" traces_line = _get_line_with_str(lines, traces_str, 0) traces_half = traces_line.split("found ")[1].strip() records["calibration_trace_count"] = int(traces_half.split(" ")[0]) error_str = "median" error_line = _get_line_with_str(lines, error_str, -1) error_half = error_line.split(error_str)[1].strip() records["calibration_median_error"] = float(error_half.split(" ")[0]) if parse_timing: timing_str = "Aggregate timing" timing_line = _get_line_with_str(lines, timing_str, 0) times = _get_time_split(timing_line) records["calibration_cpu_time"] = (times["cpu"], ) records["calibration_wall_time"] = times["wall"] return records if __name__ == "__main__": generic_main([CalibrationTest])
class SimpleMeshRendererTest(DepTest): """Unit test class for SimpleMeshRenderer. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for SimpleMeshRenderer. Raises: AssertionError: If incorrect results are produced. """ meshes_dir = "meshes" formats_str = "cubecolor,cubedisp,eqrcolor,eqrdisp,lr180,snapcolor,snapdisp,tb3dof,tbstereo" formats = formats_str.split(",") self.io_args.color = os.path.join(self.io_args.color, config.TEST_LEVEL) self.io_args.disparity = os.path.join(self.io_args.disparity_levels, config.TEST_LEVEL) for format in formats: self.io_args.output = os.path.join(self.io_args.output_root, meshes_dir, format) self.io_args.format = format self.run_app("SimpleMeshRenderer") if __name__ == "__main__": generic_main([SimpleMeshRendererTest])
import os import test_config as config from test_master_class import DepTest, generic_main class ExportPointCloudTest(DepTest): """Unit test class for ExportPointCloud. Attributes: name (str): String representation of the class name. """ def test_run(self): point_cloud_fn = "point_cloud.xyz" self.io_args.color = os.path.join(self.io_args.color, config.TEST_LEVEL) self.io_args.disparity = os.path.join(self.io_args.disparity_levels, config.TEST_LEVEL) self.io_args.output = os.path.join(self.io_args.output_root, point_cloud_fn) self.run_app("ExportPointCloud") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, point_cloud_fn), output=self.io_args.output, ) if __name__ == "__main__": generic_main([ExportPointCloudTest])
Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for ConvertToBinary. Raises: AssertionError: If incorrect results are produced. """ fused_dir = "fused" binary_dir = os.path.join(self.io_args.output_root, fused_dir) os.makedirs(binary_dir, exist_ok=True) self.io_args.color = self.io_args.color_full self.io_args.disparity = os.path.join( self.io_args.disparity_levels, config.TEST_LEVEL ) self.io_args.bin = os.path.join(self.io_args.testing_dir, "bin") self.io_args.fused = binary_dir self.io_args.cameras = config.TEST_CAM self.run_app("ConvertToBinary") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, fused_dir), output=binary_dir ) if __name__ == "__main__": generic_main([ConvertToBinaryTest])
class GenerateForegroundMasksTest(DepTest): """Unit test class for GenerateForegroundMasks. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for GenerateForegroundMasks. Raises: AssertionError: If incorrect results are produced. """ self.io_args.color = os.path.join(self.io_args.color, config.TEST_LEVEL) self.io_args.background_color = os.path.join( self.io_args.background_color, config.TEST_LEVEL) self.io_args.foreground_masks = os.path.join( self.io_args.foreground_masks, config.TEST_LEVEL) self.run_app("GenerateForegroundMasks") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, "foreground_masks_levels", config.TEST_LEVEL), output=self.io_args.foreground_masks, ) if __name__ == "__main__": generic_main([GenerateForegroundMasksTest])
"""Unit test class for ProjectEquirectsToCameras. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for ProjectEquirectsToCameras. Raises: AssertionError: If incorrect results are produced. """ projected_dir = "equirect_projections" eqr_masks = "equirect_foreground_masks" self.io_args.eqr_masks = os.path.join(self.io_args.testing_dir, eqr_masks) self.io_args.first, self.io_args.last = min_max_frame_from_data_dir( self.io_args.eqr_masks) self.io_args.output = os.path.join(self.io_args.output_root, projected_dir) self.run_app("ProjectEquirectsToCameras") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, projected_dir), output=self.io_args.output, ) if __name__ == "__main__": generic_main([ProjectEquirectsToCamerasTest])
"""Unit test class for LayerDisparities. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for LayerDisparities. Raises: AssertionError: If incorrect results are produced. """ merged_disparities = "merged_disparities" self.io_args.background_disp = os.path.join( self.io_args.background_disp, config.TEST_LEVEL) self.io_args.foreground_disp = os.path.join( self.io_args.disparity_levels, config.TEST_LEVEL) self.io_args.output = os.path.join(self.io_args.output_root, merged_disparities) self.io_args.first, self.io_args.last = min_max_frame_from_data_dir( self.io_args.foreground_disp) self.run_app("LayerDisparities") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, merged_disparities), output=self.io_args.output, ) if __name__ == "__main__": generic_main([LayerDisparitiesTest])
import test_config as config from test_master_class import DepTest, generic_main class GenerateCameraOverlapsTest(DepTest): """Unit test class for GenerateCameraOverlaps. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for GenerateCameraOverlaps. Raises: AssertionError: If incorrect results are produced. """ overlap_dir = "overlaps" self.io_args.color = os.path.join(self.io_args.color, config.TEST_LEVEL) self.io_args.output = overlap_dir self.run_app("GenerateCameraOverlaps") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, overlap_dir), output=self.io_args.output, ) if __name__ == "__main__": generic_main([GenerateCameraOverlapsTest])
"foreground_masks_in": "", "foreground_masks_out": foreground_masks_out, }, } disparity_upsample_dir = "disparity_upsample" self.io_args.color = os.path.join(self.io_args.color, output_level) self.io_args.background_disp = os.path.join( self.io_args.background_disp_levels, output_level) self.io_args.disparity = os.path.join(self.io_args.disparity_levels, input_level) for test_type, mask_paths in test_type_to_masks.items(): self.io_args.foreground_masks_in = mask_paths[ "foreground_masks_in"] self.io_args.foreground_masks_out = mask_paths[ "foreground_masks_out"] self.io_args.output = os.path.join(self.io_args.output_root, disparity_upsample_dir, test_type) self.run_app("UpsampleDisparity") self.check_against_truth( truth=os.path.join(self.io_args.truth_dir, disparity_upsample_dir, test_type), output=self.io_args.output, ) if __name__ == "__main__": generic_main([UpsampleDisparityTest])
dict[str, float]: Map of relevant metric names to their values. """ info_path = os.path.join(log_dir, "RigCompare.INFO") with open(info_path, "r") as f: lines = f.readlines() avg_results = lines[-5:] record = {} for line in avg_results: metric_value = line.split("-")[-1].strip() metric, value = metric_value.split(": ") record[metric.strip()] = float(value) return record def test_run(self): """Run test for RigCompare. Raises: AssertionError: If incorrect results are produced. """ self.io_args.reference = os.path.join(self.io_args.input_root, "rig_calibrated.json") self.run_app("RigCompare") record = self.parse_diffs(self.io_args.log_dir) self.check_metrics(record) if __name__ == "__main__": generic_main([RigCompareTest])
class RawToRgbTest(DepTest): """Unit test class for RawToRgb. Attributes: name (str): String representation of the class name. """ def test_run(self): """Run test for RawToRgb. Raises: AssertionError: If incorrect results are produced. """ processed_image = "000000.png" self.io_args.input_image_path = os.path.join(self.io_args.input_root, "000000.raw") self.io_args.output_image_path = os.path.join(self.io_args.testing_dir, processed_image) self.io_args.isp_config_path = os.path.join(self.io_args.input_root, "isp.json") self.run_app("RawToRgb") self.check_against_truth( truth=os.path.join(self.io_args.testing_dir, processed_image), output=self.io_args.output_image_path, ) if __name__ == "__main__": generic_main([RawToRgbTest])