from hyo2.qc.qctools.qt_progress import QtProgress from hyo2.qc.common.grid_callback.qt_grid_callback import QtGridCallback app = QtGui.QApplication([]) wid = QtGui.QWidget() prj = SurveyProject(output_folder=testing.output_data_folder(), progress=QtProgress(parent=wid)) prj.set_callback(QtGridCallback(progress=prj.progress)) # add BAG files bag_files = testing.input_test_files(".bag") # logger.info("S57 files: %d" % len(bag_files)) # prj.add_to_grid_list(bag_files[0]) prj.add_to_grid_list( "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\test data\\H12976\\H12976_MB_1m_MLLW_1of2.bag" ) prj.add_to_s57_list( "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\test data\\H12976\\H12976_FFF_Test.000" ) four_gb = 4294967296 one_mb = 1048576 specs = "2016" prj.clear_survey_label() prj.read_feature_file(feature_path=prj.s57_list[0]) prj.open_grid(path=prj.grid_list[0], chunk_size=four_gb) prj.designated_scan_v2(survey_scale=20000, neighborhood=True, specs=specs) saved = prj.save_designated() if saved:
) logger.debug(prj) # add a grid file # csar_files = testing.input_test_files(".csar") # print("- CSAR files: %d" % len(csar_files)) # bag_files = testing.input_test_files(".bag") # print("- BAG files: %d" % len(bag_files)) # prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\testing\\survey\\valsou_check\\test1\\H12976_1m_clip.csar") # prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\testing\\survey\\valsou_check\\test1\\H12976_1m_clip.bag") #prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\testing\\survey\\valsou_check\\test1\\H12976_VR_clip.csar") #prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\testing\\survey\\valsou_check\\test1\\H12976_VR_clip.bag") # prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\survey\\VALSOU Check\\H12881\\H12881_MB_1m_MLLW_1of5.bag") # prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\survey\\VALSOU Check\\H12881\\H12881_MB_2m_MLLW_Final.csar") prj.add_to_grid_list( "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\_issues\\VALSOU_Check_8_27_2018\\H13022_MB_50cm_MLLW_final.csar" ) prj.add_to_grid_list( "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\_issues\\VALSOU_Check_8_27_2018\\H13022_MB_1m_MLLW_final.csar" ) four_gb = 4294967296 one_mb = 1048576 with_laser = True survey_scale = 20000 specs_version = "2018" is_target_detection = True for s57_file in prj.s57_list: logger.debug("s57: %s" % s57_file)
if not os.path.isdir(output_folder): logger.error("output folder is actually not a folder: %s" % output_folder) exit(-1) logger.debug("output folder: %s" % output_folder) # create the project prj = SurveyProject(output_folder=output_folder) # add all the BAG files to the project for bag_path in bag_todo_list: prj.add_to_grid_list(bag_path) # create 3 empty lists: one for the BAGs successfully processed, one for the ones without Uncertainty lauyer, # and another for crashes bag_done_list = list() bag_without_uncertainty_list = list() bag_crash_list = list() # actually processing the BAG files for i, bag_path in enumerate(prj.grid_list): try: # Trapping Exceptions like OSError (File permissions) logger.debug(">>>>>> #%03d (%s): processing ..." % (i, bag_path))
def run_qc_tools_v3(grid_path, flier_finder, holiday_finder, holiday_finder_mode, grid_qa, survey_name, output_shp=False, output_kml=False): from hyo2.qc.survey.project import SurveyProject from hyo2.qc import __version__ print("\n-->> running QC Tools (v.%s)" % __version__) grid_folder, grid_name = os.path.split(grid_path) prj = SurveyProject(output_folder=grid_folder) # disable output formats except .000 prj.output_svp = False prj.output_shp = False prj.output_kml = False prj.add_to_grid_list(grid_path) if survey_name is None: prj.clear_survey_label() else: prj.survey_label = survey_name if flier_finder: print('running flier finder on: %s' % grid_path) prj.set_cur_grid(path=grid_path) prj.open_to_read_cur_grid() prj.find_fliers_v8(height=None) saved = prj.save_fliers() if saved: print('- found fliers: %d' % prj.number_of_fliers()) else: print('- no fliers found') if holiday_finder: print('running holiday finder on: %s' % grid_path) prj.find_holes_v4(path=grid_path, mode=holiday_finder_mode) prj.output_shp = output_shp prj.output_kml = output_kml saved = prj.save_holes() if saved: print('- found holidays: certain %d, possible %d' % (prj.number_of_certain_holes(), prj.number_of_possible_holes())) else: print('- no holidays') if grid_qa: print('running grid qa on: %s' % grid_path) prj.set_cur_grid(path=grid_path) prj.open_to_read_cur_grid() tvu_qc_layers = prj.cur_grid_tvu_qc_layers() if len(tvu_qc_layers) > 0: prj.set_cur_grid_tvu_qc_name(tvu_qc_layers[0]) ret = prj.grid_qa_v5() print("- passed? %r" % ret) if grid_qa: prj.open_gridqa_output_folder() return if holiday_finder: prj.open_holes_output_folder() return if flier_finder: prj.open_fliers_output_folder() return
wid = QtWidgets.QWidget() # create the project prj = SurveyProject(output_folder=testing.output_data_folder(), progress=QtProgress(parent=wid)) # add a CSAR file # csar_files = testing.input_test_files(".csar") # print("- CSAR files: %d" % len(csar_files)) # add a BAG file bag_files = testing.input_test_files(".bag") print("- BAG files: %d" % len(bag_files)) kluster_file = r"C:\code\kluster\kluster\test_data\srgrid_mean_auto_depth_20211127_201258" prj.add_to_grid_list(kluster_file) # prj.add_to_grid_list(csar_files[0]) # prj.add_to_grid_list(csar_files[1]) # prj.add_to_grid_list(bag_files[0]) # prj.add_to_grid_list(bag_files[1]) # prj.add_to_grid_list(bag_files[2]) # prj.add_to_grid_list(csar_file) print("%s" % (prj.grid_list, )) four_gb = 4294967296 one_mb = 1048576 force_tvu_qc = True calc_object_detection = False
app = QtWidgets.QApplication([]) wid = QtWidgets.QWidget() # create the project prj = SurveyProject(output_folder=testing.output_data_folder(), progress=QtProgress(parent=wid)) # add a CSAR file csar_files = testing.input_test_files(".csar") print("- CSAR files: %d" % len(csar_files)) # # # add a BAG file # bag_files = testing.input_test_files(".bag") # print("- BAG files: %d" % len(bag_files)) prj.add_to_grid_list(csar_files[0]) # prj.add_to_grid_list(bag_files[0]) # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/H12880_2806_2016DN149_Ranges_CUBE.csar") # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/H12880_10_1_0_Ranges_CUBE_Final.csar") # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/Test0_H12280_2806_200kHz_DN149_CalderRice.csar") # prj.add_to_grid_list("C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\_issues\\H13034_Holiday_Finder_Issue_4_23_2018\\H13034_MB_2m_MLLW_Final.csar") print("%s" % (prj.grid_list, )) #four_gb = 4294967296 # one_mb = 1048576 # for grid_path in prj.grid_list: prj.clear_survey_label() # v4 prj.find_holes_v4(path=grid_path,
# # add a CSAR file # csar_files = testing.input_test_files(".csar") # print("- CSAR files: %d" % len(csar_files)) # # # add a BAG file # bag_files = testing.input_test_files(".bag") # print("- BAG files: %d" % len(bag_files)) # prj.add_to_grid_list(csar_files[0]) # prj.add_to_grid_list(bag_files[0]) # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/H12880_2806_2016DN149_Ranges_CUBE.csar") # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/H12880_10_1_0_Ranges_CUBE_Final.csar") # prj.add_to_grid_list("V:/CARIS_VR/H12880/Surfaces/Test0_H12280_2806_200kHz_DN149_CalderRice.csar") prj.add_to_grid_list( "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\data\\_issues\\H13034_Holiday_Finder_Issue_4_23_2018\\H13034_MB_2m_MLLW_Final.csar" ) print("%s" % (prj.grid_list, )) #four_gb = 4294967296 # one_mb = 1048576 # for grid_path in prj.grid_list: prj.clear_survey_label() # v4 prj.find_holes_v4(path=grid_path, sizer="THREE_TIMES", mode="FULL_COVERAGE", local_perimeter=True, max_size=0,
s57_idx = 0 s57_files = testing.input_test_files(".000") logger.debug("- test S57 files: %d" % len(s57_files)) logger.debug("- adding test S57 file #%d" % s57_idx) prj.add_to_s57_list(s57_path=s57_files[s57_idx]) # add a grid file grid_idx = 0 if use_internal_csar: grid_files = testing.input_test_files(".csar") logger.debug("- test CSAR files: %d" % len(grid_files)) else: grid_files = testing.input_test_files(".bag") logger.debug("- test BAG files: %d" % len(grid_files)) logger.debug("- adding test grid file #%d" % grid_idx) prj.add_to_grid_list(path=grid_files[grid_idx]) else: prj.add_to_s57_list(r"C:\code\hyo2\processing\hyo2_qc\data\download\H13384_FFF_testing_subset.000") prj.add_to_grid_list(r"C:\code\hyo2\processing\hyo2_qc\data\download\H13384_50cm_MLLW_testing_subset.csar") logger.debug(prj) for s57_file in prj.s57_list: logger.debug("s57: %s" % s57_file) prj.clear_survey_label() prj.read_feature_file(feature_path=s57_file) for grid_path in prj.grid_list: logger.debug("grid: %s" % grid_path)
def run_find_fliers(self, args): if not os.path.exists(args.output_folder): raise RuntimeError('Unable to locate output folder: %s' % args.output_folder) out_folder = args.output_folder logger.debug('output folder: %s' % out_folder) # create the project prj = SurveyProject(output_folder=out_folder) if not os.path.exists(args.input_dtm): raise RuntimeError('Unable to locate input DTM: %s' % args.input_dtm) dtm_file = args.input_dtm logger.debug('input DTM: %s' % dtm_file) prj.add_to_grid_list(dtm_file) if args.s57_path is not None: if not os.path.exists(args.s57_path): raise RuntimeError('Unable to locate input S57: %s' % args.s57_path) s57_file = args.s57_path logger.debug('input S57: %s' % s57_file) prj.add_to_s57_list(s57_file) if args.enforce_height is not None: if args.enforce_height <= 0.0: raise RuntimeError('Invalid height: %s' % args.enforce_height) height_value = args.enforce_height # logger.debug('height: %s' % height_value) check_laplacian = args.check_laplacian check_curv = args.check_curv check_adjacent = args.check_adjacent check_slivers = args.check_slivers check_isolated = args.check_isolated check_edges = args.check_edges filter_designated = args.filter_designated filter_fff = args.filter_fff self._check_web_page( token='FFv8_%d%d%d%d%d%d_%d%d' % (check_laplacian, check_curv, check_adjacent, check_slivers, check_isolated, check_edges, filter_designated, filter_fff)) # actual execution for i, grid_path in enumerate(prj.grid_list): logger.debug(">>> #%d (%s)" % (i, grid_path)) prj.clear_survey_label() prj.set_cur_grid(path=grid_path) prj.open_to_read_cur_grid(chunk_size=4294967296) prj.find_fliers_v8(height=height_value, check_laplacian=check_laplacian, check_curv=check_curv, check_adjacent=check_adjacent, check_slivers=check_slivers, check_isolated=check_isolated, check_edges=check_edges, filter_fff=filter_fff, filter_designated=filter_designated) prj.close_cur_grid() prj.set_cur_grid(path=grid_path) prj.open_to_read_cur_grid(chunk_size=4294967296) prj.find_fliers_v8_apply_filters() saved = prj.save_fliers() if saved: logger.debug('Fliers saved')
app = QtWidgets.QApplication([]) wid = QtWidgets.QWidget() # create the project prj = SurveyProject(output_folder=testing.output_data_folder(), progress=QtProgress(parent=wid)) csar_files = testing.input_test_files(".csar") logger.debug("test CSAR files: %d" % len(csar_files)) bag_files = testing.input_test_files(".bag") logger.debug("test BAG files: %d" % len(bag_files)) # prj.add_to_grid_list(csar_files[0]) # prj.add_to_grid_list(bag_files[0]) prj.add_to_grid_list( r"C:\Users\gmasetti\Google Drive\QC Tools\data\survey\Find Fliers\v8_filters\test_finalized.csar" ) prj.add_to_grid_list( r"C:\Users\gmasetti\Google Drive\QC Tools\data\survey\Find Fliers\v8_filters\test_finalized.bag" ) prj.add_to_s57_list( r"C:\Users\gmasetti\Google Drive\QC Tools\data\survey\Find Fliers\v8_filters\v8_filters.000" ) logger.debug("grid list: %s" % (prj.grid_list, )) for i, grid_path in enumerate(prj.grid_list): logger.debug(">>> #%d (%s)" % (i, grid_path)) prj.clear_survey_label()
check_structure: bool = True check_metadata: bool = True check_elevation: bool = True check_uncertainty: bool = True check_tracking_list: bool = True prj = SurveyProject(output_folder=testing.output_data_folder(), progress=QtProgress(parent=wid)) if use_internal_test_files: # add a grid file grid_idx = 0 grid_files = testing.input_test_files(".bag") logger.debug("- test BAG files: %d" % len(grid_files)) logger.debug("- adding test grid file #%d" % grid_idx) prj.add_to_grid_list(path=grid_files[grid_idx]) else: prj.add_to_grid_list( r"C:\code\hyo2\processing\hyo2_qc\data\input\tiny_vr.bag") logger.debug(prj) prj.bag_checks_v1(use_nooa_nbs_profile=use_noaa_nbs_profile, check_structure=check_structure, check_metadata=check_metadata, check_elevation=check_elevation, check_uncertainty=check_uncertainty, check_tracking_list=check_tracking_list) logger.debug(prj.bag_checks_message)
# add a CSAR file # csar_files = testing.input_test_files(".csar") # print("- CSAR files: %d" % len(csar_files)) # # add a BAG file bag_files = testing.input_test_files(".bag") # print("- BAG files: %d" % len(bag_files)) # csar_file = "C:/Users/giumas/Google Drive/Testing_Datasets/ObjectDetection_H12676/H12676_10_2_2_Ranges_ObjectDetection_Holidays_finalized.csar" # csar_file = "C:/Users/giumas/Google Drive/Testing_Datasets/ObjectDetection_H12676/H12676_10_2_2_Ranges_ObjectDetection_Holidays.csar" # csar_file = "C:\\Users\\giumas\\Google Drive\\QC Tools\\test data\\H12924\\H12924_MB_1m_MLLW_Final.csar" csar_file = "C:\\Users\\gmasetti\\Google Drive\\QC Tools\\test data\\W00341\\W00341_MB_VR_MLLW_Final.csar" # prj.add_to_grid_list(csar_files[0]) # prj.add_to_grid_list(csar_files[1]) # prj.add_to_grid_list(bag_files[0]) prj.add_to_grid_list(csar_file) print("%s" % (prj.grid_list, )) four_gb = 4294967296 one_mb = 1048576 force_tvu_qc = True calc_object_detection = False calc_full_coverage = True hist_depth = True hist_density = True hist_tvu_qc = True hist_pct_res = True