def test_clm_clip_latlon(self): if os.environ.get('TRAVIS'): pass elif os.path.isfile(test_files.conus1_latlon): bbox_list = test_files.huc10190004.get('conus1_bbox') bbox = BBox(bbox_list[0], bbox_list[1], bbox_list[2], bbox_list[3]) clm_clipper = ClmClipper(bbox) latlon_formatted, latlon_data = clm_clipper.clip_latlon( test_files.conus1_latlon) clm_clipper.write_lat_lon(latlon_formatted, 'WBDHU8_latlon_test.sa', x=latlon_data.shape[2], y=latlon_data.shape[1], z=latlon_data.shape[0]) self.assertIsNone( np.testing.assert_array_equal( file_io_tools.read_file('WBDHU8_latlon_test.sa'), file_io_tools.read_file( test_files.huc10190004.get( 'conus1_latlon').as_posix())), 'writing and reading a tif gives back the same array values') os.remove('WBDHU8_latlon_test.sa') else: print( 'WARNING! Unable to run test test_clm_clip_latlon because source file not found. ' 'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!' ) pass
def test_clm_clip_latlon_padded(self): if os.environ.get('TRAVIS'): pass elif os.path.isfile(test_files.conus1_latlon): my_mask = SubsetMask(test_files.huc10190004.get('conus1_padded_mask')) bbox = my_mask.get_bbox() self.assertSequenceEqual((9, 9, 9, 9), bbox.get_padding()) clm_clipper = ClmClipper(bbox) latlon_formatted, latlon_data = clm_clipper.clip_latlon(test_files.conus1_latlon) clm_clipper.write_lat_lon(latlon_formatted, 'WBDHU8_latlon_test.sa', x=latlon_data.shape[2], y=latlon_data.shape[1], z=latlon_data.shape[0]) self.assertSequenceEqual((1, 48, 103), latlon_data.shape, 'check that clm clip when mask has padding works') os.remove('WBDHU8_latlon_test.sa') else: print('WARNING! Unable to run test test_clm_clip_latlon because source file not found. ' 'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!') pass
def test_clm_clip_latlon(self): if os.environ.get('TRAVIS'): pass elif os.path.isfile(test_files.conus1_latlon): bbox_list = test_files.huc10190004.get('conus1_bbox') bbox = BBox(bbox_list[0], bbox_list[1], bbox_list[2], bbox_list[3]) clm_clipper = ClmClipper(bbox) latlon_formatted, latlon_data = clm_clipper.clip_latlon(test_files.conus1_latlon) self.assertSequenceEqual((1, 30, 85), latlon_data.shape, 'check that clm clip when mask has no padding works') clm_clipper.write_lat_lon(latlon_formatted, 'WBDHU8_latlon_test.sa', x=latlon_data.shape[2], y=latlon_data.shape[1], z=latlon_data.shape[0]) self.assertIsNone(np.testing.assert_array_equal(file_io_tools.read_file('WBDHU8_latlon_test.sa'), file_io_tools.read_file( test_files.huc10190004.get('conus1_latlon'))), 'clipping conus1 lat/lon values regression test works') os.remove('WBDHU8_latlon_test.sa') else: print('WARNING! Unable to run test test_clm_clip_latlon because source file not found. ' 'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!') pass
def test_clm_clip_land_cover(self): if os.environ.get('TRAVIS'): pass elif os.path.isfile(test_files.conus1_latlon): bbox_list = test_files.huc10190004.get('conus1_bbox') bbox = BBox(bbox_list[0], bbox_list[1], bbox_list[2], bbox_list[3]) clm_clipper = ClmClipper(bbox) latlon_data, _ = clm_clipper.clip_latlon(test_files.conus1_latlon) land_cover_data, vegm_data = clm_clipper.clip_land_cover( lat_lon_array=latlon_data, land_cover_file=test_files.conus1_landcover) clm_clipper.write_land_cover(vegm_data, 'WBDHU8_vegm_test.dat') with open('WBDHU8_vegm_test.dat', 'r') as test_file: with open( test_files.huc10190004.get('conus1_vegm').as_posix(), 'r') as ref_file: self.assertEqual( test_file.read().split('\n'), ref_file.read().split('\n'), 'Writing vegm file matches reference for conus1') os.remove('WBDHU8_vegm_test.dat') else: print( 'WARNING! Unable to run test test_clm_clip_latlon because source file not found. ' 'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!' ) pass
def test_clm_clip_land_cover_padded(self): if os.environ.get('TRAVIS'): pass elif os.path.isfile(test_files.conus1_latlon): my_mask = SubsetMask(test_files.huc10190004.get('conus1_padded_mask')) bbox = my_mask.get_bbox() self.assertSequenceEqual((9, 9, 9, 9), bbox.get_padding()) clm_clipper = ClmClipper(bbox) latlon_data, _ = clm_clipper.clip_latlon(test_files.conus1_latlon) land_cover_data, vegm_data = clm_clipper.clip_land_cover(lat_lon_array=latlon_data, land_cover_file=test_files.conus1_landcover) clm_clipper.write_land_cover(vegm_data, 'WBDHU8_vegm_test.dat') os.remove('WBDHU8_vegm_test.dat') else: print('WARNING! Unable to run test test_clm_clip_latlon because source file not found. ' 'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!') pass
def subset_conus(input_path=None, shapefile=None, subset_tif=None, mask_value=1, conus_version=1, conus_files='.', out_dir='.', out_name=None, clip_clm=False, run_script=False, padding=(0, 0, 0, 0), attribute_name='OBJECTID', attribute_ids=None, write_tifs=False, manifest_file=conus_manifest): """subset a conus domain inputs for running a regional model Parameters ---------- input_path : str, optional path to input shapefile parts to use as mask. Only applicable if shapefile is also provided. shapefile : str, optional name of shapefile to use as mask. Required if subset_tif is not provided. subset_tif : str, optional path to tiff file containing mask. Required if shapefile is not provided. mask_value : int, optional integer value specifying the data value in the tiff file to consider as the masking value. Only applicable if subset_tif is provided. conus_version : int, optional version of the CONUS domain to use (1 or 2) (Default value = 1) conus_files : str, optional path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.') out_dir : str, optional directory to write the outputs (default .) out_name : str, optional name to give the outputs (default shapefile name) clip_clm : int, optional whether or not to clip the CLM input files too (default no) run_script : int, optional whether or not to build and return a Run object for the subset (default no) padding : tuple, optional grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0 attribute_name : str, optional attribute name defined in shapefile to select as mask default 'OBJECTID' attribute_ids : list, optional list of attribute ID's defined in shapefile to use as mask input. default [1] write_tifs : int, optional whether or not to write outputs as TIF's in addition to PFB's. (default no) Returns ------- run_script : parflow.tools.Run The Run object which can be used to execute the ParFlow model subset that was created by subset_conus """ assert any((shapefile, subset_tif)) and not all((shapefile, subset_tif)), \ 'Specify either a shapefile or a subset_tif file.' if out_name is None: out_name = shapefile or os.path.splitext( os.path.basename(subset_tif))[0] conus = Conus(version=conus_version, local_path=conus_files, manifest_file=manifest_file) if attribute_ids is None: attribute_ids = [1] # Step 1, rasterize shapefile if shapefile is not None: rasterizer = ShapefileRasterizer( input_path, shapefile, reference_dataset=conus.get_domain_tif(), no_data=TIF_NO_DATA_VALUE_OUT, output_path=out_dir, ) mask_array = rasterizer.rasterize_shapefile_to_disk( out_name=f'{out_name}_raster_from_shapefile.tif', padding=padding, attribute_name=attribute_name, attribute_ids=attribute_ids) subset_mask = rasterizer.subset_mask else: subset_mask = SubsetMask(tif_file=subset_tif, mask_value=mask_value) mask_array = subset_mask.mask_array # Step 2, Generate solid file clip = MaskClipper(subset_mask, no_data_threshold=-1) sfb = SolidFileBuilder(top=3, bottom=6, side=1).mask( clip.subset(mask_array, crop_inner=0)[0][0, :, :]) # identify the unique patch ID's assigned to the solid file #TODO: get patch defs from a class sfb.top_ids(clip.subset(conus.get_domain_mask())[0][0, :, :]) sfb.side_ids(clip.subset(conus.get_border_mask())[0][0, :, :]) top_patchIDs = np.unique(clip.subset(conus.get_domain_mask())[0][0, :, :]) side_patchIDs = np.unique(clip.subset(conus.get_border_mask())[0][0, :, :]) side_patchIDs[side_patchIDs == 0] = 2 botom_patchIDs = [6] patch_ids = np.unique( np.concatenate((top_patchIDs, side_patchIDs, botom_patchIDs))) sfb = sfb.write(os.path.join(out_dir, f'{out_name}.pfsol'), cellsize=1000, vtk=True) # Step 3. Clip all the domain data inputs bulk_clipper.clip_inputs(clip, [ os.path.join(conus.local_path, value) for key, value in conus.required_files.items() if key not in ['DOMAIN_MASK', 'CHANNELS'] ], out_dir=out_dir, tif_outs=write_tifs) # Step 4. Clip CLM inputs if clip_clm == 1: clm_clipper = ClmClipper(subset_mask.get_bbox()) latlon_formatted, latlon_data = clm_clipper.clip_latlon( os.path.join(conus.local_path, conus.optional_files.get('LAT_LON'))) clm_clipper.write_lat_lon(latlon_formatted, os.path.join(out_dir, f'{out_name}_latlon.sa'), x=latlon_data.shape[2], y=latlon_data.shape[1], z=latlon_data.shape[0]) land_cover_data, vegm_data = clm_clipper.clip_land_cover( lat_lon_array=latlon_formatted, land_cover_file=os.path.join( conus.local_path, conus.optional_files.get('LAND_COVER'))) clm_clipper.write_land_cover( vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat')) # Step 5. Generate Run Script if run_script == 1: # the Run object reads sys.argv, and this is problematic because they share a common flag -r sys.argv = ['Run'] slopex_file = os.path.join( out_dir, f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb') slopey_file = os.path.join( out_dir, f'{Path(conus.required_files.get("SLOPE_Y")).stem}_clip.pfb') solid_file = os.path.join(out_dir, f'{out_name}.pfsol') bbox = subset_mask.get_bbox() extents = bbox.get_padded_extents() NX = int(extents[3] - extents[2]) NY = int(extents[1] - extents[0]) out_name = f'{out_name}.conus{conus_version}.parking_lot' # TODO: associate model templates with models and versions, provide method to override boundary conditions run_script = parking_lot_template.get_parking_lot_model( out_name, slopex_file, slopey_file, solid_file, NX, NY) patch_names = [ conus.get_patch_name(patch_id) for patch_id in patch_ids[patch_ids > TIF_NO_DATA_VALUE_OUT] ] run_script.Geom.domain.Patches = ' '.join(patch_names) # convert patch ID's to patch names for run script for patch in patch_names: bc = parking_lot_template.get_parking_lot_model_boundary(patch) for k, v in bc.items(): # assign patch Boundary Conditions as defined by CONUS Model 1 or 2 run_script.Patch.pfset(key=f'{patch}.BCPressure.{k}', value=v) if conus_version == 1: # CONUS1 doesn't seem to work well with OverlandKinematic run_script.Patch.top.BCPressure.Type = 'OverlandFlow' run_script.validate() run_script.write(file_name=os.path.join(out_dir, out_name), file_format='pfidb') run_script.write(file_name=os.path.join(out_dir, out_name), file_format='yaml') run_script.write(file_name=os.path.join(out_dir, out_name), file_format='json') return run_script
def subset_conus(input_path, shapefile, conus_version=1, conus_files='.', out_dir='.', out_name=None, clip_clm=False, write_tcl=False, padding=(0, 0, 0, 0), attribute_name='OBJECTID', attribute_ids=None, write_tifs=False, manifest_file=conus_manifest): """subset a conus domain inputs for running a regional model Parameters ---------- input_path : str path to input shapefile parts to use as mask shapefile : str name of shapefile to use as mask conus_version : int, optional version of the CONUS domain to use (1 or 2) (Default value = 1) conus_files : str, optional path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.') out_dir : str, optional directory to write the outputs (default .) out_name : str, optional name to give the outputs (default shapefile name) clip_clm : int, optional whether or not to clip the CLM input files too (default no) write_tcl : int, optional whether or not to write a TCL file for the subset (default no) padding : tuple, optional grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0 attribute_name : str, optional attribute name defined in shapefile to select as mask default 'OBJECTID' attribute_ids : list, optional list of attribute ID's defined in shapefile to use as mask input. default [1] write_tifs : int, optional whether or not to write outputs as TIF's in addition to PFB's. (default no) Returns ------- None """ if out_name is None: out_name = shapefile conus = Conus(version=conus_version, local_path=conus_files, manifest_file=manifest_file) if attribute_ids is None: attribute_ids = [1] # Step 1, rasterize shapefile rasterizer = ShapefileRasterizer( input_path, shapefile, reference_dataset=conus.get_domain_tif(), no_data=-999, output_path=out_dir, ) rasterizer.rasterize_shapefile_to_disk( out_name=f'{out_name}_raster_from_shapefile.tif', padding=padding, attribute_name=attribute_name, attribute_ids=attribute_ids) subset_mask = rasterizer.subset_mask # Step 2, Generate solid file clip = MaskClipper(subset_mask, no_data_threshold=-1) batches = solidfile_generator.make_solid_file( clipped_mask=clip.clipped_mask, out_name=os.path.join(out_dir, out_name)) if len(batches) == 0: raise Exception("Did not make solid file correctly") # Step 3. Clip all the domain data inputs bulk_clipper.clip_inputs(clip, [ os.path.join(conus.local_path, value) for key, value in conus.required_files.items() if key not in ['DOMAIN_MASK', 'CHANNELS'] ], out_dir=out_dir, tif_outs=write_tifs) # Step 4. Clip CLM inputs if clip_clm == 1: clm_clipper = ClmClipper(subset_mask) latlon_formatted, latlon_data = clm_clipper.clip_latlon( os.path.join(conus.local_path, conus.optional_files.get('LAT_LON'))) clm_clipper.write_lat_lon(latlon_formatted, os.path.join(out_dir, f'{out_name}_latlon.sa'), x=latlon_data.shape[2], y=latlon_data.shape[1], z=latlon_data.shape[0]) land_cover_data, vegm_data = clm_clipper.clip_land_cover( lat_lon_array=latlon_formatted, land_cover_file=os.path.join( conus.local_path, conus.optional_files.get('LAND_COVER'))) clm_clipper.write_land_cover( vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat')) # Step 5. Generate TCL File if write_tcl == 1: build_tcl( os.path.join(out_dir, f'{out_name}.tcl'), parkinglot_template, out_name, os.path.join( out_dir, f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb'), os.path.join(out_dir, f'{out_name}.pfsol'), os.path.join(out_dir, 'pme.pfb'), end_time=10, batches=batches, p=2, q=1, r=1, timestep=1, constant=1)