def test_rasterize_no_data_values(self):
rasterizer = ShapefileRasterizer(self.shape_path, shapefile_name=self.shape_name,
reference_dataset=self.conus1_mask_datset, no_data=-9999999)
rasterizer.reproject_and_mask()
rasterizer.subset_mask.write_mask_to_tif(filename='testout.tif')
self.assertIsNone(
np.testing.assert_array_equal(file_io_tools.read_file(
test_files.huc10190004.get('conus1_mask_-9999999').as_posix()),
file_io_tools.read_file('testout.tif')),
'Should create a mask from CONUS1 with 1/-9999999')
os.remove('testout.tif')
def test_reproject_conus2(self):
rasterizer = ShapefileRasterizer(self.shape_path, self.shape_name, self.conus2_mask_dataset)
rasterizer.reproject_and_mask()
subset_mask = rasterizer.subset_mask
subset_mask.add_bbox_to_mask()
subset_mask.write_mask_to_tif(filename='testout.tif')
self.assertIsNone(
np.testing.assert_array_equal(file_io_tools.read_file(test_files.huc10190004.get('conus2_mask').as_posix()),
file_io_tools.read_file('testout.tif')),
'Should create a mask from CONUS2 with 1/0s')
os.remove('testout.tif')
def rasterize_shape(input_path,
shapefile,
ref_file,
out_dir='.',
out_file=None,
padding=(0, 0, 0, 0),
attribute_name=None,
attribute_ids=None):
""" rasterize a shapefile to disk in the projection and extents of the reference file
Parameters
----------
input_path : str
path to input files (shapefile set)
shapefile : str
name of shapefile dataset
ref_file : str
tif file describing the domain
out_dir : str
directory to write output to (optional) (Default value = '.')
out_file : str
filename to give output (optional) (Default value = None)
padding : tuple
padding (top,right,bottom,left) (optional) (Default value = (0)
attribute_name : str
name of shapefile attribute to select on (optional) (Default value = None)
attribute_ids : list
list of attribute ids in shapefile to select for full_dim_mask (optional) (Default value = None)
Returns
-------
None
"""
reference_dataset = file_io_tools.read_geotiff(ref_file)
rasterizer = ShapefileRasterizer(input_path,
shapefile_name=shapefile,
reference_dataset=reference_dataset,
output_path=out_dir)
rasterizer.rasterize_shapefile_to_disk(out_dir=out_dir,
out_name=out_file,
padding=padding,
attribute_ids=attribute_ids,
attribute_name=attribute_name)
def subset_conus(input_path=None,
shapefile=None,
subset_tif=None,
mask_value=1,
conus_version=1,
conus_files='.',
out_dir='.',
out_name=None,
clip_clm=False,
run_script=False,
padding=(0, 0, 0, 0),
attribute_name='OBJECTID',
attribute_ids=None,
write_tifs=False,
manifest_file=conus_manifest):
"""subset a conus domain inputs for running a regional model
Parameters
----------
input_path : str, optional
path to input shapefile parts to use as mask. Only applicable if shapefile is also provided.
shapefile : str, optional
name of shapefile to use as mask. Required if subset_tif is not provided.
subset_tif : str, optional
path to tiff file containing mask. Required if shapefile is not provided.
mask_value : int, optional
integer value specifying the data value in the tiff file to consider as the masking value.
Only applicable if subset_tif is provided.
conus_version : int, optional
version of the CONUS domain to use (1 or 2) (Default value = 1)
conus_files : str, optional
path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.')
out_dir : str, optional
directory to write the outputs (default .)
out_name : str, optional
name to give the outputs (default shapefile name)
clip_clm : int, optional
whether or not to clip the CLM input files too (default no)
run_script : int, optional
whether or not to build and return a Run object for the subset (default no)
padding : tuple, optional
grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0
attribute_name : str, optional
attribute name defined in shapefile to select as mask default 'OBJECTID'
attribute_ids : list, optional
list of attribute ID's defined in shapefile to use as mask input. default [1]
write_tifs : int, optional
whether or not to write outputs as TIF's in addition to PFB's. (default no)
Returns
-------
run_script : parflow.tools.Run
The Run object which can be used to execute the ParFlow model subset that was created by subset_conus
"""
assert any((shapefile, subset_tif)) and not all((shapefile, subset_tif)), \
'Specify either a shapefile or a subset_tif file.'
if out_name is None:
out_name = shapefile or os.path.splitext(
os.path.basename(subset_tif))[0]
conus = Conus(version=conus_version,
local_path=conus_files,
manifest_file=manifest_file)
if attribute_ids is None:
attribute_ids = [1]
# Step 1, rasterize shapefile
if shapefile is not None:
rasterizer = ShapefileRasterizer(
input_path,
shapefile,
reference_dataset=conus.get_domain_tif(),
no_data=TIF_NO_DATA_VALUE_OUT,
output_path=out_dir,
)
mask_array = rasterizer.rasterize_shapefile_to_disk(
out_name=f'{out_name}_raster_from_shapefile.tif',
padding=padding,
attribute_name=attribute_name,
attribute_ids=attribute_ids)
subset_mask = rasterizer.subset_mask
else:
subset_mask = SubsetMask(tif_file=subset_tif, mask_value=mask_value)
mask_array = subset_mask.mask_array
# Step 2, Generate solid file
clip = MaskClipper(subset_mask, no_data_threshold=-1)
sfb = SolidFileBuilder(top=3, bottom=6, side=1).mask(
clip.subset(mask_array, crop_inner=0)[0][0, :, :])
# identify the unique patch ID's assigned to the solid file
#TODO: get patch defs from a class
sfb.top_ids(clip.subset(conus.get_domain_mask())[0][0, :, :])
sfb.side_ids(clip.subset(conus.get_border_mask())[0][0, :, :])
top_patchIDs = np.unique(clip.subset(conus.get_domain_mask())[0][0, :, :])
side_patchIDs = np.unique(clip.subset(conus.get_border_mask())[0][0, :, :])
side_patchIDs[side_patchIDs == 0] = 2
botom_patchIDs = [6]
patch_ids = np.unique(
np.concatenate((top_patchIDs, side_patchIDs, botom_patchIDs)))
sfb = sfb.write(os.path.join(out_dir, f'{out_name}.pfsol'),
cellsize=1000,
vtk=True)
# Step 3. Clip all the domain data inputs
bulk_clipper.clip_inputs(clip, [
os.path.join(conus.local_path, value)
for key, value in conus.required_files.items()
if key not in ['DOMAIN_MASK', 'CHANNELS']
],
out_dir=out_dir,
tif_outs=write_tifs)
# Step 4. Clip CLM inputs
if clip_clm == 1:
clm_clipper = ClmClipper(subset_mask.get_bbox())
latlon_formatted, latlon_data = clm_clipper.clip_latlon(
os.path.join(conus.local_path,
conus.optional_files.get('LAT_LON')))
clm_clipper.write_lat_lon(latlon_formatted,
os.path.join(out_dir,
f'{out_name}_latlon.sa'),
x=latlon_data.shape[2],
y=latlon_data.shape[1],
z=latlon_data.shape[0])
land_cover_data, vegm_data = clm_clipper.clip_land_cover(
lat_lon_array=latlon_formatted,
land_cover_file=os.path.join(
conus.local_path, conus.optional_files.get('LAND_COVER')))
clm_clipper.write_land_cover(
vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat'))
# Step 5. Generate Run Script
if run_script == 1:
# the Run object reads sys.argv, and this is problematic because they share a common flag -r
sys.argv = ['Run']
slopex_file = os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb')
slopey_file = os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_Y")).stem}_clip.pfb')
solid_file = os.path.join(out_dir, f'{out_name}.pfsol')
bbox = subset_mask.get_bbox()
extents = bbox.get_padded_extents()
NX = int(extents[3] - extents[2])
NY = int(extents[1] - extents[0])
out_name = f'{out_name}.conus{conus_version}.parking_lot'
# TODO: associate model templates with models and versions, provide method to override boundary conditions
run_script = parking_lot_template.get_parking_lot_model(
out_name, slopex_file, slopey_file, solid_file, NX, NY)
patch_names = [
conus.get_patch_name(patch_id)
for patch_id in patch_ids[patch_ids > TIF_NO_DATA_VALUE_OUT]
]
run_script.Geom.domain.Patches = ' '.join(patch_names)
# convert patch ID's to patch names for run script
for patch in patch_names:
bc = parking_lot_template.get_parking_lot_model_boundary(patch)
for k, v in bc.items():
# assign patch Boundary Conditions as defined by CONUS Model 1 or 2
run_script.Patch.pfset(key=f'{patch}.BCPressure.{k}', value=v)
if conus_version == 1:
# CONUS1 doesn't seem to work well with OverlandKinematic
run_script.Patch.top.BCPressure.Type = 'OverlandFlow'
run_script.validate()
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='pfidb')
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='yaml')
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='json')
return run_script
def test_rasterizer_print_no_except(self):
rasterizer = ShapefileRasterizer(self.shape_path, self.shape_name, self.conus2_mask_dataset)
self.assertIsNone(print(rasterizer))
def test_rasterizer_bad_no_data_val0(self):
with self.assertRaises(Exception):
ShapefileRasterizer(self.shape_path, shapefile_name=self.shape_name,
reference_dataset=self.conus1_mask_datset, no_data=0)
def subset_conus(input_path,
shapefile,
conus_version=1,
conus_files='.',
out_dir='.',
out_name=None,
clip_clm=False,
write_tcl=False,
padding=(0, 0, 0, 0),
attribute_name='OBJECTID',
attribute_ids=None,
write_tifs=False,
manifest_file=conus_manifest):
"""subset a conus domain inputs for running a regional model
Parameters
----------
input_path : str
path to input shapefile parts to use as mask
shapefile : str
name of shapefile to use as mask
conus_version : int, optional
version of the CONUS domain to use (1 or 2) (Default value = 1)
conus_files : str, optional
path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.')
out_dir : str, optional
directory to write the outputs (default .)
out_name : str, optional
name to give the outputs (default shapefile name)
clip_clm : int, optional
whether or not to clip the CLM input files too (default no)
write_tcl : int, optional
whether or not to write a TCL file for the subset (default no)
padding : tuple, optional
grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0
attribute_name : str, optional
attribute name defined in shapefile to select as mask default 'OBJECTID'
attribute_ids : list, optional
list of attribute ID's defined in shapefile to use as mask input. default [1]
write_tifs : int, optional
whether or not to write outputs as TIF's in addition to PFB's. (default no)
Returns
-------
None
"""
if out_name is None:
out_name = shapefile
conus = Conus(version=conus_version,
local_path=conus_files,
manifest_file=manifest_file)
if attribute_ids is None:
attribute_ids = [1]
# Step 1, rasterize shapefile
rasterizer = ShapefileRasterizer(
input_path,
shapefile,
reference_dataset=conus.get_domain_tif(),
no_data=-999,
output_path=out_dir,
)
rasterizer.rasterize_shapefile_to_disk(
out_name=f'{out_name}_raster_from_shapefile.tif',
padding=padding,
attribute_name=attribute_name,
attribute_ids=attribute_ids)
subset_mask = rasterizer.subset_mask
# Step 2, Generate solid file
clip = MaskClipper(subset_mask, no_data_threshold=-1)
batches = solidfile_generator.make_solid_file(
clipped_mask=clip.clipped_mask,
out_name=os.path.join(out_dir, out_name))
if len(batches) == 0:
raise Exception("Did not make solid file correctly")
# Step 3. Clip all the domain data inputs
bulk_clipper.clip_inputs(clip, [
os.path.join(conus.local_path, value)
for key, value in conus.required_files.items()
if key not in ['DOMAIN_MASK', 'CHANNELS']
],
out_dir=out_dir,
tif_outs=write_tifs)
# Step 4. Clip CLM inputs
if clip_clm == 1:
clm_clipper = ClmClipper(subset_mask)
latlon_formatted, latlon_data = clm_clipper.clip_latlon(
os.path.join(conus.local_path,
conus.optional_files.get('LAT_LON')))
clm_clipper.write_lat_lon(latlon_formatted,
os.path.join(out_dir,
f'{out_name}_latlon.sa'),
x=latlon_data.shape[2],
y=latlon_data.shape[1],
z=latlon_data.shape[0])
land_cover_data, vegm_data = clm_clipper.clip_land_cover(
lat_lon_array=latlon_formatted,
land_cover_file=os.path.join(
conus.local_path, conus.optional_files.get('LAND_COVER')))
clm_clipper.write_land_cover(
vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat'))
# Step 5. Generate TCL File
if write_tcl == 1:
build_tcl(
os.path.join(out_dir, f'{out_name}.tcl'),
parkinglot_template,
out_name,
os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb'),
os.path.join(out_dir, f'{out_name}.pfsol'),
os.path.join(out_dir, 'pme.pfb'),
end_time=10,
batches=batches,
p=2,
q=1,
r=1,
timestep=1,
constant=1)