def test_clm_clip_latlon(self):
if os.environ.get('TRAVIS'):
pass
elif os.path.isfile(test_files.conus1_latlon):
bbox_list = test_files.huc10190004.get('conus1_bbox')
bbox = BBox(bbox_list[0], bbox_list[1], bbox_list[2], bbox_list[3])
clm_clipper = ClmClipper(bbox)
latlon_formatted, latlon_data = clm_clipper.clip_latlon(
test_files.conus1_latlon)
clm_clipper.write_lat_lon(latlon_formatted,
'WBDHU8_latlon_test.sa',
x=latlon_data.shape[2],
y=latlon_data.shape[1],
z=latlon_data.shape[0])
self.assertIsNone(
np.testing.assert_array_equal(
file_io_tools.read_file('WBDHU8_latlon_test.sa'),
file_io_tools.read_file(
test_files.huc10190004.get(
'conus1_latlon').as_posix())),
'writing and reading a tif gives back the same array values')
os.remove('WBDHU8_latlon_test.sa')
else:
print(
'WARNING! Unable to run test test_clm_clip_latlon because source file not found. '
'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!'
)
pass
def test_clm_clip_latlon_padded(self):
if os.environ.get('TRAVIS'):
pass
elif os.path.isfile(test_files.conus1_latlon):
my_mask = SubsetMask(test_files.huc10190004.get('conus1_padded_mask'))
bbox = my_mask.get_bbox()
self.assertSequenceEqual((9, 9, 9, 9), bbox.get_padding())
clm_clipper = ClmClipper(bbox)
latlon_formatted, latlon_data = clm_clipper.clip_latlon(test_files.conus1_latlon)
clm_clipper.write_lat_lon(latlon_formatted, 'WBDHU8_latlon_test.sa', x=latlon_data.shape[2],
y=latlon_data.shape[1], z=latlon_data.shape[0])
self.assertSequenceEqual((1, 48, 103), latlon_data.shape, 'check that clm clip when mask has padding works')
os.remove('WBDHU8_latlon_test.sa')
else:
print('WARNING! Unable to run test test_clm_clip_latlon because source file not found. '
'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!')
pass
def test_clm_clip_latlon(self):
if os.environ.get('TRAVIS'):
pass
elif os.path.isfile(test_files.conus1_latlon):
bbox_list = test_files.huc10190004.get('conus1_bbox')
bbox = BBox(bbox_list[0], bbox_list[1], bbox_list[2], bbox_list[3])
clm_clipper = ClmClipper(bbox)
latlon_formatted, latlon_data = clm_clipper.clip_latlon(test_files.conus1_latlon)
self.assertSequenceEqual((1, 30, 85), latlon_data.shape, 'check that clm clip when mask has no padding works')
clm_clipper.write_lat_lon(latlon_formatted, 'WBDHU8_latlon_test.sa', x=latlon_data.shape[2],
y=latlon_data.shape[1], z=latlon_data.shape[0])
self.assertIsNone(np.testing.assert_array_equal(file_io_tools.read_file('WBDHU8_latlon_test.sa'),
file_io_tools.read_file(
test_files.huc10190004.get('conus1_latlon'))),
'clipping conus1 lat/lon values regression test works')
os.remove('WBDHU8_latlon_test.sa')
else:
print('WARNING! Unable to run test test_clm_clip_latlon because source file not found. '
'copy conus1_Grid_Centers_Short_Deg.format.sa into test_inputs/CONUS1_Inputs to enable test!')
pass
def subset_conus(input_path=None,
shapefile=None,
subset_tif=None,
mask_value=1,
conus_version=1,
conus_files='.',
out_dir='.',
out_name=None,
clip_clm=False,
run_script=False,
padding=(0, 0, 0, 0),
attribute_name='OBJECTID',
attribute_ids=None,
write_tifs=False,
manifest_file=conus_manifest):
"""subset a conus domain inputs for running a regional model
Parameters
----------
input_path : str, optional
path to input shapefile parts to use as mask. Only applicable if shapefile is also provided.
shapefile : str, optional
name of shapefile to use as mask. Required if subset_tif is not provided.
subset_tif : str, optional
path to tiff file containing mask. Required if shapefile is not provided.
mask_value : int, optional
integer value specifying the data value in the tiff file to consider as the masking value.
Only applicable if subset_tif is provided.
conus_version : int, optional
version of the CONUS domain to use (1 or 2) (Default value = 1)
conus_files : str, optional
path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.')
out_dir : str, optional
directory to write the outputs (default .)
out_name : str, optional
name to give the outputs (default shapefile name)
clip_clm : int, optional
whether or not to clip the CLM input files too (default no)
run_script : int, optional
whether or not to build and return a Run object for the subset (default no)
padding : tuple, optional
grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0
attribute_name : str, optional
attribute name defined in shapefile to select as mask default 'OBJECTID'
attribute_ids : list, optional
list of attribute ID's defined in shapefile to use as mask input. default [1]
write_tifs : int, optional
whether or not to write outputs as TIF's in addition to PFB's. (default no)
Returns
-------
run_script : parflow.tools.Run
The Run object which can be used to execute the ParFlow model subset that was created by subset_conus
"""
assert any((shapefile, subset_tif)) and not all((shapefile, subset_tif)), \
'Specify either a shapefile or a subset_tif file.'
if out_name is None:
out_name = shapefile or os.path.splitext(
os.path.basename(subset_tif))[0]
conus = Conus(version=conus_version,
local_path=conus_files,
manifest_file=manifest_file)
if attribute_ids is None:
attribute_ids = [1]
# Step 1, rasterize shapefile
if shapefile is not None:
rasterizer = ShapefileRasterizer(
input_path,
shapefile,
reference_dataset=conus.get_domain_tif(),
no_data=TIF_NO_DATA_VALUE_OUT,
output_path=out_dir,
)
mask_array = rasterizer.rasterize_shapefile_to_disk(
out_name=f'{out_name}_raster_from_shapefile.tif',
padding=padding,
attribute_name=attribute_name,
attribute_ids=attribute_ids)
subset_mask = rasterizer.subset_mask
else:
subset_mask = SubsetMask(tif_file=subset_tif, mask_value=mask_value)
mask_array = subset_mask.mask_array
# Step 2, Generate solid file
clip = MaskClipper(subset_mask, no_data_threshold=-1)
sfb = SolidFileBuilder(top=3, bottom=6, side=1).mask(
clip.subset(mask_array, crop_inner=0)[0][0, :, :])
# identify the unique patch ID's assigned to the solid file
#TODO: get patch defs from a class
sfb.top_ids(clip.subset(conus.get_domain_mask())[0][0, :, :])
sfb.side_ids(clip.subset(conus.get_border_mask())[0][0, :, :])
top_patchIDs = np.unique(clip.subset(conus.get_domain_mask())[0][0, :, :])
side_patchIDs = np.unique(clip.subset(conus.get_border_mask())[0][0, :, :])
side_patchIDs[side_patchIDs == 0] = 2
botom_patchIDs = [6]
patch_ids = np.unique(
np.concatenate((top_patchIDs, side_patchIDs, botom_patchIDs)))
sfb = sfb.write(os.path.join(out_dir, f'{out_name}.pfsol'),
cellsize=1000,
vtk=True)
# Step 3. Clip all the domain data inputs
bulk_clipper.clip_inputs(clip, [
os.path.join(conus.local_path, value)
for key, value in conus.required_files.items()
if key not in ['DOMAIN_MASK', 'CHANNELS']
],
out_dir=out_dir,
tif_outs=write_tifs)
# Step 4. Clip CLM inputs
if clip_clm == 1:
clm_clipper = ClmClipper(subset_mask.get_bbox())
latlon_formatted, latlon_data = clm_clipper.clip_latlon(
os.path.join(conus.local_path,
conus.optional_files.get('LAT_LON')))
clm_clipper.write_lat_lon(latlon_formatted,
os.path.join(out_dir,
f'{out_name}_latlon.sa'),
x=latlon_data.shape[2],
y=latlon_data.shape[1],
z=latlon_data.shape[0])
land_cover_data, vegm_data = clm_clipper.clip_land_cover(
lat_lon_array=latlon_formatted,
land_cover_file=os.path.join(
conus.local_path, conus.optional_files.get('LAND_COVER')))
clm_clipper.write_land_cover(
vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat'))
# Step 5. Generate Run Script
if run_script == 1:
# the Run object reads sys.argv, and this is problematic because they share a common flag -r
sys.argv = ['Run']
slopex_file = os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb')
slopey_file = os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_Y")).stem}_clip.pfb')
solid_file = os.path.join(out_dir, f'{out_name}.pfsol')
bbox = subset_mask.get_bbox()
extents = bbox.get_padded_extents()
NX = int(extents[3] - extents[2])
NY = int(extents[1] - extents[0])
out_name = f'{out_name}.conus{conus_version}.parking_lot'
# TODO: associate model templates with models and versions, provide method to override boundary conditions
run_script = parking_lot_template.get_parking_lot_model(
out_name, slopex_file, slopey_file, solid_file, NX, NY)
patch_names = [
conus.get_patch_name(patch_id)
for patch_id in patch_ids[patch_ids > TIF_NO_DATA_VALUE_OUT]
]
run_script.Geom.domain.Patches = ' '.join(patch_names)
# convert patch ID's to patch names for run script
for patch in patch_names:
bc = parking_lot_template.get_parking_lot_model_boundary(patch)
for k, v in bc.items():
# assign patch Boundary Conditions as defined by CONUS Model 1 or 2
run_script.Patch.pfset(key=f'{patch}.BCPressure.{k}', value=v)
if conus_version == 1:
# CONUS1 doesn't seem to work well with OverlandKinematic
run_script.Patch.top.BCPressure.Type = 'OverlandFlow'
run_script.validate()
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='pfidb')
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='yaml')
run_script.write(file_name=os.path.join(out_dir, out_name),
file_format='json')
return run_script
def subset_conus(input_path,
shapefile,
conus_version=1,
conus_files='.',
out_dir='.',
out_name=None,
clip_clm=False,
write_tcl=False,
padding=(0, 0, 0, 0),
attribute_name='OBJECTID',
attribute_ids=None,
write_tifs=False,
manifest_file=conus_manifest):
"""subset a conus domain inputs for running a regional model
Parameters
----------
input_path : str
path to input shapefile parts to use as mask
shapefile : str
name of shapefile to use as mask
conus_version : int, optional
version of the CONUS domain to use (1 or 2) (Default value = 1)
conus_files : str, optional
path to the CONUS source input files listed in conus_manifest.yaml (Default value = '.')
out_dir : str, optional
directory to write the outputs (default .)
out_name : str, optional
name to give the outputs (default shapefile name)
clip_clm : int, optional
whether or not to clip the CLM input files too (default no)
write_tcl : int, optional
whether or not to write a TCL file for the subset (default no)
padding : tuple, optional
grid cells of no_data to add around domain mask. CSS Style (top, right, bottom, left) default 0
attribute_name : str, optional
attribute name defined in shapefile to select as mask default 'OBJECTID'
attribute_ids : list, optional
list of attribute ID's defined in shapefile to use as mask input. default [1]
write_tifs : int, optional
whether or not to write outputs as TIF's in addition to PFB's. (default no)
Returns
-------
None
"""
if out_name is None:
out_name = shapefile
conus = Conus(version=conus_version,
local_path=conus_files,
manifest_file=manifest_file)
if attribute_ids is None:
attribute_ids = [1]
# Step 1, rasterize shapefile
rasterizer = ShapefileRasterizer(
input_path,
shapefile,
reference_dataset=conus.get_domain_tif(),
no_data=-999,
output_path=out_dir,
)
rasterizer.rasterize_shapefile_to_disk(
out_name=f'{out_name}_raster_from_shapefile.tif',
padding=padding,
attribute_name=attribute_name,
attribute_ids=attribute_ids)
subset_mask = rasterizer.subset_mask
# Step 2, Generate solid file
clip = MaskClipper(subset_mask, no_data_threshold=-1)
batches = solidfile_generator.make_solid_file(
clipped_mask=clip.clipped_mask,
out_name=os.path.join(out_dir, out_name))
if len(batches) == 0:
raise Exception("Did not make solid file correctly")
# Step 3. Clip all the domain data inputs
bulk_clipper.clip_inputs(clip, [
os.path.join(conus.local_path, value)
for key, value in conus.required_files.items()
if key not in ['DOMAIN_MASK', 'CHANNELS']
],
out_dir=out_dir,
tif_outs=write_tifs)
# Step 4. Clip CLM inputs
if clip_clm == 1:
clm_clipper = ClmClipper(subset_mask)
latlon_formatted, latlon_data = clm_clipper.clip_latlon(
os.path.join(conus.local_path,
conus.optional_files.get('LAT_LON')))
clm_clipper.write_lat_lon(latlon_formatted,
os.path.join(out_dir,
f'{out_name}_latlon.sa'),
x=latlon_data.shape[2],
y=latlon_data.shape[1],
z=latlon_data.shape[0])
land_cover_data, vegm_data = clm_clipper.clip_land_cover(
lat_lon_array=latlon_formatted,
land_cover_file=os.path.join(
conus.local_path, conus.optional_files.get('LAND_COVER')))
clm_clipper.write_land_cover(
vegm_data, os.path.join(out_dir, f'{out_name}_vegm.dat'))
# Step 5. Generate TCL File
if write_tcl == 1:
build_tcl(
os.path.join(out_dir, f'{out_name}.tcl'),
parkinglot_template,
out_name,
os.path.join(
out_dir,
f'{Path(conus.required_files.get("SLOPE_X")).stem}_clip.pfb'),
os.path.join(out_dir, f'{out_name}.pfsol'),
os.path.join(out_dir, 'pme.pfb'),
end_time=10,
batches=batches,
p=2,
q=1,
r=1,
timestep=1,
constant=1)